add ctags/improve tags support

--HG--
branch : pmacs2

ctags.py

@@ -0,0 +1,127 @@
+import os
+import re
+from stat import ST_MTIME
+from subprocess import Popen, PIPE, STDOUT
+
+class CtagsRunError(Exception): pass
+class CtagsParseError(Exception): pass
+
+# ctags --languages=... -R -excmd=number
+
+class TagManager(object):
+    lang  = None
+    prune = set(['SCCS', 'RCS', 'CVS', '.svn', '.hg', '.git', '.bzr'])
+    exts  = set()
+
+    def __init__(self, base='.'):
+        self.db = None
+        self.base  = base
+        self.path  = os.path.join(self.base, 'tags')
+        self.update()
+
+    def has(self, name):
+        return name in self.db.tags
+
+    def get(self, name):
+        return self.db.tags.get(name, [])
+
+    def update(self):
+        if self.is_outdated():
+            self.generate()
+        self.db = Ctags(self.path)
+        self.db.parse()
+
+    def generate(self):
+        args = ['ctags', '-f', self.path, '-L-', '--excmd=number']
+        if self.lang:
+            args.append('--language-force=%s' % self.lang)
+            
+        pipe = Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+        indata = '\n'.join(self.get_paths()) + '\n'
+        outdata = pipe.communicate(indata)
+        if pipe.returncode != 0:
+            raise CtagsRunError("%r: %r" % (args, outdata))
+
+    def get_paths(self):
+        return list(self._walk())
+
+    def is_outdated(self):
+        if not os.path.exists(self.path):
+            return True
+        mtime = os.stat(self.path)[ST_MTIME]
+        itr = self._walk(mtime)
+        try:
+            itr.next()
+            return True
+        except StopIteration:
+            return False
+
+    def _walk(self, mtime=0):
+        paths = []
+        for root, dirs, files in os.walk(self.base):
+            for d in dirs:
+                if d in self.prune:
+                    dirs.remove(d)
+            for f in files:
+                path = os.path.join(root, f)
+                if not self._match(path):
+                    continue
+                elif os.stat(path)[ST_MTIME] < mtime:
+                    continue
+                else:
+                    yield os.path.join(root, f)
+        raise StopIteration
+
+    def _match(self, path):
+        return os.path.splitext(path)[1] in self.exts
+
+class Ctags(object):
+    line_re = re.compile(r'^(.+?)\t(.+?)\t(.+?);"(.+)$')
+
+    def __init__(self, path):
+        self.path = path
+        self.records = []
+        self.tags = {}
+
+    def get(self, tag):
+        return self.tags.get(tag, [])
+
+    def parse(self):
+        for line in open(self.path, 'r'):
+            if line.startswith('!'): continue
+
+            m = self.line_re.match(line)
+            if not m: raise CtagsParseError("%r" % line)
+
+            name, path, ex, _ = m.groups()
+            self.add(CtagRecord(name, path, ex))
+
+    def add(self, r):
+        self.records.append(r)
+        self.tags.setdefault(r.name, [])
+        self.tags[r.name].append(r)
+
+class CtagRecord(object):
+    def __init__(self, name, path, ex):
+        if not ex.isdigit():
+            raise Exception("excmd %r unsupported; use --excmd=number" % ex)
+        self.name = name
+        self.path = path
+        self.line = int(ex)
+
+    def __repr__(self):
+        return "CtagRecord(%r, %r, %d)" % (self.name, self.path, self.line)
+
+if __name__ == '__main__':
+    import sys
+    from pprint import pprint
+
+    ctags = Ctags(sys.argv[1])
+    ctags.parse()
+
+    if len(sys.argv) > 2:
+        print "records for %r:" % sys.argv[2]
+        for r in ctags.get(sys.argv[2]):
+            print r
+    else:
+        pprint(ctags.records)

etags.py

@@ -9,35 +9,33 @@ import re
 from stat import ST_MTIME
 from subprocess import Popen, PIPE, STDOUT
 
-
 class EtagsRunError(Exception): pass
 
-
 class TagManager(object):
     lang  = None
     prune = set(['SCCS', 'RCS', 'CVS', '.svn', '.hg', '.git', '.bzr'])
     exts  = set()
 
     def __init__(self, base='.'):
-        self.etags = None
+        self.db = None
         self.base  = base
         self.path  = os.path.join(self.base, 'TAGS')
         self.update()
 
     def has(self, name):
-        return name in self.etags.tag_map
+        return name in self.db.tags
 
     def get(self, name):
-        return self.etags.tag_map.get(name, [])
+        return self.db.get(name, [])
 
     def list(self):
-        return self.etags.record_list
+        return self.db.records
 
     def update(self):
         if self.is_outdated():
             self.run()
-        self.etags = Etags(self.path)
+        self.db = Etags(self.path)
-        self.etags.parse()
+        self.db.parse()
 
     def run(self):
         lf      = '--language-force=%s' % self.lang
@@ -86,19 +84,16 @@ class Etags(object):
     def __init__(self, fname=None):
         self.fname = fname
         self.rawdata = None
-        self.record_list = []
+        self.records = []
-        self.tag_map = {}
+        self.tags = {}
 
     def _load(self):
         fd = file(self.fname, 'r')
         self.rawdata = fd.read()
         fd.close()
 
-    def lookup(self, tag):
+    def get(self, name):
-        return self.tag_map[tag]
+        return self.tags.get(name, [])
-
-    def __getitem__(self, tag):
-        return self.lookup(tag)
 
     def parse(self, fname=None):
         """
@@ -116,21 +111,20 @@ class Etags(object):
                 i = self._parse_block(data, i+2)
 
     def _add_record(self, record):
-        self.record_list.append(record)
+        self.records.append(record)
         name = record.name
         if name is None:
             return
-        self.tag_map.setdefault(name, [])
+        self.tags.setdefault(name, [])
-        self.tag_map[name].append(record)
+        self.tags[name].append(record)
 
     def _parse_block(self, data, i):
         n = data[i:].find('\n') + i
         l = data[i:n]
         try:
             filename, size = l.split(',')
-        except ValueError:
+        except ValueError, e:
-            print i
+            raise Exception("parse failed(%s): %r %r %r" % (i, l, e, data))
-            raise
         size = int(size)
         subblock = data[n+1:n+size+1]
 
@@ -185,6 +179,6 @@ if __name__ == '__main__':
     etags.parse()
 
     if len(sys.argv) > 2:
-        print etags[sys.argv[2]]
+        print etags.get(sys.argv[2])
     else:
-        pprint(etags.record_list)
+        pprint(etags.records)

method/git.py

@@ -177,7 +177,7 @@ class GitBlame(VcBlame):
     # rev, user, date, [time], [timezone], [date-str], content
     num_fields = 3
     #line_re = re.compile(r'^\^*([0-9a-f]+) \(([a-zA-Z0-9_ ]+|Not Committed Yet) +([-0-9]+) [:0-9]+ +[-\+]\d{4} +\d+\) (.*)\n$')
-    line_re = re.compile(r'^\^*([0-9a-f]+) +(.+) +\(([a-zA-Z0-9_ ]+|Not Committed Yet) +([-0-9]+) [:0-9]+ +[-\+]\d{4} +\d+\) (.*)\n$')
+    line_re = re.compile(r'^\^*([0-9a-f]+) +(.*)\((.+) +([-0-9]+) [:0-9]+ +[-\+]\d{4} +\d+\) (.*)\n$')
     prefix_fmt = '[g:d:*]%*s [c:d:*]%-*s [b:d:*]%*s[d:d:*]'
     pretest_err_msg = 'Git is not installed'
     _is_method = True

method/tags.py

@@ -31,7 +31,7 @@ class TagBase(Method):
         if not cwd.endswith('/'):
             cwd += '/'
 
-        tpls = [(r.path.replace(cwd, ''), r.line, r.defn) for r in records]
+        tpls = [(r.path.replace(cwd, ''), r.line, r.name) for r in records]
         data = '\n'.join(['%s:%d:%s' % tpl for tpl in tpls]) + '\n'
         return data
 
@@ -47,7 +47,7 @@ class SearchTags(TagBase):
         if base is None:
             return
 
-        records = [r for r in m.etags.record_list if query in r.name]
+        records = [r for r in m.etags.records if query in r.name]
 
         if len(records) == 0:
             w.set_error('no records matched query %r' % query)
@@ -94,7 +94,7 @@ class InitTags(Method):
         m = w.mode.tagcls(base)
         w.application.state['tags'][base] = m
         w.buffer.settings[w.mode.name]['tag-base'] = base
-        ntag, nrec = len(m.etags.tag_map), len(m.etags.record_list)
+        ntag, nrec = len(m.db.tags), len(m.db.records)
         w.set_error('%s: loaded %d names (%d records)' % (m.path, ntag, nrec))
         
     def _execute(self, w, **vargs):
@@ -112,7 +112,7 @@ class InitTags(Method):
             m = a.state['tags'][t]
             if m.is_outdated():
                 m.update()
-                ntag, nrec = len(m.etags.tag_map), len(m.etags.record_list)
+                ntag, nrec = len(m.db.tags), len(m.db.records)
                 fmt = '%s: updated %d names (%d records)'
                 w.set_error(fmt % (m.path, ntag, nrec))
             else:

mode/markdown.py

@@ -58,6 +58,7 @@ MarkdownGrammar.rules = [
     RegionRule('md.bold', r'\*', MarkdownGrammar, r'\*'),
     RegionRule('md.tt', r'`', MarkdownGrammar, r'`'),
 
+    PatternRule('md.escaped', r'\\.'),
     PatternRule('md.word', r'[a-zA-Z\'"\-]+'),
 
     PatternRule('spaces', '\s+'),

mode/scala.py

@@ -11,6 +11,8 @@ import urllib2
 import os
 import re
 from subprocess import Popen, PIPE, STDOUT
+#from etags import TagManager
+from ctags import TagManager
 
 chr1 = '[a-zA-Z_]'
 chr2 = '[a-zA-Z_0-9]'
@@ -22,12 +24,32 @@ NestedCommentGrammar.rules = [
     PatternRule('data', r'(?:[^\*]|\*(?!/))+'),
 ]
 
+class String3Grammar(Grammar):
+    rules = [
+        PatternRule('data', r'[^"]+'),
+    ]
+
 class StringGrammar(Grammar):
     rules = [
         PatternRule('escaped', r"\\u[0-9A-Fa-f]{4}|\\[0-7]{1,3}|\\[btnfr\"'\\]"),
         PatternRule('data', r'[^\\"]+'),
     ]
 
+class IString3Grammar(Grammar):
+    rules = [
+        PatternRule('interp', r'\$[a-zA-Z0-9_]+'),
+        PatternRule('escaped', r'\$\$'),
+        PatternRule('data', r'[^$"]+'),
+    ]
+
+class IStringGrammar(Grammar):
+    rules = [
+        PatternRule('interp', r'\$[a-zA-Z0-9_]+'),
+        PatternRule('escaped', r'\$\$'),
+        PatternRule('escaped', r"\\u[0-9A-Fa-f]{4}|\\[0-7]{1,3}|\\[btnfr\"'\\]"),
+        PatternRule('data', r'[^\\$"]+'),
+    ]
+
 class SubTypeGrammar(Grammar): pass
 SubTypeGrammar.rules = [
     RegionRule('sub', r'\[', SubTypeGrammar, r'\]'),
@@ -49,7 +71,7 @@ class ScalaGrammar(Grammar):
                          'delimiter', 'scala.type'),
         PatternMatchRule('x', r'(?<=[a-zA-Z0-9_ ])(:)( +)([a-zA-Z0-9_]+)',
                          'delimiter', 'spaces', 'scala.type'),
-        PatternMatchRule('x', r'(?<=[^a-zA-Z0-9_])(new|extends|with)( +)([^{}0-9:\[\(\n\t ][^:\[\(\n\t ]*)',
+        PatternMatchRule('x', r'(?<=[^a-zA-Z0-9_])(new|extends|with)( +)([^{}()0-9:\[\n\t ][^:\[\]{}()\n\t ]*)',
                          'scala.reserved', 'spaces', 'scala.type'),
 
         # class, object, and trait
@@ -58,7 +80,9 @@ class ScalaGrammar(Grammar):
         PatternRule('scala.trait', '(?<=(?<![a-zA-Z0-9_])trait )[^0-9:\[\](){}\n\t ][^:\[\]{}()\n\t ]*'),
 
         # method names
-        PatternRule('scala.def', '(?<=(?<![a-zA-Z0-9_])def )[^0-9:\[\](){}\n\t ][^:\[\]{}()\n\t ]*'),
+        ###PatternRule('scala.def', '(?<=(?<![a-zA-Z0-9_])def )[^0-9\[\](){}\n\t ][^\[\]{}()\n\t ]*(?=[:\[\{\( \t\n])'),
+        PatternRule('scala.def', '(?<=(?<![a-zA-Z0-9_])def )[a-zA-Z_][a-zA-Z0-9_]*[^a-zA-Z0-9_\[\]{}()\n\t ]*(?=[:\[{( \t\n])'),
+        PatternRule('scala.def', '(?<=(?<![a-zA-Z0-9_])def )[^a-zA-Z0-9_\[\]{}()\n\t ]+(?=[:\[{( \t\n])'),
 
         # package names
         PatternRule('scala.package', '(?<=(?<![a-zA-Z0-9_])package )[a-zA-Z0-9_.]+'),
@@ -75,18 +99,20 @@ class ScalaGrammar(Grammar):
         PatternRule('eol', r'\n'),
 
         # these are some constants that show up a lot
-        PatternRule('scala.pseudo', '(?:true|null|false)(?!%s)' % word),
+        PatternRule('scala.pseudo', '(?:true|null|false)(?!%s)' % chr2),
 
-        PatternRule('scala.reserved', '(?:yield|with|while|var|val|type|true|try|trait|throw|this|super|sealed|return|protected|private|package|override|object|null|new|match|macro|lazy|import|implicit|if|forSome|for|finally|final|false|extends|else|do|def|class|catch|case object|case class|case|abstract)(?!%s)' % word),
+        PatternRule('scala.reserved', '(?:yield|with|while|var|val|type|true|try|trait|throw|this|super|sealed|return|protected|private|package|override|object|null|new|match|macro|lazy|import|implicit|if|forSome|for|finally|final|false|extends|else|do|def|class|catch|case object|case class|case|abstract)(?!%s)' % chr2),
 
-        PatternRule('scala.float', r'[0-9]+\.[0-9]*[Ff]?'), # FIXME
+        PatternRule('scala.float', r'[0-9]+\.[0-9]*(?:[eE][0-9]+)?[FfDd]?'), # FIXME
 
         PatternRule('scala.integer', '(?:0|[1-9])[0-9]*[Ll]?'),
         PatternRule('scala.integer', '0x[0-9A-Fa-f]+[Ll]?'),
         PatternRule('scala.integer', '0[0-7]+[Ll]?'),
 
         PatternRule('scala.char', r"'(?:[^'\\]|\\u[0-9A-Fa-f]{4}|\\[0-7]{1,3}|\\[btnfr\"'\\])'"),
-        RegionRule('scala.string', '"""', Grammar, '"""+'),
+        RegionRule('scala.string', '[a-zA-Z_][a-zA-Z0-9_]*"""', IString3Grammar, '"""+'),
+        RegionRule('scala.string', '[a-zA-Z_][a-zA-Z0-9_]*"', IStringGrammar, '"'),
+        RegionRule('scala.string', '"""', String3Grammar, '"""+'),
         RegionRule('scala.string', '"', StringGrammar, '"'),
         PatternRule('scala.symbol', "'[a-zA-Z_][a-zA-Z0-9_]*"),
 
@@ -332,6 +358,10 @@ class ScalaShowClasspath(ScalaSetClasspath):
         w.set_error(w.application.config.get('scala.cp'))
         w.application.config['scala.cp'].extend(vargs['lib'].split(':'))
 
+class ScalaTagManager(TagManager):
+    lang  = 'scala'
+    exts  = set(['.scala', '.sbt'])
+
 # white is for delimiters, operators, numbers
 default = ('default', 'default')
 
@@ -365,9 +395,10 @@ hi_blue = ('blue225', 'default')
 
 class Scala(Fundamental):
     name        = 'Scala'
-    extensions  = ['.scala']
+    extensions  = ['.scala', '.sbt']
     tabwidth    = 2
     tabbercls   = ScalaTabber
+    tagcls      = ScalaTagManager
     grammar     = ScalaGrammar
     commentc    = '//'
 
@@ -405,6 +436,7 @@ class Scala(Fundamental):
         'scala.def':  hi_blue,
         'scala.type': hi_magenta,
 
+        'scala.string.interp': hi_yellow,
     }
 
     _bindings   = {

tab.py

@@ -413,9 +413,11 @@ class StackTabber2(Tabber):
             if all_closed:
                 self._save_curr_level()
 
+        at_eol = i + start == end
+
         # if we need to end at eof and we're at the EOF and we have a control
         # token, then we need to pop it.
-        if self.end_at_eof and i == end - start and self._match('control'):
+        if self.end_at_eof and at_eol and self._match('control'):
             self.stack.pop()
 
         # if we are in a control test and this closes the test, then we need to
@@ -424,6 +426,11 @@ class StackTabber2(Tabber):
             t.string in self.close_test_tokens.get(t.name, empty):
             self.stack[-1].name = 'control'
 
+        # FIXME: we shouldn't be getting the type from the language
+        if (self.end_at_eof and at_eol and self.stack and
+            self.stack[-1].type_ == 'else'):
+            self._pop_while('continue', 'control', 'pre-control')
+
         # if we don't want implicit continuation, return.
         if self.end_at_eof:
             return
@@ -531,7 +538,7 @@ class StackTabber2(Tabber):
             if i == start:
                 self._save_curr_level()
             self._pop_while('continue');
-            self._append_unless('pre-control', name, self._get_next_level(), y)
+            self._append_unless('pre-control', s, self._get_next_level(), y)
         elif s in self.case_tokens.get(name, empty):
             if top is not None and top.name in ['case', 'case-eol']:
                 self._pop('case', 'case-eol')