From 0806e467fe5c0cb9304a31297e1ceaac8b0a9a2e Mon Sep 17 00:00:00 2001 From: Erik Osheim Date: Wed, 5 Dec 2012 14:33:51 -0500 Subject: [PATCH] add ctags/improve tags support --HG-- branch : pmacs2 --- ctags.py | 127 +++++++++++++++++++++++++++++++++++++++++++++++ etags.py | 40 +++++++-------- method/git.py | 2 +- method/tags.py | 8 +-- mode/markdown.py | 1 + mode/scala.py | 46 ++++++++++++++--- tab.py | 11 +++- 7 files changed, 198 insertions(+), 37 deletions(-) create mode 100755 ctags.py diff --git a/ctags.py b/ctags.py new file mode 100755 index 0000000..dd1ea69 --- /dev/null +++ b/ctags.py @@ -0,0 +1,127 @@ +import os +import re +from stat import ST_MTIME +from subprocess import Popen, PIPE, STDOUT + +class CtagsRunError(Exception): pass +class CtagsParseError(Exception): pass + +# ctags --languages=... -R -excmd=number + +class TagManager(object): + lang = None + prune = set(['SCCS', 'RCS', 'CVS', '.svn', '.hg', '.git', '.bzr']) + exts = set() + + def __init__(self, base='.'): + self.db = None + self.base = base + self.path = os.path.join(self.base, 'tags') + self.update() + + def has(self, name): + return name in self.db.tags + + def get(self, name): + return self.db.tags.get(name, []) + + def update(self): + if self.is_outdated(): + self.generate() + self.db = Ctags(self.path) + self.db.parse() + + def generate(self): + args = ['ctags', '-f', self.path, '-L-', '--excmd=number'] + if self.lang: + args.append('--language-force=%s' % self.lang) + + pipe = Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT) + indata = '\n'.join(self.get_paths()) + '\n' + outdata = pipe.communicate(indata) + if pipe.returncode != 0: + raise CtagsRunError("%r: %r" % (args, outdata)) + + def get_paths(self): + return list(self._walk()) + + def is_outdated(self): + if not os.path.exists(self.path): + return True + mtime = os.stat(self.path)[ST_MTIME] + itr = self._walk(mtime) + try: + itr.next() + return True + except StopIteration: + return False + + def _walk(self, mtime=0): + paths = [] + for root, dirs, files in os.walk(self.base): + for d in dirs: + if d in self.prune: + dirs.remove(d) + for f in files: + path = os.path.join(root, f) + if not self._match(path): + continue + elif os.stat(path)[ST_MTIME] < mtime: + continue + else: + yield os.path.join(root, f) + raise StopIteration + + def _match(self, path): + return os.path.splitext(path)[1] in self.exts + +class Ctags(object): + line_re = re.compile(r'^(.+?)\t(.+?)\t(.+?);"(.+)$') + + def __init__(self, path): + self.path = path + self.records = [] + self.tags = {} + + def get(self, tag): + return self.tags.get(tag, []) + + def parse(self): + for line in open(self.path, 'r'): + if line.startswith('!'): continue + + m = self.line_re.match(line) + if not m: raise CtagsParseError("%r" % line) + + name, path, ex, _ = m.groups() + self.add(CtagRecord(name, path, ex)) + + def add(self, r): + self.records.append(r) + self.tags.setdefault(r.name, []) + self.tags[r.name].append(r) + +class CtagRecord(object): + def __init__(self, name, path, ex): + if not ex.isdigit(): + raise Exception("excmd %r unsupported; use --excmd=number" % ex) + self.name = name + self.path = path + self.line = int(ex) + + def __repr__(self): + return "CtagRecord(%r, %r, %d)" % (self.name, self.path, self.line) + +if __name__ == '__main__': + import sys + from pprint import pprint + + ctags = Ctags(sys.argv[1]) + ctags.parse() + + if len(sys.argv) > 2: + print "records for %r:" % sys.argv[2] + for r in ctags.get(sys.argv[2]): + print r + else: + pprint(ctags.records) diff --git a/etags.py b/etags.py index e9e2d4b..bd9a09c 100644 --- a/etags.py +++ b/etags.py @@ -9,35 +9,33 @@ import re from stat import ST_MTIME from subprocess import Popen, PIPE, STDOUT - class EtagsRunError(Exception): pass - class TagManager(object): lang = None prune = set(['SCCS', 'RCS', 'CVS', '.svn', '.hg', '.git', '.bzr']) exts = set() def __init__(self, base='.'): - self.etags = None + self.db = None self.base = base self.path = os.path.join(self.base, 'TAGS') self.update() def has(self, name): - return name in self.etags.tag_map + return name in self.db.tags def get(self, name): - return self.etags.tag_map.get(name, []) + return self.db.get(name, []) def list(self): - return self.etags.record_list + return self.db.records def update(self): if self.is_outdated(): self.run() - self.etags = Etags(self.path) - self.etags.parse() + self.db = Etags(self.path) + self.db.parse() def run(self): lf = '--language-force=%s' % self.lang @@ -86,19 +84,16 @@ class Etags(object): def __init__(self, fname=None): self.fname = fname self.rawdata = None - self.record_list = [] - self.tag_map = {} + self.records = [] + self.tags = {} def _load(self): fd = file(self.fname, 'r') self.rawdata = fd.read() fd.close() - def lookup(self, tag): - return self.tag_map[tag] - - def __getitem__(self, tag): - return self.lookup(tag) + def get(self, name): + return self.tags.get(name, []) def parse(self, fname=None): """ @@ -116,21 +111,20 @@ class Etags(object): i = self._parse_block(data, i+2) def _add_record(self, record): - self.record_list.append(record) + self.records.append(record) name = record.name if name is None: return - self.tag_map.setdefault(name, []) - self.tag_map[name].append(record) + self.tags.setdefault(name, []) + self.tags[name].append(record) def _parse_block(self, data, i): n = data[i:].find('\n') + i l = data[i:n] try: filename, size = l.split(',') - except ValueError: - print i - raise + except ValueError, e: + raise Exception("parse failed(%s): %r %r %r" % (i, l, e, data)) size = int(size) subblock = data[n+1:n+size+1] @@ -185,6 +179,6 @@ if __name__ == '__main__': etags.parse() if len(sys.argv) > 2: - print etags[sys.argv[2]] + print etags.get(sys.argv[2]) else: - pprint(etags.record_list) + pprint(etags.records) diff --git a/method/git.py b/method/git.py index ac6e861..0b42a3e 100644 --- a/method/git.py +++ b/method/git.py @@ -177,7 +177,7 @@ class GitBlame(VcBlame): # rev, user, date, [time], [timezone], [date-str], content num_fields = 3 #line_re = re.compile(r'^\^*([0-9a-f]+) \(([a-zA-Z0-9_ ]+|Not Committed Yet) +([-0-9]+) [:0-9]+ +[-\+]\d{4} +\d+\) (.*)\n$') - line_re = re.compile(r'^\^*([0-9a-f]+) +(.+) +\(([a-zA-Z0-9_ ]+|Not Committed Yet) +([-0-9]+) [:0-9]+ +[-\+]\d{4} +\d+\) (.*)\n$') + line_re = re.compile(r'^\^*([0-9a-f]+) +(.*)\((.+) +([-0-9]+) [:0-9]+ +[-\+]\d{4} +\d+\) (.*)\n$') prefix_fmt = '[g:d:*]%*s [c:d:*]%-*s [b:d:*]%*s[d:d:*]' pretest_err_msg = 'Git is not installed' _is_method = True diff --git a/method/tags.py b/method/tags.py index 9921e6c..c629b21 100644 --- a/method/tags.py +++ b/method/tags.py @@ -31,7 +31,7 @@ class TagBase(Method): if not cwd.endswith('/'): cwd += '/' - tpls = [(r.path.replace(cwd, ''), r.line, r.defn) for r in records] + tpls = [(r.path.replace(cwd, ''), r.line, r.name) for r in records] data = '\n'.join(['%s:%d:%s' % tpl for tpl in tpls]) + '\n' return data @@ -47,7 +47,7 @@ class SearchTags(TagBase): if base is None: return - records = [r for r in m.etags.record_list if query in r.name] + records = [r for r in m.etags.records if query in r.name] if len(records) == 0: w.set_error('no records matched query %r' % query) @@ -94,7 +94,7 @@ class InitTags(Method): m = w.mode.tagcls(base) w.application.state['tags'][base] = m w.buffer.settings[w.mode.name]['tag-base'] = base - ntag, nrec = len(m.etags.tag_map), len(m.etags.record_list) + ntag, nrec = len(m.db.tags), len(m.db.records) w.set_error('%s: loaded %d names (%d records)' % (m.path, ntag, nrec)) def _execute(self, w, **vargs): @@ -112,7 +112,7 @@ class InitTags(Method): m = a.state['tags'][t] if m.is_outdated(): m.update() - ntag, nrec = len(m.etags.tag_map), len(m.etags.record_list) + ntag, nrec = len(m.db.tags), len(m.db.records) fmt = '%s: updated %d names (%d records)' w.set_error(fmt % (m.path, ntag, nrec)) else: diff --git a/mode/markdown.py b/mode/markdown.py index c22cccb..868754c 100644 --- a/mode/markdown.py +++ b/mode/markdown.py @@ -58,6 +58,7 @@ MarkdownGrammar.rules = [ RegionRule('md.bold', r'\*', MarkdownGrammar, r'\*'), RegionRule('md.tt', r'`', MarkdownGrammar, r'`'), + PatternRule('md.escaped', r'\\.'), PatternRule('md.word', r'[a-zA-Z\'"\-]+'), PatternRule('spaces', '\s+'), diff --git a/mode/scala.py b/mode/scala.py index 74f9986..bdd6e46 100644 --- a/mode/scala.py +++ b/mode/scala.py @@ -11,6 +11,8 @@ import urllib2 import os import re from subprocess import Popen, PIPE, STDOUT +#from etags import TagManager +from ctags import TagManager chr1 = '[a-zA-Z_]' chr2 = '[a-zA-Z_0-9]' @@ -22,12 +24,32 @@ NestedCommentGrammar.rules = [ PatternRule('data', r'(?:[^\*]|\*(?!/))+'), ] +class String3Grammar(Grammar): + rules = [ + PatternRule('data', r'[^"]+'), + ] + class StringGrammar(Grammar): rules = [ PatternRule('escaped', r"\\u[0-9A-Fa-f]{4}|\\[0-7]{1,3}|\\[btnfr\"'\\]"), PatternRule('data', r'[^\\"]+'), ] +class IString3Grammar(Grammar): + rules = [ + PatternRule('interp', r'\$[a-zA-Z0-9_]+'), + PatternRule('escaped', r'\$\$'), + PatternRule('data', r'[^$"]+'), + ] + +class IStringGrammar(Grammar): + rules = [ + PatternRule('interp', r'\$[a-zA-Z0-9_]+'), + PatternRule('escaped', r'\$\$'), + PatternRule('escaped', r"\\u[0-9A-Fa-f]{4}|\\[0-7]{1,3}|\\[btnfr\"'\\]"), + PatternRule('data', r'[^\\$"]+'), + ] + class SubTypeGrammar(Grammar): pass SubTypeGrammar.rules = [ RegionRule('sub', r'\[', SubTypeGrammar, r'\]'), @@ -49,7 +71,7 @@ class ScalaGrammar(Grammar): 'delimiter', 'scala.type'), PatternMatchRule('x', r'(?<=[a-zA-Z0-9_ ])(:)( +)([a-zA-Z0-9_]+)', 'delimiter', 'spaces', 'scala.type'), - PatternMatchRule('x', r'(?<=[^a-zA-Z0-9_])(new|extends|with)( +)([^{}0-9:\[\(\n\t ][^:\[\(\n\t ]*)', + PatternMatchRule('x', r'(?<=[^a-zA-Z0-9_])(new|extends|with)( +)([^{}()0-9:\[\n\t ][^:\[\]{}()\n\t ]*)', 'scala.reserved', 'spaces', 'scala.type'), # class, object, and trait @@ -58,7 +80,9 @@ class ScalaGrammar(Grammar): PatternRule('scala.trait', '(?<=(?