diff --git a/highlight2.py b/highlight2.py index bf66340..4cdb1a6 100644 --- a/highlight2.py +++ b/highlight2.py @@ -44,3 +44,6 @@ class Highlighter: self.lexer.lex(lines, y=0, x=0) for token in self.lexer: self.tokens[token.y].append(token) + + def update(self, lines): + \ No newline at end of file diff --git a/lex2.py b/lex2.py index d43e199..f08a597 100755 --- a/lex2.py +++ b/lex2.py @@ -4,7 +4,7 @@ valid_name_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$') reserved_names = ['start', 'middle', 'end', 'null'] class Token(object): - def __init__(self, name, y, x, s, **vargs): + def __init__(self, name, rule, y, x, s, **vargs): self.name = name self.y = y self.x = x @@ -17,7 +17,7 @@ class Token(object): s = self.string else: s = self.string[:10] + '...' - return "" % (self.name, self.y, self.x, s) + return "" % (self.name, self.rule, self.y, self.x, s) def render(self): return (self,) @@ -26,7 +26,7 @@ class Rule: def match(self, lexer, context=[], d={}): raise Exception, "%s rule cannot match!" % self.name def make_token(self, lexer, s, name, **vargs): - return Token(name, lexer.y, lexer.x, s, **vargs) + return Token(name, self, lexer.y, lexer.x, s, **vargs) class ConstantRule(Rule): def __init__(self, name, constant): @@ -34,10 +34,11 @@ class ConstantRule(Rule): assert name not in reserved_names, "reserved rule name: %r" % name self.name = name self.constant = constant - def match(self, lexer, context=[], d={}): + def match(self, lexer, context=[], d={}, parent=None): if lexer.lines[lexer.y][lexer.x:].startswith(self.constant): name = '.'.join(context + [self.name]) - lexer.add_token(self.make_token(lexer, self.constant, name)) + lexer.add_token(self.make_token(lexer, self.constant, name, + parent=parent)) lexer.x += len(self.constant) return True else: @@ -50,11 +51,12 @@ class PatternRule(Rule): self.name = name self.pattern = pattern self.re = re.compile(pattern) - def match(self, lexer, context=[], d={}): + def match(self, lexer, context=[], d={}, parent=None): m = self.re.match(lexer.lines[lexer.y], lexer.x) if m: name = '.'.join(context + [self.name]) - lexer.add_token(self.make_token(lexer, m.group(0), name)) + lexer.add_token(self.make_token(lexer, m.group(0), name, + parent=parent)) lexer.x += len(m.group(0)) return True else: @@ -68,7 +70,7 @@ class ContextPatternRule(Rule): self.pattern = pattern self.fallback = fallback self.fallback_re = re.compile(fallback) - def match(self, lexer, context=[], d={}): + def match(self, lexer, context=[], d={}, parent=None): try: r = re.compile(self.pattern % d) except KeyError: @@ -76,7 +78,8 @@ class ContextPatternRule(Rule): m = r.match(lexer.lines[lexer.y], lexer.x) if m: name = '.'.join(context + [self.name]) - lexer.add_token(self.make_token(lexer, m.group(0), name)) + lexer.add_token(self.make_token(lexer, m.group(0), name, + parent=parent)) lexer.x += len(m.group(0)) return True else: @@ -91,9 +94,9 @@ class RegionRule(Rule): self.grammar = grammar self.end = end self.start_re = re.compile(start) - def _add_from_regex(self, context, name, lexer, m): + def _add_from_regex(self, context, name, lexer, m, parent=None): t_name = '.'.join(context + [self.name, name]) - t = self.make_token(lexer, m.group(0), t_name) + t = self.make_token(lexer, m.group(0), t_name, parent=parent) lexer.add_token(t) lexer.x += len(m.group(0)) def match(self, lexer, context=[], d={}): @@ -120,7 +123,7 @@ class RegionRule(Rule): # if this line is empty, then we will skip it, but here weinsert # an empty null token just so we have something if len(lexer.lines[lexer.y]) == 0: - null_t = Token(null_t_name, lexer.y, lexer.x, '') + null_t = Token(null_t_name, None, lexer.y, lexer.x, '') lexer.add_token(null_t) null_t = None @@ -150,7 +153,7 @@ class RegionRule(Rule): # create if it isn't set). if not found: if null_t is None: - null_t = Token(null_t_name, lexer.y, lexer.x, '') + null_t = Token(null_t_name, None, lexer.y, lexer.x, '') lexer.add_token(null_t) null_t.add_to_string(lexer.lines[lexer.y][lexer.x]) lexer.x += 1 @@ -184,9 +187,9 @@ class DualRegionRule(Rule): self.grammar2 = grammar2 self.end = end self.start_re = re.compile(start) - def _add_from_regex(self, context, name, lexer, m): + def _add_from_regex(self, context, name, lexer, m, parent=None): t_name = '.'.join(context + [self.name, name]) - t = self.make_token(lexer, m.group(0), t_name) + t = self.make_token(lexer, m.group(0), t_name, parent=parent) lexer.add_token(t) lexer.x += len(m.group(0)) def match(self, lexer, context=[], d={}): @@ -211,7 +214,7 @@ class DualRegionRule(Rule): # if this line is empty, then we will skip it, but here weinsert # an empty null token just so we have something if len(lexer.lines[lexer.y]) == 0: - null_t = Token(null_t_name, lexer.y, lexer.x, '') + null_t = Token(null_t_name, None, lexer.y, lexer.x, '') lexer.add_token(null_t) null_t = None @@ -241,7 +244,7 @@ class DualRegionRule(Rule): # create if it isn't set). if not found: if null_t is None: - null_t = Token(null_t_name, lexer.y, lexer.x, '') + null_t = Token(null_t_name, None, lexer.y, lexer.x, '') lexer.add_token(null_t) null_t.add_to_string(lexer.lines[lexer.y][lexer.x]) lexer.x += 1 @@ -271,7 +274,7 @@ class DualRegionRule(Rule): # if this line is empty, then we will skip it, but here weinsert # an empty null token just so we have something if len(lexer.lines[lexer.y]) == 0: - null_t = Token(null_t_name, lexer.y, lexer.x, '') + null_t = Token(null_t_name, None, lexer.y, lexer.x, '') lexer.add_token(null_t) null_t = None @@ -300,7 +303,7 @@ class DualRegionRule(Rule): # create if it isn't set). if not found: if null_t is None: - null_t = Token(null_t_name, lexer.y, lexer.x, '') + null_t = Token(null_t_name, None, lexer.y, lexer.x, '') lexer.add_token(null_t) null_t.add_to_string(lexer.lines[lexer.y][lexer.x]) lexer.x += 1 @@ -366,7 +369,7 @@ class Lexer: assert self.tokens, "AAAAA %s" % repr(self.tokens) return self.tokens.pop(0) if null_t is None: - null_t = Token(null_t_name, self.y, self.x, '') + null_t = Token(null_t_name, None, self.y, self.x, '') self.add_token(null_t) null_t.add_to_string(line[self.x]) self.x += 1