some fixes

--HG--
branch : pmacs2

highlight2.py

@@ -44,3 +44,6 @@ class Highlighter:
         self.lexer.lex(lines, y=0, x=0)
         for token in self.lexer:
             self.tokens[token.y].append(token)
+
+    def update(self, lines):
+        

lex2.py

@@ -4,7 +4,7 @@ valid_name_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
 reserved_names = ['start', 'middle', 'end', 'null']
 
 class Token(object):
-    def __init__(self, name, y, x, s, **vargs):
+    def __init__(self, name, rule, y, x, s, **vargs):
         self.name = name
         self.y = y
         self.x = x
@@ -17,7 +17,7 @@ class Token(object):
             s = self.string
         else:
             s = self.string[:10] + '...'
-        return "<Token(%r, %d, %d, %r)>" % (self.name, self.y, self.x, s)
+        return "<Token(%r, %r, %d, %d, %r)>" % (self.name, self.rule, self.y, self.x, s)
     def render(self):
         return (self,)
 
@@ -26,7 +26,7 @@ class Rule:
     def match(self, lexer, context=[], d={}):
         raise Exception, "%s rule cannot match!" % self.name
     def make_token(self, lexer, s, name, **vargs):
-        return Token(name, lexer.y, lexer.x, s, **vargs)
+        return Token(name, self, lexer.y, lexer.x, s, **vargs)
 
 class ConstantRule(Rule):
     def __init__(self, name, constant):
@@ -34,10 +34,11 @@ class ConstantRule(Rule):
         assert name not in reserved_names, "reserved rule name: %r" % name
         self.name     = name
         self.constant = constant
-    def match(self, lexer, context=[], d={}):
+    def match(self, lexer, context=[], d={}, parent=None):
         if lexer.lines[lexer.y][lexer.x:].startswith(self.constant):
             name = '.'.join(context + [self.name])
-            lexer.add_token(self.make_token(lexer, self.constant, name))
+            lexer.add_token(self.make_token(lexer, self.constant, name,
+                                            parent=parent))
             lexer.x += len(self.constant)
             return True
         else:
@@ -50,11 +51,12 @@ class PatternRule(Rule):
         self.name    = name
         self.pattern = pattern
         self.re      = re.compile(pattern)
-    def match(self, lexer, context=[], d={}):
+    def match(self, lexer, context=[], d={}, parent=None):
         m = self.re.match(lexer.lines[lexer.y], lexer.x)
         if m:
             name = '.'.join(context + [self.name])
-            lexer.add_token(self.make_token(lexer, m.group(0), name))
+            lexer.add_token(self.make_token(lexer, m.group(0), name,
+                                            parent=parent))
             lexer.x += len(m.group(0))
             return True
         else:
@@ -68,7 +70,7 @@ class ContextPatternRule(Rule):
         self.pattern = pattern
         self.fallback = fallback
         self.fallback_re = re.compile(fallback)
-    def match(self, lexer, context=[], d={}):
+    def match(self, lexer, context=[], d={}, parent=None):
         try:
             r = re.compile(self.pattern % d)
         except KeyError:
@@ -76,7 +78,8 @@ class ContextPatternRule(Rule):
         m = r.match(lexer.lines[lexer.y], lexer.x)
         if m:
             name = '.'.join(context + [self.name])
-            lexer.add_token(self.make_token(lexer, m.group(0), name))
+            lexer.add_token(self.make_token(lexer, m.group(0), name,
+                                            parent=parent))
             lexer.x += len(m.group(0))
             return True
         else:
@@ -91,9 +94,9 @@ class RegionRule(Rule):
         self.grammar  = grammar
         self.end      = end
         self.start_re = re.compile(start)
-    def _add_from_regex(self, context, name, lexer, m):
+    def _add_from_regex(self, context, name, lexer, m, parent=None):
         t_name = '.'.join(context + [self.name, name])
-        t      = self.make_token(lexer, m.group(0), t_name)
+        t      = self.make_token(lexer, m.group(0), t_name, parent=parent)
         lexer.add_token(t)
         lexer.x += len(m.group(0))
     def match(self, lexer, context=[], d={}):
@@ -120,7 +123,7 @@ class RegionRule(Rule):
                 # if this line is empty, then we will skip it, but here weinsert
                 # an empty null token just so we have something
                 if len(lexer.lines[lexer.y]) == 0:
-                    null_t = Token(null_t_name, lexer.y, lexer.x, '')                    
+                    null_t = Token(null_t_name, None, lexer.y, lexer.x, '')                    
                     lexer.add_token(null_t)
                     null_t = None
 
@@ -150,7 +153,7 @@ class RegionRule(Rule):
                     # create if it isn't set).
                     if not found:
                         if null_t is None:
-                            null_t = Token(null_t_name, lexer.y, lexer.x, '')
+                            null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
                             lexer.add_token(null_t)
                         null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
                         lexer.x += 1
@@ -184,9 +187,9 @@ class DualRegionRule(Rule):
         self.grammar2 = grammar2
         self.end      = end
         self.start_re = re.compile(start)
-    def _add_from_regex(self, context, name, lexer, m):
+    def _add_from_regex(self, context, name, lexer, m, parent=None):
         t_name = '.'.join(context + [self.name, name])
-        t      = self.make_token(lexer, m.group(0), t_name)
+        t      = self.make_token(lexer, m.group(0), t_name, parent=parent)
         lexer.add_token(t)
         lexer.x += len(m.group(0))
     def match(self, lexer, context=[], d={}):
@@ -211,7 +214,7 @@ class DualRegionRule(Rule):
                 # if this line is empty, then we will skip it, but here weinsert
                 # an empty null token just so we have something
                 if len(lexer.lines[lexer.y]) == 0:
-                    null_t = Token(null_t_name, lexer.y, lexer.x, '')                    
+                    null_t = Token(null_t_name, None, lexer.y, lexer.x, '')                    
                     lexer.add_token(null_t)
                     null_t = None
 
@@ -241,7 +244,7 @@ class DualRegionRule(Rule):
                     # create if it isn't set).
                     if not found:
                         if null_t is None:
-                            null_t = Token(null_t_name, lexer.y, lexer.x, '')
+                            null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
                             lexer.add_token(null_t)
                         null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
                         lexer.x += 1
@@ -271,7 +274,7 @@ class DualRegionRule(Rule):
                 # if this line is empty, then we will skip it, but here weinsert
                 # an empty null token just so we have something
                 if len(lexer.lines[lexer.y]) == 0:
-                    null_t = Token(null_t_name, lexer.y, lexer.x, '')                    
+                    null_t = Token(null_t_name, None, lexer.y, lexer.x, '')                    
                     lexer.add_token(null_t)
                     null_t = None
 
@@ -300,7 +303,7 @@ class DualRegionRule(Rule):
                     # create if it isn't set).
                     if not found:
                         if null_t is None:
-                            null_t = Token(null_t_name, lexer.y, lexer.x, '')
+                            null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
                             lexer.add_token(null_t)
                         null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
                         lexer.x += 1
@@ -366,7 +369,7 @@ class Lexer:
                         assert self.tokens, "AAAAA %s" % repr(self.tokens)
                         return self.tokens.pop(0)
                 if null_t is None:
-                    null_t = Token(null_t_name, self.y, self.x, '')
+                    null_t = Token(null_t_name, None, self.y, self.x, '')
                     self.add_token(null_t)
                 null_t.add_to_string(line[self.x])
                 self.x += 1