--HG--

branch : pmacs2

highlight2.py

@@ -61,7 +61,9 @@ class Highlighter:
     # ======================
     def relex(self, lines, y1, x1, y2, x2):
         # start the relexing process
-        self.lexer.lex(lines, y1, 0)
+        #self.lexer.lex(lines, y1, 0)
+        rulecontexts = self.line_contexts[y1]
+        self.lexer.resume(lines, y1, 0, rulecontexts)
 
         # these keep track of the current y coordinate, the current token index
         # on line[y], and the current "new token", respectively.
@@ -108,7 +110,7 @@ class Highlighter:
                 self.tokens[y].insert(i, new_token)
                 i += 1
                 getnext = True
-            elif old_token == new_token:
+            elif '.' not in old_token.name and old_token == new_token:
                 # if they match, then leave the old one alone
                 i += 1
                 getnext = True

lex2.py

@@ -14,6 +14,7 @@ class RuleContext:
         self.y       = y
         self.x       = x
         self.rule    = rule
+        self.flag    = flag
         self.context = context
         self.matchd  = matchd
 
@@ -121,17 +122,32 @@ class RegionRule(Rule):
         t      = self.make_token(lexer, m.group(0), t_name, grammar=grammar)
         lexer.add_token(t)
         lexer.x += len(m.group(0))
-    def restart(self, lexer, rulecontext):
-        pass
-    def match(self, lexer, context=[], d={}):
-        m = self.start_re.match(lexer.lines[lexer.y], lexer.x)
-        # see if we can match our start token
-        if m:
 
-            # ok, so create our start token, and get ready to start reading data
+    def resume(self, lexer, context, flag, d, rulecontexts):
-            d = m.groupdict()
+        assert rulecontexts, "can't resume without rulecontexts!"
+        self._match(lexer, context, d, None, rulecontexts)
+        return True
+
+    def match(self, lexer, context=[], d={}):
+        # see if we can match our start token
+        m = self.start_re.match(lexer.lines[lexer.y], lexer.x)
+        if m:
+            # region was match, so let's do this
+            return self._match(lexer, context, m.groupdict(), m, [])
+        else:
+            # region was not matched; we never started. so return false
+            return False
+
+    def _match(self, lexer, context, d, m, rulecontext=[]):
+        # if we have been given rulecontext, then we are going to "resume" a
+        # parse that can already be assumed to have started
+        reenter = len(rulecontext) > 0
+        assert m or reenter
+
+        # first let's do some bookkeeping
         lexer.context.append(RuleContext(lexer.y, lexer.x, self, 'start',
                                          list(context), dict(d)))
+        if m is not None:
             self._add_from_regex(context, 'start', lexer, m, lexer.grammar)
         null_t_name = '.'.join(context + [self.name, 'null'])
         null_t = None
@@ -147,9 +163,10 @@ class RegionRule(Rule):
         done = False
         while not done and lexer.y < len(lexer.lines):
             old_y = lexer.y
+
             # if this line is empty, then we skip it, but here we insert
             # an empty null token just so we have something
-                if len(lexer.lines[lexer.y]) == 0:
+            if not reenter and len(lexer.lines[lexer.y]) == 0:
                 null_t = Token(null_t_name, None, lexer.y, lexer.x, '')                    
                 lexer.add_token(null_t)
                 null_t = None
@@ -157,6 +174,18 @@ class RegionRule(Rule):
             # ok, as long as we haven't found the end token, and have more
             # data on the current line to read, we will process tokens
             while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]):
+                # if we are reentering mid-parse, then that takes precedence
+                if reenter:
+                    reenter = False
+                    rule2 = rulecontext[0].rule
+                    context2 = rulecontext[0].context
+                    d2 = rulecontext[0].matchd
+                    assert rule2.resume(lexer, context2, d2, rulecontext[1:]), \
+                        "%r %r %r %r" % (lexer, context2, d2, rulecontext[1:])
+                    found  = True
+                    null_t = None
+                    break
+
                 # if we are looking for an end token, then see if we've
                 # found it. if so, then we are done!
                 if self.end:
@@ -201,9 +230,6 @@ class RegionRule(Rule):
         # alright, we're finally done procesing the region, so return true
         lexer.context.pop(-1)
         return True
-        else:
-            # region was not matched; we never started. so return false
-            return False
 
 class DualRegionRule(Rule):
     def __init__(self, name, start, grammar1, middle, grammar2, end):
@@ -221,27 +247,58 @@ class DualRegionRule(Rule):
         t      = self.make_token(lexer, m.group(0), t_name, grammar=grammar)
         lexer.add_token(t)
         lexer.x += len(m.group(0))
+
+    def resume(self, lexer, context, flag, d, rulecontexts):
+        if flag == 'start':
+            d2 = self._match_first(lexer, context, d, None, rulecontexts)
+            d3 = dict(d.items() + d2.items())
+            self._match_second(lexer, context, d3, None, rulecontexts)
+            return True
+        elif flag == 'middle':
+            self._match_second(lexer, context, flag, d, None, rulecontexts)
+            return True
+        else:
+            raise Exception, "invalid flag %r" % flag
+
     def match(self, lexer, context=[], d={}):
-        m1 = self.start_re.match(lexer.lines[lexer.y], lexer.x)
+        # see if we can match our start token
-        # see if we can match out start token
+        m = self.start_re.match(lexer.lines[lexer.y], lexer.x)
-        if m1:
+        if m:
+            # region was match, so let's do this
+            d1 = m.groupdict()
+            d2 = self._match_first(lexer, context, d1, m, [])
+            d3 = dict(d1.items() + d2.items())
+            self._match_second(lexer, context, d3, None, [])
+            return True
+        else:
+            # region was not matched; we never started. so return false
+            return False
+
+    def _match_first(self, lexer, context, d1, m1, rulecontext=[]):
+        # if we have been given rulecontext, then we are going to "resume" a
+        # parse that can already be assumed to have started
+        reenter = len(rulecontext) > 0
+        assert m1 or reenter
+
+        # first let's do some bookkeeping
+        lexer.context.append(RuleContext(lexer.y, lexer.x, self, 'start',
+                                         list(context), dict(d1)))
+
         # ok, so create our start token, and get ready to start reading data
+        if m1 is not None:
             self._add_from_regex(context, 'start', lexer, m1, lexer.grammar)
         null_t_name = '.'.join(context + [self.name, 'null'])
         null_t = None
 
-            d1 = m1.groupdict()
-            lexer.context.append(RuleContext(lexer.y, lexer.x, self, 'start',
-                                             list(context), dict(d1)))
-            d2 = {}
-
         middle_re = re.compile(self.middle % d1)
+        d2 = {}
 
         # ok, so as long as we aren't done (we haven't found an end token),
         # keep reading input
         done = False
         while not done and lexer.y < len(lexer.lines):
             old_y = lexer.y
+
             # if this line is empty, then we will skip it, but here weinsert
             # an empty null token just so we have something
             if len(lexer.lines[lexer.y]) == 0:
@@ -252,6 +309,18 @@ class DualRegionRule(Rule):
             # ok, as long as we haven't found the end token, and have more
             # data on the current line to read, we will process tokens
             while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]):
+                # if we are reentering mid-parse, then that takes precedence
+                if reenter:
+                    reenter  = False
+                    xrule    = rulecontext[0].rule
+                    xcontext = rulecontext[0].context
+                    xd       = rulecontext[0].matchd
+                    assert rule2.resume(lexer, xcontext, xd, rulecontext[1:]), \
+                        "%r %r %r %r" % (lexer, xcontext, xd, rulecontext[1:])
+                    found  = True
+                    null_t = None
+                    break
+
                 # see if we have found the middle token. if so, we can then
                 # proceed to "stage 2"
                 m2 = middle_re.match(lexer.lines[lexer.y], lexer.x)
@@ -292,12 +361,19 @@ class DualRegionRule(Rule):
                 lexer.save_context()
                 lexer.y += 1
                 lexer.x = 0
+        lexer.context.pop(-1)
+        return d2
+
+    def _match_second(self, lexer, context, d3, m, rulecontext=[]):
+        # if we have been given rulecontext, then we are going to "resume" a
+        # parse that can already be assumed to have started
+        reenter = len(rulecontext) > 0
 
         # ok stage 2 is like stage 1, only we are looking for end tokens
         # instead of middle tokens
-            d3 = dict(d1.items() + d2.items())
+        null_t_name = '.'.join(context + [self.name, 'null'])
+        null_t = None
         end_re = re.compile(self.end % d3)
-            lexer.context.pop(-1)
         lexer.context.append(RuleContext(lexer.y, lexer.x, self, 'middle',
                                          list(context), dict(d3)))
 
@@ -306,6 +382,19 @@ class DualRegionRule(Rule):
         done = False
         while not done and lexer.y < len(lexer.lines):
             old_y = lexer.y
+
+            # if we are reentering mid-parse, then that takes precedence
+            if reenter:
+                reenter  = False
+                xrule    = rulecontext[0].rule
+                xcontext = rulecontext[0].context
+                xd       = rulecontext[0].matchd
+                assert rule2.resume(lexer, xcontext, xd, rulecontext[1:]), \
+                    "%r %r %r %r" % (lexer, xcontext, xd, rulecontext[1:])
+                found  = True
+                null_t = None
+                break
+
             # if this line is empty, then we will skip it, but here weinsert
             # an empty null token just so we have something
             if len(lexer.lines[lexer.y]) == 0:
@@ -359,9 +448,147 @@ class DualRegionRule(Rule):
         # alright, we're finally done processing; return true
         lexer.context.pop(-1)
         return True
-        else:
+
-            # dual region was not matched; we never started. so return false
+#    def matchOLD(self, lexer, context=[], d={}):
-            return False
+#        m1 = self.start_re.match(lexer.lines[lexer.y], lexer.x)
+#        # see if we can match out start token
+#        if m1:
+#            # ok, so create our start token, and get ready to start reading data
+#            self._add_from_regex(context, 'start', lexer, m1, lexer.grammar)
+#            null_t_name = '.'.join(context + [self.name, 'null'])
+#            null_t = None
+#
+#            d1 = m1.groupdict()
+#            lexer.context.append(RuleContext(lexer.y, lexer.x, self, 'start',
+#                                             list(context), dict(d1)))
+#            d2 = {}
+#            middle_re = re.compile(self.middle % d1)
+#
+#            # ok, so as long as we aren't done (we haven't found an end token),
+#            # keep reading input
+#            done = False
+#            while not done and lexer.y < len(lexer.lines):
+#                old_y = lexer.y
+#                # if this line is empty, then we will skip it, but here weinsert
+#                # an empty null token just so we have something
+#                if len(lexer.lines[lexer.y]) == 0:
+#                    null_t = Token(null_t_name, None, lexer.y, lexer.x, '')                    
+#                    lexer.add_token(null_t)
+#                    null_t = None
+#
+#                # ok, as long as we haven't found the end token, and have more
+#                # data on the current line to read, we will process tokens
+#                while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]):
+#                    # see if we have found the middle token. if so, we can then
+#                    # proceed to "stage 2"
+#                    m2 = middle_re.match(lexer.lines[lexer.y], lexer.x)
+#                    if m2:
+#                        d2 = m2.groupdict()
+#                        self._add_from_regex(context, 'middle', lexer, m2, None)
+#                        done = True
+#                        break
+#
+#                    # ok, we need to check all our rules now, in order. if we
+#                    # find a token, note that we found one and exit the loop
+#                    found = False
+#                    for rule in self.grammar1.rules:
+#                        if rule.match(lexer, context + [self.name], d1):
+#                            found = True
+#                            null_t = None
+#                            break
+#
+#                    # if we never found a token, then we need to add another
+#                    # character to the current null token (which we should
+#                    # create if it isn't set).
+#                    if not found:
+#                        if null_t is None:
+#                            null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
+#                            lexer.add_token(null_t)
+#                        null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
+#                        lexer.x += 1
+#
+#                # ok, since we're soon going to be on a different line (or
+#                # already are), we want a new null token. so forget about the
+#                # current one.
+#                null_t = None
+#
+#                # if we're still on the same line at this point (and not done)
+#                # then that means we're finished with the line and should move
+#                # on to the next one here
+#                if not done and old_y == lexer.y:
+#                    lexer.save_context()
+#                    lexer.y += 1
+#                    lexer.x = 0
+#
+#            # ok stage 2 is like stage 1, only we are looking for end tokens
+#            # instead of middle tokens
+#            d3 = dict(d1.items() + d2.items())
+#            end_re = re.compile(self.end % d3)
+#            lexer.context.pop(-1)
+#            lexer.context.append(RuleContext(lexer.y, lexer.x, self, 'middle',
+#                                             list(context), dict(d3)))
+#
+#            # ok, so as long as we aren't done (we haven't found an end token),
+#            # keep reading input
+#            done = False
+#            while not done and lexer.y < len(lexer.lines):
+#                old_y = lexer.y
+#                # if this line is empty, then we will skip it, but here weinsert
+#                # an empty null token just so we have something
+#                if len(lexer.lines[lexer.y]) == 0:
+#                    null_t = Token(null_t_name, None, lexer.y, lexer.x, '')                    
+#                    lexer.add_token(null_t)
+#                    null_t = None
+#
+#                # ok, as long as we haven't found the end token, and have more
+#                # data on the current line to read, we will process tokens
+#                while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]):
+#                    # see if we have found the middle token. if so, we can then
+#                    # proceed to "stage 2"
+#                    m3 = end_re.match(lexer.lines[lexer.y], lexer.x)
+#                    if m3:
+#                        self._add_from_regex(context, 'end', lexer, m3, None)
+#                        done = True
+#                        break
+#
+#                    # ok, we need to check all our rules now, in order. if we
+#                    # find a token, note that we found one and exit the loop
+#                    found = False
+#                    for rule in self.grammar2.rules:
+#                        if rule.match(lexer, context + [self.name], d3):
+#                            found = True
+#                            null_t = None
+#                            break
+#
+#                    # if we never found a token, then we need to add another
+#                    # character to the current null token (which we should
+#                    # create if it isn't set).
+#                    if not found:
+#                        if null_t is None:
+#                            null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
+#                            lexer.add_token(null_t)
+#                        null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
+#                        lexer.x += 1
+#
+#                # ok, since we're soon going to be on a different line (or
+#                # already are), we want a new null token. so forget about the
+#                # current one.
+#                null_t = None
+#
+#                # if we're still on the same line at this point (and not done)
+#                # then that means we're finished with the line and should move
+#                # on to the next one here
+#                if not done and old_y == lexer.y:
+#                    lexer.save_context()
+#                    lexer.y += 1
+#                    lexer.x = 0
+#
+#            # alright, we're finally done processing; return true
+#            lexer.context.pop(-1)
+#            return True
+#        else:
+#            # dual region was not matched; we never started. so return false
+#            return False
 
 class Grammar:
     rules = []
@@ -394,6 +621,17 @@ class Lexer:
         self.context       = []
         self.line_contexts = {}
 
+    def resume(self, lines, y=0, x=0, rulecontexts=[]):
+        if len(rulecontexts) == 0:
+            self.lex(lines, y, x)
+        else:
+            self.y = y
+            self.x = x
+            self.lines = lines
+            self.tokens = []
+            rc = rulecontexts[0]
+            rc.rule.resume(self, rc.context, rc.flag, rc.matchd, rulecontexts[1:])
+
     def __iter__(self):
         if self.lines is None:
             raise Exception, "no lines to lex"
@@ -406,6 +644,9 @@ class Lexer:
         null_t_name = 'null'
         null_t      = None
 
+        if self.tokens:
+            return self.tokens.pop(0)
+
         while self.y < len(self.lines):
             line = self.lines[self.y]
             while self.x < len(line):