From 7e23af56e3eac89a77f158a162910124127725b8 Mon Sep 17 00:00:00 2001
From: moculus <none@none>
Date: Sun, 15 Jul 2007 19:07:36 +0000
Subject: [PATCH] major awesomeness for lex3

--HG--
branch : pmacs2
---
 code_examples/Reporting2.pm |   2 +-
 lex2.py                     | 552 ------------------------------------
 lex3.py                     | 290 ++++++++++---------
 mode_blame.py               |  21 +-
 mode_css.py                 |  50 ++--
 mode_javascript.py          |  34 +--
 mode_life.py                |   2 -
 mode_nasm.py                |  34 +--
 mode_perl.py                |  22 +-
 mode_sh.py                  |   1 -
 mode_sql.py                 |   2 +-
 mode_text.py                |  10 +-
 mode_tt.py                  |  16 +-
 regex.py                    |   5 +
 14 files changed, 249 insertions(+), 792 deletions(-)
 delete mode 100755 lex2.py

diff --git a/code_examples/Reporting2.pm b/code_examples/Reporting2.pm
index 15e2730..6bef348 100644
--- a/code_examples/Reporting2.pm
+++ b/code_examples/Reporting2.pm
@@ -4,7 +4,7 @@ my $bar =~ s/foob/blag/g;
 
 sub foo {
     bar()
-    unless 9;
+        unless 9 && 3;
 }
 
 my $foo = {
diff --git a/lex2.py b/lex2.py
deleted file mode 100755
index bbc9a7c..0000000
--- a/lex2.py
+++ /dev/null
@@ -1,552 +0,0 @@
-import re
-import util
-
-valid_name_re  = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
-full_name_re   = re.compile('^([a-zA-Z_]+)([0-9]*)$')
-reserved_names = ['start', 'middle', 'end', 'null']
-
-class Token(object):
-    def __init__(self, name, rule=None, y=0, x=0, s="", parent=None, matchd={}):
-        self.name    = name
-        self.rule    = rule
-        self.y       = y
-        self.x       = x
-        self.string  = s
-        self.parent  = parent
-        self.matchd  = matchd
-    def parents(self):
-        if self.parent is not None:
-            parents = self.parent.parents()
-            parents.append(self.parent)
-            return parents
-        else:
-            return []
-    def domain(self):
-        if self.parent is not None:
-            names = self.parent.domain()
-        else:
-            names = []
-        if self.name != 'middle':
-            names.append(self.rule.name)
-        return names
-    def fqlist(self):
-        if self.parent is not None:
-            names = self.parent.domain()
-        else:
-            names = []
-        if self.name == 'start':
-            names.append(self.rule.name)
-        names.append(self.name)
-        return names
-    def fqname(self):
-        names = self.fqlist()
-        return '.'.join(names)
-    def copy(self):
-        return Token(self.name, self.rule, self.y, self.x, self.string,
-                     self.parent, self.matchd)
-    def add_to_string(self, s):
-        self.string += s
-    def end_x(self):
-        return self.x + len(self.string)
-    def __eq__(self, other):
-        return (self.y == other.y and self.x == other.x
-                and self.name == other.name and self.parent is other.parent and
-                self.string == other.string)
-    def __repr__(self):
-        if len(self.string) < 10:
-            s = self.string
-        else:
-            s = self.string[:10] + '...'
-        fields = (self.fqname(), self.rule, self.y, self.x, s)
-        return "<Token(%r, %r, %d, %d, %r)>" % fields
-
-class Rule:
-    def __init__(self, name):
-        assert valid_name_re.match(name), 'invalid name %r' % name
-        assert name not in reserved_names, "reserved rule name: %r" % name
-        self.name = name
-    def match(self, lexer, parent):
-        raise Exception, "%s rule cannot match!" % self.name
-    def make_token(self, lexer, s, name, parent=None, matchd={}):
-        return Token(name, self, lexer.y, lexer.x, s, parent, matchd)
-    def _get_line(self, lexer):
-        return lexer.lines[lexer.y] + '\n'
-    def _set_group(self, group):
-        if group is None:
-            self.group = self.name
-        else:
-            self.group = group
-
-class ConstantRule(Rule):
-    def __init__(self, name, constant, group=None):
-        Rule.__init__(self, name)
-        self.constant = constant
-        self.length   = len(self.constant)
-        self._set_group(group)
-    def match(self, lexer, parent):
-        line = self._get_line(lexer)
-        if line[lexer.x:].startswith(self.constant):
-            token = self.make_token(lexer, self.constant, self.name, parent)
-            lexer.add_token(token)
-            lexer.x += self.length
-            return True
-        else:
-            return False
-
-class PatternRule(Rule):
-    def __init__(self, name, pattern, group=None):
-        Rule.__init__(self, name)
-        self.pattern = pattern
-        self._compile()
-        self._set_group(group)
-    def _compile(self):
-        self.re = re.compile(self.pattern)
-    def _match(self, lexer, parent, m):
-        s = m.group(0)
-        token = self.make_token(lexer, s, self.name, parent)
-        lexer.add_token(token)
-        lexer.x += len(s)
-    def match(self, lexer, parent):
-        line = self._get_line(lexer)
-        m = self.re.match(line, lexer.x)
-        if m:
-            self._match(lexer, parent, m)
-            return True
-        else:
-            return False
-
-class NocasePatternRule(PatternRule):
-    def _compile(self):
-        self.re = re.compile(self.pattern, re.IGNORECASE)
-
-class ContextPatternRule(PatternRule):
-    def __init__(self, name, pattern, fallback, group=None):
-        Rule.__init__(self, name)
-        self.pattern     = pattern
-        self.fallback    = fallback
-        self.fallback_re = re.compile(fallback)
-        self._set_group(group)
-    def match(self, lexer, parent):
-        try:
-            r = re.compile(self.pattern % parent.matchd)
-        except KeyError:
-            r = self.fallback_re
-        line = self._get_line(lexer)
-        m = r.match(line, lexer.x)
-        if m:
-            self._match(lexer, parent, m)
-            return True
-        else:
-            return False
-
-class RegionRule(Rule):
-    def __init__(self, name, start, grammar, end, group=None):
-        Rule.__init__(self, name)
-        self.start    = start
-        self.grammar  = grammar
-        self.end      = end
-        self.start_re = self._compile_start()
-        self._set_group(group)
-
-    def _compile_start(self):
-        return re.compile(self.start)
-    def _compile_end(self, d):
-        return re.compile(self.end % d)
-
-    def resume(self, lexer, toresume):
-        assert toresume, "can't resume without tokens to resume!"
-        self._match(lexer, None, None, toresume)
-        return True
-
-    def match(self, lexer, parent):
-        line = self._get_line(lexer)
-        m = self.start_re.match(line, lexer.x)
-        if m:
-            self._match(lexer, parent, m, [])
-            return True
-        else:
-            return False
-
-    def _add_from_regex(self, name, lexer, parent, m, matchd={}):
-        s = m.group(0)
-        token = self.make_token(lexer, s, name, parent, matchd)
-        lexer.add_token(token)
-        lexer.x += len(s)
-        return token
-
-    def _match(self, lexer, parent, m, toresume=[]):
-        # we either need a match object, or a token to resume
-        assert m or len(toresume) > 0
-
-        if m:
-            # if we had a match, then it becomes the parent, and we save its
-            # subgroup dict
-            d = m.groupdict()
-            parent = self._add_from_regex('start', lexer, parent, m, d)
-        else:
-            # otherwise, we should be resuming the start token, so let's pull
-            # the relevant info out of the token
-            parent = toresume[0]
-            d = parent.matchd
-            assert parent.name == 'start'
-        null_t = None
-
-        # this determines whether we are still reentering. if len(toresume) == 1
-        # then it means that we have been reentering but will not continue, so
-        # reenter will be false.
-        reenter = len(toresume) > 1
-
-        # if we have an end regex, then build it here. notice that it can
-        # reference named groups from the start token. if we have no end,
-        # well, then, we're never getting out of here alive!
-        if self.end:
-            end_re = self._compile_end(d)
-
-        # ok, so as long as we aren't done (we haven't found an end token),
-        # keep reading input
-        done = False
-        while not done and lexer.y < len(lexer.lines):
-            old_y = lexer.y
-
-            # ok, as long as we haven't found the end token, and have more
-            # data on the current line to read, we will process tokens
-            while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1:
-                # if we are reentering mid-parse, then that takes precedence
-                if reenter:
-                    reenter = False
-                    rule2 = toresume[1].rule
-                    rule2.resume(lexer, toresume[1:])
-                    null_t = None
-                if lexer.y >= len(lexer.lines):
-                    return True
-                elif lexer.x >= len(lexer.lines[lexer.y]) + 1:
-                    lexer.y += 1
-                    lexer.x = 0
-
-                line = self._get_line(lexer)
-
-                # if we are looking for an end token, then see if we've
-                # found it. if so, then we are done!
-                if self.end:
-                    m = end_re.match(line, lexer.x)
-                    if m:
-                        self._add_from_regex('end', lexer, parent, m, {})
-                        done = True
-                        break
-
-                # ok, we need to check all our rules now, in order. if we
-                # find a token, note that we found one and exit the loop
-                found = False
-                for rule in self.grammar.rules:
-                    if rule.match(lexer, parent):
-                        found  = True
-                        null_t = None
-                        break
-
-
-                # if we never found a token, then we need to add another
-                # character to the current null token (which we should
-                # create if it isn't set).
-                if not found:
-                    if null_t is None:
-                        null_t = Token('null', None, lexer.y, lexer.x, '', parent)
-                        lexer.add_token(null_t)
-                    if lexer.x < len(line):
-                        null_t.add_to_string(line[lexer.x])
-                        lexer.x += 1
-
-            # ok, since we're soon going to be on a different line (or
-            # already are), we want a new null token. so forget about the
-            # current one (i.e. stop adding to it).
-            null_t = None
-
-            # if we're still on the same line at this point (and not done)
-            # then that means we're finished with the line and should move
-            # on to the next one here
-            if not done and old_y == lexer.y:
-                lexer.y += 1
-                lexer.x = 0
-
-        return True
-
-class NocaseRegionRule(RegionRule):
-    def _compile_start(self):
-        return re.compile(self.start, re.IGNORECASE)
-    def _compile_end(self, d):
-        return re.compile(self.end % d, re.IGNORECASE)
-
-class DualRegionRule(Rule):
-    def __init__(self, name, start, grammar1, middle, grammar2, end, group=None):
-        Rule.__init__(self, name)
-        self.start    = start
-        self.grammar1 = grammar1
-        self.middle   = middle
-        self.grammar2 = grammar2
-        self.end      = end
-        self.start_re = self._compile_start()
-        self._set_group(group)
-
-    def _compile_start(self):
-        return re.compile(self.start)
-    def _compile_middle(self, d):
-        return re.compile(self.middle % d)
-    def _compile_end(self, d):
-        return re.compile(self.end % d)
-
-    def _add_from_regex(self, name, lexer, parent, m, matchd={}):
-        s = m.group(0)
-        token = self.make_token(lexer, s, name, parent, matchd)
-        lexer.add_token(token)
-        lexer.x += len(s)
-        return token
-    def resume(self, lexer, toresume):
-        assert toresume, "can't resume without tokens to resume!"
-        token = toresume[0]
-        if token.name == 'start':
-            t2 = self._match_first(lexer, token, toresume)
-            if t2 is not None:
-                t3 = self._match_second(lexer, t2, [])
-            return True
-        elif token.name == 'middle':
-            t3 = self._match_second(lexer, token, toresume)
-        else:
-            raise Exception, "invalid flag %r" % flag
-        return True
-    def match(self, lexer, parent):
-        # see if we can match our start token
-        line = self._get_line(lexer)
-        m = self.start_re.match(line, lexer.x)
-        if m:
-            t1 = self._add_from_regex('start', lexer, parent, m, m.groupdict())
-            t2 = self._match_first(lexer, t1, [])
-            if t2 is not None:
-                t3 = self._match_second(lexer, t2, [])
-            return True
-        else:
-            # region was not matched; we never started. so return false
-            return False
-
-    def _match_first(self, lexer, parent, toresume=[]):
-        reenter = len(toresume) > 1
-        if reenter:
-            assert parent is toresume[0]
-        d1 = parent.matchd
-        assert parent.name == 'start'
-        null_t = None
-        middle_re = self._compile_middle(d1)
-        d2 = {}
-
-        # ok, so as long as we aren't done (we haven't found an end token),
-        # keep reading input
-        t2   = None
-        done = False
-        while not done and lexer.y < len(lexer.lines):
-            old_y = lexer.y
-
-            # ok, as long as we haven't found the end token, and have more
-            # data on the current line to read, we will process tokens
-            while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1:
-                # if we are reentering mid-parse, then that takes precedence
-                if reenter:
-                    reenter = False
-                    rule2 = toresume[1].rule
-                    rule2.resume(lexer, toresume[1:])
-                    null_t = None
-
-                line = self._get_line(lexer)
-
-                # see if we have found the middle token. if so, we can then
-                # proceed to "stage 2"
-                m2 = middle_re.match(line, lexer.x)
-                if m2:
-                    d2 = dict(d1.items() + m2.groupdict().items())
-                    t2 = self._add_from_regex('middle', lexer, parent, m2, d2)
-                    done = True
-                    break
-
-                # ok, we need to check all our rules now, in order. if we
-                # find a token, note that we found one and exit the loop
-                found = False
-                for rule in self.grammar1.rules:
-                    if rule.match(lexer, parent):
-                        found = True
-                        null_t = None
-                        break
-
-                # if we never found a token, then we need to add another
-                # character to the current null token (which we should
-                # create if it isn't set).
-                if not found:
-                    if null_t is None:
-                        null_t = Token('null', None, lexer.y, lexer.x, '', parent)
-                        lexer.add_token(null_t)
-                    null_t.add_to_string(line[lexer.x])
-                    lexer.x += 1
-
-            # ok, since we're soon going to be on a different line (or
-            # already are), we want a new null token. so forget about the
-            # current one.
-            null_t = None
-
-            # if we're still on the same line at this point (and not done)
-            # then that means we're finished with the line and should move
-            # on to the next one here
-            if not done and old_y == lexer.y:
-                lexer.y += 1
-                lexer.x = 0
-        return t2
-
-    def _match_second(self, lexer, parent, toresume=[]):
-        reenter = len(toresume) > 1
-        if reenter:
-            assert parent is toresume[0]
-        assert parent.name == 'middle'
-        d3 = parent.matchd
-        null_t = None
-        end_re = self._compile_end(d3)
-
-        # ok, so as long as we aren't done (we haven't found an end token),
-        # keep reading input
-        t3   = None
-        done = False
-        while not done and lexer.y < len(lexer.lines):
-            old_y = lexer.y
-
-            # if we are reentering mid-parse, then that takes precedence
-            if reenter:
-                reenter = False
-                rule2 = toresume[1].rule
-                rule2.resume(lexer, toresume[1:])
-                null_t = None
-
-            # ok, as long as we haven't found the end token, and have more
-            # data on the current line to read, we will process tokens
-            while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1:
-                # see if we have found the middle token. if so, we can then
-                # proceed to "stage 2"
-                line = self._get_line(lexer)
-                m3 = end_re.match(line, lexer.x)
-                if m3:
-                    t3 = self._add_from_regex('end', lexer, parent, m3, {})
-                    done = True
-                    break
-
-                # ok, we need to check all our rules now, in order. if we
-                # find a token, note that we found one and exit the loop
-                found = False
-                for rule in self.grammar2.rules:
-                    if rule.match(lexer, parent):
-                        found = True
-                        null_t = None
-                        break
-
-                # if we never found a token, then we need to add another
-                # character to the current null token (which we should
-                # create if it isn't set).
-                if not found:
-                    if null_t is None:
-                        null_t = Token('null', None, lexer.y, lexer.x, '', parent)
-                        lexer.add_token(null_t)
-                    null_t.add_to_string(line[lexer.x])
-                    lexer.x += 1
-
-            # ok, since we're soon going to be on a different line (or
-            # already are), we want a new null token. so forget about the
-            # current one.
-            null_t = None
-
-            # if we're still on the same line at this point (and not done)
-            # then that means we're finished with the line and should move
-            # on to the next one here
-            if not done and old_y == lexer.y:
-                lexer.y += 1
-                lexer.x = 0
-
-        # alright, we're finally done processing; return true
-        return t3
-
-class Grammar:
-    rules = []
-    def __init__(self):
-        for rule in self.rules:
-            if hasattr(rule, 'grammar') and rule.grammar is None:
-                rule.grammar = self
-            if hasattr(rule, 'grammar1') and rule.grammar is None:
-                rule.grammar = self
-            if hasattr(rule, 'grammar2') and rule.grammar is None:
-                rule.grammar = self
-grammar = Grammar()
-
-class Lexer:
-    def __init__(self, name, grammar):
-        self.name    = name
-        self.grammar = grammar
-        self.y       = 0
-        self.x       = 0
-        self.lines   = None
-        self.tokens  = []
-
-    def add_token(self, t):
-        self.tokens.append(t)
-
-    def lex(self, lines, y=0, x=0):
-        self.y      = y
-        self.x      = x
-        self.lines  = lines
-        self.tokens = []
-
-    def resume(self, lines, y, x, token):
-        self.y      = y
-        self.x      = x
-        self.lines  = lines
-        self.tokens = []
-        toresume = token.parents()
-
-        # this is a special case for the "middle" rule of a dual region rule
-        i = 0
-        while i < len(toresume):
-            if i > 0 and toresume[i].name == 'middle' and toresume[i-1].name == 'start':
-                del toresume[i-1]
-            else:
-                i += 1
-
-        if toresume:
-            toresume[0].rule.resume(self, toresume)
-
-    def __iter__(self):
-        if self.lines is None:
-            raise Exception, "no lines to lex"
-        return self
-
-    def next(self):
-        null_t = None
-        if self.tokens:
-            return self.tokens.pop(0)
-        while self.y < len(self.lines):
-            line = self.lines[self.y] + '\n'
-            while self.x < len(line):
-                curr_t = None
-                for rule in self.grammar.rules:
-                    if rule.match(self, None):
-                        assert self.tokens, "match rendered no tokens?"
-                        return self.tokens.pop(0)
-                if null_t is None:
-                    null_t = Token('null', None, self.y, self.x, '')
-                    self.add_token(null_t)
-                null_t.add_to_string(line[self.x])
-                self.x += 1
-            null_t = None
-            self.y += 1
-            self.x = 0
-        if self.tokens:
-            return self.tokens.pop(0)
-        else:
-            raise StopIteration
-
-class NocaseDualRegionRule(DualRegionRule):
-    def _compile_start(self):
-        return re.compile(self.start, re.IGNORECASE)
-    def _compile_middle(self, d):
-        return re.compile(self.middle % d, re.IGNORECASE)
-    def _compile_end(self, d):
-        return re.compile(self.end % d, re.IGNORECASE)
diff --git a/lex3.py b/lex3.py
index a6cf02c..a0fca3a 100755
--- a/lex3.py
+++ b/lex3.py
@@ -1,19 +1,16 @@
 import re
-import util
-
-valid_name_re  = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
-full_name_re   = re.compile('^([a-zA-Z_]+)([0-9]*)$')
-reserved_names = ['start', 'middle', 'end', 'null']
+import regex, util
 
 class Token(object):
-    def __init__(self, name, rule=None, y=0, x=0, s="", parent=None, matchd={}):
-        self.name    = name
-        self.rule    = rule
-        self.y       = y
-        self.x       = x
-        self.string  = s
-        self.parent  = parent
-        self.matchd  = matchd
+    def __init__(self, name, rule, y, x, s, parent=None, matchd={}, link=None):
+        self.name   = name
+        self.rule   = rule
+        self.y      = y
+        self.x      = x
+        self.string = s
+        self.parent = parent
+        self.matchd = matchd
+        self.link   = link
         assert parent is None or hasattr(parent, 'name'), 'oh no %r' % parent
     def parents(self):
         if self.parent is not None:
@@ -27,7 +24,7 @@ class Token(object):
             names = self.parent.domain()
         else:
             names = []
-        if self.name != 'middle':
+        if self.link and not self.link.startswith('middle'):
             names.append(self.rule.name)
         return names
     def fqlist(self):
@@ -35,7 +32,7 @@ class Token(object):
             names = self.parent.domain()
         else:
             names = []
-        if self.name == 'start':
+        if self.link == 'start':
             names.append(self.rule.name)
         names.append(self.name)
         return names
@@ -63,27 +60,24 @@ class Token(object):
 
 class Rule:
     reflags = 0
-    def __init__(self, name, group=None):
-        assert valid_name_re.match(name), 'invalid name %r' % name
-        assert name not in reserved_names, "reserved rule name: %r" % name
+    def __init__(self, name):
+        assert regex.valid_token_name.match(name), 'invalid name %r' % name
+        assert not regex.reserved_token_names.match(name), \
+            "rule name %r is reserved and thus illegal" % name
         self.name = name
-        if group is None:
-            self.group = name
-        else:
-            self.group = group
     def match(self, lexer, parent):
         raise Exception, "not implemented"
     def lex(self, lexer, parent, match):
         raise Exception, "not implemented"
-    def make_token(self, lexer, s, name, parent=None, matchd={}):
-        t = Token(name, self, lexer.y, lexer.x, s, parent, matchd)
+    def make_token(self, lexer, s, name, parent=None, matchd={}, link=None):
+        t = Token(name, self, lexer.y, lexer.x, s, parent, matchd, link)
         lexer.x += len(s)
         return t
     def get_line(self, lexer):
         return lexer.lines[lexer.y] + '\n'
 
 class PatternRule(Rule):
-    def __init__(self, name, pattern, group=None):
+    def __init__(self, name, pattern):
         Rule.__init__(self, name)
         self.pattern = pattern
         self.re = re.compile(self.pattern, self.reflags)
@@ -98,7 +92,7 @@ class NocasePatternRule(PatternRule):
     reflags = re.IGNORECASE
 
 class ContextPatternRule(PatternRule):
-    def __init__(self, name, pattern, fallback, group=None):
+    def __init__(self, name, pattern, fallback):
         Rule.__init__(self, name)
         self.pattern     = pattern
         self.fallback_re = re.compile(fallback, self.reflags)
@@ -111,85 +105,187 @@ class ContextPatternRule(PatternRule):
 class NocaseContextPatternRule(ContextPatternRule):
     reflags = re.IGNORECASE
 
-class RegionRule(Rule):
-    def __init__(self, name, start, grammar, end, group=None):
+class PatternGroupRule(PatternRule):
+    def __init__(self, name, *args):
+        assert args and len(args) % 2 == 0
+        i = 0
+        pairs = []
+        while i < len(args):
+            tokname, pattern = args[i], args[i+1]
+            pairs.append((name, re.compile(pattern, self.reflags)))
+            i += 2
         Rule.__init__(self, name)
-        self.grammar  = grammar
-        self.end      = end
-        self.start_re = re.compile(start, self.reflags)
+        self.pairs = tuple(pairs)
+    def match(self, lexer, parent):
+        x       = lexer.x
+        matches = []
+        line    = self.get_line(lexer)
+        for (tokname, tokre) in self.pairs:
+            m = tokre.match(line, x)
+            if m:
+                x += len(m.group(0))
+                matches.append((tokname, m))
+            else:
+                return []
+        assert len(matches) == len(self.pairs)
+        return matches
+    def lex(self, lexer, parent, matches):
+        if matches:
+            for (tokname, m) in matches:
+                yield self.make_token(lexer, m.group(0), tokname, parent, m.groupdict())
+        raise StopIteration
+
+class RegionRule(Rule):
+    def __init__(self, name, *args):
+        Rule.__init__(self, name)
+        assert len(args) > 1
+        args = list(args)
+        self.pairs    = []
+        self.start_re = re.compile(args.pop(0), self.reflags)
+        while len(args) > 1:
+            grammar = args.pop(0)
+            pattern = args.pop(0)
+            assert hasattr(grammar, 'rules'), repr(grammar)
+            assert type(pattern) == type(''), repr(pattern)
+            self.pairs.append((grammar, pattern))
+        if len(args) == 1:
+            self.pairs.append((grammar, None))
     def match(self, lexer, parent):
         return self.start_re.match(self.get_line(lexer), lexer.x)
     def lex(self, lexer, parent, m):
-        t1 = self.make_token(lexer, m.group(0), 'start', parent, m.groupdict())
-        yield t1
-        if self.end:
-            stopre = re.compile(self.end % t1.matchd, self.reflags)
-        else:
-            stopre = None
-        for t2 in self._lex(lexer, [t1], 'end', stopre, self.grammar):
-            yield t2
+        assert m
+        # ok, so since we had a match, we need to create our start token, who
+        # will be the ancestor to all other tokens matched in this region
+        matchd = m.groupdict()
+        parent = self.make_token(lexer, m.group(0), 'start', parent, matchd, 'start')
+        yield parent
+
+        # now we will loop over the different pairs of grammars/stop-patterns in
+        # this region, and return the resulting token; we start at 0
+        for tok in self._lex_loop(lexer, [parent], matchd, 0):
+            yield tok
         raise StopIteration
     def resume(self, lexer, toresume):
-        assert toresume
-        t1 = toresume[0]
-        assert t1.name
-        if self.end:
-            stopre = re.compile(self.end % t1.matchd, self.reflags)
+        assert toresume, "can't resume without tokens to resume!"
+        # ok, so we need to figure out in which of the grammars of our region
+        # we are resuming. to do this we calculate i, a position in our list
+        # of grammar/stop-pattern pairs
+        if toresume[0].link == 'start':
+            i = 0
         else:
-            stopre = None
-        for t2 in self._lex(lexer, [t1], 'end', stopre, self.grammar):
-            yield t2
-        raise StopIteration
+            m = regex.middle_token_name.match(toresume[0].link)
+            assert m
+            i = int(m.group(1)) + 1
+            assert i > 0 and i < len(self.pairs)
 
+        # now we will loop over the different pairs of grammars/stop-patterns in
+        # this region, and return the resulting token; we start at i
+        for tok in self._lex_loop(lexer, toresume, toresume[0].matchd, i):
+            yield tok
+        raise StopIteration
+    def _lex_loop(self, lexer, toresume, matchd, i):
+        # we need to loop over our grammar/stop-pattern pairs
+        while i < len(self.pairs):
+            # for each one, we will compile our stop-regex, and figure out the
+            # name of the stop token to be created if this stop-regex matches.
+            grammar = self.pairs[i][0]
+            if self.pairs[i][1]:
+                stopre = re.compile(self.pairs[i][1] % matchd, self.reflags)
+            else:
+                stopre = None
+            if i == len(self.pairs) - 1:
+                tokname = 'end'
+            else:
+                tokname = 'middle%d' % i
+
+            # ok, so now loop over all the tokens in the current grammar, until
+            # the stop-token (if any) is found, and return each result as we get
+            # it.
+            for tok in self._lex(lexer, toresume, tokname, stopre, grammar):
+                yield tok
+
+            # ok, so now either we found the stop-token, and have a new parent
+            # for future tokens (if any), or we are done.
+            if tok.name == tokname:
+                toresume = [tok]
+                matchd.update(tok.matchd)
+            else:
+                raise StopIteration
+            i += 1
+        # assuming we make it through all our grammars, and find the end-token,
+        # then we need to signal that we are done.
+        raise StopIteration
     def _lex(self, lexer, toresume, stopname, stopre, grammar):
         assert toresume
         parent  = toresume[0]
         reenter = len(toresume) > 1
         null_t  = None
 
+        # ok, so there are only two way we want to exit this loop: either we
+        # lex the whole document, or we encounter the stop-token.
         done = False
         while not done and lexer.y < len(lexer.lines):
+            line  = self.get_line(lexer)
             old_y = lexer.y
-            line = self.get_line(lexer)
             while not done and lexer.y == old_y and lexer.x < len(line):
+                # ok, so reenter gets priority, since the current input might be
+                # intended for nested grammar. so handle it here
                 if reenter:
                     reenter = False
                     for t in toresume[1].rule.resume(lexer, toresume[1:]):
                         yield t
+                # since we might have changed our x/y coordinates, we need to
+                # do some checks here, and maybe finish or change our coordintes
                 if lexer.y >= len(lexer.lines):
                     raise StopIteration
                 elif lexer.x >= len(line):
                     lexer.y += 1
                     lexer.x = 0
+
+                # ok, so get the *now* current line
                 line = self.get_line(lexer)
 
                 if stopre:
+                    # if we are looking for a stop-token, do that check now
                     m = stopre.match(line, lexer.x)
                     if m:
                         if null_t:
+                            # if we have a null token waiting, return it first.
                             yield null_t
                             null_t = None
-                        yield self.make_token(lexer, m.group(0), stopname, parent, m.groupdict())
+                        # ok, now return the stop-token, and signal that we are
+                        # done and no more input is to be consumed
+                        yield self.make_token(lexer, m.group(0), stopname,
+                                              parent, m.groupdict(), stopname)
                         done = True
                         break
 
                 m = None
+                # start checking our rules to see if we can match the input
                 for rule in grammar.rules:
                     m = rule.match(lexer, parent)
                     if m:
+                        # ok great, we have a match
                         if null_t:
+                            # if we have a null token waiting, return it first.
                             yield null_t
                             null_t = None
+                        # ok, now for every token this rules has created, we
+                        # return them, one by one.
                         for t in rule.lex(lexer, parent, m):
                             yield t
                         break
 
                 if not m:
-                    if lexer.x < len(line):
-                        if not null_t:
-                            null_t = Token('null', None, lexer.y, lexer.x, '', parent)
-                        null_t.add_to_string(line[lexer.x])
-                        lexer.x += 1
+                    # we didn't find a match on a rule, so add this character to
+                    # the current null token (creating a new one if necessary);
+                    if not null_t:
+                        null_t = Token('null', None, lexer.y, lexer.x, '', parent)
+                    null_t.add_to_string(line[lexer.x])
+                    lexer.x += 1
+
+            # ok, we are at the end of a line of input. so, if we have a null
+            # token waiting, now is the time to return it
             if null_t:
                 yield null_t
                 null_t = None
@@ -200,86 +296,8 @@ class RegionRule(Rule):
 class NocaseRegionRule(RegionRule):
     reflags = re.IGNORECASE
 
-class DualRegionRule(RegionRule):
-    def __init__(self, name, start, grammar1, middle, grammar2, end, group=None):
-        Rule.__init__(self, name)
-        self.start_re = re.compile(start, self.reflags)
-        self.grammar1 = grammar1
-        self.middle   = middle
-        self.grammar2 = grammar2
-        self.end      = end
-    def match(self, lexer, parent):
-        return self.start_re.match(self.get_line(lexer), lexer.x)
-    def lex(self, lexer, parent, m):
-        assert m
-        t1 = self.make_token(lexer, m.group(0), 'start', parent, m.groupdict())
-        yield t1
-
-        t2 = None
-        if self.middle:
-            stopre = re.compile(self.middle % t1.matchd, self.reflags)
-        else:
-            stopre = None
-        for t2 in self._lex(lexer, [t1], 'middle', stopre, self.grammar1):
-            yield t2
-
-        if t2 is not None and t2.name == 'middle':
-            if self.end:
-                d = dict(t2.matchd)
-                if t1:
-                    d.update(t1.matchd)
-                stopre = re.compile(self.end % d, self.reflags)
-            else:
-                stopre = None
-            for t3 in self._lex(lexer, [t2], 'end', stopre, self.grammar2):
-                yield t3
-
-        raise StopIteration
-    def resume(self, lexer, toresume):
-        assert toresume, "can't resume without tokens to resume!"
-        t1 = t2 = None
-        if toresume[0].name == 'start':
-            t1 = toresume[0]
-            assert t1.name
-        elif toresume[0].name == 'middle':
-            t2 = toresume[0]
-            assert t2.name
-        else:
-            raise Exception, "invalid name %r" % toresume[0].name
-
-        if t1 is not None:
-            #assert t1.name == 'start'
-            if self.middle:
-                stopre = re.compile(self.middle, self.reflags)
-            else:
-                stopre = None
-            for t2 in self._lex_first(lexer, toresume, 'middle', stopre):
-                yield t2
-            toresume = [t2]
-        if t2 is not None:
-            assert t2.name == 'middle'
-            if self.end:
-                stopre = re.compile(self.end, self.reflags)
-            else:
-                stopre = None
-            for t3 in self._lex_second(lexer, toresume, 'end', stopre):
-                yield t3
-            #toresume = [t3]
-        raise StopIteration
-class NocaseDualRegionRule(DualRegionRule):
-    reflags = re.IGNORECASE
-
 class Grammar:
     rules = []
-    def __init__(self):
-        # XYZ maybe this is unnecessary
-        for rule in self.rules:
-            if hasattr(rule, 'grammar') and rule.grammar is None:
-                rule.grammar = self
-            if hasattr(rule, 'grammar1') and rule.grammar is None:
-                rule.grammar = self
-            if hasattr(rule, 'grammar2') and rule.grammar is None:
-                rule.grammar = self
 grammar = Grammar()
 
 class Lexer:
@@ -307,10 +325,9 @@ class Lexer:
         self.tokens = []
         toresume = token.parents()
 
-        # this is a special case for the "middle" rule of a dual region rule
         i = 1
         while i < len(toresume):
-            if toresume[i].name == 'middle' and toresume[i-1].name == 'start':
+            if toresume[i].link and toresume[i].link != 'start':
                 del toresume[i-1]
             else:
                 i += 1
@@ -339,6 +356,8 @@ class Lexer:
                             yield t
                         break
 
+                if self.y >= len(self.lines):
+                    break
                 line = self.get_line()
                 if not m:
                     if self.x < len(line):
@@ -351,3 +370,4 @@ class Lexer:
             self.y += 1
             self.x = 0
         raise StopIteration
+
diff --git a/mode_blame.py b/mode_blame.py
index e5858af..3d0aa9f 100644
--- a/mode_blame.py
+++ b/mode_blame.py
@@ -1,32 +1,21 @@
 import color, mode2
 
 from point2 import Point
-from lex3 import Grammar, PatternRule, RegionRule, DualRegionRule
+from lex3 import Grammar, PatternRule, RegionRule
 
 class MetadataGrammar(Grammar):
     rules = [
-        PatternRule(
-            name=r'username',
-            pattern='[a-zA-Z0-9_]+',
-        ),
+        PatternRule(r'username', r'[a-zA-Z0-9_]+'),
     ]
 
 class BlameGrammar(Grammar):
     rules = [
-        RegionRule(
-            name=r'metadata',
-            start=r'^[0-9\.]+',
-            grammar=MetadataGrammar(),
-            end=r'[0-9]{4}-[0-9]{2}-[0-9]{2}',
-        ),
-        PatternRule(
-            name=r'data',
-            pattern=r'.+$',
-        ),
+        RegionRule(r'metadata', r'^[0-9\.]+', MetadataGrammar, r'[0-9]{4}-[0-9]{2}-[0-9]{2}'),
+        PatternRule(r'data', r'.+$'),
     ]
 
 class Blame(mode2.Fundamental):
-    grammar = BlameGrammar()
+    grammar = BlameGrammar
     def __init__(self, w):
         mode2.Fundamental.__init__(self, w)
         self.colors = {
diff --git a/mode_css.py b/mode_css.py
index f0e7bac..3f75c0e 100644
--- a/mode_css.py
+++ b/mode_css.py
@@ -4,45 +4,45 @@ from point2 import Point
 
 class StringGrammar(Grammar):
     rules = [
-        PatternRule(name=r'octal', pattern=r'\\[0-7]{3}'),
-        PatternRule(name=r'escaped', pattern=r'\\.'),
+        PatternRule(r'octal', r'\\[0-7]{3}'),
+        PatternRule(r'escaped', r'\\.'),
     ]
 
 class KeywordGrammar(Grammar):
     rules = [
-        PatternRule(name=r'octal', pattern=r'\\[0-7]{3}'),
-        PatternRule(name=r'escaped', pattern=r'\\.'),
-        RegionRule(name='string', start="'", grammar=StringGrammar(), end="'"),
-        RegionRule(name='string', start='"', grammar=StringGrammar(), end='"'),
+        PatternRule(r'octal', r'\\[0-7]{3}'),
+        PatternRule(r'escaped', r'\\.'),
+        RegionRule('string', "'", StringGrammar, r"'"),
+        RegionRule('string', '"', StringGrammar, r'"'),
     ]
 
 class CSSGrammar(Grammar):
     rules = [
-        RegionRule(name=r'comment', start='/\*', grammar=Grammar(), end='\*/'),
-        RegionRule(name=r'htmlcomment', start='<!--', grammar=Grammar(), end='-->'),
-        NocasePatternRule(name=r'dimension', pattern=r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)[-a-z_][-a-z0-9_]*'),
-        NocasePatternRule(name=r'percentage', pattern=r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)%%'),
-        NocasePatternRule(name=r'length', pattern=r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)(?:em|ex|px|in|cm|mm|pt|pc)'),
-        NocasePatternRule(name=r'hash', pattern=r'#[-a-z0-9_]+'),
-        NocasePatternRule(name=r'real', pattern=r'[+-]?[0-9]*\.[0-9]+'),
-        NocasePatternRule(name=r'int', pattern=r'[+-]?[0-9]+'),
-        NocasePatternRule(name=r'rule', pattern=r'@(?:page|media|import)'),
-        NocasePatternRule(name=r'color', pattern=r'(?:aqua|black|blue|fuchsia|gray|green|lime|maroon|navy|olive|orange|purple|red|silver|teal|white|yellow|#[0-9]{6}|#[0-9]{3})'),
+        RegionRule(r'comment', '/\*', Grammar, '\*/'),
+        RegionRule(r'htmlcomment', '<!--', Grammar, '-->'),
+        NocasePatternRule(r'dimension', r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)[-a-z_][-a-z0-9_]*'),
+        NocasePatternRule(r'percentage', r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)%%'),
+        NocasePatternRule(r'length', r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)(?:em|ex|px|in|cm|mm|pt|pc)'),
+        NocasePatternRule(r'hash', r'#[-a-z0-9_]+'),
+        NocasePatternRule(r'real', r'[+-]?[0-9]*\.[0-9]+'),
+        NocasePatternRule(r'int', r'[+-]?[0-9]+'),
+        NocasePatternRule(r'rule', r'@(?:page|media|import)'),
+        NocasePatternRule(r'color', r'(?:aqua|black|blue|fuchsia|gray|green|lime|maroon|navy|olive|orange|purple|red|silver|teal|white|yellow|#[0-9]{6}|#[0-9]{3})'),
 
-        NocasePatternRule(name=r'keyword', pattern=r'(?:url|rgb|counter)'),
-        NocaseRegionRule(name=r'keyword', start='(?:(?<=url)|(?<=rgb)|(?<=counter))\(', grammar=KeywordGrammar(), end='\)'),
+        NocasePatternRule(r'keyword', r'(?:url|rgb|counter)'),
+        NocaseRegionRule(r'keyword', '(?:(?<=url)|(?<=rgb)|(?<=counter))\(', KeywordGrammar, '\)'),
 
-        NocasePatternRule(name=r'label', pattern=r"\.?[-a-zA-Z0-9_]+(?= *{)"),
+        NocasePatternRule(r'label', r"\.?[-a-zA-Z0-9_]+(?= *{)"),
 
-        NocasePatternRule(name=r'ident', pattern=r"-?[a-z_][-a-z0-9_]*"),
-        NocasePatternRule(name=r'name', pattern=r"[-a-z0-9_]+"),
-        NocasePatternRule(name=r'delimiter', pattern=r'[:;,{}()\[\]]|~=|\|=|='),
-        RegionRule(name='string', start="'", grammar=StringGrammar(), end="'"),
-        RegionRule(name='string', start='"', grammar=StringGrammar(), end='"'),
+        NocasePatternRule(r'ident', r"-?[a-z_][-a-z0-9_]*"),
+        NocasePatternRule(r'name', r"[-a-z0-9_]+"),
+        NocasePatternRule(r'delimiter', r'[:;,{}()\[\]]|~=|\|=|='),
+        RegionRule(r'string', "'", StringGrammar, r"'"),
+        RegionRule(r'string', '"', StringGrammar, r'"'),
     ]
 
 class CSS(mode2.Fundamental):
-    grammar = CSSGrammar()
+    grammar = CSSGrammar
     def __init__(self, w):
         mode2.Fundamental.__init__(self, w)
         self.add_bindings('close-paren', (')',))
diff --git a/mode_javascript.py b/mode_javascript.py
index 6ad6e16..a7699ad 100644
--- a/mode_javascript.py
+++ b/mode_javascript.py
@@ -5,33 +5,33 @@ from mode_python import StringGrammar
 
 class JavascriptGrammar(Grammar):
     rules = [
-        PatternRule(name=r'comment', pattern=r'//.*$'),
-        RegionRule(name=r'comment', start='/\*', grammar=Grammar, end='\*/'),
-        PatternRule(name=r'continuation', pattern=r'\\(?= *$)'),
-        PatternRule(name=r'function', pattern=r"(?<=function )[a-zA-Z_][a-zA-Z0-9_]*"),
-        PatternRule(name=r'class', pattern=r"(?<=class )[a-zA-Z_][a-zA-Z0-9_]*"),
+        PatternRule(r'comment', r'//.*$'),
+        RegionRule(r'comment', '/\*', Grammar, '\*/'),
+        PatternRule(r'continuation', r'\\(?= *$)'),
+        PatternRule(r'function', r"(?<=function )[a-zA-Z_][a-zA-Z0-9_]*"),
+        PatternRule(r'class', r"(?<=class )[a-zA-Z_][a-zA-Z0-9_]*"),
         
-        PatternRule(name=r'reserved', pattern=r'(?:as|break|case|catch|class|const|continue|default|delete|do|else|export|extends|false|finally|for|function|if|import|in|instanceof|is|namespace|new|null|package|private|public|return|super|switch|this|throw|true|try|typeof|use|var|void|while|with)(?![a-zA-Z0-9_])'),
-        PatternRule(name=r'reserved', pattern=r'(?:abstract|debugger|enum|goto|implements|interface|native|protected|synchronized|throws|transient|volatile)(?![a-zA-Z0-9_])'),
-        PatternRule(name=r'nonreserved', pattern=r'(?:get|include|set)(?![a-zA-Z0-9_])'),
+        PatternRule(r'reserved', r'(?:as|break|case|catch|class|const|continue|default|delete|do|else|export|extends|false|finally|for|function|if|import|in|instanceof|is|namespace|new|null|package|private|public|return|super|switch|this|throw|true|try|typeof|use|var|void|while|with)(?![a-zA-Z0-9_])'),
+        PatternRule(r'reserved', r'(?:abstract|debugger|enum|goto|implements|interface|native|protected|synchronized|throws|transient|volatile)(?![a-zA-Z0-9_])'),
+        PatternRule(r'nonreserved', r'(?:get|include|set)(?![a-zA-Z0-9_])'),
 
-        PatternRule(name=r"method", pattern=r"(?<=\.)[a-zA-Z_][a-zA-Z0-9_]*(?= *\()"),
-        PatternRule(name=r'identifier', pattern=r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        PatternRule(r"method", r"(?<=\.)[a-zA-Z_][a-zA-Z0-9_]*(?= *\()"),
+        PatternRule(r'identifier', r"[a-zA-Z_][a-zA-Z0-9_]*"),
 
-        PatternRule(name=r'integer', pattern=r"(?:0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
-        PatternRule(name=r'float', pattern=r"[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"),
+        PatternRule(r'integer', r"(?:0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
+        PatternRule(r'float', r"[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"),
 
         # fucking javascript!
         # their lexer grammar requires one-token look-behind in order to know
         # whether a "/" starts a literal regex, or is part of a mathematical
         # expression/assignment. so for now we will require either a space or $
         # after / in order to *not* treat it as a regex. dammit!
-        PatternRule(name=r'delimiter', pattern=r'%=|&&=|&=|\(|\)|\*=|\+=|,|-=|\.{3}|\.|/=(?= |$)|::|:|;|<<=|>>=|>>>=|\?|\[|\]|^=|^^=|\{|\}|\|=|\|\|='),
-        PatternRule(name=r'operator', pattern=r'!==|!=|!|%|&&|&|\*|\+\+|\+|--|-|/(?= |$)|<<=|<<|<=|<|===|==|=|>>>=|>>>|>>=|>>|>=|>|\\'),
+        PatternRule(r'delimiter', r'%=|&&=|&=|\(|\)|\*=|\+=|,|-=|\.{3}|\.|/=(?= |$)|::|:|;|<<=|>>=|>>>=|\?|\[|\]|^=|^^=|\{|\}|\|=|\|\|='),
+        PatternRule(r'operator', r'!==|!=|!|%|&&|&|\*|\+\+|\+|--|-|/(?= |$)|<<=|<<|<=|<|===|==|=|>>>=|>>>|>>=|>>|>=|>|\\'),
 
-        RegionRule(name='regex', start="/", grammar=StringGrammar, end="/"),
-        RegionRule(name='string', start="'", grammar=StringGrammar, end="'"),
-        RegionRule(name='string', start='"', grammar=StringGrammar, end='"'),
+        RegionRule('regex', "/", StringGrammar, "/"),
+        RegionRule('string', "'", StringGrammar, "'"),
+        RegionRule('string', '"', StringGrammar, '"'),
     ]
 
 class JavascriptTabber(tab2.StackTabber):
diff --git a/mode_life.py b/mode_life.py
index 59e8107..d86f417 100644
--- a/mode_life.py
+++ b/mode_life.py
@@ -1,8 +1,6 @@
 import re, sets, string, sys
 import color, commands, default, method, mode2, regex, tab2
 from point2 import Point
-from lex3 import Grammar, PatternRule, ContextPatternRule, \
-    RegionRule, DualRegionRule
 
 class Life(mode2.Fundamental):
     def __init__(self, w):
diff --git a/mode_nasm.py b/mode_nasm.py
index 97ad478..7568b14 100644
--- a/mode_nasm.py
+++ b/mode_nasm.py
@@ -3,31 +3,31 @@ from lex3 import Grammar, PatternRule, RegionRule
 
 class StringGrammar(Grammar):
     rules = [
-        PatternRule(name=r'octal', pattern=r'\\[0-7]{3}'),
-        PatternRule(name=r'escaped', pattern=r'\\.'),
+        PatternRule(r'octal', r'\\[0-7]{3}'),
+        PatternRule(r'escaped', r'\\.'),
     ]
 
 class NasmGrammar(Grammar):
     rules = [
-        PatternRule(name=r'keyword', pattern=r"(?:section|global|extern)(?![a-zA-Z_])"),
-        PatternRule(name=r'macros', pattern=r"%(?:define|undef|assign|strlen|macro|endmacro|if|elif|else|endif|ifdef|ifndef|include|push|pop|stacksize)(?![a-zA-Z_])"),
-        PatternRule(name=r'instructions', pattern=r"(?:jeq|jne|ja|jmp|push|pushad|pushfd|call|ret|sub|add|pop|popa|popad|popfd|call|and|cwd|cdq|cmp|cmpxchg|cpuid|div|divpd|enter|leave|fadd|fld|fmul|fsqrt|fsub|hlt|imul|inc|int|int3|lea|mov|movd|mul|neg|not|nop|or|sal|sar|shl|shr|shld|shrd|syscall|sysenter|sysexit|test|xchg|xadd|xor)(?![a-zA-Z_])"),
-        PatternRule(name=r'registers', pattern=r"(?:eax|ax|ah|al|ebx|bx|bh|bl|ecx|cx|ch|cl|esi|edi|esp|ebp)(?![a-zA-Z_])"),
-        PatternRule(name=r'prefix', pattern=r"(?:dword|word|lock)(?![a-zA-Z_])"),
-        PatternRule(name=r'label', pattern=r"[a-zA-Z_.][a-zA-Z0-9_.]*:"),
-        PatternRule(name=r"identifier", pattern=r"[a-zA-Z_][a-zA-Z0-9_]*"),
-        PatternRule(name=r"integer", pattern=r"(0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
-        PatternRule(name=r"float", pattern=r"[0-9]+\.[0-9]*|\.[0-9]+|([0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"),
-        RegionRule(name=r'string', start=r'"""', grammar=StringGrammar(), end=r'"""'),
-        RegionRule(name=r'string', start=r"'''", grammar=StringGrammar(), end=r"'''"),
-        RegionRule(name=r'string', start=r'"', grammar=StringGrammar(), end=r'"'),
-        RegionRule(name=r'string', start=r"'", grammar=StringGrammar(), end=r"'"),
-        PatternRule(name=r'comment', pattern=r';.*$'),
+        PatternRule(r'keyword', r"(?:section|global|extern)(?![a-zA-Z_])"),
+        PatternRule(r'macros', r"%(?:define|undef|assign|strlen|macro|endmacro|if|elif|else|endif|ifdef|ifndef|include|push|pop|stacksize)(?![a-zA-Z_])"),
+        PatternRule(r'instructions', r"(?:jeq|jne|ja|jmp|push|pushad|pushfd|call|ret|sub|add|pop|popa|popad|popfd|call|and|cwd|cdq|cmp|cmpxchg|cpuid|div|divpd|enter|leave|fadd|fld|fmul|fsqrt|fsub|hlt|imul|inc|int|int3|lea|mov|movd|mul|neg|not|nop|or|sal|sar|shl|shr|shld|shrd|syscall|sysenter|sysexit|test|xchg|xadd|xor)(?![a-zA-Z_])"),
+        PatternRule(r'registers', r"(?:eax|ax|ah|al|ebx|bx|bh|bl|ecx|cx|ch|cl|esi|edi|esp|ebp)(?![a-zA-Z_])"),
+        PatternRule(r'prefix', r"(?:dword|word|lock)(?![a-zA-Z_])"),
+        PatternRule(r'label', r"[a-zA-Z_.][a-zA-Z0-9_.]*:"),
+        PatternRule(r"identifier", r"[a-zA-Z_][a-zA-Z0-9_]*"),
+        PatternRule(r"integer", r"(0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
+        PatternRule(r"float", r"[0-9]+\.[0-9]*|\.[0-9]+|([0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"),
+        RegionRule(r'string', r'"""', StringGrammar, r'"""'),
+        RegionRule(r'string', r"'''", StringGrammar, r"'''"),
+        RegionRule(r'string', r'"', StringGrammar, r'"'),
+        RegionRule(r'string', r"'", StringGrammar, r"'"),
+        PatternRule(r'comment', r';.*$'),
     ]
 
 
 class Nasm(mode2.Fundamental):
-    grammar = NasmGrammar()
+    grammar = NasmGrammar
     def __init__(self, w):
         mode2.Fundamental.__init__(self, w)
         self.colors = {
diff --git a/mode_perl.py b/mode_perl.py
index 8873d1e..4e2878f 100644
--- a/mode_perl.py
+++ b/mode_perl.py
@@ -1,7 +1,7 @@
 import re, sets, string, sys
 import color, commands, default, method, mode2, regex, tab2
 from point2 import Point
-from lex3 import Grammar, PatternRule, ContextPatternRule, RegionRule, DualRegionRule
+from lex3 import Grammar, PatternRule, ContextPatternRule, RegionRule
 from method import Argument, Method
 
 class PodGrammar(Grammar):
@@ -45,7 +45,7 @@ class PerlGrammar(Grammar):
         PatternRule(r'hash_key', r'[A-Za-z0-9_]+(?= *=>)'),
         PatternRule(r'length', r"\$#[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"),
         PatternRule(r'cast', r'[\$\@\%\^\&](?= *{)'),
-        PatternRule(r'scalar', r"\$[][><ab/'\"_@\?#\$!%^|&*()](?![A-Za-z0-9_])"),
+        PatternRule(r'scalar', r"\$[\[\]<>ab/'\"_@\?#\$!%^|&*()](?![A-Za-z0-9_])"),
         PatternRule(r'array', r"@_"),
         PatternRule(r'function', r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*(?=-> *\()"),
         PatternRule(r'scalar', r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"),
@@ -59,12 +59,12 @@ class PerlGrammar(Grammar):
         RegionRule(r'match', r'm(?P<delim>#)', StringGrammar, r'#[a-z]*'),
 
         # replace regexes
-        DualRegionRule(r'replace', r's *(?P<delim>[^ a-zA-Z0-9_])', StringGrammar, r'%(delim)s', StringGrammar, r'%(delim)s[a-z]*'),
-        DualRegionRule(r'replace', r's(?P<delim>#)', StringGrammar, r'#', StringGrammar, r'#[a-z]*'),
+        RegionRule(r'replace', r's *(?P<delim>[^ a-zA-Z0-9_])', StringGrammar, r'%(delim)s', StringGrammar, r'%(delim)s[a-z]*'),
+        RegionRule(r'replace', r's(?P<delim>#)', StringGrammar, r'#', StringGrammar, r'#[a-z]*'),
 
         # translate operator
-        DualRegionRule(r'translate', r'(?:y|tr) *(?P<delim>[^ a-zA-Z0-9_])', Grammar, r'%(delim)s', Grammar, r'%(delim)s[a-z]*'),
-        DualRegionRule(r'translate', r'(?:y|tr)#', Grammar, r'#', Grammar, r'#[a-z]*'),
+        RegionRule(r'translate', r'(?:y|tr) *(?P<delim>[^ a-zA-Z0-9_])', Grammar, r'%(delim)s', Grammar, r'%(delim)s[a-z]*'),
+        RegionRule(r'translate', r'(?:y|tr)#', Grammar, r'#', Grammar, r'#[a-z]*'),
 
         # some more basic stuff
         PatternRule(r'package', r"(?<=package )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*"),
@@ -255,7 +255,7 @@ class Perl(mode2.Fundamental):
             
             # replace regex
             'replace.start':   color.build('cyan', 'default'),
-            'replace.middle':  color.build('cyan', 'default'),
+            'replace.middle0': color.build('cyan', 'default'),
             'replace.end':     color.build('cyan', 'default'),
             'replace.null':    color.build('cyan', 'default'),
             'replace.escaped': color.build('magenta', 'default'),
@@ -266,10 +266,10 @@ class Perl(mode2.Fundamental):
             'replace.cast':    color.build('yellow', 'default'),
             
             # translate regex
-            'translate.start':  color.build('magenta', 'default'),
-            'translate.middle': color.build('magenta', 'default'),
-            'translate.end':    color.build('magenta', 'default'),
-            'translate.null':   color.build('magenta', 'default'),
+            'translate.start':   color.build('magenta', 'default'),
+            'translate.middle0': color.build('magenta', 'default'),
+            'translate.end':     color.build('magenta', 'default'),
+            'translate.null':    color.build('magenta', 'default'),
         }
         
         # perl-specific
diff --git a/mode_sh.py b/mode_sh.py
index 36d4827..ecf9698 100644
--- a/mode_sh.py
+++ b/mode_sh.py
@@ -17,7 +17,6 @@ class ShGrammar(Grammar):
         PatternRule(r'operator', r"(?:-eq|-ne|-gt|-lt|-ge|-le| = | != )"),
         PatternRule(r'delimiter', r";;|[\[\]\(\);\{\}|&><]"),
         RegionRule(r'eval', '`', StringGrammar, '`'),
-        #RegionRule(r'neval', r'\$\(', None, r'\)'),
         RegionRule(r'neval', r'\$\(', StringGrammar, r'\)'),
         PatternRule(r'variable', r"(?:^|(?<= ))[a-zA-Z_][a-zA-Z_][a-zA-Z0-9_]*(?==)"),
         PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
diff --git a/mode_sql.py b/mode_sql.py
index 2abb41f..541a6d2 100644
--- a/mode_sql.py
+++ b/mode_sql.py
@@ -1,5 +1,5 @@
 import color, mode2, tab2
-from lex3 import Grammar, PatternRule, NocasePatternRule, RegionRule, NocaseRegionRule, DualRegionRule, NocaseDualRegionRule
+from lex3 import Grammar, PatternRule, NocasePatternRule, RegionRule, NocaseRegionRule
 from mode_python import StringGrammar
 
 class PlPgSqlGrammar(Grammar):
diff --git a/mode_text.py b/mode_text.py
index 5471e3e..8b1e022 100644
--- a/mode_text.py
+++ b/mode_text.py
@@ -3,8 +3,7 @@ from lex3 import Token, Rule, PatternRule, RegionRule, Grammar
 
 class WordRule(PatternRule):
     def __init__(self):
-        PatternRule.__init__(self, name=r'word',
-                             pattern=r"[a-zA-Z][a-zA-Z-\']*[a-zA-Z](?=$|[^a-zA-Z0-9-_])")
+        PatternRule.__init__(self, r'word', pattern=r"[a-zA-Z][a-zA-Z-\']*[a-zA-Z](?=$|[^a-zA-Z0-9-_])")
     def _spelled_ok(self, word):
         if ispell.can_spell():
             speller = ispell.get_speller()
@@ -22,15 +21,14 @@ class WordRule(PatternRule):
 
 class ContinuedRule(RegionRule):
     def __init__(self):
-        RegionRule.__init__(self,  name=r'cont', start=r'[a-zA-Z0-9_]+- *$',
-                            grammar=Grammar(), end=r'^ *(?:[^ ]+|$)')
+        RegionRule.__init__(self, r'cont', r'[a-zA-Z0-9_]+- *$', Grammar, r'^ *(?:[^ ]+|$)')
 
 class TextGrammar(Grammar):
     rules = [
         ContinuedRule(),
         WordRule(),
-        PatternRule(name=r'punct', pattern=r'[^a-zA-Z0-9_]'),
-        PatternRule(name=r'stuff', pattern=r'[a-zA-Z0-9_]+'),
+        PatternRule(r'punct', r'[^a-zA-Z0-9_]'),
+        PatternRule(r'stuff', r'[a-zA-Z0-9_]+'),
     ]
 
 class Text(mode2.Fundamental):
diff --git a/mode_tt.py b/mode_tt.py
index 64fcb1b..a163928 100644
--- a/mode_tt.py
+++ b/mode_tt.py
@@ -3,18 +3,18 @@ from lex3 import Grammar, PatternRule, RegionRule
 
 class TagGrammar(Grammar):
     rules = [
-        RegionRule(name=r'string', start=r'(?P<tag>["\'])', grammar=Grammar(), end=r'%(tag)s'),
-        PatternRule(name=r'namespace', pattern=r'[a-zA-Z_]+:'),
-        PatternRule(name=r'attrname', pattern=r'[^ =>\n]+(?==)'),
-        PatternRule(name=r'name', pattern=r'[^ =>\n]+'),
+        RegionRule(r'string', r'(?P<tag>["\'])', Grammar, r'%(tag)s'),
+        PatternRule(r'namespace', pattern=r'[a-zA-Z_]+:'),
+        PatternRule(r'attrname', pattern=r'[^ =>\n]+(?==)'),
+        PatternRule(r'name', pattern=r'[^ =>\n]+'),
     ]
 
 class TemplateGrammar(Grammar):
     rules = [
-        RegionRule(name=r'comment', start=r'<!--', grammar=Grammar(), end=r'-->'),
-        RegionRule(name=r'template', start=r'\[\%', grammar=Grammar(), end=r'%%\]'),
-        RegionRule(name=r'opentag', start=r'<', grammar=TagGrammar(), end=r'/?>'),
-        PatternRule(name=r'closetag', pattern=r'< */ *[ =>\n]+ *>'),
+        RegionRule(r'comment', r'<!--', Grammar, r'-->'),
+        RegionRule(r'template', r'\[\%', Grammar, r'%%\]'),
+        RegionRule(r'opentag', r'<', TagGrammar, r'/?>'),
+        PatternRule(r'closetag', pattern=r'< */ *[ =>\n]+ *>'),
     ]
 
 class Template(mode2.Fundamental):
diff --git a/regex.py b/regex.py
index b560021..37bcd70 100644
--- a/regex.py
+++ b/regex.py
@@ -1,5 +1,10 @@
 import re
 
+# lexing
+reserved_token_names = re.compile(r'^(?:rules|null|start|end|middle[0-9]*)$')
+valid_token_name     = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
+middle_token_name    = re.compile(r'^middle([0-9]+)$')
+
 # meta regexes
 meta_chars = re.compile(r'([\.\^\$\*\+\?\{\}\(\)\[\]\|\"\'\\,])')