diff --git a/lex.py b/lex.py
new file mode 100755
index 0000000..6c9ac41
--- /dev/null
+++ b/lex.py
@@ -0,0 +1,455 @@
+import curses, re
+import regex, util
+from point import Point
+
+class Token(object):
+    def __init__(self, name, rule, y, x, s, color=None, parent=None, matchd={}, link=None):
+        self.name   = name
+        self.rule   = rule
+        self.y      = y
+        self.x      = x
+        self.string = s
+        self.color  = color
+        self.parent = parent
+        self.matchd = matchd
+        self.link   = link
+        assert parent is None or hasattr(parent, 'name'), 'oh no %r' % parent
+    def parents(self):
+        if self.parent is not None:
+            parents = self.parent.parents()
+            parents.append(self.parent)
+            return parents
+        else:
+            return []
+    def domain(self):
+        if self.parent is not None:
+            names = self.parent.domain()
+        else:
+            names = []
+        if self.link and not self.link.startswith('middle'):
+            names.append(self.rule.name)
+        return names
+    def fqlist(self):
+        if self.parent is not None:
+            names = self.parent.domain()
+        else:
+            names = []
+        if self.link == 'start':
+            names.append(self.rule.name)
+        names.append(self.name)
+        return names
+    def fqname(self):
+        names = self.fqlist()
+        return '.'.join(names)
+    def copy(self):
+        return Token(self.name, self.rule, self.y, self.x, self.string,
+                     self.color, self.parent, self.matchd, self.link)
+    def add_to_string(self, s):
+        self.string += s
+    def end_x(self):
+        return self.x + len(self.string)
+    def __eq__(self, other):
+        return (self.y == other.y and self.x == other.x
+                and self.name == other.name and self.parent is other.parent and
+                self.string == other.string)
+    def __repr__(self):
+        if len(self.string) < 10:
+            s = self.string
+        else:
+            s = self.string[:10] + '...'
+        fields = (self.fqname(), self.rule, self.y, self.x, s)
+        return "<Token(%r, %r, %d, %d, %r)>" % fields
+
+class Rule:
+    reflags = 0
+    def __init__(self, name):
+        assert regex.valid_token_name.match(name), 'invalid name %r' % name
+        assert not regex.reserved_token_names.match(name), \
+            "rule name %r is reserved and thus illegal" % name
+        self.name = name
+    def match(self, lexer, parent):
+        raise Exception, "not implemented"
+    def lex(self, lexer, parent, match):
+        raise Exception, "not implemented"
+    def make_token(self, lexer, s, name, parent=None, matchd={}, link=None):
+        t = Token(name, self, lexer.y, lexer.x, s, None, parent, matchd, link)
+        t.color = lexer.get_color(t)
+        lexer.x += len(s)
+        if lexer.x > len(lexer.lines[lexer.y]):
+            lexer.x = 0
+            lexer.y += 1
+        return t
+    def get_line(self, lexer, y=None):
+        if y is None:
+            return lexer.lines[lexer.y] + '\n'
+        else:
+            return lexer.lines[y] + '\n'
+
+class PatternRule(Rule):
+    def __init__(self, name, pattern):
+        Rule.__init__(self, name)
+        self.pattern = pattern
+        self.re = re.compile(self.pattern, self.reflags)
+    def match(self, lexer, parent):
+        return self.re.match(self.get_line(lexer), lexer.x)
+    def lex(self, lexer, parent, m):
+        if m:
+            yield self.make_token(lexer, m.group(0), self.name, parent, m.groupdict())
+        raise StopIteration
+class NocasePatternRule(PatternRule):
+    reflags = re.IGNORECASE
+
+class OverrideError(Exception):
+    pass
+class OverridePatternRule(PatternRule):
+    def lex(self, lexer, parent, m):
+        if m:
+            d = m.groupdict()
+            if lexer.action == 'lex':
+                a = lexer.mode.window.application
+                try:
+                    names   = d['mode'].split('.')
+                    modecls = a.globals()[names.pop(0)]
+                    for name in names:
+                        modecls = getattr(modecls, name)
+                    mode = modecls(None)
+                    if hasattr(mode, 'grammar') and hasattr(mode, 'colors'):
+                        lexer.mode.gstack['%s.start' % d['token']] = mode
+                    else:
+                        raise OverrideError, "argh: %r" % mode
+                except (KeyError, AttributeError, OverrideError):
+                    pass
+            yield self.make_token(lexer, m.group(0), self.name, parent, d)
+        raise StopIteration
+
+class ContextPatternRule(PatternRule):
+    def __init__(self, name, pattern, fallback):
+        Rule.__init__(self, name)
+        self.pattern     = pattern
+        self.fallback_re = re.compile(fallback, self.reflags)
+    def match(self, lexer, parent):
+        try:
+            r = re.compile(self.pattern % parent.matchd)
+        except KeyError:
+            r = self.fallback_re
+        return r.match(self.get_line(lexer), lexer.x)
+class NocaseContextPatternRule(ContextPatternRule):
+    reflags = re.IGNORECASE
+
+class PatternGroupRule(PatternRule):
+    def __init__(self, name, *args):
+        assert args and len(args) % 2 == 0
+        i = 0
+        pairs = []
+        while i < len(args):
+            tokname, pattern = args[i], args[i+1]
+            pairs.append((tokname, re.compile(pattern, self.reflags)))
+            i += 2
+        Rule.__init__(self, name)
+        self.pairs = tuple(pairs)
+    def match(self, lexer, parent):
+        (x, y) = (lexer.x, lexer.y)
+        matches = []
+        for (tokname, tokre) in self.pairs:
+            if y >= len(lexer.lines):
+                return []
+            line = self.get_line(lexer, y)
+            m    = tokre.match(line, x)
+            if m:
+                x += len(m.group(0))
+                if x >= len(line):
+                    x = 0
+                    y += 1
+                matches.append((tokname, m))
+            else:
+                return []
+        assert len(matches) == len(self.pairs)
+        return matches
+    def lex(self, lexer, parent, matches):
+        if matches:
+            for (tokname, m) in matches:
+                yield self.make_token(lexer, m.group(0), tokname, parent, m.groupdict())
+        raise StopIteration
+
+class RegionRule(Rule):
+    def __init__(self, name, *args):
+        Rule.__init__(self, name)
+        assert len(args) > 1
+        args = list(args)
+        self.pairs    = []
+        self.start_re = re.compile(args.pop(0), self.reflags)
+        while len(args) > 1:
+            grammar = args.pop(0)
+            pattern = args.pop(0)
+            assert hasattr(grammar, 'rules'), repr(grammar)
+            assert type(pattern) == type(''), repr(pattern)
+            self.pairs.append((grammar, pattern))
+        if len(args) == 1:
+            self.pairs.append((grammar, None))
+
+    def match(self, lexer, parent):
+        return self.start_re.match(self.get_line(lexer), lexer.x)
+    def lex(self, lexer, parent, m):
+        assert m
+        # ok, so since we had a match, we need to create our start token, who
+        # will be the ancestor to all other tokens matched in this region
+        matchd = m.groupdict()
+        parent = self.make_token(lexer, m.group(0), 'start', parent, matchd, 'start')
+        yield parent
+
+        # now we will loop over the different pairs of grammars/stop-patterns in
+        # this region, and return the resulting token; we start at 0
+        for tok in self._lex_loop(lexer, [parent], matchd, 0):
+            yield tok
+        raise StopIteration
+    def resume(self, lexer, toresume):
+        assert toresume, "can't resume without tokens to resume!"
+        # ok, so we need to figure out in which of the grammars of our region
+        # we are resuming. to do this we calculate i, a position in our list
+        # of grammar/stop-pattern pairs
+        if toresume[0].link == 'start':
+            i = 0
+        else:
+            m = regex.middle_token_name.match(toresume[0].link)
+            assert m
+            i = int(m.group(1)) + 1
+            assert i > 0 and i < len(self.pairs)
+
+        # now we will loop over the different pairs of grammars/stop-patterns in
+        # this region, and return the resulting token; we start at i
+        for tok in self._lex_loop(lexer, toresume, toresume[0].matchd, i):
+            yield tok
+        raise StopIteration
+    def _lex_loop(self, lexer, toresume, matchd, i):
+        # we need to loop over our grammar/stop-pattern pairs
+        while i < len(self.pairs):
+            # for each one, we will compile our stop-regex, and figure out the
+            # name of the stop token to be created if this stop-regex matches.
+            fqname = toresume[0].fqname()
+            p = Point(toresume[0].x, toresume[0].y)
+            if fqname in lexer.mode.ghist and p in lexer.mode.ghist[fqname]:
+                mode = lexer.mode.ghist[fqname][p]
+                grammar = mode.grammar
+            elif fqname in lexer.mode.gstack:
+                mode = lexer.mode.gstack[fqname]
+                grammar = mode.grammar
+                lexer.mode.ghist.setdefault(fqname, {})
+                lexer.mode.ghist[fqname][p] = mode
+                del lexer.mode.gstack[fqname]
+            else:
+                mode = lexer.mode
+                grammar = self.pairs[i][0]
+            lexer.mstack.append(mode)
+
+            if self.pairs[i][1]:
+                stopre = re.compile(self.pairs[i][1] % matchd, self.reflags)
+            else:
+                stopre = None
+            if i == len(self.pairs) - 1:
+                tokname = 'end'
+            else:
+                tokname = 'middle%d' % i
+
+            # ok, so now loop over all the tokens in the current grammar, until
+            # the stop-token (if any) is found, and return each result as we get
+            # it.
+            tok = None
+            for tok in self._lex(lexer, toresume, tokname, stopre, grammar):
+                yield tok
+
+            # ok, so now either we found the stop-token, and have a new parent
+            # for future tokens (if any), or we are done.
+            if tok is not None and tok.name == tokname:
+                toresume = [tok]
+                matchd.update(tok.matchd)
+            else:
+                raise StopIteration
+            # this should have already gotten done by _lex
+            #lexer.mstack.pop(-1)
+            i += 1
+        # assuming we make it through all our grammars, and find the end-token,
+        # then we need to signal that we are done.
+        raise StopIteration
+    def _lex(self, lexer, toresume, stopname, stopre, grammar):
+        assert toresume
+        parent  = toresume[0]
+        reenter = len(toresume) > 1
+        null_t  = None
+
+        # ok, so there are only two way we want to exit this loop: either we
+        # lex the whole document, or we encounter the stop-token.
+        done = False
+        while not done and lexer.y < len(lexer.lines):
+            line  = self.get_line(lexer)
+            old_y = lexer.y
+            while not done and lexer.y == old_y and lexer.x < len(line):
+                # ok, so reenter gets priority, since the current input might be
+                # intended for nested grammar. so handle it here
+                if reenter:
+                    reenter = False
+                    for t in toresume[1].rule.resume(lexer, toresume[1:]):
+                        yield t
+                # since we might have changed our x/y coordinates, we need to
+                # do some checks here, and maybe finish or change our coordintes
+                if lexer.y >= len(lexer.lines):
+                    raise StopIteration
+                elif lexer.x >= len(line):
+                    lexer.y += 1
+                    lexer.x = 0
+
+                # ok, so get the *now* current line
+                line = self.get_line(lexer)
+
+                if stopre:
+                    # if we are looking for a stop-token, do that check now
+                    m = stopre.match(line, lexer.x)
+                    if m:
+                        if null_t:
+                            # if we have a null token waiting, return it first.
+                            yield null_t
+                            null_t = None
+                        # ok, now return the stop-token, and signal that we are
+                        # done and no more input is to be consumed
+                        lexer.mstack.pop(-1)
+                        yield self.make_token(lexer, m.group(0), stopname,
+                                              parent, m.groupdict(), stopname)
+                        done = True
+                        break
+
+                m = None
+                # start checking our rules to see if we can match the input
+                for rule in grammar.rules:
+                    m = rule.match(lexer, parent)
+                    if m:
+                        # ok great, we have a match
+                        if null_t:
+                            # if we have a null token waiting, return it first.
+                            yield null_t
+                            null_t = None
+                        # ok, now for every token this rules has created, we
+                        # return them, one by one.
+                        for t in rule.lex(lexer, parent, m):
+                            yield t
+                        break
+
+                if not m:
+                    # we didn't find a match on a rule, so add this character to
+                    # the current null token (creating a new one if necessary);
+                    if not null_t:
+                        null_t = Token('null', None, lexer.y, lexer.x, '', None, parent)
+                        null_t.color = lexer.get_color(null_t)
+                    null_t.add_to_string(line[lexer.x])
+                    lexer.x += 1
+
+            # ok, we are at the end of a line of input. so, if we have a null
+            # token waiting, now is the time to return it
+            if null_t:
+                yield null_t
+                null_t = None
+            if not done and old_y == lexer.y:
+                lexer.y += 1
+                lexer.x = 0
+        raise StopIteration
+class NocaseRegionRule(RegionRule):
+    reflags = re.IGNORECASE
+
+class Grammar:
+    rules = []
+grammar = Grammar()
+
+class Lexer:
+    def __init__(self, mode, grammar):
+        self.mode    = mode
+        self.mstack  = []
+        self.grammar = grammar
+        self.y       = 0
+        self.x       = 0
+        self.lines   = None
+        assert self.grammar.rules
+    def get_line(self):
+        return self.lines[self.y] + '\n'
+    def lex(self, lines, y=0, x=0):
+        self.action      = 'lex'
+        self.y           = y
+        self.x           = x
+        self.lines       = lines
+        self.mstack      = []
+        self.mode.ghist  = {}
+        self.mode.gstack = {}
+        for t in self._lex():
+            yield t
+        del self.action
+        raise StopIteration
+    def resume(self, lines, y, x, token):
+        self.action = 'resume'
+        self.y      = y
+        self.x      = x
+        self.lines  = lines
+        self.mstack = []
+        toresume    = token.parents()
+
+        i = 1
+        while i < len(toresume):
+            if toresume[i].link and toresume[i].link != 'start':
+                del toresume[i-1]
+            else:
+                i += 1
+
+        if toresume:
+            for t in toresume[0].rule.resume(self, toresume):
+                yield t
+        for t in self._lex():
+            yield t
+        del self.action
+        raise StopIteration
+
+    def _lex(self):
+        parent = None
+        while self.y < len(self.lines):
+            null_t = None
+            line = self.get_line()
+            while self.x < len(line):
+                m = None
+                for rule in self.grammar.rules:
+                    m = rule.match(self, parent)
+                    if m:
+                        if null_t:
+                            yield null_t
+                            null_t = None
+                        for t in rule.lex(self, parent, m):
+                            yield t
+                        break
+
+                if self.y >= len(self.lines):
+                    break
+                line = self.get_line()
+                if not m:
+                    if self.x < len(line):
+                        if null_t is None:
+                            null_t = Token('null', None, self.y, self.x, '', None, parent)
+                            null_t.color = self.get_color(null_t)
+                        null_t.add_to_string(line[self.x])
+                        self.x += 1
+            if null_t:
+                yield null_t
+            self.y += 1
+            self.x = 0
+        raise StopIteration
+
+    def get_color(self, token):
+        fqlist = token.fqlist()
+        if self.mstack:
+            mode = self.mstack[-1]
+        else:
+            mode = self.mode
+        v = list(mode.default_color)
+        for j in range(0, len(fqlist)):
+            name = '.'.join(fqlist[j:])
+            if name in mode.colors:
+                assert type(mode.colors[name]) == type(()), repr(mode)
+                v = list(mode.colors[name])
+                break
+        #if DARK_BACKGROUND:
+        if True:
+            v.append('bold')
+        return v