pmacs3/lex2.py

import re

valid_name_re  = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
reserved_names = ['start', 'middle', 'end', 'null']

class Token(object):
    def __init__(self, name, rule=None, y=0, x=0, s="", parent=None, matchd={}):
        self.name    = name
        self.rule    = rule
        self.y       = y
        self.x       = x
        self.string  = s
        self.parent  = parent
        self.matchd  = matchd
    def parents(self):
        if self.parent is not None:
            parents = self.parent.parents()
            parents.append(self.parent)
            return parents
        else:
            return []
    def domain(self):
        if self.parent is not None:
            names = self.parent.domain()
        else:
            names = []
        names.append(self.rule.name)
        return names
    def fqlist(self):
        if self.parent is not None:
            names = self.parent.domain()
        else:
            names = []
        if self.name == 'start':
            names.append(self.rule.name)
        names.append(self.name)
        return names
    def fqname(self):
        names = self.fqlist()
        return '.'.join(names)
    def copy(self):
        return Token(self.name, self.rule, self.y, self.x, self.string,
                     self.parent, self.matchd)
    def add_to_string(self, s):
        self.string += s
    def end_x(self):
        return self.x + len(self.string)
    def __eq__(self, other):
        return (self.y == other.y and self.x == other.x
                and self.name == other.name and self.parent is other.parent and
                self.string == other.string)
    def __repr__(self):
        if len(self.string) < 10:
            s = self.string
        else:
            s = self.string[:10] + '...'
        fields = (self.fqname(), self.rule, self.y, self.x, s)
        return "<Token(%r, %r, %d, %d, %r)>" % fields

class Rule:
    name = 'abstract'
    def match(self, lexer, parent):
        raise Exception, "%s rule cannot match!" % self.name
    def make_token(self, lexer, s, name, parent=None, matchd={}):
        return Token(name, self, lexer.y, lexer.x, s, parent, matchd)
    def _get_line(self, lexer):
        return lexer.lines[lexer.y] + '\n'
    def _set_group(self, group):
        if group is None:
            self.group = self.name
        else:
            self.group = group

class ConstantRule(Rule):
    def __init__(self, name, constant, group=None):
        assert valid_name_re.match(name), 'invalid name %r' % name
        assert name not in reserved_names, "reserved rule name: %r" % name
        self.name     = name
        self.constant = constant
        self.length   = len(self.constant)
        self._set_group(group)
    def match(self, lexer, parent):
        line = self._get_line(lexer)
        if line[lexer.x:].startswith(self.constant):
            token = self.make_token(lexer, self.constant, self.name, parent)
            lexer.add_token(token)
            lexer.x += self.length
            return True
        else:
            return False

class PatternRule(Rule):
    def __init__(self, name, pattern, group=None):
        assert valid_name_re.match(name), 'invalid name %r' % name
        assert name not in reserved_names, "reserved rule name: %r" % name
        self.name    = name
        self.pattern = pattern
        self._compile()
        self._set_group(group)
    def _compile(self):
        self.re = re.compile(self.pattern)
    def _match(self, lexer, parent, m):
        s = m.group(0)
        token = self.make_token(lexer, s, self.name, parent)
        lexer.add_token(token)
        lexer.x += len(s)
    def match(self, lexer, parent):
        line = self._get_line(lexer)
        m = self.re.match(line, lexer.x)
        if m:
            self._match(lexer, parent, m)
            return True
        else:
            return False

class NocasePatternRule(PatternRule):
    def _compile(self):
        self.re = re.compile(self.pattern, re.IGNORECASE)

class ContextPatternRule(PatternRule):
    def __init__(self, name, pattern, fallback, group=None):
        assert valid_name_re.match(name), 'invalid name %r' % name
        assert name not in reserved_names, "reserved rule name: %r" % name
        self.name        = name
        self.pattern     = pattern
        self.fallback    = fallback
        self.fallback_re = re.compile(fallback)
        self._set_group(group)
    def match(self, lexer, parent):
        try:
            r = re.compile(self.pattern % parent.matchd)
        except KeyError:
            r = self.fallback_re
        line = self._get_line(lexer)
        m = r.match(line, lexer.x)
        if m:
            self._match(lexer, parent, m)
            return True
        else:
            return False

class RegionRule(Rule):
    def __init__(self, name, start, grammar, end, group=None):
        assert valid_name_re.match(name), 'invalid name %r' % name
        assert name not in reserved_names, "reserved rule name: %r" % name
        self.name     = name
        self.start    = start
        self.grammar  = grammar
        self.end      = end
        self.start_re = self._compile_start()
        self._set_group(group)

    def _compile_start(self):
        return re.compile(self.start)
    def _compile_end(self, d):
        return re.compile(self.end % d)

    def resume(self, lexer, toresume):
        #raise Exception, "%r %r" % (lexer, toresume) #XYZ
        assert toresume, "can't resume without tokens to resume!"
        self._match(lexer, None, None, toresume)
        return True

    def match(self, lexer, parent):
        line = self._get_line(lexer)
        m = self.start_re.match(line, lexer.x)
        if m:
            self._match(lexer, parent, m, [])
            return True
        else:
            return False

    def _add_from_regex(self, name, lexer, parent, m, matchd={}):
        s = m.group(0)
        token = self.make_token(lexer, s, name, parent, matchd)
        lexer.add_token(token)
        lexer.x += len(s)
        return token

    def _match(self, lexer, parent, m, toresume=[]):
        # we either need a match object, or a token to resume
        assert m or len(toresume) > 0

        if m:
            # if we had a match, then it becomes the parent, and we save its
            # subgroup dict
            d = m.groupdict()
            parent = self._add_from_regex('start', lexer, parent, m, d)
        else:
            # otherwise, we should be resuming the start token, so let's pull
            # the relevant info out of the token
            parent = toresume[0]
            d = parent.matchd
            assert parent.name == 'start'
        null_t = None

        # this determines whether we are still reentering. if len(toresume) == 1
        # then it means that we have been reentering but will not continue, so
        # reenter will be false.
        reenter = len(toresume) > 1

        # if we have an end regex, then build it here. notice that it can
        # reference named groups from the start token. if we have no end,
        # well, then, we're never getting out of here alive!
        if self.end:
            #end_re = re.compile(self.end % d)
            end_re = self._compile_end(d)

        # ok, so as long as we aren't done (we haven't found an end token),
        # keep reading input
        done = False
        while not done and lexer.y < len(lexer.lines):
            old_y = lexer.y

            # if this line is empty, then we skip it, but here we insert
            # an empty null token just so we have something
            #if not reenter and len(lexer.lines[lexer.y]) == 0:
            #    null_t = Token('null', None, lexer.y, lexer.x, '', parent)
            #    lexer.add_token(null_t)
            #    null_t = None

            # ok, as long as we haven't found the end token, and have more
            # data on the current line to read, we will process tokens
            #while (not done and lexer.y == old_y and
            #       lexer.x < len(lexer.lines[lexer.y])):
            while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1:
                # if we are reentering mid-parse, then that takes precedence
                if reenter:
                    reenter = False
                    rule2 = toresume[1].rule
                    rule2.resume(lexer, toresume[1:])
                    null_t = None
                if lexer.y >= len(lexer.lines):
                    return True
                #elif lexer.x >= len(lexer.lines[lexer.y]):
                elif lexer.x >= len(lexer.lines[lexer.y]) + 1:
                    lexer.y += 1
                    lexer.x = 0

                line = self._get_line(lexer)

                # if we are looking for an end token, then see if we've
                # found it. if so, then we are done!
                if self.end:
                    m = end_re.match(line, lexer.x)
                    if m:
                        self._add_from_regex('end', lexer, parent, m, {})
                        done = True
                        break

                # ok, we need to check all our rules now, in order. if we
                # find a token, note that we found one and exit the loop
                found = False
                for rule in self.grammar.rules:
                    if rule.match(lexer, parent):
                        found  = True
                        null_t = None
                        break


                # if we never found a token, then we need to add another
                # character to the current null token (which we should
                # create if it isn't set).
                if not found:
                    if null_t is None:
                        null_t = Token('null', None, lexer.y, lexer.x, '', parent)
                        lexer.add_token(null_t)
                    #if len(lexer.lines[lexer.y]) > lexer.x:
                    if lexer.x < len(line):
                        #null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
                        null_t.add_to_string(line[lexer.x])
                        lexer.x += 1

            # ok, since we're soon going to be on a different line (or
            # already are), we want a new null token. so forget about the
            # current one (i.e. stop adding to it).
            null_t = None

            # if we're still on the same line at this point (and not done)
            # then that means we're finished with the line and should move
            # on to the next one here
            if not done and old_y == lexer.y:
                lexer.y += 1
                lexer.x = 0

        return True

class NocaseRegionRule(RegionRule):
    def _compile_start(self):
        return re.compile(self.start, re.IGNORECASE)
    def _compile_end(self, d):
        return re.compile(self.end % d, re.IGNORECASE)

class DualRegionRule(Rule):
    def __init__(self, name, start, grammar1, middle, grammar2, end, group=None):
        assert valid_name_re.match(name), 'invalid name %r' % name
        assert name not in reserved_names, "reserved rule name: %r" % name
        self.name     = name
        self.start    = start
        self.grammar1 = grammar1
        self.middle   = middle
        self.grammar2 = grammar2
        self.end      = end
        self.start_re = re.compile(start)
        self._set_group(group)
    def _add_from_regex(self, name, lexer, parent, m, matchd={}):
        s = m.group(0)
        token = self.make_token(lexer, s, name, parent, matchd)
        lexer.add_token(token)
        lexer.x += len(s)
        return token
    def resume(self, lexer, toresume):
        assert toresume, "can't resume without tokens to resume!"
        token = toresume[0]
        if token.name == 'start':
            t2 = self._match_first(lexer, token, toresume)
            if t2 is not None:
                t3 = self._match_second(lexer, t2, [])
            return True
        elif token.name == 'middle':
            t3 = self._match_second(lexer, token, toresume)
        else:
            raise Exception, "invalid flag %r" % flag
        return True
    def match(self, lexer, parent):
        # see if we can match our start token
        line = self._get_line(lexer)
        m = self.start_re.match(line, lexer.x)
        if m:
            t1 = self._add_from_regex('start', lexer, parent, m, m.groupdict())
            t2 = self._match_first(lexer, t1, [])
            if t2 is not None:
                t3 = self._match_second(lexer, t2, [])
            return True
        else:
            # region was not matched; we never started. so return false
            return False

    def _match_first(self, lexer, parent, toresume=[]):
        reenter = len(toresume) > 1
        if reenter:
            assert parent is toresume[0]
        d1 = parent.matchd
        assert parent.name == 'start'
        null_t = None
        middle_re = re.compile(self.middle % d1)
        d2 = {}

        # ok, so as long as we aren't done (we haven't found an end token),
        # keep reading input
        t2   = None
        done = False
        while not done and lexer.y < len(lexer.lines):
            old_y = lexer.y

            # if this line is empty, then we will skip it, but here we insert
            # an empty null token just so we have something
            #if len(lexer.lines[lexer.y]) == 0:
            #    null_t = Token('null', None, lexer.y, lexer.x, '', parent)
            #    lexer.add_token(null_t)
            #    null_t = None

            # ok, as long as we haven't found the end token, and have more
            # data on the current line to read, we will process tokens
            #while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]):
            while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1:
                # if we are reentering mid-parse, then that takes precedence
                if reenter:
                    raise Exception, "aw damn1"
                    #reenter  = False
                    #xrule    = rulecontext[0].rule
                    #xd       = rulecontext[0].matchd
                    #assert rule2.resume(lexer, xcontext, xd, rulecontext[1:]), \
                    #    "%r %r %r %r" % (lexer, xcontext, xd, rulecontext[1:])
                    #found  = True
                    #null_t = None
                    #break

                line = self._get_line(lexer)

                # see if we have found the middle token. if so, we can then
                # proceed to "stage 2"
                m2 = middle_re.match(line, lexer.x)
                if m2:
                    d2 = dict(d1.items() + m2.groupdict().items())
                    t2 = self._add_from_regex('middle', lexer, parent, m2, d2)
                    done = True
                    break

                # ok, we need to check all our rules now, in order. if we
                # find a token, note that we found one and exit the loop
                found = False
                for rule in self.grammar1.rules:
                    if rule.match(lexer, parent):
                        found = True
                        null_t = None
                        break

                # if we never found a token, then we need to add another
                # character to the current null token (which we should
                # create if it isn't set).
                if not found:
                    if null_t is None:
                        null_t = Token('null', None, lexer.y, lexer.x, '', parent)
                        lexer.add_token(null_t)
                    #null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
                    null_t.add_to_string(line[lexer.x])
                    lexer.x += 1

            # ok, since we're soon going to be on a different line (or
            # already are), we want a new null token. so forget about the
            # current one.
            null_t = None

            # if we're still on the same line at this point (and not done)
            # then that means we're finished with the line and should move
            # on to the next one here
            if not done and old_y == lexer.y:
                lexer.y += 1
                lexer.x = 0
        return t2

    def _match_second(self, lexer, parent, toresume=[]):
        reenter = len(toresume) > 1
        if reenter:
            assert parent is toresume[0]
        assert parent.name == 'middle'
        #assert parent.name == 'middle'
        d3 = parent.matchd
        null_t = None
        end_re = re.compile(self.end % d3)

        # ok, so as long as we aren't done (we haven't found an end token),
        # keep reading input
        t3   = None
        done = False
        while not done and lexer.y < len(lexer.lines):
            old_y = lexer.y

            # if we are reentering mid-parse, then that takes precedence
            if reenter:
                raise Exception, "aw damn2"
                #reenter  = False
                #xrule    = rulecontext[0].rule
                #xd       = rulecontext[0].matchd
                #assert rule2.resume(lexer, xcontext, xd, rulecontext[1:]), \
                #    "%r %r %r %r" % (lexer, xcontext, xd, rulecontext[1:])
                #found  = True
                #null_t = None
                #break

            # if this line is empty, then we will skip it, but here weinsert
            # an empty null token just so we have something
            #if len(lexer.lines[lexer.y]) == 0:
            #    null_t = Token('null', None, lexer.y, lexer.x, '', parent)
            #    lexer.add_token(null_t)
            #    null_t = None

            # ok, as long as we haven't found the end token, and have more
            # data on the current line to read, we will process tokens
            while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1:
                # see if we have found the middle token. if so, we can then
                # proceed to "stage 2"
                line = self._get_line(lexer)
                m3 = end_re.match(line, lexer.x)
                if m3:
                    t3 = self._add_from_regex('end', lexer, parent, m3, {})
                    done = True
                    break

                # ok, we need to check all our rules now, in order. if we
                # find a token, note that we found one and exit the loop
                found = False
                for rule in self.grammar2.rules:
                    if rule.match(lexer, parent):
                        found = True
                        null_t = None
                        break

                # if we never found a token, then we need to add another
                # character to the current null token (which we should
                # create if it isn't set).
                if not found:
                    if null_t is None:
                        null_t = Token('null', None, lexer.y, lexer.x, '', parent)
                        lexer.add_token(null_t)
                    #null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
                    null_t.add_to_string(line[lexer.x])
                    lexer.x += 1

            # ok, since we're soon going to be on a different line (or
            # already are), we want a new null token. so forget about the
            # current one.
            null_t = None

            # if we're still on the same line at this point (and not done)
            # then that means we're finished with the line and should move
            # on to the next one here
            if not done and old_y == lexer.y:
                lexer.y += 1
                lexer.x = 0

        # alright, we're finally done processing; return true
        return t3

class Grammar:
    rules = []
    def __init__(self):
        for rule in self.rules:
            if hasattr(rule, 'grammar') and rule.grammar is None:
                rule.grammar = self

grammars = {}
grammars['null'] = Grammar()
crash = False

def add(name, grammar):
    global crash, grammars
    if crash and name in grammars:
        raise Exception, "oh no! already have a grammar for %r" %name
    else:
        grammars[name] = grammar
            

def get(name):
    global crash, grammars
    try:
        return grammars[name]
    except KeyError:
        if crash:
            raise
        elif name == 'null':
            return Grammar()
        else:
            return get('null')

class Lexer:
    def __init__(self, name, grammar):
        self.name    = name
        self.grammar = grammar
        self.y       = 0
        self.x       = 0
        self.lines   = None
        self.tokens  = []

    def add_token(self, t):
        self.tokens.append(t)

    def lex(self, lines, y=0, x=0):
        self.y      = y
        self.x      = x
        self.lines  = lines
        self.tokens = []

    def resume(self, lines, y, x, token):
        #raise Exception, "%r %r" % (self, token) #XYZ
        self.y      = y
        self.x      = x
        self.lines  = lines
        self.tokens = []
        toresume = token.parents()
        if toresume:
            toresume[0].rule.resume(self, toresume)
        #else:
        #    raise Exception, "dammmmit"

    def __iter__(self):
        if self.lines is None:
            raise Exception, "no lines to lex"
        return self

    def next(self):
        null_t      = None

        if self.tokens:
            return self.tokens.pop(0)

        while self.y < len(self.lines):
            #line = self.lines[self.y] 
            line = self.lines[self.y] + '\n'
            while self.x < len(line):
                curr_t = None
                for rule in self.grammar.rules:
                    if rule.match(self, None):
                        assert self.tokens, "match rendered no tokens?"
                        return self.tokens.pop(0)
                if null_t is None:
                    null_t = Token('null', None, self.y, self.x, '')
                    self.add_token(null_t)
                #assert line[self.x] != '\n', "DAMN"
                #assert line[self.x] != '$', "DAMN"
                null_t.add_to_string(line[self.x])
                self.x += 1
            null_t = None
            self.y += 1
            self.x = 0

        if self.tokens:
            return self.tokens.pop(0)
        else:
            raise StopIteration