YES

--HG--
branch : pmacs2

lex2.py

@@ -1,12 +1,15 @@
 import re
 
-class Token:
+valid_name_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
-    def __init__(self, rule, y, x, s, role='single'):
+reserved_names = ['start', 'null', 'end']
-        self.rule = rule
+
+class Token(object):
+    def __init__(self, name, y, x, s, **vargs):
+        self.name = name
         self.y = y
         self.x = x
         self.string = s
-        self.role = role
+        self.vargs = vargs
     def add_to_string(self, s):
         self.string += s
     def __repr__(self):
@@ -14,203 +17,394 @@ class Token:
             s = self.string
         else:
             s = self.string[:10] + '...'
-        return "<Token(%r, %d, %d, %r)>" % (self.rule, self.y, self.x, s)
+        return "<Token(%r, %d, %d, %r)>" % (self.name, self.y, self.x, s)
+    def render(self):
+        return (self,)
 
 class Rule:
-    def __init__(self):
+    name = 'abstract'
-        self.name = 'null'
+    def match(self, lexer, context=[]):
-    def add_token(self, lexer, s, role='single'):
+        raise Exception, "%s rule cannot match!" % self.name
-        t = Token(self, lexer.y, lexer.x, s, role)
+    def make_token(self, lexer, s, name, **vargs):
-        lexer.curr_tokens.append(t)
+        return Token(name, lexer.y, lexer.x, s, **vargs)
-        lexer.x += len(s)
-    def add_to_last_token(self, lexer, s):
-        assert lexer.curr_tokens
-        lexer.curr_tokens[-1].add_to_string(s)
-        lexer.x += len(s)
-    def match(self):
-        raise Exception, "not implemented"
-
-class NullRule(Rule):
-    def __init__(self):
-        self.name = 'null'
-    def match(self):
-        raise Exception, "null rule does not match!"
-
-class NewlineRule(Rule):
-    def __init__(self):
-        self.name = 'newline'
-    def match(self):
-        raise Exception, "newline rule does not match!"
 
 class ConstantRule(Rule):
-    def __init__(self, name="unnamed_constant", const="foo"):
+    def __init__(self, name, constant):
+        assert valid_name_re.match(name), 'invalid name %r' % name
+        assert name not in reserved_names, "reserved rule name: %r" % name
         self.name  = name
-        self.const = const
+        self.constant = constant
-    def match(self, lexer):
+    def match(self, lexer, context=[]):
-        if lexer.lines[lexer.y][lexer.x:].startswith(self.const):
+        if lexer.lines[lexer.y][lexer.x:].startswith(self.constant):
-            self.add_token(lexer, self.const)
+            lexer.add_token(self.make_token(lexer, self.constant, self.name))
+            lexer.x += len(self.constant)
             return True
         else:
             return False
 
-class RegexRule(Rule):
+class PatternRule(Rule):
-    def __init__(self, name="unnamed_regex", expr="[^ ]+"):
+    def __init__(self, name, pattern):
+        assert valid_name_re.match(name), 'invalid name %r' % name
+        assert name not in reserved_names, "reserved rule name: %r" % name
         self.name    = name
-        self.expr = expr
+        self.pattern = pattern
-        self.re = re.compile(expr)
+        self.re      = re.compile(pattern)
-    def match(self, lexer):
+    def match(self, lexer, context=[]):
         m = self.re.match(lexer.lines[lexer.y], lexer.x)
         if m:
-            self.add_token(lexer, m.group(0))
+            lexer.add_token(self.make_token(lexer, m.group(0), self.name))
+            lexer.x += len(m.group(0))
             return True
         else:
             return False
 
 class RegionRule(Rule):
-    def __init__(self, name, start, mid, end):
+    def __init__(self, name, start, grammar, end):
+        assert valid_name_re.match(name), 'invalid name %r' % name
+        assert name not in reserved_names, "reserved rule name: %r" % name
         self.name     = name
+        self.start    = start
+        self.grammar  = grammar
+        self.end      = end
         self.start_re = re.compile(start)
-        self.mid_re = re.compile(mid)
+    def _add_from_regex(self, context, name, lexer, m):
-        self.end_re = re.compile(end)
+        t_name = '.'.join(context + [self.name, name])
-    def match(self, lexer):
+        t      = self.make_token(lexer, m.group(0), t_name)
-        lt = lexer.last_token
+        lexer.add_token(t)
-        l = lexer.lines[lexer.y]
+        lexer.x += len(m.group(0))
-        if lt is not None and lt.rule.name == self.name and lt.role != 'end':
+    def match(self, lexer, context=[]):
-            saw_mid = False
+        m = self.start_re.match(lexer.lines[lexer.y], lexer.x)
-            while lexer.x < len(l):
-                m_end = self.end_re.match(l, lexer.x)
-                if m_end:
-                    self.add_token(lexer, m_end.group(0), 'end')
-                    return True
-                m_mid = self.mid_re.match(l, lexer.x)
-                if m_mid:
-                    s = m_mid.group(0)
-                else:
-                    s = l[lexer.x]
-                if saw_mid:
-                    self.add_to_last_token(lexer, s)
-                else:
-                    self.add_token(lexer, s, 'mid')
-                    saw_mid = True
-            return True
-        else:
-            m = self.start_re.match(l, lexer.x)
         if m:
-                self.add_token(lexer, m.group(0), 'start')
+            self._add_from_regex(context, 'start', lexer, m)
+
+            null_t_name = '.'.join(context + [self.name, 'null'])
+            null_t = None
+
+            if self.end:
+                end_re = re.compile(self.end % m.groupdict())
+
+            done = False
+            while not done and lexer.y < len(lexer.lines):
+                line = lexer.lines[lexer.y]
+                if len(line) == 0:
+                    null_t = Token(null_t_name, lexer.y, lexer.x, '')                    
+                    lexer.add_token(null_t)
+                while not done and lexer.x < len(line):
+                    if self.end:
+                        m = end_re.match(line, lexer.x)
+                        if m:
+                            self._add_from_regex(context, 'end', lexer, m)
+                            done = True
+                            continue
+
+                    found = False
+                    for rule in self.grammar.rules:
+                        if rule.match(lexer, context + [self.name]):
+                            found = True
+                            null_t = None
+                            break
+                    if not found:
+                        if null_t is None:
+                            null_t = Token(null_t_name, lexer.y, lexer.x, '')
+                            lexer.add_token(null_t)
+                        null_t.add_to_string(line[lexer.x])
+                        lexer.x += 1
+                    
+                null_t = None
+                lexer.y += 1
+                lexer.x = 0
             return True
         else:
             return False
 
-class DynamicRegionRule(Rule):
+class Grammar:
-    def __init__(self, name, start, mid, end_fmt):
+    rules = []
-        self.name = name
+
-        self.start_re = re.compile(start)
+class NullGrammar(Grammar):
-        self.mid_re = re.compile(mid)
+    pass
-        self.end_fmt = end_fmt
+
-    def add_token(self, lexer, s, role, end_re):
+class PodGrammar(Grammar):
-        t = Token(self, lexer.y, lexer.x, s, role)
+    pass
-        t.end_re = end_re
+
-        lexer.curr_tokens.append(t)
+class StringGrammar(Grammar):
-        lexer.x += len(s)
+    rules = [
-    def match(self, lexer):
+        PatternRule('escaped', '\\.'),
-        lt = lexer.last_token
+    ]
-        l = lexer.lines[lexer.y]
+
-        if lt is not None and lt.rule.name == self.name and lt.role != 'end':
+class TestGrammar(Grammar):
-            saw_mid = False
+    rules = [
-            while lexer.x < len(l):
+        RegionRule(
-                m_end = self.end_re.match(l, lexer.x)
+            name='heredoc',
-                if m_end:
+            start="<< *(?P<heredoc>[a-zA-Z0-9_]+) *;",
-                    self.add_token(lexer, m_end.group(0), 'end', None)
+            grammar=StringGrammar(),
-                    return True
+            end='^%(heredoc)s$',
-                m_mid = self.mid_re.match(l, lexer.x)
+        ),
-                if m_mid:
+        RegionRule(
-                    s = m_mid.group(0)
+            name='string1',
-                else:
+            start='"',
-                    s = l[lexer.x]
+            grammar=StringGrammar(),
-                if saw_mid:
+            end='"',
-                    self.add_to_last_token(lexer, s)
+        ),
-                else:
+        RegionRule(
-                    self.add_token(lexer, s, 'mid', lt.end_re)
+            name='string2',
-                    saw_mid = True
+            start="'",
-            return True
+            grammar=StringGrammar(),
-        else:
+            end="'",
-            m = self.start_re.match(l, lexer.x)
+        ),
-            if m:
+        PatternRule(
-                end_re = re.compile(self.end_fmt % m.groups())
+            name='word',
-                self.add_token(lexer, m.group(0), 'start', end_re)
+            pattern='[^ \t\n]+',
-                return True
+        ),
-            else:
+    ]
-                return False
+
+class PerlGrammar(Grammar):
+    rules = [
+        RegionRule(
+            name='heredoc',
+            start="<< *(?P<heredoc>[a-zA-Z0-9_]+) *;",
+            grammar=StringGrammar(),
+            end='^%(heredoc)s$',
+        ),
+        RegionRule(
+            name='endblock',
+            start="^__END__|__DATA__ *$",
+            grammar=NullGrammar(),
+            end='',
+        ),
+        RegionRule(
+            name='pod',
+            start='^=[a-zA-Z0-9_]+',
+            grammar=PodGrammar(),
+            end='^=cut',
+        ),
+        PatternRule(
+            name='comment',
+            pattern='#.*$',
+        ),
+        RegionRule(
+            name='string1',
+            start='"',
+            grammar=StringGrammar(),
+            end='"',
+        ),
+        RegionRule(
+            name='string2',
+            start="'",
+            grammar=StringGrammar(),
+            end="'",
+        ),
+        RegionRule(
+            name='evalstring',
+            start="`",
+            grammar=StringGrammar(),
+            end="`",
+        ),
+        PatternRule(
+            name='number',
+            pattern='0?\.[0-9]+|[0-9]+(?:\.[0-9]+)?',
+        ),
+        PatternRule(
+            name='keyword',
+            pattern="(?<!->)(?:STDIN|STDERR|STDOUT|and|cmp|continue|do|else|elsif|eq|eval|foreach|for|if|last|my|next|ne|not|or|our|package|require|return|sub|undef|unless|until|use|while)(?![a-zA-Z0-9_])",
+        ),
+        PatternRule(
+            name='hash_bareword_index',
+            pattern='(?<={) *[A-Za-z0-9_]+(?=})',
+        ),
+        PatternRule(
+            name='literal_hash_bareword_index',
+            pattern='[A-Za-z0-9_]+(?= *=>)',
+        ),
+        PatternRule(
+            name='length_scalar',
+            pattern=r"\$#[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*",
+        ),
+        PatternRule(
+            name='system_scalar',
+            pattern=r"\$[][><ab/'\"_@\?#\$!%^|&*()](?![A-Za-z0-9_])",
+        ),
+        PatternRule(
+            name='system_array',
+            pattern="@_",
+        ),
+        PatternRule(
+            name='scalar',
+            pattern="\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*",
+        ),
+        PatternRule(
+            name='array',
+            pattern="@\$*[A-Za-z_](?:[A-Za-z0-9_]|::)*",
+        ),
+        PatternRule(
+            name='hash',
+            pattern="%\$*[A-Za-z_](?:[A-Za-z0-9_]|::)*",
+        ),
+        PatternRule(
+            name='dereference',
+            pattern="[@%\$&\*](?={)",
+        ),
+        RegionRule(
+            name='quoted_region1',
+            start='q[rqwx]? *(?P<delim>[^ #])',
+            grammar=Grammar(),
+            end='%(delim)s',
+        ),
+        RegionRule(
+            name='quoted_region2',
+            start='q[rqwx]?#',
+            grammar=Grammar(),
+            end='#',
+        ),
+        RegionRule(
+            name='bracket_quoted_region1',
+            start='q[rqwx]? *\(',
+            grammar=Grammar(),
+            end='\)',
+        ),
+        RegionRule(
+            name='bracket_quoted_region2',
+            start='q[rqwx]? *{',
+            grammar=Grammar(),
+            end='}',
+        ),
+        RegionRule(
+            name='bracket_quoted_region3',
+            start='q[rqwx]? *<',
+            grammar=Grammar(),
+            end='>',
+        ),
+        RegionRule(
+            name='bracket_quoted_region4',
+            start='q[rqwx]? *\[',
+            grammar=Grammar(),
+            end='\]',
+        ),
+        RegionRule(
+            name='implicit_match_regex',
+            start='(?:(?<==~)|(?<=!~)|(?<=\()) */',
+            grammar=StringGrammar(),
+            end='/',
+        ),
+        RegionRule(
+            name='explicit_match_regex1',
+            start='m *(?P<delim>[^ #])',
+            grammar=StringGrammar(),
+            end='%(delim)s',
+        ),
+        RegionRule(
+            name='explicit_match_regex1',
+            start='m#',
+            grammar=StringGrammar(),
+            end='#',
+        ),
+        # we officially don't support the bullshit s{a}{b} thing perl has going.
+        # those guys are on crack. we only support things like s#a#b# or s/a/b/.
+        # same comments as above apply
+        #{'name': 'replace regex',
+        # 'expr': r"""(?:y|tr|s)([^<[{(A-Za-z0-9 \t\n])(?:\\.|[^\\])*?\1(?:\\.|[^\\])*?\1[a-z]*""",
+        # 'action': lex.make_token},
+
+        PatternRule(
+            name='package',
+            pattern="(?<=package )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*",
+        ),
+        PatternRule(
+            name='sub',
+            pattern="(?<=sub )[a-zA-Z_][a-zA-Z_0-9]*(?=[ \n]*{)",
+        ),
+        PatternRule(
+            name='use',
+            pattern="(?<=use )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*",
+        ),
+        PatternRule(
+            name='label',
+            pattern='[a-zA-Z_][a-zA-Z0-9_]*:',
+        ),
+        PatternRule(
+            name='instance_method',
+            pattern="(?<=->)[a-zA-Z_][a-zA-Z_0-9]*",
+        ),
+        PatternRule(
+            name='static_method',
+            pattern="&?(?:[a-zA-Z_][a-zA-Z_0-9]*::)+[a-zA-Z_][a-zA-Z_0-9]*",
+        ),
+        PatternRule(
+            name='builtin_method',
+            pattern="(?<!->)&?(?:write|warn|wantarray|waitpid|wait|vec|values|utime|use|untie|unshift|unpack|unlink|undef|umask|ucfirst|uc|truncate|times|time|tied|tie|telldir|tell|syswrite|system|sysseek|sysread|sysopen|syscall|symlink|substr|sub|study|stat|srand|sqrt|sprintf|split|splice|sort|socketpair|socket|sleep|sin|shutdown|shmwrite|shmread|shmget|shmctl|shift|setsockopt|setservent|setpwent|setprotoent|setpriority|setpgrp|setnetent|sethostent|setgrent|send|semop|semget|semctl|select|seekdir|seek|scalar|rmdir|rindex|rewinddir|reverse|return|reset|require|rename|ref|redo|recv|readpipe|readlink|readline|readdir|read|rand|quotemeta|push|prototype|printf|print|pos|pop|pipe|package|pack|our|ord|opendir|open|oct|no|next|my|msgsnd|msgrcv|msgget|msgctl|mkdir|map|lstat|log|lock|localtime|local|listen|link|length|lcfirst|lc|last|kill|keys|join|ioctl|int|index|import|hex|grep|goto|gmtime|glob|getsockopt|getsockname|getservent|getservbyport|getservbyname|getpwuid|getpwnam|getpwent|getprotoent|getprotobynumber|getprotobyname|getpriority|getppid|getpgrp|getpeername|getnetent|getnetbyname|getnetbyaddr|getlogin|gethostent|gethostbyname|gethostbyaddr|getgrnam|getgrgid|getgrent|getc|formline|format|fork|flock|fileno|fcntl|exp|exit|exists|exec|eval|eof|endservent|endpwent|endprotoent|endnetent|endhostent|endgrent|each|dump|do|die|delete|defined|dbmopen|dbmclose|crypt|cos|continue|connect|closedir|close|chroot|chr|chown|chop|chomp|chmod|chdir|caller|bless|binmode|bind|atan2|alarm|accept|abs)(?![a-zA-Z0-9_])",
+        ),
+        PatternRule(
+            name='method',
+            pattern="&(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*",
+        ),
+        PatternRule(
+            name='ref_method',
+            pattern="&\$(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*",
+        ),
+        PatternRule(
+            name='bareword_method',
+            pattern="(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]* *\(",
+        ),
+        PatternRule(
+            name='delimiter',
+            pattern="\(|\)|\[|\]|{|}|,|;|->|=>|=|\?|(?<!:):(?!=:)",
+        ),
+        PatternRule(
+            name='unary_operator',
+            pattern="\+=|-=|\*=|/=|//=|%=|&=\|\^=|>>=|<<=|\*\*=",
+        ),
+        PatternRule(
+            name='operator',
+            pattern="\+|<=>|<>|<<|<=|<|-|>>|>=|>|\*\*|&|\*|\||/|\^|==|//|~|=~|!~|!=|%|!|\.",
+        ),
+        PatternRule(
+            name='bareword',
+            pattern='(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*',
+        ),
+    ]
 
 class Lexer:
-    rules = [
+    def __init__(self, name, grammar):
-        RegionRule('heredoc', "<< *([a-zA-Z0-9_]+) *;", '.', '^%s$'),
+        self.name        = name
-        RegionRule('string1', '"', '\\.|.', '"'),
+        self.grammar     = grammar
-        RegexRule('word'),
-    ]
-    null = NullRule()
-    newline = NewlineRule()
-    def __init__(self):
-        self.lines = None
         self.y           = 0
         self.x           = 0
-        self.last_token = None
+        self.lines       = None
-        self.curr_tokens = []
+        self.tokens = []
 
-    def lex(self, lines, y=0, x=0, last_token=None, next_token=None):
+    def add_token(self, t):
-        self.lines = lines
+        self.tokens.append(t)
+
+    def lex(self, lines, y=0, x=0):
         self.y           = y
         self.x           = x
-        self.last_token = None
+        self.lines       = lines
-        self.curr_tokens = []
+        self.tokens = []
 
     def __iter__(self):
         if self.lines is None:
             raise Exception, "no lines to lex"
         return self
 
-    def match(self):
-        for rule in self.rules:
-            match = rule.match(self)
-            if match:
-                assert self.curr_tokens
-                return True
-        return False
-
-    def add_to_null_token(self):
-        c = self.lines[self.y][self.x]
-        if self.curr_tokens:
-            assert self.curr_tokens[0].rule.name == 'null', self.curr_tokens[0].rule.name
-            self.curr_tokens[0].add_to_string(c)
-        else:
-            self.curr_tokens.append(self.make_null_token(c))
-        self.x += 1
-
-    def make_null_token(self, c):
-        return Token(self.null, self.y, self.x, c)
-    def make_newline_token(self):
-        return Token(self.newline, self.y, self.x, '\n')
-
-    def pop_curr_token(self):
-        t = self.curr_tokens.pop(0)
-        self.last_token = t
-        return t
-
     def next(self):
-        if self.curr_tokens:
+        null_t_name = 'null'
-            return self.pop_curr_token()
+        null_t      = None
 
         while self.y < len(self.lines):
-            while self.x < len(self.lines[self.y]):
+            line = self.lines[self.y]
-                t = self.match()
+            while self.x < len(line):
-                if t:
+                for rule in self.grammar.rules:
-                    return self.pop_curr_token()
+                    if rule.match(self):
-                else:
+                        assert self.tokens, "AAAAA %s" % repr(self.tokens)
-                    self.add_to_null_token()
+                        return self.tokens.pop(0)
+                if null_t is None:
+                    null_t = Token(null_t_name, self.y, self.x, '')
+                    self.add_token(null_t)
+                null_t.add_to_string(line[self.x])
+                self.x += 1
             self.y += 1
             self.x = 0
-            #self.curr_tokens.append(self.make_newline_token())
-            if self.curr_tokens:
-                return self.pop_curr_token()
 
+        if self.tokens:
+            return self.tokens.pop(0)
+        else:
             raise StopIteration

test.py

@@ -9,7 +9,8 @@ lines = [
     '"this is a double-quoted string" and this is not...',
     "we're done",
 ]
-l = lex2.Lexer()
+g = lex2.TestGrammar()
+l = lex2.Lexer('lexer', g)
 l.lex(lines)
 for t in l:
-    print '%-20r %r' % (t.rule.name, t.string)
+    print '%-20r %r' % (t.name, t.string)

test2.py

@@ -0,0 +1,18 @@
+import sys
+import lex2
+
+paths = sys.argv[1:]
+for path in paths:
+    f = open(path, 'r')
+    data = f.read()
+    f.close()
+
+    lines = data.split('\n')
+
+    grammar = lex2.PerlGrammar()
+    lexer = lex2.Lexer('lexer', grammar)
+
+    lexer.lex(lines)
+    print path
+    for token in lexer:
+        print '%-20s| %s' % (token.name, token.string)