From 8d33c8b0e2a177469b501a878c7b83b50f464a3c Mon Sep 17 00:00:00 2001 From: moculus Date: Wed, 28 Mar 2007 22:38:32 +0000 Subject: [PATCH] oh no --HG-- branch : pmacs2 --- highlight2.py | 24 ++++++++++++ lex2.py | 15 +++++--- lex2_perl.py | 14 ++++--- test3.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 highlight2.py create mode 100644 test3.py diff --git a/highlight2.py b/highlight2.py new file mode 100644 index 0000000..f2a3966 --- /dev/null +++ b/highlight2.py @@ -0,0 +1,24 @@ +class Highlighter: + def __init__(self, lexer): + self.lexer = lexer + self.tokens = [] + + def display(self): + for group in self.tokens: + for token in group: + sys.stdout.write(token.string) + sys.stdout.write('\n') + sys.stdout.write('\n') + + def highlight(self, lines): + self.tokens = [[] for l in lines] + self.lexer.lex(lines, y=0, x=0) + y = 0 + for token in self.lexer: + assert y <= token.y, "oh no %d %d" % (y, token.y) + assert token.y < len(lines), "uh oh %d %d" % (token.y, len(lines)) + self.tokens[token.y].append(token) + y = token.y + + def update(self): + pass diff --git a/lex2.py b/lex2.py index 593c742..9053148 100755 --- a/lex2.py +++ b/lex2.py @@ -111,10 +111,11 @@ class RegionRule(Rule): lexer.add_token(null_t) null_t.add_to_string(line[lexer.x]) lexer.x += 1 - + null_t = None - lexer.y += 1 - lexer.x = 0 + if not done: + lexer.y += 1 + lexer.x = 0 return True else: return False @@ -211,8 +212,9 @@ class DualRegionRule(Rule): lexer.x += 1 null_t = None - lexer.y += 1 - lexer.x = 0 + if not done: + lexer.y += 1 + lexer.x = 0 return True else: return False @@ -250,9 +252,12 @@ class Lexer: while self.y < len(self.lines): line = self.lines[self.y] while self.x < len(line): + curr_t = None + #print 'Checking(%d) %r' % (self.x, line[self.x:]) for rule in self.grammar.rules: if rule.match(self): assert self.tokens, "AAAAA %s" % repr(self.tokens) + #print 'Returning(%d)' % self.x return self.tokens.pop(0) if null_t is None: null_t = Token(null_t_name, self.y, self.x, '') diff --git a/lex2_perl.py b/lex2_perl.py index b2983b0..aaa15ca 100755 --- a/lex2_perl.py +++ b/lex2_perl.py @@ -31,9 +31,13 @@ class StringGrammar(Grammar): pattern=r'\\.', ), PatternRule( - name=r'hash_bareword_index', - pattern=r'(?<={) *[A-Za-z0-9_]+(?=})', + name=r'hash_deref', + pattern=r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*(?:->{(?:[a-zA-Z_][a-zA-Z_0-9]*|'(?:\\.|[^'\\])*'|\"(\\.|[^\\\"])*\")})+", ), + #PatternRule( + # name=r'hash_bareword_index', + # pattern=r'(?<={) *[A-Za-z0-9_]+(?=})', + #), PatternRule( name=r'length_scalar', pattern=r"\$#[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*", @@ -93,7 +97,7 @@ class PerlGrammar(Grammar): RegionRule( name=r'string2', start=r"'", - grammar=StringGrammar(), + grammar=Grammar(), end=r"'", ), RegionRule( @@ -190,7 +194,7 @@ class PerlGrammar(Grammar): ), RegionRule( name=r'explicit_match_regex1', - start=r'm *(?P[^ #])', + start=r'm *(?P[^ #a-zA-Z0-9_])', grammar=StringGrammar(), end=r'%(delim)s', ), @@ -230,7 +234,7 @@ class PerlGrammar(Grammar): ), PatternRule( name=r'label', - pattern=r'[a-zA-Z_][a-zA-Z0-9_]*:', + pattern=r'[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', ), PatternRule( name=r'instance_method', diff --git a/test3.py b/test3.py new file mode 100644 index 0000000..6fb4fcb --- /dev/null +++ b/test3.py @@ -0,0 +1,101 @@ +import sys +import lex2, lex2_perl + +color_list = [] +color_list.extend(['\033[3%dm' % x for x in range(0, 8)]) +color_list.extend(['\033[3%d;1m' % x for x in range(0, 8)]) +color_list.append('\033[0m') + +color_names = [ + 'black', 'dred', 'dgreen', 'brown', 'dblue', 'dpurple', 'dcyan', 'lgrey', + 'dgrey', 'lred', 'lgreen', 'yellow', 'lblue', 'lpurple', 'lcyan', 'white', + 'unset', +] + +color_dict ={} +for i in range(0, len(color_list)): + color_dict[color_names[i]] = color_list[i] + +token_colors = { + 'null': 'white', + 'delimiter': 'white', + 'pod.start': 'lred', + 'pod.null': 'lred', + 'pod.end': 'lred', + 'pod.header': 'lpurple', + 'sub': 'lcyan', + 'number': 'white', + 'operator': 'white', + 'heredoc': 'lgreen', + 'endblock': 'lred', + 'pod': 'lred', + 'comment': 'lred', + 'string1': 'lgreen', + 'string1.start': 'lgreen', + 'string1.null': 'lgreen', + 'string1.escaped': 'lpurple', + 'string1.scalar': 'yellow', + 'string1.system_scalar': 'yellow', + 'string1.hash_deref': 'yellow', + 'string1.hash_bareword_index': 'lgreen', + 'string1.end': 'lgreen', + 'string2': 'lgreen', + 'string2.start': 'lgreen', + 'string2.null': 'lgreen', + 'string2.end': 'lgreen', + 'evalstring': 'lcyan', + 'default_string': 'lgreen', + 'keyword': 'lpurple', + 'length_scalar': 'yellow', + 'system_scalar': 'yellow', + 'system_array': 'yellow', + 'scalar': 'yellow', + 'dereference': 'yellow', + 'array': 'yellow', + 'hash': 'yellow', + 'hash_bareword_index': 'lgreen', + 'quoted_region': 'lcyan', + 'match_regex': 'lcyan', + 'replace_regex': 'lcyan', + 'literal_hash_bareword_index': 'lgreen', + 'interpolated_scalar': 'yellow', + 'interpolated_system_scalar': 'yellow', + 'interpolated_array': 'yellow', + 'interpolated_system_array': 'yellow', + 'interpolated_hash': 'yellow', + 'label': 'lcyan', + 'package': 'lcyan', + 'use': 'lcyan', + 'method': 'lcyan', + 'methodref': 'lcyan', + 'method_declaration': 'lcyan', + 'instance_method': 'lcyan', + 'static_method': 'lcyan', + 'builtin_method': 'lpurple', + 'bareword_method': 'lcyan', + 'bareword': 'yellow', + 'bizzaro': 'lpurple', +} + +paths = sys.argv[1:] +for path in paths: + f = open(path, 'r') + data = f.read() + f.close() + + lines = data.split('\n') + + grammar = lex2_perl.PerlGrammar() + lexer = lex2.Lexer('lexer', grammar) + + lexer.lex(lines) + y = 0 + for token in lexer: + while token.y > y: + sys.stdout.write('\n') + y += 1 + #color_name = token_colors.get(token.name, 'white') + color_name = token_colors[token.name] + sys.stdout.write(color_dict[color_name]) + sys.stdout.write(token.string) + sys.stdout.write('\n') \ No newline at end of file