d6
/
pmacs3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

some fixes 

--HG--
branch : pmacs2
This commit is contained in:
moculus 2007-04-01 23:08:36 +00:00
parent ce6199eb58
commit 9be1839db9
2 changed files with 26 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
highlight2.py
View File

@ -44,3 +44,6 @@ class Highlighter:
self.lexer.lex(lines, y=0, x=0) self.lexer.lex(lines, y=0, x=0)
for token in self.lexer: for token in self.lexer:
self.tokens[token.y].append(token) self.tokens[token.y].append(token)
def update(self, lines):

43
lex2.py
View File

@ -4,7 +4,7 @@ valid_name_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
reserved_names = ['start', 'middle', 'end', 'null'] reserved_names = ['start', 'middle', 'end', 'null']
class Token(object): class Token(object):
def __init__(self, name, y, x, s, **vargs): def __init__(self, name, rule, y, x, s, **vargs):
self.name = name self.name = name
self.y = y self.y = y
self.x = x self.x = x
@ -17,7 +17,7 @@ class Token(object):
s = self.string s = self.string
else: else:
s = self.string[:10] + '...' s = self.string[:10] + '...'
return "<Token(%r, %d, %d, %r)>" % (self.name, self.y, self.x, s) return "<Token(%r, %r, %d, %d, %r)>" % (self.name, self.rule, self.y, self.x, s)
def render(self): def render(self):
return (self,) return (self,)
@ -26,7 +26,7 @@ class Rule:
def match(self, lexer, context=[], d={}): def match(self, lexer, context=[], d={}):
raise Exception, "%s rule cannot match!" % self.name raise Exception, "%s rule cannot match!" % self.name
def make_token(self, lexer, s, name, **vargs): def make_token(self, lexer, s, name, **vargs):
return Token(name, lexer.y, lexer.x, s, **vargs) return Token(name, self, lexer.y, lexer.x, s, **vargs)
class ConstantRule(Rule): class ConstantRule(Rule):
def __init__(self, name, constant): def __init__(self, name, constant):
@ -34,10 +34,11 @@ class ConstantRule(Rule):
assert name not in reserved_names, "reserved rule name: %r" % name assert name not in reserved_names, "reserved rule name: %r" % name
self.name = name self.name = name
self.constant = constant self.constant = constant
def match(self, lexer, context=[], d={}): def match(self, lexer, context=[], d={}, parent=None):
if lexer.lines[lexer.y][lexer.x:].startswith(self.constant): if lexer.lines[lexer.y][lexer.x:].startswith(self.constant):
name = '.'.join(context + [self.name]) name = '.'.join(context + [self.name])
lexer.add_token(self.make_token(lexer, self.constant, name)) lexer.add_token(self.make_token(lexer, self.constant, name,
parent=parent))
lexer.x += len(self.constant) lexer.x += len(self.constant)
return True return True
else: else:
@ -50,11 +51,12 @@ class PatternRule(Rule):
self.name = name self.name = name
self.pattern = pattern self.pattern = pattern
self.re = re.compile(pattern) self.re = re.compile(pattern)
def match(self, lexer, context=[], d={}): def match(self, lexer, context=[], d={}, parent=None):
m = self.re.match(lexer.lines[lexer.y], lexer.x) m = self.re.match(lexer.lines[lexer.y], lexer.x)
if m: if m:
name = '.'.join(context + [self.name]) name = '.'.join(context + [self.name])
lexer.add_token(self.make_token(lexer, m.group(0), name)) lexer.add_token(self.make_token(lexer, m.group(0), name,
parent=parent))
lexer.x += len(m.group(0)) lexer.x += len(m.group(0))
return True return True
else: else:
@ -68,7 +70,7 @@ class ContextPatternRule(Rule):
self.pattern = pattern self.pattern = pattern
self.fallback = fallback self.fallback = fallback
self.fallback_re = re.compile(fallback) self.fallback_re = re.compile(fallback)
def match(self, lexer, context=[], d={}): def match(self, lexer, context=[], d={}, parent=None):
try: try:
r = re.compile(self.pattern % d) r = re.compile(self.pattern % d)
except KeyError: except KeyError:
@ -76,7 +78,8 @@ class ContextPatternRule(Rule):
m = r.match(lexer.lines[lexer.y], lexer.x) m = r.match(lexer.lines[lexer.y], lexer.x)
if m: if m:
name = '.'.join(context + [self.name]) name = '.'.join(context + [self.name])
lexer.add_token(self.make_token(lexer, m.group(0), name)) lexer.add_token(self.make_token(lexer, m.group(0), name,
parent=parent))
lexer.x += len(m.group(0)) lexer.x += len(m.group(0))
return True return True
else: else:
@ -91,9 +94,9 @@ class RegionRule(Rule):
self.grammar = grammar self.grammar = grammar
self.end = end self.end = end
self.start_re = re.compile(start) self.start_re = re.compile(start)
def _add_from_regex(self, context, name, lexer, m): def _add_from_regex(self, context, name, lexer, m, parent=None):
t_name = '.'.join(context + [self.name, name]) t_name = '.'.join(context + [self.name, name])
t = self.make_token(lexer, m.group(0), t_name) t = self.make_token(lexer, m.group(0), t_name, parent=parent)
lexer.add_token(t) lexer.add_token(t)
lexer.x += len(m.group(0)) lexer.x += len(m.group(0))
def match(self, lexer, context=[], d={}): def match(self, lexer, context=[], d={}):
@ -120,7 +123,7 @@ class RegionRule(Rule):
# if this line is empty, then we will skip it, but here weinsert # if this line is empty, then we will skip it, but here weinsert
# an empty null token just so we have something # an empty null token just so we have something
if len(lexer.lines[lexer.y]) == 0: if len(lexer.lines[lexer.y]) == 0:
null_t = Token(null_t_name, lexer.y, lexer.x, '') null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
lexer.add_token(null_t) lexer.add_token(null_t)
null_t = None null_t = None
@ -150,7 +153,7 @@ class RegionRule(Rule):
# create if it isn't set). # create if it isn't set).
if not found: if not found:
if null_t is None: if null_t is None:
null_t = Token(null_t_name, lexer.y, lexer.x, '') null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
lexer.add_token(null_t) lexer.add_token(null_t)
null_t.add_to_string(lexer.lines[lexer.y][lexer.x]) null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
lexer.x += 1 lexer.x += 1
@ -184,9 +187,9 @@ class DualRegionRule(Rule):
self.grammar2 = grammar2 self.grammar2 = grammar2
self.end = end self.end = end
self.start_re = re.compile(start) self.start_re = re.compile(start)
def _add_from_regex(self, context, name, lexer, m): def _add_from_regex(self, context, name, lexer, m, parent=None):
t_name = '.'.join(context + [self.name, name]) t_name = '.'.join(context + [self.name, name])
t = self.make_token(lexer, m.group(0), t_name) t = self.make_token(lexer, m.group(0), t_name, parent=parent)
lexer.add_token(t) lexer.add_token(t)
lexer.x += len(m.group(0)) lexer.x += len(m.group(0))
def match(self, lexer, context=[], d={}): def match(self, lexer, context=[], d={}):
@ -211,7 +214,7 @@ class DualRegionRule(Rule):
# if this line is empty, then we will skip it, but here weinsert # if this line is empty, then we will skip it, but here weinsert
# an empty null token just so we have something # an empty null token just so we have something
if len(lexer.lines[lexer.y]) == 0: if len(lexer.lines[lexer.y]) == 0:
null_t = Token(null_t_name, lexer.y, lexer.x, '') null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
lexer.add_token(null_t) lexer.add_token(null_t)
null_t = None null_t = None
@ -241,7 +244,7 @@ class DualRegionRule(Rule):
# create if it isn't set). # create if it isn't set).
if not found: if not found:
if null_t is None: if null_t is None:
null_t = Token(null_t_name, lexer.y, lexer.x, '') null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
lexer.add_token(null_t) lexer.add_token(null_t)
null_t.add_to_string(lexer.lines[lexer.y][lexer.x]) null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
lexer.x += 1 lexer.x += 1
@ -271,7 +274,7 @@ class DualRegionRule(Rule):
# if this line is empty, then we will skip it, but here weinsert # if this line is empty, then we will skip it, but here weinsert
# an empty null token just so we have something # an empty null token just so we have something
if len(lexer.lines[lexer.y]) == 0: if len(lexer.lines[lexer.y]) == 0:
null_t = Token(null_t_name, lexer.y, lexer.x, '') null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
lexer.add_token(null_t) lexer.add_token(null_t)
null_t = None null_t = None
@ -300,7 +303,7 @@ class DualRegionRule(Rule):
# create if it isn't set). # create if it isn't set).
if not found: if not found:
if null_t is None: if null_t is None:
null_t = Token(null_t_name, lexer.y, lexer.x, '') null_t = Token(null_t_name, None, lexer.y, lexer.x, '')
lexer.add_token(null_t) lexer.add_token(null_t)
null_t.add_to_string(lexer.lines[lexer.y][lexer.x]) null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
lexer.x += 1 lexer.x += 1
@ -366,7 +369,7 @@ class Lexer:
assert self.tokens, "AAAAA %s" % repr(self.tokens) assert self.tokens, "AAAAA %s" % repr(self.tokens)
return self.tokens.pop(0) return self.tokens.pop(0)
if null_t is None: if null_t is None:
null_t = Token(null_t_name, self.y, self.x, '') null_t = Token(null_t_name, None, self.y, self.x, '')
self.add_token(null_t) self.add_token(null_t)
null_t.add_to_string(line[self.x]) null_t.add_to_string(line[self.x])
self.x += 1 self.x += 1