d6
/
pmacs3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed bugs in perl and lex3 

--HG--
branch : pmacs2
This commit is contained in:
moculus 2007-07-15 14:26:28 +00:00
parent 01c101d1ad
commit 7d2f7d5a6c
2 changed files with 45 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
lex3.py
View File

@ -91,7 +91,7 @@ class PatternRule(Rule):
return self.re.match(self.get_line(lexer), lexer.x) return self.re.match(self.get_line(lexer), lexer.x)
def lex(self, lexer, parent, m): def lex(self, lexer, parent, m):
if m: if m:
yield self.make_token(lexer, m.group(0), self.name, parent) yield self.make_token(lexer, m.group(0), self.name, parent, m.groupdict())
raise StopIteration raise StopIteration
class NocasePatternRule(PatternRule): class NocasePatternRule(PatternRule):
@ -126,7 +126,7 @@ class RegionRule(Rule):
stopre = re.compile(self.end % t1.matchd, self.reflags) stopre = re.compile(self.end % t1.matchd, self.reflags)
else: else:
stopre = None stopre = None
for t2 in self._lex(lexer, [t1], 'end', stopre): for t2 in self._lex(lexer, [t1], 'end', stopre, self.grammar):
yield t2 yield t2
raise StopIteration raise StopIteration
def resume(self, lexer, toresume): def resume(self, lexer, toresume):
@ -137,11 +137,11 @@ class RegionRule(Rule):
stopre = re.compile(self.end % t1.matchd, self.reflags) stopre = re.compile(self.end % t1.matchd, self.reflags)
else: else:
stopre = None stopre = None
for t2 in self._lex(lexer, [t1], 'end', stopre): for t2 in self._lex(lexer, [t1], 'end', stopre, self.grammar):
yield t2 yield t2
raise StopIteration raise StopIteration
def _lex(self, lexer, toresume, stopname, stopre): def _lex(self, lexer, toresume, stopname, stopre, grammar):
assert toresume assert toresume
parent = toresume[0] parent = toresume[0]
reenter = len(toresume) > 1 reenter = len(toresume) > 1
@ -169,12 +169,12 @@ class RegionRule(Rule):
if null_t: if null_t:
yield null_t yield null_t
null_t = None null_t = None
yield self.make_token(lexer, m.group(0), stopname, parent) yield self.make_token(lexer, m.group(0), stopname, parent, m.groupdict())
done = True done = True
break break
m = None m = None
for rule in self.grammar.rules: for rule in grammar.rules:
m = rule.match(lexer, parent) m = rule.match(lexer, parent)
if m: if m:
if null_t: if null_t:
@ -185,9 +185,9 @@ class RegionRule(Rule):
break break
if not m: if not m:
if not null_t:
null_t = Token('null', None, lexer.y, lexer.x, '', parent)
if lexer.x < len(line): if lexer.x < len(line):
if not null_t:
null_t = Token('null', None, lexer.y, lexer.x, '', parent)
null_t.add_to_string(line[lexer.x]) null_t.add_to_string(line[lexer.x])
lexer.x += 1 lexer.x += 1
if null_t: if null_t:
@ -217,18 +217,21 @@ class DualRegionRule(RegionRule):
t2 = None t2 = None
if self.middle: if self.middle:
stopre = re.compile(self.middle % t1.groupdict(), self.reflags) stopre = re.compile(self.middle % t1.matchd, self.reflags)
else: else:
stopre = None stopre = None
for t2 in self._lex(lexer, [t1], 'middle', stopre): for t2 in self._lex(lexer, [t1], 'middle', stopre, self.grammar1):
yield t2 yield t2
if t2 is not None and t2.name == 'middle': if t2 is not None and t2.name == 'middle':
if self.end: if self.end:
stopre = re.compile(self.end % t2.groupdict(), self.reflags) d = dict(t2.matchd)
if t1:
d.update(t1.matchd)
stopre = re.compile(self.end % d, self.reflags)
else: else:
stopre = None stopre = None
for t3 in self._lex(lexer, [t2], 'end', stopre): for t3 in self._lex(lexer, [t2], 'end', stopre, self.grammar2):
yield t3 yield t3
raise StopIteration raise StopIteration
@ -286,6 +289,7 @@ class Lexer:
self.y = 0 self.y = 0
self.x = 0 self.x = 0
self.lines = None self.lines = None
assert self.grammar.rules
def get_line(self): def get_line(self):
return self.lines[self.y] + '\n' return self.lines[self.y] + '\n'
def lex(self, lines, y=0, x=0): def lex(self, lines, y=0, x=0):
@ -319,9 +323,9 @@ class Lexer:
raise StopIteration raise StopIteration
def _lex(self): def _lex(self):
null_t = None
parent = None parent = None
while self.y < len(self.lines): while self.y < len(self.lines):
null_t = None
line = self.get_line() line = self.get_line()
while self.x < len(line): while self.x < len(line):
m = None m = None
@ -335,15 +339,15 @@ class Lexer:
yield t yield t
break break
line = self.get_line()
if not m: if not m:
if not null_t:
null_t = Token('null', None, self.y, self.x, '', parent)
if self.x < len(line): if self.x < len(line):
if null_t is None:
null_t = Token('null', None, self.y, self.x, '', parent)
null_t.add_to_string(line[self.x]) null_t.add_to_string(line[self.x])
self.x += 1 self.x += 1
if null_t: if null_t:
yield null_t yield null_t
null_t = None
self.y += 1 self.y += 1
self.x = 0 self.x = 0
raise StopIteration raise StopIteration

64
mode_perl.py
View File

@ -30,7 +30,7 @@ class PerlGrammar(Grammar):
RegionRule(r'heredoc', r"<<(?P<heredoc>[a-zA-Z0-9_]+) *;", StringGrammar, r'^%(heredoc)s$'), RegionRule(r'heredoc', r"<<(?P<heredoc>[a-zA-Z0-9_]+) *;", StringGrammar, r'^%(heredoc)s$'),
RegionRule(r'heredoc', r'<< *"(?P<heredoc>[a-zA-Z0-9_]+)" *;', StringGrammar, r'^%(heredoc)s$'), RegionRule(r'heredoc', r'<< *"(?P<heredoc>[a-zA-Z0-9_]+)" *;', StringGrammar, r'^%(heredoc)s$'),
RegionRule(r'heredoc', r"<< *'(?P<heredoc>[a-zA-Z0-9_]+)' *;", Grammar, r'^%(heredoc)s$'), RegionRule(r'heredoc', r"<< *'(?P<heredoc>[a-zA-Z0-9_]+)' *;", Grammar, r'^%(heredoc)s$'),
RegionRule(r'eval_heredoc', r"<< *`(?P<heredoc>[a-zA-Z0-9_]+)` *;", StringGrammar, r'^%(heredoc)s$'), RegionRule(r'evaldoc', r"<< *`(?P<heredoc>[a-zA-Z0-9_]+)` *;", StringGrammar, r'^%(heredoc)s$'),
RegionRule(r'endblock', r"^__END__|__DATA__ *$", Grammar, r''), RegionRule(r'endblock', r"^__END__|__DATA__ *$", Grammar, r''),
RegionRule(r'pod', r'^=[a-zA-Z0-9_]+', PodGrammar, r'^=cut'), RegionRule(r'pod', r'^=[a-zA-Z0-9_]+', PodGrammar, r'^=cut'),
@ -124,37 +124,31 @@ class PerlTabber(tab2.StackTabber):
fqname = token.fqname() fqname = token.fqname()
if fqname == 'delimiter' and token.string == ';': if fqname == 'delimiter' and token.string == ';':
self._opt_pop('cont') self._opt_pop('cont')
elif fqname == 'heredoc1.start': elif fqname == 'heredoc.start':
self._opt_append('heredoc1', None) self._opt_append('heredoc', None)
elif fqname == 'heredoc1.end': elif fqname == 'heredoc.end':
self._opt_pop('heredoc1') self._opt_pop('heredoc')
self._opt_pop('cont') self._opt_pop('cont')
elif fqname == 'heredoc2.start': elif fqname == 'evaldoc.start':
self._opt_append('heredoc2', None) self._opt_append('evaldoc', None)
elif fqname == 'heredoc2.end': elif fqname == 'evaldoc.end':
self._opt_pop('heredoc2') self._opt_pop('evaldoc')
self._opt_pop('cont')
elif fqname == 'eval_heredoc.start':
self._opt_append('eval_heredoc', None)
elif fqname == 'eval_heredoc.end':
self._opt_pop('eval_heredoc')
self._opt_pop('cont') self._opt_pop('cont')
elif fqname == 'pod.start': elif fqname == 'pod.start':
self._opt_append('pod', None) self._opt_append('pod', None)
elif fqname == 'pod.end': elif fqname == 'pod.end':
self._opt_pop('pod') self._opt_pop('pod')
currlvl = 0 currlvl = 0
elif fqname == 'string1.start' or fqname == 'string2.start': elif fqname == 'string.start':
self._opt_append('string', None) self._opt_append('string', None)
elif fqname == 'string1.end' or fqname == 'string2.end': elif fqname == 'string.end':
self._opt_pop('string') self._opt_pop('string')
if self.is_rightmost_token(y, i): if self.is_rightmost_token(y, i):
self._opt_append('cont', currlvl + 4) self._opt_append('cont', currlvl + 4)
if self.is_rightmost_token(y, i): if self.is_rightmost_token(y, i):
if(not fqname.startswith('pod') and if(not fqname.startswith('pod') and
not fqname.startswith('heredoc') and not fqname.startswith('heredoc') and
not fqname.startswith('string1') and not fqname.startswith('string') and
not fqname.startswith('string2') and
not fqname.startswith('endblock') and not fqname.startswith('endblock') and
not fqname == 'comment' and not fqname == 'comment' and
not fqname == 'null' and not fqname == 'null' and
@ -217,16 +211,13 @@ class Perl(mode2.Fundamental):
'require': color.build('cyan', 'default'), 'require': color.build('cyan', 'default'),
'method': color.build('cyan', 'default'), 'method': color.build('cyan', 'default'),
# heredoc # heredoc/evaldoc
'heredoc1.start': color.build('green', 'default'), 'heredoc.start': color.build('green', 'default'),
'heredoc1.null': color.build('green', 'default'), 'heredoc.null': color.build('green', 'default'),
'heredoc1.end': color.build('green', 'default'), 'heredoc.end': color.build('green', 'default'),
'heredoc2.start': color.build('green', 'default'), 'evaldoc.start': color.build('cyan', 'default'),
'heredoc2.null': color.build('green', 'default'), 'evaldoc.null': color.build('cyan', 'default'),
'heredoc2.end': color.build('green', 'default'), 'evaldoc.end': color.build('cyan', 'default'),
'eval_heredoc.start': color.build('cyan', 'default'),
'eval_heredoc.null': color.build('cyan', 'default'),
'eval_heredoc.end': color.build('cyan', 'default'),
# pod # pod
'pod.start': color.build('red', 'default'), 'pod.start': color.build('red', 'default'),
@ -236,17 +227,12 @@ class Perl(mode2.Fundamental):
'pod.entry.end': color.build('magenta', 'default'), 'pod.entry.end': color.build('magenta', 'default'),
'pod.end': color.build('red', 'default'), 'pod.end': color.build('red', 'default'),
# "" strings # strings
'string1.start': color.build('green', 'default'), 'string.start': color.build('green', 'default'),
'string1.null': color.build('green', 'default'), 'string.null': color.build('green', 'default'),
'string1.escaped': color.build('magenta', 'default'), 'string.escaped': color.build('magenta', 'default'),
'string1.deref': color.build('yellow', 'default'), 'string.deref': color.build('yellow', 'default'),
'string1.end': color.build('green', 'default'), 'string.end': color.build('green', 'default'),
# '' strings
'string2.start': color.build('green', 'default'),
'string2.null': color.build('green', 'default'),
'string2.end': color.build('green', 'default'),
# `` strings # `` strings
'evalstring.start': color.build('cyan', 'default'), 'evalstring.start': color.build('cyan', 'default'),