528 lines
20 KiB
Python
Executable File
528 lines
20 KiB
Python
Executable File
import re
|
|
|
|
valid_name_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
|
|
reserved_names = ['start', 'middle', 'end', 'null']
|
|
|
|
class Token(object):
|
|
def __init__(self, name, rule=None, y=0, x=0, s="", parent=None, matchd={}):
|
|
self.name = name
|
|
self.rule = rule
|
|
self.y = y
|
|
self.x = x
|
|
self.string = s
|
|
self.parent = parent
|
|
self.matchd = matchd
|
|
def parents(self):
|
|
if self.parent is not None:
|
|
parents = self.parent.parents()
|
|
parents.append(self.parent)
|
|
return parents
|
|
else:
|
|
return []
|
|
def domain(self):
|
|
names = []
|
|
if self.parent is not None:
|
|
names.extend(self.parent.domain())
|
|
names.append(self.rule.name)
|
|
return names
|
|
def fqlist(self):
|
|
names = []
|
|
if self.parent is not None:
|
|
names.extend(self.parent.domain())
|
|
names.append(self.name)
|
|
return names
|
|
def fqname(self):
|
|
if self.name == 'start':
|
|
names = self.domain()
|
|
names.append(self.name)
|
|
else:
|
|
names = self.fqlist()
|
|
return '.'.join(names)
|
|
def copy(self):
|
|
return Token(self.name, self.rule, self.y, self.x, self.string,
|
|
self.parent, self.matchd)
|
|
def add_to_string(self, s):
|
|
self.string += s
|
|
def end_x(self):
|
|
return self.x + len(self.string)
|
|
def __eq__(self, other):
|
|
return (self.y == other.y and self.x == other.x
|
|
and self.name == other.name and self.parent is other.parent and
|
|
self.string == other.string)
|
|
def __repr__(self):
|
|
if len(self.string) < 10:
|
|
s = self.string
|
|
else:
|
|
s = self.string[:10] + '...'
|
|
fields = (self.fqname(), self.rule, self.y, self.x, s)
|
|
return "<Token(%r, %r, %d, %d, %r)>" % fields
|
|
|
|
class Rule:
|
|
name = 'abstract'
|
|
def match(self, lexer, parent):
|
|
raise Exception, "%s rule cannot match!" % self.name
|
|
def make_token(self, lexer, s, name, parent=None, matchd={}):
|
|
return Token(name, self, lexer.y, lexer.x, s, parent, matchd)
|
|
|
|
class ConstantRule(Rule):
|
|
def __init__(self, name, constant):
|
|
assert valid_name_re.match(name), 'invalid name %r' % name
|
|
assert name not in reserved_names, "reserved rule name: %r" % name
|
|
self.name = name
|
|
self.constant = constant
|
|
self.lenth = len(self.constant)
|
|
def match(self, lexer, parent):
|
|
if lexer.lines[lexer.y][lexer.x:].startswith(self.constant):
|
|
token = self.make_token(lexer, self.constant, self.name, parent)
|
|
lexer.add_token(token)
|
|
lexer.x += self.length
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
class PatternRule(Rule):
|
|
def __init__(self, name, pattern):
|
|
assert valid_name_re.match(name), 'invalid name %r' % name
|
|
assert name not in reserved_names, "reserved rule name: %r" % name
|
|
self.name = name
|
|
self.pattern = pattern
|
|
self.re = re.compile(pattern)
|
|
def _match(self, lexer, parent, m):
|
|
s = m.group(0)
|
|
token = self.make_token(lexer, s, self.name, parent)
|
|
lexer.add_token(token)
|
|
lexer.x += len(s)
|
|
def match(self, lexer, parent):
|
|
m = self.re.match(lexer.lines[lexer.y], lexer.x)
|
|
if m:
|
|
self._match(lexer, parent, m)
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
class ContextPatternRule(PatternRule):
|
|
def __init__(self, name, pattern, fallback):
|
|
assert valid_name_re.match(name), 'invalid name %r' % name
|
|
assert name not in reserved_names, "reserved rule name: %r" % name
|
|
self.name = name
|
|
self.pattern = pattern
|
|
self.fallback = fallback
|
|
self.fallback_re = re.compile(fallback)
|
|
def match(self, lexer, parent):
|
|
try:
|
|
r = re.compile(self.pattern % parent.matchd)
|
|
except KeyError:
|
|
r = self.fallback_re
|
|
m = r.match(lexer.lines[lexer.y], lexer.x)
|
|
if m:
|
|
self._match(lexer, parent, m)
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
class RegionRule(Rule):
|
|
def __init__(self, name, start, grammar, end):
|
|
assert valid_name_re.match(name), 'invalid name %r' % name
|
|
assert name not in reserved_names, "reserved rule name: %r" % name
|
|
self.name = name
|
|
self.start = start
|
|
self.grammar = grammar
|
|
self.end = end
|
|
self.start_re = re.compile(start)
|
|
|
|
def resume(self, lexer, toresume):
|
|
assert toresume, "can't resume without tokens to resume!"
|
|
self._match(lexer, None, None, toresume)
|
|
return True
|
|
|
|
def match(self, lexer, parent):
|
|
m = self.start_re.match(lexer.lines[lexer.y], lexer.x)
|
|
if m:
|
|
self._match(lexer, parent, m, [])
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def _add_from_regex(self, name, lexer, parent, m, matchd={}):
|
|
s = m.group(0)
|
|
token = self.make_token(lexer, s, name, parent, matchd)
|
|
lexer.add_token(token)
|
|
lexer.x += len(s)
|
|
return token
|
|
|
|
def _match(self, lexer, parent, m, toresume=[]):
|
|
# we either need a match object, or a token to resume
|
|
assert m or len(toresume) > 0
|
|
|
|
if m:
|
|
# if we had a match, then it becomes the parent, and we save its
|
|
# subgroup dict
|
|
d = m.groupdict()
|
|
parent = self._add_from_regex('start', lexer, parent, m, d)
|
|
else:
|
|
# otherwise, we should be resuming the start token, so let's pull
|
|
# the relevant info out of the token
|
|
parent = toresume[0]
|
|
d = parent.matchd
|
|
assert parent.name == 'start'
|
|
null_t = None
|
|
|
|
# this determines whether we are still reentering. if len(toresume) == 1
|
|
# then it means that we have been reentering but will not continue, so
|
|
# reenter will be false.
|
|
reenter = len(toresume) > 1
|
|
|
|
# if we have an end regex, then build it here. notice that it can
|
|
# reference named groups from the start token. if we have no end,
|
|
# well, then, we're never getting out of here alive!
|
|
if self.end:
|
|
end_re = re.compile(self.end % d)
|
|
|
|
# ok, so as long as we aren't done (we haven't found an end token),
|
|
# keep reading input
|
|
done = False
|
|
while not done and lexer.y < len(lexer.lines):
|
|
old_y = lexer.y
|
|
|
|
# if this line is empty, then we skip it, but here we insert
|
|
# an empty null token just so we have something
|
|
if not reenter and len(lexer.lines[lexer.y]) == 0:
|
|
null_t = Token('null', None, lexer.y, lexer.x, '', parent)
|
|
lexer.add_token(null_t)
|
|
null_t = None
|
|
|
|
# ok, as long as we haven't found the end token, and have more
|
|
# data on the current line to read, we will process tokens
|
|
while (not done and lexer.y == old_y and
|
|
lexer.x < len(lexer.lines[lexer.y])):
|
|
# if we are reentering mid-parse, then that takes precedence
|
|
if reenter:
|
|
reenter = False
|
|
rule2 = toresume[1].rule
|
|
rule2.resume(lexer, toresume[1:])
|
|
found = True
|
|
null_t = None
|
|
break
|
|
|
|
# if we are looking for an end token, then see if we've
|
|
# found it. if so, then we are done!
|
|
if self.end:
|
|
m = end_re.match(lexer.lines[lexer.y], lexer.x)
|
|
if m:
|
|
self._add_from_regex('end', lexer, parent, m, {})
|
|
done = True
|
|
break
|
|
|
|
# ok, we need to check all our rules now, in order. if we
|
|
# find a token, note that we found one and exit the loop
|
|
found = False
|
|
for rule in self.grammar.rules:
|
|
if rule.match(lexer, parent):
|
|
found = True
|
|
null_t = None
|
|
break
|
|
|
|
# if we never found a token, then we need to add another
|
|
# character to the current null token (which we should
|
|
# create if it isn't set).
|
|
if not found:
|
|
if null_t is None:
|
|
null_t = Token('null', None, lexer.y, lexer.x, '', parent)
|
|
lexer.add_token(null_t)
|
|
null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
|
|
lexer.x += 1
|
|
|
|
# ok, since we're soon going to be on a different line (or
|
|
# already are), we want a new null token. so forget about the
|
|
# current one (i.e. stop adding to it).
|
|
null_t = None
|
|
|
|
# if we're still on the same line at this point (and not done)
|
|
# then that means we're finished with the line and should move
|
|
# on to the next one here
|
|
if not done and old_y == lexer.y:
|
|
lexer.y += 1
|
|
lexer.x = 0
|
|
|
|
return True
|
|
|
|
class DualRegionRule(Rule):
|
|
def __init__(self, name, start, grammar1, middle, grammar2, end):
|
|
assert valid_name_re.match(name), 'invalid name %r' % name
|
|
assert name not in reserved_names, "reserved rule name: %r" % name
|
|
self.name = name
|
|
self.start = start
|
|
self.grammar1 = grammar1
|
|
self.middle = middle
|
|
self.grammar2 = grammar2
|
|
self.end = end
|
|
self.start_re = re.compile(start)
|
|
def _add_from_regex(self, name, lexer, parent, m, matchd={}):
|
|
s = m.group(0)
|
|
token = self.make_token(lexer, s, name, parent, matchd)
|
|
lexer.add_token(token)
|
|
lexer.x += len(s)
|
|
return token
|
|
def resume(self, lexer, toresume):
|
|
assert toresume
|
|
token = toresume[0]
|
|
d = token.matchd
|
|
if token.name == 'start':
|
|
stoken = toresume[0]
|
|
mtoken = self._match_first(lexer, stoken, None, toresume)
|
|
self._match_second(lexer, mtoken, [])
|
|
elif token.name == 'middle':
|
|
d3 = token.matchd
|
|
self._match_second(lexer, token.parent, d3, toresume)
|
|
else:
|
|
raise Exception, "invalid flag %r" % flag
|
|
return True
|
|
def match(self, lexer, parent):
|
|
# see if we can match our start token
|
|
m = self.start_re.match(lexer.lines[lexer.y], lexer.x)
|
|
if m:
|
|
# region was match, so let's do this
|
|
d1 = m.groupdict()
|
|
d2 = self._match_first(lexer, parent, m, [])
|
|
d3 = dict(d1.items() + d2.items())
|
|
self._match_second(lexer, parent, d3, None, [])
|
|
return True
|
|
else:
|
|
# region was not matched; we never started. so return false
|
|
return False
|
|
|
|
def _match_first(self, lexer, context, d1, m1, toresume=[]):
|
|
# if we have been given rulecontext, then we are going to "resume" a
|
|
# parse that can already be assumed to have started
|
|
reenter = len(toresume) > 1
|
|
assert m1 or reenter
|
|
|
|
# ok, so create our start token, and get ready to start reading data
|
|
if m1 is not None:
|
|
self._add_from_regex('start', lexer, parent, m1, m1.groupdict())
|
|
null_t = None
|
|
|
|
middle_re = re.compile(self.middle % d1)
|
|
d2 = {}
|
|
|
|
# ok, so as long as we aren't done (we haven't found an end token),
|
|
# keep reading input
|
|
done = False
|
|
while not done and lexer.y < len(lexer.lines):
|
|
old_y = lexer.y
|
|
|
|
# if this line is empty, then we will skip it, but here weinsert
|
|
# an empty null token just so we have something
|
|
if len(lexer.lines[lexer.y]) == 0:
|
|
null_t = Token('null', None, lexer.y, lexer.x, '')
|
|
lexer.add_token(null_t)
|
|
null_t = None
|
|
|
|
# ok, as long as we haven't found the end token, and have more
|
|
# data on the current line to read, we will process tokens
|
|
while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]):
|
|
# if we are reentering mid-parse, then that takes precedence
|
|
if reenter:
|
|
raise Exception, "aw damn"
|
|
#reenter = False
|
|
#xrule = rulecontext[0].rule
|
|
#xd = rulecontext[0].matchd
|
|
#assert rule2.resume(lexer, xcontext, xd, rulecontext[1:]), \
|
|
# "%r %r %r %r" % (lexer, xcontext, xd, rulecontext[1:])
|
|
#found = True
|
|
#null_t = None
|
|
#break
|
|
|
|
# see if we have found the middle token. if so, we can then
|
|
# proceed to "stage 2"
|
|
m2 = middle_re.match(lexer.lines[lexer.y], lexer.x)
|
|
if m2:
|
|
d2 = m2.groupdict()
|
|
self._add_from_regex('middle', lexer, parent, m2, {})
|
|
done = True
|
|
break
|
|
|
|
# ok, we need to check all our rules now, in order. if we
|
|
# find a token, note that we found one and exit the loop
|
|
found = False
|
|
for rule in self.grammar1.rules:
|
|
if rule.match(lexer, parent):
|
|
found = True
|
|
null_t = None
|
|
break
|
|
|
|
# if we never found a token, then we need to add another
|
|
# character to the current null token (which we should
|
|
# create if it isn't set).
|
|
if not found:
|
|
if null_t is None:
|
|
null_t = Token('null', None, lexer.y, lexer.x, '')
|
|
lexer.add_token(null_t)
|
|
null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
|
|
lexer.x += 1
|
|
|
|
# ok, since we're soon going to be on a different line (or
|
|
# already are), we want a new null token. so forget about the
|
|
# current one.
|
|
null_t = None
|
|
|
|
# if we're still on the same line at this point (and not done)
|
|
# then that means we're finished with the line and should move
|
|
# on to the next one here
|
|
if not done and old_y == lexer.y:
|
|
lexer.save_context()
|
|
lexer.y += 1
|
|
lexer.x = 0
|
|
return d2
|
|
|
|
def _match_second(self, lexer, context, d3, m, toresume=[]):
|
|
# if we have been given rulecontext, then we are going to "resume" a
|
|
# parse that can already be assumed to have started
|
|
reenter = len(toresume) > 1
|
|
|
|
# ok stage 2 is like stage 1, only we are looking for end tokens
|
|
# instead of middle tokens
|
|
null_t = None
|
|
end_re = re.compile(self.end % d3)
|
|
|
|
# ok, so as long as we aren't done (we haven't found an end token),
|
|
# keep reading input
|
|
done = False
|
|
while not done and lexer.y < len(lexer.lines):
|
|
old_y = lexer.y
|
|
|
|
# if we are reentering mid-parse, then that takes precedence
|
|
if reenter:
|
|
raise Exception, "aw damn"
|
|
#reenter = False
|
|
#xrule = rulecontext[0].rule
|
|
#xd = rulecontext[0].matchd
|
|
#assert rule2.resume(lexer, xcontext, xd, rulecontext[1:]), \
|
|
# "%r %r %r %r" % (lexer, xcontext, xd, rulecontext[1:])
|
|
#found = True
|
|
#null_t = None
|
|
#break
|
|
|
|
# if this line is empty, then we will skip it, but here weinsert
|
|
# an empty null token just so we have something
|
|
if len(lexer.lines[lexer.y]) == 0:
|
|
null_t = Token('null', None, lexer.y, lexer.x, '')
|
|
lexer.add_token(null_t)
|
|
null_t = None
|
|
|
|
# ok, as long as we haven't found the end token, and have more
|
|
# data on the current line to read, we will process tokens
|
|
while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]):
|
|
# see if we have found the middle token. if so, we can then
|
|
# proceed to "stage 2"
|
|
m3 = end_re.match(lexer.lines[lexer.y], lexer.x)
|
|
if m3:
|
|
self._add_from_regex('end', lexer, parent, m3, {})
|
|
done = True
|
|
break
|
|
|
|
# ok, we need to check all our rules now, in order. if we
|
|
# find a token, note that we found one and exit the loop
|
|
found = False
|
|
for rule in self.grammar2.rules:
|
|
if rule.match(lexer, parent):
|
|
found = True
|
|
null_t = None
|
|
break
|
|
|
|
# if we never found a token, then we need to add another
|
|
# character to the current null token (which we should
|
|
# create if it isn't set).
|
|
if not found:
|
|
if null_t is None:
|
|
null_t = Token('null', None, lexer.y, lexer.x, '')
|
|
lexer.add_token(null_t)
|
|
null_t.add_to_string(lexer.lines[lexer.y][lexer.x])
|
|
lexer.x += 1
|
|
|
|
# ok, since we're soon going to be on a different line (or
|
|
# already are), we want a new null token. so forget about the
|
|
# current one.
|
|
null_t = None
|
|
|
|
# if we're still on the same line at this point (and not done)
|
|
# then that means we're finished with the line and should move
|
|
# on to the next one here
|
|
if not done and old_y == lexer.y:
|
|
lexer.save_context()
|
|
lexer.y += 1
|
|
lexer.x = 0
|
|
|
|
# alright, we're finally done processing; return true
|
|
return True
|
|
|
|
class Grammar:
|
|
rules = []
|
|
def __init__(self):
|
|
for rule in self.rules:
|
|
if hasattr(rule, 'grammar') and rule.grammar is None:
|
|
rule.grammar = self
|
|
|
|
class Lexer:
|
|
def __init__(self, name, grammar):
|
|
self.name = name
|
|
self.grammar = grammar
|
|
self.y = 0
|
|
self.x = 0
|
|
self.lines = None
|
|
self.tokens = []
|
|
|
|
def add_token(self, t):
|
|
self.tokens.append(t)
|
|
|
|
def lex(self, lines, y=0, x=0):
|
|
self.y = y
|
|
self.x = x
|
|
self.lines = lines
|
|
self.tokens = []
|
|
|
|
def resume(self, lines, y, x, token):
|
|
self.y = y
|
|
self.x = x
|
|
#self.x = 0
|
|
self.lines = lines
|
|
self.tokens = []
|
|
|
|
if token:
|
|
toresume = token.parents()
|
|
if toresume:
|
|
raise Exception, "aw damn"
|
|
|
|
def __iter__(self):
|
|
if self.lines is None:
|
|
raise Exception, "no lines to lex"
|
|
return self
|
|
|
|
def next(self):
|
|
null_t = None
|
|
|
|
if self.tokens:
|
|
return self.tokens.pop(0)
|
|
|
|
while self.y < len(self.lines):
|
|
line = self.lines[self.y]
|
|
while self.x < len(line):
|
|
curr_t = None
|
|
for rule in self.grammar.rules:
|
|
if rule.match(self, None):
|
|
assert self.tokens, "match rendered no tokens?"
|
|
return self.tokens.pop(0)
|
|
if null_t is None:
|
|
null_t = Token('null', None, self.y, self.x, '')
|
|
self.add_token(null_t)
|
|
null_t.add_to_string(line[self.x])
|
|
self.x += 1
|
|
null_t = None
|
|
self.y += 1
|
|
self.x = 0
|
|
|
|
if self.tokens:
|
|
return self.tokens.pop(0)
|
|
else:
|
|
raise StopIteration
|