diff --git a/code_examples/Reporting2.pm b/code_examples/Reporting2.pm index 15e2730..6bef348 100644 --- a/code_examples/Reporting2.pm +++ b/code_examples/Reporting2.pm @@ -4,7 +4,7 @@ my $bar =~ s/foob/blag/g; sub foo { bar() - unless 9; + unless 9 && 3; } my $foo = { diff --git a/lex2.py b/lex2.py deleted file mode 100755 index bbc9a7c..0000000 --- a/lex2.py +++ /dev/null @@ -1,552 +0,0 @@ -import re -import util - -valid_name_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$') -full_name_re = re.compile('^([a-zA-Z_]+)([0-9]*)$') -reserved_names = ['start', 'middle', 'end', 'null'] - -class Token(object): - def __init__(self, name, rule=None, y=0, x=0, s="", parent=None, matchd={}): - self.name = name - self.rule = rule - self.y = y - self.x = x - self.string = s - self.parent = parent - self.matchd = matchd - def parents(self): - if self.parent is not None: - parents = self.parent.parents() - parents.append(self.parent) - return parents - else: - return [] - def domain(self): - if self.parent is not None: - names = self.parent.domain() - else: - names = [] - if self.name != 'middle': - names.append(self.rule.name) - return names - def fqlist(self): - if self.parent is not None: - names = self.parent.domain() - else: - names = [] - if self.name == 'start': - names.append(self.rule.name) - names.append(self.name) - return names - def fqname(self): - names = self.fqlist() - return '.'.join(names) - def copy(self): - return Token(self.name, self.rule, self.y, self.x, self.string, - self.parent, self.matchd) - def add_to_string(self, s): - self.string += s - def end_x(self): - return self.x + len(self.string) - def __eq__(self, other): - return (self.y == other.y and self.x == other.x - and self.name == other.name and self.parent is other.parent and - self.string == other.string) - def __repr__(self): - if len(self.string) < 10: - s = self.string - else: - s = self.string[:10] + '...' - fields = (self.fqname(), self.rule, self.y, self.x, s) - return "" % fields - -class Rule: - def __init__(self, name): - assert valid_name_re.match(name), 'invalid name %r' % name - assert name not in reserved_names, "reserved rule name: %r" % name - self.name = name - def match(self, lexer, parent): - raise Exception, "%s rule cannot match!" % self.name - def make_token(self, lexer, s, name, parent=None, matchd={}): - return Token(name, self, lexer.y, lexer.x, s, parent, matchd) - def _get_line(self, lexer): - return lexer.lines[lexer.y] + '\n' - def _set_group(self, group): - if group is None: - self.group = self.name - else: - self.group = group - -class ConstantRule(Rule): - def __init__(self, name, constant, group=None): - Rule.__init__(self, name) - self.constant = constant - self.length = len(self.constant) - self._set_group(group) - def match(self, lexer, parent): - line = self._get_line(lexer) - if line[lexer.x:].startswith(self.constant): - token = self.make_token(lexer, self.constant, self.name, parent) - lexer.add_token(token) - lexer.x += self.length - return True - else: - return False - -class PatternRule(Rule): - def __init__(self, name, pattern, group=None): - Rule.__init__(self, name) - self.pattern = pattern - self._compile() - self._set_group(group) - def _compile(self): - self.re = re.compile(self.pattern) - def _match(self, lexer, parent, m): - s = m.group(0) - token = self.make_token(lexer, s, self.name, parent) - lexer.add_token(token) - lexer.x += len(s) - def match(self, lexer, parent): - line = self._get_line(lexer) - m = self.re.match(line, lexer.x) - if m: - self._match(lexer, parent, m) - return True - else: - return False - -class NocasePatternRule(PatternRule): - def _compile(self): - self.re = re.compile(self.pattern, re.IGNORECASE) - -class ContextPatternRule(PatternRule): - def __init__(self, name, pattern, fallback, group=None): - Rule.__init__(self, name) - self.pattern = pattern - self.fallback = fallback - self.fallback_re = re.compile(fallback) - self._set_group(group) - def match(self, lexer, parent): - try: - r = re.compile(self.pattern % parent.matchd) - except KeyError: - r = self.fallback_re - line = self._get_line(lexer) - m = r.match(line, lexer.x) - if m: - self._match(lexer, parent, m) - return True - else: - return False - -class RegionRule(Rule): - def __init__(self, name, start, grammar, end, group=None): - Rule.__init__(self, name) - self.start = start - self.grammar = grammar - self.end = end - self.start_re = self._compile_start() - self._set_group(group) - - def _compile_start(self): - return re.compile(self.start) - def _compile_end(self, d): - return re.compile(self.end % d) - - def resume(self, lexer, toresume): - assert toresume, "can't resume without tokens to resume!" - self._match(lexer, None, None, toresume) - return True - - def match(self, lexer, parent): - line = self._get_line(lexer) - m = self.start_re.match(line, lexer.x) - if m: - self._match(lexer, parent, m, []) - return True - else: - return False - - def _add_from_regex(self, name, lexer, parent, m, matchd={}): - s = m.group(0) - token = self.make_token(lexer, s, name, parent, matchd) - lexer.add_token(token) - lexer.x += len(s) - return token - - def _match(self, lexer, parent, m, toresume=[]): - # we either need a match object, or a token to resume - assert m or len(toresume) > 0 - - if m: - # if we had a match, then it becomes the parent, and we save its - # subgroup dict - d = m.groupdict() - parent = self._add_from_regex('start', lexer, parent, m, d) - else: - # otherwise, we should be resuming the start token, so let's pull - # the relevant info out of the token - parent = toresume[0] - d = parent.matchd - assert parent.name == 'start' - null_t = None - - # this determines whether we are still reentering. if len(toresume) == 1 - # then it means that we have been reentering but will not continue, so - # reenter will be false. - reenter = len(toresume) > 1 - - # if we have an end regex, then build it here. notice that it can - # reference named groups from the start token. if we have no end, - # well, then, we're never getting out of here alive! - if self.end: - end_re = self._compile_end(d) - - # ok, so as long as we aren't done (we haven't found an end token), - # keep reading input - done = False - while not done and lexer.y < len(lexer.lines): - old_y = lexer.y - - # ok, as long as we haven't found the end token, and have more - # data on the current line to read, we will process tokens - while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1: - # if we are reentering mid-parse, then that takes precedence - if reenter: - reenter = False - rule2 = toresume[1].rule - rule2.resume(lexer, toresume[1:]) - null_t = None - if lexer.y >= len(lexer.lines): - return True - elif lexer.x >= len(lexer.lines[lexer.y]) + 1: - lexer.y += 1 - lexer.x = 0 - - line = self._get_line(lexer) - - # if we are looking for an end token, then see if we've - # found it. if so, then we are done! - if self.end: - m = end_re.match(line, lexer.x) - if m: - self._add_from_regex('end', lexer, parent, m, {}) - done = True - break - - # ok, we need to check all our rules now, in order. if we - # find a token, note that we found one and exit the loop - found = False - for rule in self.grammar.rules: - if rule.match(lexer, parent): - found = True - null_t = None - break - - - # if we never found a token, then we need to add another - # character to the current null token (which we should - # create if it isn't set). - if not found: - if null_t is None: - null_t = Token('null', None, lexer.y, lexer.x, '', parent) - lexer.add_token(null_t) - if lexer.x < len(line): - null_t.add_to_string(line[lexer.x]) - lexer.x += 1 - - # ok, since we're soon going to be on a different line (or - # already are), we want a new null token. so forget about the - # current one (i.e. stop adding to it). - null_t = None - - # if we're still on the same line at this point (and not done) - # then that means we're finished with the line and should move - # on to the next one here - if not done and old_y == lexer.y: - lexer.y += 1 - lexer.x = 0 - - return True - -class NocaseRegionRule(RegionRule): - def _compile_start(self): - return re.compile(self.start, re.IGNORECASE) - def _compile_end(self, d): - return re.compile(self.end % d, re.IGNORECASE) - -class DualRegionRule(Rule): - def __init__(self, name, start, grammar1, middle, grammar2, end, group=None): - Rule.__init__(self, name) - self.start = start - self.grammar1 = grammar1 - self.middle = middle - self.grammar2 = grammar2 - self.end = end - self.start_re = self._compile_start() - self._set_group(group) - - def _compile_start(self): - return re.compile(self.start) - def _compile_middle(self, d): - return re.compile(self.middle % d) - def _compile_end(self, d): - return re.compile(self.end % d) - - def _add_from_regex(self, name, lexer, parent, m, matchd={}): - s = m.group(0) - token = self.make_token(lexer, s, name, parent, matchd) - lexer.add_token(token) - lexer.x += len(s) - return token - def resume(self, lexer, toresume): - assert toresume, "can't resume without tokens to resume!" - token = toresume[0] - if token.name == 'start': - t2 = self._match_first(lexer, token, toresume) - if t2 is not None: - t3 = self._match_second(lexer, t2, []) - return True - elif token.name == 'middle': - t3 = self._match_second(lexer, token, toresume) - else: - raise Exception, "invalid flag %r" % flag - return True - def match(self, lexer, parent): - # see if we can match our start token - line = self._get_line(lexer) - m = self.start_re.match(line, lexer.x) - if m: - t1 = self._add_from_regex('start', lexer, parent, m, m.groupdict()) - t2 = self._match_first(lexer, t1, []) - if t2 is not None: - t3 = self._match_second(lexer, t2, []) - return True - else: - # region was not matched; we never started. so return false - return False - - def _match_first(self, lexer, parent, toresume=[]): - reenter = len(toresume) > 1 - if reenter: - assert parent is toresume[0] - d1 = parent.matchd - assert parent.name == 'start' - null_t = None - middle_re = self._compile_middle(d1) - d2 = {} - - # ok, so as long as we aren't done (we haven't found an end token), - # keep reading input - t2 = None - done = False - while not done and lexer.y < len(lexer.lines): - old_y = lexer.y - - # ok, as long as we haven't found the end token, and have more - # data on the current line to read, we will process tokens - while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1: - # if we are reentering mid-parse, then that takes precedence - if reenter: - reenter = False - rule2 = toresume[1].rule - rule2.resume(lexer, toresume[1:]) - null_t = None - - line = self._get_line(lexer) - - # see if we have found the middle token. if so, we can then - # proceed to "stage 2" - m2 = middle_re.match(line, lexer.x) - if m2: - d2 = dict(d1.items() + m2.groupdict().items()) - t2 = self._add_from_regex('middle', lexer, parent, m2, d2) - done = True - break - - # ok, we need to check all our rules now, in order. if we - # find a token, note that we found one and exit the loop - found = False - for rule in self.grammar1.rules: - if rule.match(lexer, parent): - found = True - null_t = None - break - - # if we never found a token, then we need to add another - # character to the current null token (which we should - # create if it isn't set). - if not found: - if null_t is None: - null_t = Token('null', None, lexer.y, lexer.x, '', parent) - lexer.add_token(null_t) - null_t.add_to_string(line[lexer.x]) - lexer.x += 1 - - # ok, since we're soon going to be on a different line (or - # already are), we want a new null token. so forget about the - # current one. - null_t = None - - # if we're still on the same line at this point (and not done) - # then that means we're finished with the line and should move - # on to the next one here - if not done and old_y == lexer.y: - lexer.y += 1 - lexer.x = 0 - return t2 - - def _match_second(self, lexer, parent, toresume=[]): - reenter = len(toresume) > 1 - if reenter: - assert parent is toresume[0] - assert parent.name == 'middle' - d3 = parent.matchd - null_t = None - end_re = self._compile_end(d3) - - # ok, so as long as we aren't done (we haven't found an end token), - # keep reading input - t3 = None - done = False - while not done and lexer.y < len(lexer.lines): - old_y = lexer.y - - # if we are reentering mid-parse, then that takes precedence - if reenter: - reenter = False - rule2 = toresume[1].rule - rule2.resume(lexer, toresume[1:]) - null_t = None - - # ok, as long as we haven't found the end token, and have more - # data on the current line to read, we will process tokens - while not done and lexer.y == old_y and lexer.x < len(lexer.lines[lexer.y]) + 1: - # see if we have found the middle token. if so, we can then - # proceed to "stage 2" - line = self._get_line(lexer) - m3 = end_re.match(line, lexer.x) - if m3: - t3 = self._add_from_regex('end', lexer, parent, m3, {}) - done = True - break - - # ok, we need to check all our rules now, in order. if we - # find a token, note that we found one and exit the loop - found = False - for rule in self.grammar2.rules: - if rule.match(lexer, parent): - found = True - null_t = None - break - - # if we never found a token, then we need to add another - # character to the current null token (which we should - # create if it isn't set). - if not found: - if null_t is None: - null_t = Token('null', None, lexer.y, lexer.x, '', parent) - lexer.add_token(null_t) - null_t.add_to_string(line[lexer.x]) - lexer.x += 1 - - # ok, since we're soon going to be on a different line (or - # already are), we want a new null token. so forget about the - # current one. - null_t = None - - # if we're still on the same line at this point (and not done) - # then that means we're finished with the line and should move - # on to the next one here - if not done and old_y == lexer.y: - lexer.y += 1 - lexer.x = 0 - - # alright, we're finally done processing; return true - return t3 - -class Grammar: - rules = [] - def __init__(self): - for rule in self.rules: - if hasattr(rule, 'grammar') and rule.grammar is None: - rule.grammar = self - if hasattr(rule, 'grammar1') and rule.grammar is None: - rule.grammar = self - if hasattr(rule, 'grammar2') and rule.grammar is None: - rule.grammar = self -grammar = Grammar() - -class Lexer: - def __init__(self, name, grammar): - self.name = name - self.grammar = grammar - self.y = 0 - self.x = 0 - self.lines = None - self.tokens = [] - - def add_token(self, t): - self.tokens.append(t) - - def lex(self, lines, y=0, x=0): - self.y = y - self.x = x - self.lines = lines - self.tokens = [] - - def resume(self, lines, y, x, token): - self.y = y - self.x = x - self.lines = lines - self.tokens = [] - toresume = token.parents() - - # this is a special case for the "middle" rule of a dual region rule - i = 0 - while i < len(toresume): - if i > 0 and toresume[i].name == 'middle' and toresume[i-1].name == 'start': - del toresume[i-1] - else: - i += 1 - - if toresume: - toresume[0].rule.resume(self, toresume) - - def __iter__(self): - if self.lines is None: - raise Exception, "no lines to lex" - return self - - def next(self): - null_t = None - if self.tokens: - return self.tokens.pop(0) - while self.y < len(self.lines): - line = self.lines[self.y] + '\n' - while self.x < len(line): - curr_t = None - for rule in self.grammar.rules: - if rule.match(self, None): - assert self.tokens, "match rendered no tokens?" - return self.tokens.pop(0) - if null_t is None: - null_t = Token('null', None, self.y, self.x, '') - self.add_token(null_t) - null_t.add_to_string(line[self.x]) - self.x += 1 - null_t = None - self.y += 1 - self.x = 0 - if self.tokens: - return self.tokens.pop(0) - else: - raise StopIteration - -class NocaseDualRegionRule(DualRegionRule): - def _compile_start(self): - return re.compile(self.start, re.IGNORECASE) - def _compile_middle(self, d): - return re.compile(self.middle % d, re.IGNORECASE) - def _compile_end(self, d): - return re.compile(self.end % d, re.IGNORECASE) diff --git a/lex3.py b/lex3.py index a6cf02c..a0fca3a 100755 --- a/lex3.py +++ b/lex3.py @@ -1,19 +1,16 @@ import re -import util - -valid_name_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$') -full_name_re = re.compile('^([a-zA-Z_]+)([0-9]*)$') -reserved_names = ['start', 'middle', 'end', 'null'] +import regex, util class Token(object): - def __init__(self, name, rule=None, y=0, x=0, s="", parent=None, matchd={}): - self.name = name - self.rule = rule - self.y = y - self.x = x - self.string = s - self.parent = parent - self.matchd = matchd + def __init__(self, name, rule, y, x, s, parent=None, matchd={}, link=None): + self.name = name + self.rule = rule + self.y = y + self.x = x + self.string = s + self.parent = parent + self.matchd = matchd + self.link = link assert parent is None or hasattr(parent, 'name'), 'oh no %r' % parent def parents(self): if self.parent is not None: @@ -27,7 +24,7 @@ class Token(object): names = self.parent.domain() else: names = [] - if self.name != 'middle': + if self.link and not self.link.startswith('middle'): names.append(self.rule.name) return names def fqlist(self): @@ -35,7 +32,7 @@ class Token(object): names = self.parent.domain() else: names = [] - if self.name == 'start': + if self.link == 'start': names.append(self.rule.name) names.append(self.name) return names @@ -63,27 +60,24 @@ class Token(object): class Rule: reflags = 0 - def __init__(self, name, group=None): - assert valid_name_re.match(name), 'invalid name %r' % name - assert name not in reserved_names, "reserved rule name: %r" % name + def __init__(self, name): + assert regex.valid_token_name.match(name), 'invalid name %r' % name + assert not regex.reserved_token_names.match(name), \ + "rule name %r is reserved and thus illegal" % name self.name = name - if group is None: - self.group = name - else: - self.group = group def match(self, lexer, parent): raise Exception, "not implemented" def lex(self, lexer, parent, match): raise Exception, "not implemented" - def make_token(self, lexer, s, name, parent=None, matchd={}): - t = Token(name, self, lexer.y, lexer.x, s, parent, matchd) + def make_token(self, lexer, s, name, parent=None, matchd={}, link=None): + t = Token(name, self, lexer.y, lexer.x, s, parent, matchd, link) lexer.x += len(s) return t def get_line(self, lexer): return lexer.lines[lexer.y] + '\n' class PatternRule(Rule): - def __init__(self, name, pattern, group=None): + def __init__(self, name, pattern): Rule.__init__(self, name) self.pattern = pattern self.re = re.compile(self.pattern, self.reflags) @@ -98,7 +92,7 @@ class NocasePatternRule(PatternRule): reflags = re.IGNORECASE class ContextPatternRule(PatternRule): - def __init__(self, name, pattern, fallback, group=None): + def __init__(self, name, pattern, fallback): Rule.__init__(self, name) self.pattern = pattern self.fallback_re = re.compile(fallback, self.reflags) @@ -111,85 +105,187 @@ class ContextPatternRule(PatternRule): class NocaseContextPatternRule(ContextPatternRule): reflags = re.IGNORECASE -class RegionRule(Rule): - def __init__(self, name, start, grammar, end, group=None): +class PatternGroupRule(PatternRule): + def __init__(self, name, *args): + assert args and len(args) % 2 == 0 + i = 0 + pairs = [] + while i < len(args): + tokname, pattern = args[i], args[i+1] + pairs.append((name, re.compile(pattern, self.reflags))) + i += 2 Rule.__init__(self, name) - self.grammar = grammar - self.end = end - self.start_re = re.compile(start, self.reflags) + self.pairs = tuple(pairs) + def match(self, lexer, parent): + x = lexer.x + matches = [] + line = self.get_line(lexer) + for (tokname, tokre) in self.pairs: + m = tokre.match(line, x) + if m: + x += len(m.group(0)) + matches.append((tokname, m)) + else: + return [] + assert len(matches) == len(self.pairs) + return matches + def lex(self, lexer, parent, matches): + if matches: + for (tokname, m) in matches: + yield self.make_token(lexer, m.group(0), tokname, parent, m.groupdict()) + raise StopIteration + +class RegionRule(Rule): + def __init__(self, name, *args): + Rule.__init__(self, name) + assert len(args) > 1 + args = list(args) + self.pairs = [] + self.start_re = re.compile(args.pop(0), self.reflags) + while len(args) > 1: + grammar = args.pop(0) + pattern = args.pop(0) + assert hasattr(grammar, 'rules'), repr(grammar) + assert type(pattern) == type(''), repr(pattern) + self.pairs.append((grammar, pattern)) + if len(args) == 1: + self.pairs.append((grammar, None)) def match(self, lexer, parent): return self.start_re.match(self.get_line(lexer), lexer.x) def lex(self, lexer, parent, m): - t1 = self.make_token(lexer, m.group(0), 'start', parent, m.groupdict()) - yield t1 - if self.end: - stopre = re.compile(self.end % t1.matchd, self.reflags) - else: - stopre = None - for t2 in self._lex(lexer, [t1], 'end', stopre, self.grammar): - yield t2 + assert m + # ok, so since we had a match, we need to create our start token, who + # will be the ancestor to all other tokens matched in this region + matchd = m.groupdict() + parent = self.make_token(lexer, m.group(0), 'start', parent, matchd, 'start') + yield parent + + # now we will loop over the different pairs of grammars/stop-patterns in + # this region, and return the resulting token; we start at 0 + for tok in self._lex_loop(lexer, [parent], matchd, 0): + yield tok raise StopIteration def resume(self, lexer, toresume): - assert toresume - t1 = toresume[0] - assert t1.name - if self.end: - stopre = re.compile(self.end % t1.matchd, self.reflags) + assert toresume, "can't resume without tokens to resume!" + # ok, so we need to figure out in which of the grammars of our region + # we are resuming. to do this we calculate i, a position in our list + # of grammar/stop-pattern pairs + if toresume[0].link == 'start': + i = 0 else: - stopre = None - for t2 in self._lex(lexer, [t1], 'end', stopre, self.grammar): - yield t2 - raise StopIteration + m = regex.middle_token_name.match(toresume[0].link) + assert m + i = int(m.group(1)) + 1 + assert i > 0 and i < len(self.pairs) + # now we will loop over the different pairs of grammars/stop-patterns in + # this region, and return the resulting token; we start at i + for tok in self._lex_loop(lexer, toresume, toresume[0].matchd, i): + yield tok + raise StopIteration + def _lex_loop(self, lexer, toresume, matchd, i): + # we need to loop over our grammar/stop-pattern pairs + while i < len(self.pairs): + # for each one, we will compile our stop-regex, and figure out the + # name of the stop token to be created if this stop-regex matches. + grammar = self.pairs[i][0] + if self.pairs[i][1]: + stopre = re.compile(self.pairs[i][1] % matchd, self.reflags) + else: + stopre = None + if i == len(self.pairs) - 1: + tokname = 'end' + else: + tokname = 'middle%d' % i + + # ok, so now loop over all the tokens in the current grammar, until + # the stop-token (if any) is found, and return each result as we get + # it. + for tok in self._lex(lexer, toresume, tokname, stopre, grammar): + yield tok + + # ok, so now either we found the stop-token, and have a new parent + # for future tokens (if any), or we are done. + if tok.name == tokname: + toresume = [tok] + matchd.update(tok.matchd) + else: + raise StopIteration + i += 1 + # assuming we make it through all our grammars, and find the end-token, + # then we need to signal that we are done. + raise StopIteration def _lex(self, lexer, toresume, stopname, stopre, grammar): assert toresume parent = toresume[0] reenter = len(toresume) > 1 null_t = None + # ok, so there are only two way we want to exit this loop: either we + # lex the whole document, or we encounter the stop-token. done = False while not done and lexer.y < len(lexer.lines): + line = self.get_line(lexer) old_y = lexer.y - line = self.get_line(lexer) while not done and lexer.y == old_y and lexer.x < len(line): + # ok, so reenter gets priority, since the current input might be + # intended for nested grammar. so handle it here if reenter: reenter = False for t in toresume[1].rule.resume(lexer, toresume[1:]): yield t + # since we might have changed our x/y coordinates, we need to + # do some checks here, and maybe finish or change our coordintes if lexer.y >= len(lexer.lines): raise StopIteration elif lexer.x >= len(line): lexer.y += 1 lexer.x = 0 + + # ok, so get the *now* current line line = self.get_line(lexer) if stopre: + # if we are looking for a stop-token, do that check now m = stopre.match(line, lexer.x) if m: if null_t: + # if we have a null token waiting, return it first. yield null_t null_t = None - yield self.make_token(lexer, m.group(0), stopname, parent, m.groupdict()) + # ok, now return the stop-token, and signal that we are + # done and no more input is to be consumed + yield self.make_token(lexer, m.group(0), stopname, + parent, m.groupdict(), stopname) done = True break m = None + # start checking our rules to see if we can match the input for rule in grammar.rules: m = rule.match(lexer, parent) if m: + # ok great, we have a match if null_t: + # if we have a null token waiting, return it first. yield null_t null_t = None + # ok, now for every token this rules has created, we + # return them, one by one. for t in rule.lex(lexer, parent, m): yield t break if not m: - if lexer.x < len(line): - if not null_t: - null_t = Token('null', None, lexer.y, lexer.x, '', parent) - null_t.add_to_string(line[lexer.x]) - lexer.x += 1 + # we didn't find a match on a rule, so add this character to + # the current null token (creating a new one if necessary); + if not null_t: + null_t = Token('null', None, lexer.y, lexer.x, '', parent) + null_t.add_to_string(line[lexer.x]) + lexer.x += 1 + + # ok, we are at the end of a line of input. so, if we have a null + # token waiting, now is the time to return it if null_t: yield null_t null_t = None @@ -200,86 +296,8 @@ class RegionRule(Rule): class NocaseRegionRule(RegionRule): reflags = re.IGNORECASE -class DualRegionRule(RegionRule): - def __init__(self, name, start, grammar1, middle, grammar2, end, group=None): - Rule.__init__(self, name) - self.start_re = re.compile(start, self.reflags) - self.grammar1 = grammar1 - self.middle = middle - self.grammar2 = grammar2 - self.end = end - def match(self, lexer, parent): - return self.start_re.match(self.get_line(lexer), lexer.x) - def lex(self, lexer, parent, m): - assert m - t1 = self.make_token(lexer, m.group(0), 'start', parent, m.groupdict()) - yield t1 - - t2 = None - if self.middle: - stopre = re.compile(self.middle % t1.matchd, self.reflags) - else: - stopre = None - for t2 in self._lex(lexer, [t1], 'middle', stopre, self.grammar1): - yield t2 - - if t2 is not None and t2.name == 'middle': - if self.end: - d = dict(t2.matchd) - if t1: - d.update(t1.matchd) - stopre = re.compile(self.end % d, self.reflags) - else: - stopre = None - for t3 in self._lex(lexer, [t2], 'end', stopre, self.grammar2): - yield t3 - - raise StopIteration - def resume(self, lexer, toresume): - assert toresume, "can't resume without tokens to resume!" - t1 = t2 = None - if toresume[0].name == 'start': - t1 = toresume[0] - assert t1.name - elif toresume[0].name == 'middle': - t2 = toresume[0] - assert t2.name - else: - raise Exception, "invalid name %r" % toresume[0].name - - if t1 is not None: - #assert t1.name == 'start' - if self.middle: - stopre = re.compile(self.middle, self.reflags) - else: - stopre = None - for t2 in self._lex_first(lexer, toresume, 'middle', stopre): - yield t2 - toresume = [t2] - if t2 is not None: - assert t2.name == 'middle' - if self.end: - stopre = re.compile(self.end, self.reflags) - else: - stopre = None - for t3 in self._lex_second(lexer, toresume, 'end', stopre): - yield t3 - #toresume = [t3] - raise StopIteration -class NocaseDualRegionRule(DualRegionRule): - reflags = re.IGNORECASE - class Grammar: rules = [] - def __init__(self): - # XYZ maybe this is unnecessary - for rule in self.rules: - if hasattr(rule, 'grammar') and rule.grammar is None: - rule.grammar = self - if hasattr(rule, 'grammar1') and rule.grammar is None: - rule.grammar = self - if hasattr(rule, 'grammar2') and rule.grammar is None: - rule.grammar = self grammar = Grammar() class Lexer: @@ -307,10 +325,9 @@ class Lexer: self.tokens = [] toresume = token.parents() - # this is a special case for the "middle" rule of a dual region rule i = 1 while i < len(toresume): - if toresume[i].name == 'middle' and toresume[i-1].name == 'start': + if toresume[i].link and toresume[i].link != 'start': del toresume[i-1] else: i += 1 @@ -339,6 +356,8 @@ class Lexer: yield t break + if self.y >= len(self.lines): + break line = self.get_line() if not m: if self.x < len(line): @@ -351,3 +370,4 @@ class Lexer: self.y += 1 self.x = 0 raise StopIteration + diff --git a/mode_blame.py b/mode_blame.py index e5858af..3d0aa9f 100644 --- a/mode_blame.py +++ b/mode_blame.py @@ -1,32 +1,21 @@ import color, mode2 from point2 import Point -from lex3 import Grammar, PatternRule, RegionRule, DualRegionRule +from lex3 import Grammar, PatternRule, RegionRule class MetadataGrammar(Grammar): rules = [ - PatternRule( - name=r'username', - pattern='[a-zA-Z0-9_]+', - ), + PatternRule(r'username', r'[a-zA-Z0-9_]+'), ] class BlameGrammar(Grammar): rules = [ - RegionRule( - name=r'metadata', - start=r'^[0-9\.]+', - grammar=MetadataGrammar(), - end=r'[0-9]{4}-[0-9]{2}-[0-9]{2}', - ), - PatternRule( - name=r'data', - pattern=r'.+$', - ), + RegionRule(r'metadata', r'^[0-9\.]+', MetadataGrammar, r'[0-9]{4}-[0-9]{2}-[0-9]{2}'), + PatternRule(r'data', r'.+$'), ] class Blame(mode2.Fundamental): - grammar = BlameGrammar() + grammar = BlameGrammar def __init__(self, w): mode2.Fundamental.__init__(self, w) self.colors = { diff --git a/mode_css.py b/mode_css.py index f0e7bac..3f75c0e 100644 --- a/mode_css.py +++ b/mode_css.py @@ -4,45 +4,45 @@ from point2 import Point class StringGrammar(Grammar): rules = [ - PatternRule(name=r'octal', pattern=r'\\[0-7]{3}'), - PatternRule(name=r'escaped', pattern=r'\\.'), + PatternRule(r'octal', r'\\[0-7]{3}'), + PatternRule(r'escaped', r'\\.'), ] class KeywordGrammar(Grammar): rules = [ - PatternRule(name=r'octal', pattern=r'\\[0-7]{3}'), - PatternRule(name=r'escaped', pattern=r'\\.'), - RegionRule(name='string', start="'", grammar=StringGrammar(), end="'"), - RegionRule(name='string', start='"', grammar=StringGrammar(), end='"'), + PatternRule(r'octal', r'\\[0-7]{3}'), + PatternRule(r'escaped', r'\\.'), + RegionRule('string', "'", StringGrammar, r"'"), + RegionRule('string', '"', StringGrammar, r'"'), ] class CSSGrammar(Grammar): rules = [ - RegionRule(name=r'comment', start='/\*', grammar=Grammar(), end='\*/'), - RegionRule(name=r'htmlcomment', start=''), - NocasePatternRule(name=r'dimension', pattern=r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)[-a-z_][-a-z0-9_]*'), - NocasePatternRule(name=r'percentage', pattern=r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)%%'), - NocasePatternRule(name=r'length', pattern=r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)(?:em|ex|px|in|cm|mm|pt|pc)'), - NocasePatternRule(name=r'hash', pattern=r'#[-a-z0-9_]+'), - NocasePatternRule(name=r'real', pattern=r'[+-]?[0-9]*\.[0-9]+'), - NocasePatternRule(name=r'int', pattern=r'[+-]?[0-9]+'), - NocasePatternRule(name=r'rule', pattern=r'@(?:page|media|import)'), - NocasePatternRule(name=r'color', pattern=r'(?:aqua|black|blue|fuchsia|gray|green|lime|maroon|navy|olive|orange|purple|red|silver|teal|white|yellow|#[0-9]{6}|#[0-9]{3})'), + RegionRule(r'comment', '/\*', Grammar, '\*/'), + RegionRule(r'htmlcomment', ''), + NocasePatternRule(r'dimension', r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)[-a-z_][-a-z0-9_]*'), + NocasePatternRule(r'percentage', r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)%%'), + NocasePatternRule(r'length', r'[+-]?(?:[0-9]+|[0-9]*\.[0-9]+)(?:em|ex|px|in|cm|mm|pt|pc)'), + NocasePatternRule(r'hash', r'#[-a-z0-9_]+'), + NocasePatternRule(r'real', r'[+-]?[0-9]*\.[0-9]+'), + NocasePatternRule(r'int', r'[+-]?[0-9]+'), + NocasePatternRule(r'rule', r'@(?:page|media|import)'), + NocasePatternRule(r'color', r'(?:aqua|black|blue|fuchsia|gray|green|lime|maroon|navy|olive|orange|purple|red|silver|teal|white|yellow|#[0-9]{6}|#[0-9]{3})'), - NocasePatternRule(name=r'keyword', pattern=r'(?:url|rgb|counter)'), - NocaseRegionRule(name=r'keyword', start='(?:(?<=url)|(?<=rgb)|(?<=counter))\(', grammar=KeywordGrammar(), end='\)'), + NocasePatternRule(r'keyword', r'(?:url|rgb|counter)'), + NocaseRegionRule(r'keyword', '(?:(?<=url)|(?<=rgb)|(?<=counter))\(', KeywordGrammar, '\)'), - NocasePatternRule(name=r'label', pattern=r"\.?[-a-zA-Z0-9_]+(?= *{)"), + NocasePatternRule(r'label', r"\.?[-a-zA-Z0-9_]+(?= *{)"), - NocasePatternRule(name=r'ident', pattern=r"-?[a-z_][-a-z0-9_]*"), - NocasePatternRule(name=r'name', pattern=r"[-a-z0-9_]+"), - NocasePatternRule(name=r'delimiter', pattern=r'[:;,{}()\[\]]|~=|\|=|='), - RegionRule(name='string', start="'", grammar=StringGrammar(), end="'"), - RegionRule(name='string', start='"', grammar=StringGrammar(), end='"'), + NocasePatternRule(r'ident', r"-?[a-z_][-a-z0-9_]*"), + NocasePatternRule(r'name', r"[-a-z0-9_]+"), + NocasePatternRule(r'delimiter', r'[:;,{}()\[\]]|~=|\|=|='), + RegionRule(r'string', "'", StringGrammar, r"'"), + RegionRule(r'string', '"', StringGrammar, r'"'), ] class CSS(mode2.Fundamental): - grammar = CSSGrammar() + grammar = CSSGrammar def __init__(self, w): mode2.Fundamental.__init__(self, w) self.add_bindings('close-paren', (')',)) diff --git a/mode_javascript.py b/mode_javascript.py index 6ad6e16..a7699ad 100644 --- a/mode_javascript.py +++ b/mode_javascript.py @@ -5,33 +5,33 @@ from mode_python import StringGrammar class JavascriptGrammar(Grammar): rules = [ - PatternRule(name=r'comment', pattern=r'//.*$'), - RegionRule(name=r'comment', start='/\*', grammar=Grammar, end='\*/'), - PatternRule(name=r'continuation', pattern=r'\\(?= *$)'), - PatternRule(name=r'function', pattern=r"(?<=function )[a-zA-Z_][a-zA-Z0-9_]*"), - PatternRule(name=r'class', pattern=r"(?<=class )[a-zA-Z_][a-zA-Z0-9_]*"), + PatternRule(r'comment', r'//.*$'), + RegionRule(r'comment', '/\*', Grammar, '\*/'), + PatternRule(r'continuation', r'\\(?= *$)'), + PatternRule(r'function', r"(?<=function )[a-zA-Z_][a-zA-Z0-9_]*"), + PatternRule(r'class', r"(?<=class )[a-zA-Z_][a-zA-Z0-9_]*"), - PatternRule(name=r'reserved', pattern=r'(?:as|break|case|catch|class|const|continue|default|delete|do|else|export|extends|false|finally|for|function|if|import|in|instanceof|is|namespace|new|null|package|private|public|return|super|switch|this|throw|true|try|typeof|use|var|void|while|with)(?![a-zA-Z0-9_])'), - PatternRule(name=r'reserved', pattern=r'(?:abstract|debugger|enum|goto|implements|interface|native|protected|synchronized|throws|transient|volatile)(?![a-zA-Z0-9_])'), - PatternRule(name=r'nonreserved', pattern=r'(?:get|include|set)(?![a-zA-Z0-9_])'), + PatternRule(r'reserved', r'(?:as|break|case|catch|class|const|continue|default|delete|do|else|export|extends|false|finally|for|function|if|import|in|instanceof|is|namespace|new|null|package|private|public|return|super|switch|this|throw|true|try|typeof|use|var|void|while|with)(?![a-zA-Z0-9_])'), + PatternRule(r'reserved', r'(?:abstract|debugger|enum|goto|implements|interface|native|protected|synchronized|throws|transient|volatile)(?![a-zA-Z0-9_])'), + PatternRule(r'nonreserved', r'(?:get|include|set)(?![a-zA-Z0-9_])'), - PatternRule(name=r"method", pattern=r"(?<=\.)[a-zA-Z_][a-zA-Z0-9_]*(?= *\()"), - PatternRule(name=r'identifier', pattern=r"[a-zA-Z_][a-zA-Z0-9_]*"), + PatternRule(r"method", r"(?<=\.)[a-zA-Z_][a-zA-Z0-9_]*(?= *\()"), + PatternRule(r'identifier', r"[a-zA-Z_][a-zA-Z0-9_]*"), - PatternRule(name=r'integer', pattern=r"(?:0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"), - PatternRule(name=r'float', pattern=r"[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"), + PatternRule(r'integer', r"(?:0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"), + PatternRule(r'float', r"[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"), # fucking javascript! # their lexer grammar requires one-token look-behind in order to know # whether a "/" starts a literal regex, or is part of a mathematical # expression/assignment. so for now we will require either a space or $ # after / in order to *not* treat it as a regex. dammit! - PatternRule(name=r'delimiter', pattern=r'%=|&&=|&=|\(|\)|\*=|\+=|,|-=|\.{3}|\.|/=(?= |$)|::|:|;|<<=|>>=|>>>=|\?|\[|\]|^=|^^=|\{|\}|\|=|\|\|='), - PatternRule(name=r'operator', pattern=r'!==|!=|!|%|&&|&|\*|\+\+|\+|--|-|/(?= |$)|<<=|<<|<=|<|===|==|=|>>>=|>>>|>>=|>>|>=|>|\\'), + PatternRule(r'delimiter', r'%=|&&=|&=|\(|\)|\*=|\+=|,|-=|\.{3}|\.|/=(?= |$)|::|:|;|<<=|>>=|>>>=|\?|\[|\]|^=|^^=|\{|\}|\|=|\|\|='), + PatternRule(r'operator', r'!==|!=|!|%|&&|&|\*|\+\+|\+|--|-|/(?= |$)|<<=|<<|<=|<|===|==|=|>>>=|>>>|>>=|>>|>=|>|\\'), - RegionRule(name='regex', start="/", grammar=StringGrammar, end="/"), - RegionRule(name='string', start="'", grammar=StringGrammar, end="'"), - RegionRule(name='string', start='"', grammar=StringGrammar, end='"'), + RegionRule('regex', "/", StringGrammar, "/"), + RegionRule('string', "'", StringGrammar, "'"), + RegionRule('string', '"', StringGrammar, '"'), ] class JavascriptTabber(tab2.StackTabber): diff --git a/mode_life.py b/mode_life.py index 59e8107..d86f417 100644 --- a/mode_life.py +++ b/mode_life.py @@ -1,8 +1,6 @@ import re, sets, string, sys import color, commands, default, method, mode2, regex, tab2 from point2 import Point -from lex3 import Grammar, PatternRule, ContextPatternRule, \ - RegionRule, DualRegionRule class Life(mode2.Fundamental): def __init__(self, w): diff --git a/mode_nasm.py b/mode_nasm.py index 97ad478..7568b14 100644 --- a/mode_nasm.py +++ b/mode_nasm.py @@ -3,31 +3,31 @@ from lex3 import Grammar, PatternRule, RegionRule class StringGrammar(Grammar): rules = [ - PatternRule(name=r'octal', pattern=r'\\[0-7]{3}'), - PatternRule(name=r'escaped', pattern=r'\\.'), + PatternRule(r'octal', r'\\[0-7]{3}'), + PatternRule(r'escaped', r'\\.'), ] class NasmGrammar(Grammar): rules = [ - PatternRule(name=r'keyword', pattern=r"(?:section|global|extern)(?![a-zA-Z_])"), - PatternRule(name=r'macros', pattern=r"%(?:define|undef|assign|strlen|macro|endmacro|if|elif|else|endif|ifdef|ifndef|include|push|pop|stacksize)(?![a-zA-Z_])"), - PatternRule(name=r'instructions', pattern=r"(?:jeq|jne|ja|jmp|push|pushad|pushfd|call|ret|sub|add|pop|popa|popad|popfd|call|and|cwd|cdq|cmp|cmpxchg|cpuid|div|divpd|enter|leave|fadd|fld|fmul|fsqrt|fsub|hlt|imul|inc|int|int3|lea|mov|movd|mul|neg|not|nop|or|sal|sar|shl|shr|shld|shrd|syscall|sysenter|sysexit|test|xchg|xadd|xor)(?![a-zA-Z_])"), - PatternRule(name=r'registers', pattern=r"(?:eax|ax|ah|al|ebx|bx|bh|bl|ecx|cx|ch|cl|esi|edi|esp|ebp)(?![a-zA-Z_])"), - PatternRule(name=r'prefix', pattern=r"(?:dword|word|lock)(?![a-zA-Z_])"), - PatternRule(name=r'label', pattern=r"[a-zA-Z_.][a-zA-Z0-9_.]*:"), - PatternRule(name=r"identifier", pattern=r"[a-zA-Z_][a-zA-Z0-9_]*"), - PatternRule(name=r"integer", pattern=r"(0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"), - PatternRule(name=r"float", pattern=r"[0-9]+\.[0-9]*|\.[0-9]+|([0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"), - RegionRule(name=r'string', start=r'"""', grammar=StringGrammar(), end=r'"""'), - RegionRule(name=r'string', start=r"'''", grammar=StringGrammar(), end=r"'''"), - RegionRule(name=r'string', start=r'"', grammar=StringGrammar(), end=r'"'), - RegionRule(name=r'string', start=r"'", grammar=StringGrammar(), end=r"'"), - PatternRule(name=r'comment', pattern=r';.*$'), + PatternRule(r'keyword', r"(?:section|global|extern)(?![a-zA-Z_])"), + PatternRule(r'macros', r"%(?:define|undef|assign|strlen|macro|endmacro|if|elif|else|endif|ifdef|ifndef|include|push|pop|stacksize)(?![a-zA-Z_])"), + PatternRule(r'instructions', r"(?:jeq|jne|ja|jmp|push|pushad|pushfd|call|ret|sub|add|pop|popa|popad|popfd|call|and|cwd|cdq|cmp|cmpxchg|cpuid|div|divpd|enter|leave|fadd|fld|fmul|fsqrt|fsub|hlt|imul|inc|int|int3|lea|mov|movd|mul|neg|not|nop|or|sal|sar|shl|shr|shld|shrd|syscall|sysenter|sysexit|test|xchg|xadd|xor)(?![a-zA-Z_])"), + PatternRule(r'registers', r"(?:eax|ax|ah|al|ebx|bx|bh|bl|ecx|cx|ch|cl|esi|edi|esp|ebp)(?![a-zA-Z_])"), + PatternRule(r'prefix', r"(?:dword|word|lock)(?![a-zA-Z_])"), + PatternRule(r'label', r"[a-zA-Z_.][a-zA-Z0-9_.]*:"), + PatternRule(r"identifier", r"[a-zA-Z_][a-zA-Z0-9_]*"), + PatternRule(r"integer", r"(0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"), + PatternRule(r"float", r"[0-9]+\.[0-9]*|\.[0-9]+|([0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"), + RegionRule(r'string', r'"""', StringGrammar, r'"""'), + RegionRule(r'string', r"'''", StringGrammar, r"'''"), + RegionRule(r'string', r'"', StringGrammar, r'"'), + RegionRule(r'string', r"'", StringGrammar, r"'"), + PatternRule(r'comment', r';.*$'), ] class Nasm(mode2.Fundamental): - grammar = NasmGrammar() + grammar = NasmGrammar def __init__(self, w): mode2.Fundamental.__init__(self, w) self.colors = { diff --git a/mode_perl.py b/mode_perl.py index 8873d1e..4e2878f 100644 --- a/mode_perl.py +++ b/mode_perl.py @@ -1,7 +1,7 @@ import re, sets, string, sys import color, commands, default, method, mode2, regex, tab2 from point2 import Point -from lex3 import Grammar, PatternRule, ContextPatternRule, RegionRule, DualRegionRule +from lex3 import Grammar, PatternRule, ContextPatternRule, RegionRule from method import Argument, Method class PodGrammar(Grammar): @@ -45,7 +45,7 @@ class PerlGrammar(Grammar): PatternRule(r'hash_key', r'[A-Za-z0-9_]+(?= *=>)'), PatternRule(r'length', r"\$#[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"), PatternRule(r'cast', r'[\$\@\%\^\&](?= *{)'), - PatternRule(r'scalar', r"\$[][>ab/'\"_@\?#\$!%^|&*()](?![A-Za-z0-9_])"), PatternRule(r'array', r"@_"), PatternRule(r'function', r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*(?=-> *\()"), PatternRule(r'scalar', r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"), @@ -59,12 +59,12 @@ class PerlGrammar(Grammar): RegionRule(r'match', r'm(?P#)', StringGrammar, r'#[a-z]*'), # replace regexes - DualRegionRule(r'replace', r's *(?P[^ a-zA-Z0-9_])', StringGrammar, r'%(delim)s', StringGrammar, r'%(delim)s[a-z]*'), - DualRegionRule(r'replace', r's(?P#)', StringGrammar, r'#', StringGrammar, r'#[a-z]*'), + RegionRule(r'replace', r's *(?P[^ a-zA-Z0-9_])', StringGrammar, r'%(delim)s', StringGrammar, r'%(delim)s[a-z]*'), + RegionRule(r'replace', r's(?P#)', StringGrammar, r'#', StringGrammar, r'#[a-z]*'), # translate operator - DualRegionRule(r'translate', r'(?:y|tr) *(?P[^ a-zA-Z0-9_])', Grammar, r'%(delim)s', Grammar, r'%(delim)s[a-z]*'), - DualRegionRule(r'translate', r'(?:y|tr)#', Grammar, r'#', Grammar, r'#[a-z]*'), + RegionRule(r'translate', r'(?:y|tr) *(?P[^ a-zA-Z0-9_])', Grammar, r'%(delim)s', Grammar, r'%(delim)s[a-z]*'), + RegionRule(r'translate', r'(?:y|tr)#', Grammar, r'#', Grammar, r'#[a-z]*'), # some more basic stuff PatternRule(r'package', r"(?<=package )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*"), @@ -255,7 +255,7 @@ class Perl(mode2.Fundamental): # replace regex 'replace.start': color.build('cyan', 'default'), - 'replace.middle': color.build('cyan', 'default'), + 'replace.middle0': color.build('cyan', 'default'), 'replace.end': color.build('cyan', 'default'), 'replace.null': color.build('cyan', 'default'), 'replace.escaped': color.build('magenta', 'default'), @@ -266,10 +266,10 @@ class Perl(mode2.Fundamental): 'replace.cast': color.build('yellow', 'default'), # translate regex - 'translate.start': color.build('magenta', 'default'), - 'translate.middle': color.build('magenta', 'default'), - 'translate.end': color.build('magenta', 'default'), - 'translate.null': color.build('magenta', 'default'), + 'translate.start': color.build('magenta', 'default'), + 'translate.middle0': color.build('magenta', 'default'), + 'translate.end': color.build('magenta', 'default'), + 'translate.null': color.build('magenta', 'default'), } # perl-specific diff --git a/mode_sh.py b/mode_sh.py index 36d4827..ecf9698 100644 --- a/mode_sh.py +++ b/mode_sh.py @@ -17,7 +17,6 @@ class ShGrammar(Grammar): PatternRule(r'operator', r"(?:-eq|-ne|-gt|-lt|-ge|-le| = | != )"), PatternRule(r'delimiter', r";;|[\[\]\(\);\{\}|&><]"), RegionRule(r'eval', '`', StringGrammar, '`'), - #RegionRule(r'neval', r'\$\(', None, r'\)'), RegionRule(r'neval', r'\$\(', StringGrammar, r'\)'), PatternRule(r'variable', r"(?:^|(?<= ))[a-zA-Z_][a-zA-Z_][a-zA-Z0-9_]*(?==)"), PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"), diff --git a/mode_sql.py b/mode_sql.py index 2abb41f..541a6d2 100644 --- a/mode_sql.py +++ b/mode_sql.py @@ -1,5 +1,5 @@ import color, mode2, tab2 -from lex3 import Grammar, PatternRule, NocasePatternRule, RegionRule, NocaseRegionRule, DualRegionRule, NocaseDualRegionRule +from lex3 import Grammar, PatternRule, NocasePatternRule, RegionRule, NocaseRegionRule from mode_python import StringGrammar class PlPgSqlGrammar(Grammar): diff --git a/mode_text.py b/mode_text.py index 5471e3e..8b1e022 100644 --- a/mode_text.py +++ b/mode_text.py @@ -3,8 +3,7 @@ from lex3 import Token, Rule, PatternRule, RegionRule, Grammar class WordRule(PatternRule): def __init__(self): - PatternRule.__init__(self, name=r'word', - pattern=r"[a-zA-Z][a-zA-Z-\']*[a-zA-Z](?=$|[^a-zA-Z0-9-_])") + PatternRule.__init__(self, r'word', pattern=r"[a-zA-Z][a-zA-Z-\']*[a-zA-Z](?=$|[^a-zA-Z0-9-_])") def _spelled_ok(self, word): if ispell.can_spell(): speller = ispell.get_speller() @@ -22,15 +21,14 @@ class WordRule(PatternRule): class ContinuedRule(RegionRule): def __init__(self): - RegionRule.__init__(self, name=r'cont', start=r'[a-zA-Z0-9_]+- *$', - grammar=Grammar(), end=r'^ *(?:[^ ]+|$)') + RegionRule.__init__(self, r'cont', r'[a-zA-Z0-9_]+- *$', Grammar, r'^ *(?:[^ ]+|$)') class TextGrammar(Grammar): rules = [ ContinuedRule(), WordRule(), - PatternRule(name=r'punct', pattern=r'[^a-zA-Z0-9_]'), - PatternRule(name=r'stuff', pattern=r'[a-zA-Z0-9_]+'), + PatternRule(r'punct', r'[^a-zA-Z0-9_]'), + PatternRule(r'stuff', r'[a-zA-Z0-9_]+'), ] class Text(mode2.Fundamental): diff --git a/mode_tt.py b/mode_tt.py index 64fcb1b..a163928 100644 --- a/mode_tt.py +++ b/mode_tt.py @@ -3,18 +3,18 @@ from lex3 import Grammar, PatternRule, RegionRule class TagGrammar(Grammar): rules = [ - RegionRule(name=r'string', start=r'(?P["\'])', grammar=Grammar(), end=r'%(tag)s'), - PatternRule(name=r'namespace', pattern=r'[a-zA-Z_]+:'), - PatternRule(name=r'attrname', pattern=r'[^ =>\n]+(?==)'), - PatternRule(name=r'name', pattern=r'[^ =>\n]+'), + RegionRule(r'string', r'(?P["\'])', Grammar, r'%(tag)s'), + PatternRule(r'namespace', pattern=r'[a-zA-Z_]+:'), + PatternRule(r'attrname', pattern=r'[^ =>\n]+(?==)'), + PatternRule(r'name', pattern=r'[^ =>\n]+'), ] class TemplateGrammar(Grammar): rules = [ - RegionRule(name=r'comment', start=r''), - RegionRule(name=r'template', start=r'\[\%', grammar=Grammar(), end=r'%%\]'), - RegionRule(name=r'opentag', start=r'<', grammar=TagGrammar(), end=r'/?>'), - PatternRule(name=r'closetag', pattern=r'< */ *[ =>\n]+ *>'), + RegionRule(r'comment', r''), + RegionRule(r'template', r'\[\%', Grammar, r'%%\]'), + RegionRule(r'opentag', r'<', TagGrammar, r'/?>'), + PatternRule(r'closetag', pattern=r'< */ *[ =>\n]+ *>'), ] class Template(mode2.Fundamental): diff --git a/regex.py b/regex.py index b560021..37bcd70 100644 --- a/regex.py +++ b/regex.py @@ -1,5 +1,10 @@ import re +# lexing +reserved_token_names = re.compile(r'^(?:rules|null|start|end|middle[0-9]*)$') +valid_token_name = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$') +middle_token_name = re.compile(r'^middle([0-9]+)$') + # meta regexes meta_chars = re.compile(r'([\.\^\$\*\+\?\{\}\(\)\[\]\|\"\'\\,])')