diff --git a/lex.py b/lex.py index 17d4cda..b97eb59 100755 --- a/lex.py +++ b/lex.py @@ -368,6 +368,12 @@ class Lexer(object): assert self.grammar.rules def get_line(self): return self.lines[self.y] + '\n' + def lex_all(self, lines): + lextokens = [[] for l in lines] + for t in lexer.lex(lines): + lextokens[t.y].append(t) + return lextokens + def lex(self, lines, y=0, x=0): self.action = 'lex' self.y = y