import lex2

lines = [
    'this is the first line',
    '<< EOF;'
    '   abracadra hocus pocus',
    '   EOF',
    'EOF',
    '"this is a double-quoted string" and this is not...',
    "we're done",
]
l = lex2.Lexer()
l.lex(lines)
for t in l:
    print '%-20r %r' % (t.rule.name, t.string)