s = "gwejgwe
gewjgewjgwe
gewjgewjgwe
gejwgew
gewgewjge"

import lex2

lines = [
    'this is the first    line',
    '<< EOF;'
    '   abracadra hocus pocus',
    '   EOF',
    'EOF',
    '"this is a double-quoted string" and this is not...',
    "we're done",
]
g = lex2.TestGrammar()
l = lex2.Lexer('lexer', g)
l.lex(lines)
for t in l:
    print '%-20r %r' % (t.name, t.string)