general mode clean-up

--HG-- branch : pmacs2
2009-03-29 02:20:37 +00:00 · 2009-03-29 02:20:37 +00:00 · fe11590c64
parent a971586fed
commit fe11590c64
14 changed files with 235 additions and 279 deletions
--- a/mode/c.py
+++ b/mode/c.py
@ -1,8 +1,7 @@
 import os, re
 from subprocess import Popen, PIPE, STDOUT
 import color, default, method, method.shell, mode, tab
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule, OverridePatternRule
-from lex import PatternMatchRule
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule, OverridePatternRule
 from mode.python import StringGrammar2

 class CommentGrammar(Grammar):
@ -62,10 +61,6 @@ class CGrammar(Grammar):
        PatternRule(r'char', r"'.'|'\\.'|'\\[0-7]{3}'"),
        PatternMatchRule(r'x', r'(# *include)( +)(.+)(\n|$)',
                         r'macro.start', r'spaces', r'header', r'macro.end'),
-        #PatternGroupRule(r'includegrp', r'macro.start', r'# *include', r'spaces',
-        #                 #r' +', r'header', r'< *[-A-Za-z/0-9_.]+ *>|" *[-A-Za-z/0-9_.]+ *"',
-        #                 r' +', r'header', r'< *[-A-Za-z/0-9_.]+ *>|" *[-A-Za-z/0-9_.]+ *"|[A-Za-z0-9_]+',
-        #                 'macro.end', r'\n$'),
        PatternRule(r'identifier', r"[a-zA-Z_][a-zA-Z0-9_]*"),
        OverridePatternRule(r'comment', r'/\* *@@:(?P<token>[.a-zA-Z0-9_]+):(?P<mode>[.a-zA-Z0-9_]+) *\*/$'),
        OverridePatternRule(r'comment', r'// *@@:(?P<token>[.a-zA-Z0-9_]+):(?P<mode>[.a-zA-Z0-9_]+) *$'),
--- a/mode/dir.py
+++ b/mode/dir.py
@ -1,7 +1,7 @@
 import commands, dirutil, grp, method, mode, os.path, pwd, re
 import buffer, buffer.fs
 import window
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule
 from point import Point
 from method import Method, Argument

@ -14,17 +14,14 @@ class PermGrammar(Grammar):
        PatternRule(r'perm_exec', r'x'),
    ]

+ds = r'([^ ]+)( +)([^ ]+)( +)([^ ]+)( +)([A-Za-z]{3} [ 0-9]{2} [0-9]{2}:[0-9]{2})( +)([^\n]+)'
 class PathGrammar(Grammar):
    rules = [
        RegionRule(r'dir_perm', r'(?<=^.)', PermGrammar, r' '),
-        PatternGroupRule(
-            r'dir_fields',
-            r'dir_owner', r'[^ ]+ +',
-            r'dir_group', r'[^ ]+ +',
-            r'dir_size', r'[^ ]+ +',
-            r'dir_mtime', r'[A-Za-z]{3} [ 0-9]{2} [0-9]{2}:[0-9]{2} +',
-            r'dir_name', r'[^\n]*',
-        ),
+        PatternMatchRule(r'x', ds,
+                         r'dir_fields', r'spaces', r'dir_owner', r'spaces',
+                         r'dir_group', r'spaces', r'dir_size', r'spaces',
+                         r'dir_mtime', r'spaces', r'dir_name', r'spaces'),
    ]

 class DirGrammar(Grammar):
--- a/mode/forth.py
+++ b/mode/forth.py
@ -1,8 +1,8 @@
 import time
 import tab
 from mode import Fundamental
-from lex import Grammar, PatternRule, RegionRule, NocasePatternRule, \
-    NocaseRegionRule, NocasePatternGroupRule
+from lex import Grammar, PatternRule, RegionRule, NocasePatternRule
+from lex import NocaseRegionRule, NocasePatternMatchRule
 from mode.python import StringGrammar2
 from mode.pipe import Pipe
 from method.shell import Interact
@ -47,8 +47,8 @@ class ForthGrammar(Grammar):
        NocasePatternRule(r'builtin', r'(?:assembler|code|end-code|;code|flush-icache|c,)(?= |\n|$)'),

        # xyz
-        NocasePatternGroupRule(r'declaration', r'delimiter', r':',
-                               r'spaces', r' +', r'function', r'[^ ]+'),
+        NocasePatternMatchRule(r'x', r'(:)( +)([^ ]+)', r'delimiter', r'spaces',
+                               r'function'),
        PatternRule(r'delimiter', r"[:;\[\]]"),
        NocasePatternRule(r'number', r"'[a-z](?= |$)"),
        NocasePatternRule(r'number', r'%?-?[0-1]+\.?(?= |$)'),
--- a/mode/fstab.py
+++ b/mode/fstab.py
@ -1,20 +1,26 @@
-import color, mode
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from mode import Fundamental
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule
+
+fields = (
+    (r'fstab_device', r'^ *(?:[^# \n\t]+|\\.)+'),
+    (r'fstab_point', r' *(?:[^# \n\t]+|\\.)+'),
+    (r'fstab_type', r' *(?:[^# \n\t]+|\\.)+'),
+    (r'fstab_options', r' *(?:[^# \n\t]+|\\.)+'),
+    (r'fstab_dump', r' *(?:[^# \n\t]+|\\.)+'),
+    (r'fstab_pass', r' *(?:[^# \n\t]+|\\.)+'),
+    (r'fstab_error', r'[^\n]*'),
+    (r'fstab_eol', r'\n'),
+)
+fregex = ''.join(['(%s)' % tpl[1] for tpl in fields])
+fnames = [tpl[0] for tpl in fields]

 class FstabGrammar(Grammar):
    rules = [
        PatternRule(r'comment', r'#.*$'),
-        PatternGroupRule(r'fstab_stanza',
-                         r'fstab_device', r'^ *(?:[^# \n\t]+|\\.)+',
-                         r'fstab_point', r' *(?:[^# \n\t]+|\\.)+',
-                         r'fstab_type', r' *(?:[^# \n\t]+|\\.)+',
-                         r'fstab_options', r' *(?:[^# \n\t]+|\\.)+',
-                         r'fstab_dump', r' *(?:[^# \n\t]+|\\.)+',
-                         r'fstab_pass', r' *(?:[^# \n\t]+|\\.)+',
-                         r'fstab_error', r'[^\n]*', r'fstab_eol', r'\n'),
+        PatternMatchRule(r'x', fregex, *fnames),
    ]

-class Fstab(mode.Fundamental):
+class Fstab(Fundamental):
    name      = 'fstab'
    basenames = ['fstab']
    grammar   = FstabGrammar
--- a/mode/java.py
+++ b/mode/java.py
@ -1,13 +1,13 @@
 import color, mode, tab
 import context
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule
 from mode.python import StringGrammar2
 from mode.c import CTabber2
 from parse import Any, And, Or, Optional, Name, Match, Matchs

 class CommentGrammar(Grammar):
    rules = [
-        PatternGroupRule(r'xyz', 'javadoc', r'@[a-z]+', r'spaces', r' +', r'javaname', r'[^ ]+'),
+        PatternMatchRule(r'x', r'(@[a-z]+)( +)([^ ]+)', r'javadoc', r'spaces', r'javaname'),
        PatternRule(r"javadoc", r"@[a-z]+"),
        PatternRule(r"data", r"(?:[^@*]|\*(?!/))+"),
    ]
--- a/mode/javascript.py
+++ b/mode/javascript.py
@ -1,5 +1,5 @@
 import color, mode, tab
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule
 from point import Point

 class StringGrammar1(Grammar):
@ -30,17 +30,12 @@ class JavascriptGrammar(Grammar):
        RegionRule(r'comment', '/\*', Grammar, '\*/'),
        PatternRule(r'continuation', r'\\(?= *$)'),

-        PatternGroupRule(r'func_def', 'js_reserved', r'function', r'spaces', r' +', r'js_function', r'[a-zA-Z_][a-zA-Z0-9_]*'),
-        PatternGroupRule(r'class_def', 'js_reserved', r'class', r'spaces', r' +', r'js_class', r'[a-zA-Z_][a-zA-Z0-9_]*'),
-        PatternGroupRule(r'class_new', r'js_reserved', r'new', r'spaces', r' +', r'js_class', r'[a-zA-Z_][a-zA-Z0-9_]*'),
+        PatternMatchRule(r'x', r'(function)( +)([a-zA-Z_][a-zA-Z0-9_]*)', 'js_reserved', r'spaces', r'js_function'),
+        PatternMatchRule(r'x', r'(class|new)( +)([a-zA-Z_][a-zA-Z0-9_]*)', 'js_reserved', r'spaces', r'js_class'),
        
        PatternRule(r'js_reserved', r'(?:abstract|as|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|false|finally|for|function|get|goto|if|import|implements|include|instanceof|interface|in|is|namespace|native|new|null|package|private|protected|public|return|set|super|switch|synchronized|this|throws|throw|transient|true|try|typeof|use|var|void|volatile|while|with)(?![a-zA-Z0-9_])'),

-        PatternGroupRule(r'func_use', r"js_function", r"[a-zA-Z_][a-zA-Z0-9_]", r'spaces', ' +',
-                         r'delimiter', r"\("),
-        PatternRule(r"js_function", r"[a-zA-Z_][a-zA-Z0-9_]*(?= *\()"),
        PatternRule(r'identifier', r"[a-zA-Z_][a-zA-Z0-9_]*"),
-
        PatternRule(r'integer', r"(?:0|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
        PatternRule(r'float', r"[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"),

--- a/mode/lily.py
+++ b/mode/lily.py
@ -1,5 +1,5 @@
 import color, mode
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule
 from mode.scheme import SchemeGrammar
 from mode.python import StringGrammar1, StringGrammar2
 import regex
@ -17,13 +17,11 @@ class LilyGrammar(Grammar):
        RegionRule(r'comment', r'%\{', CommentGrammar, r'%\}'),
        RegionRule(r'string', r'#?"', StringGrammar2, r'"'),
        PatternRule(r'boolean', r'#?#[tf]'),
-        PatternGroupRule(r'd1', r'directive', r'\\new', r'spaces', r' +', r'lily_class', r'[a-zA-Z_]+'),
-        PatternGroupRule(r'd2', r'directive', r'\\set', r'spaces', r' +', r'lily_var', r'[a-zA-Z_]+'),
-        PatternGroupRule(r'd3', r'directive', r'\\override', r'spaces', r' +', r'lily_var', r'[a-zA-Z_]+'),
+        PatternMatchRule(r'x', r'(\\new)( +)([a-zA-Z_]+)', r'directive', r'spaces', r'lily_class'),
+        PatternMatchRule(r'x', r'(\\set|\\override)( +)([a-zA-Z_]+)', r'directive', r'spaces', r'lily_var'),
        PatternRule(r'tie', r'~'),
        PatternRule(r'augmentation', r'\.+'),
        PatternRule(r'directive', r'\\[a-zA-Z_][a-zA-Z_0-9]*'),
-        #PatternRule(r'dynamics', r'\\[a-z]+'),
        PatternRule(r'delimiter', r'(?:[={}]|<<|>>)'),
        RegionRule(r'text', r'^"', StringGrammar2, r'"'),
        RegionRule(r'markup', r'(?<=\\markup) *{', MarkupGrammar, '}'),
--- a/mode/make.py
+++ b/mode/make.py
@ -1,5 +1,5 @@
 import color, mode
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from lex import Grammar, PatternRule, RegionRule
 from mode.sh import ShGrammar, Sh

 class StringGrammar1(Grammar):
--- a/mode/mbox.py
+++ b/mode/mbox.py
@ -1,7 +1,7 @@
 import commands, dirutil, grp, mailbox, method, mode, os.path, pwd, re, sys
 import buffer, default, window
 from mode.mutt import MuttGrammar
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule
 from point import Point
 from buffer import Buffer
 from method import Method, Argument, arg
@ -192,13 +192,11 @@ class MboxOpenPath(Method):

 class MailListGrammar(Grammar):
    rules = [
-        PatternGroupRule(
-            r'xyz', r'index', r'^ *[0-9]+', r'spaces', r' ',
-            r'flag', r'.', r'spaces', r' ',
-            r'month', r'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec',
-            r'spaces', r' +', r'day', r'[0-9]+', r'spaces', r' +',
-            r'sender', r'.{16}', r'spaces', r' +', r'size', r'[0-9]+',
-            r'spaces', ' +', r'subject', r'.+$',
+        PatternMatchRule(
+            r'x', r'^( *)([0-9]+)( )(.)( )(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)( +)([0-9]+)( +)(.{16})( +)([0-9]+)( +)(.+)$',
+            r'spaces', r'index', r'spaces', r'flag', r'spaces', r'month',
+            r'spaces', r'day', r'spaces', r'sender', r'spaces', r'size',
+            r'spaces', r'subject'
        ),
    ]

--- a/mode/mp3.py
+++ b/mode/mp3.py
@ -1,7 +1,7 @@
 import commands, dirutil, grp, mailbox, method, mode, os.path, pwd, re, sys
 import buffer, default, util, window
 from mode.mutt import MuttGrammar
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
+from lex import Grammar, PatternRule, RegionRule
 from point import Point
 from buffer import Buffer
 from method import Method, Argument, arg
--- a/mode/perl.py
+++ b/mode/perl.py
@ -2,8 +2,8 @@ import os, re, string, sys
 from subprocess import Popen, PIPE, STDOUT
 import buffer, color, commands, completer, context, method, mode, regex, tab
 from point import Point
-from lex import Grammar, PatternRule, ContextPatternRule, RegionRule, \
-    OverridePatternRule, PatternGroupRule
+from lex import Grammar, PatternRule, ContextPatternRule, RegionRule
+from lex import OverridePatternRule, PatternMatchRule
 from method import Argument, Method, WrapParagraph
 from tab import StackTabber, StackTabber2
 from parse import Any, And, Or, Optional, Name, Match, Matchs
@ -21,20 +21,32 @@ class PodGrammar(Grammar):
        RegionRule(r'entry', r'(?<=^=encoding) +.*$', PodDataGrammar, '^\n$'),
    ]

+strg1 = r"'(?:\\.|[^'\\])*'"
+strg2 = r'"(?:\\.|[^"\\])*"'
+wchr1 = '[a-zA-Z_]'
+wchr2 = '[a-zA-Z0-9_]'
+hword = wchr2 + '+'
+word1 = wchr1 + wchr2 + '*'
+word2 = '(?:' + word1 + '::)*' + word1
+pname = '[.a-zA-Z0-9_]+'
+
 def _make_string_rules(forbidden):
    rules = [
        PatternRule(r'octal', r'\\[0-7]{3}'),
        PatternRule(r'escaped', r'\\.'),
-        PatternRule(r'deref', r"\$+[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*(?:(?:->)?{\$?(?:[a-zA-Z_][a-zA-Z_0-9]*|'(?:\\.|[^'\\])*'|\"(\\.|[^\\\"])*\")}|(?:->)?\[\$?[0-9a-zA-Z_]+\])+"),
-        PatternRule(r'length', r"\$#[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"),
-        PatternRule(r'scalar', r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"),
-        PatternRule(r'cast', r"[\$\@\%\&]{.*?}"),
-        PatternRule(r'array', r"@\$*[A-Za-z_](?:[A-Za-z0-9_]|::)*"),
+        PatternRule(r'deref', r"\$+" + word2 + "(?:"+
+                    "(?:->)?{\$?(?:" + hword + "|" + strg1 + "|" + strg2 + ")}|" +
+                    "(?:->)?\[\$?"+hword+"\]"+
+                    ")+"),
+        PatternRule(r'length', r"\$#" + word2),
+        PatternRule(r'scalar', r"\$\$*" + word2),
+        PatternRule(r'cast', r"[\$\@\%\&]{.+?}"),
+        PatternRule(r'array', r"@\$*" + word2),
    ]
    return rules

 class QuotedWords(Grammar): rules = [
-    PatternRule(r'data', r'[a-zA-Z0-9_]+'),
+    PatternRule(r'data', hword),
    PatternRule(r'eol', r'\n'),
    PatternRule(r'spaces', r' +'),
 ]
@ -45,75 +57,63 @@ class NoAngle(Grammar):     rules = [PatternRule(r'data', r'[^>]+')]
 class NoHash(Grammar):      rules = [PatternRule(r'data', r'[^#]+')]
 class DataGrammar(Grammar): rules = [PatternRule(r'data', r'.+')]

-class StrictStringGrammar(Grammar):
-    rules = [
-        PatternRule(r'escaped', r"\\'"),
-    ]
-class StringGrammar(Grammar):
-    rules = _make_string_rules('"')
+class StrictStringGrammar(Grammar): rules = [PatternRule(r'escaped', r"\\'")]
+class StringGrammar(Grammar): rules = _make_string_rules('"')
+class EvalGrammar(Grammar): rules = _make_string_rules('`')

-class EvalGrammar(Grammar):
-    rules = _make_string_rules('`')
+class TranslateGrammar1(Grammar): rules = [PatternRule(r'data', r"(?:\\.|[^\\/])")]
+class TranslateGrammar2(Grammar): rules = [PatternRule(r'data', r"(?:\\.|[^\\#])")]
+class TranslateGrammarX(Grammar): rules = [PatternRule(r'data', r"(?:\\.|[^\\%(delim)s])")]

-class TranslateGrammar1(Grammar):
-    rules = [PatternRule(r'data', r"(?:\\.|[^\\/])")]
-class TranslateGrammar2(Grammar):
-    rules = [PatternRule(r'data', r"(?:\\.|[^\\#])")]
-class TranslateGrammarX(Grammar):
-    rules = [PatternRule(r'data', r"(?:\\.|[^\\%(delim)s])")]
+class MatchGrammar1(Grammar): rules = _make_string_rules('/')
+class MatchGrammar2(Grammar): rules = _make_string_rules('#')
+class MatchGrammar3(Grammar): rules = _make_string_rules(')')
+class MatchGrammar4(Grammar): rules = _make_string_rules(']')
+class MatchGrammar5(Grammar): rules = _make_string_rules('}')
+class MatchGrammar6(Grammar): rules = _make_string_rules('>')

-class MatchGrammar1(Grammar):
-    rules = _make_string_rules('/')
-class MatchGrammar2(Grammar):
-    rules = _make_string_rules('#')
-class MatchGrammar3(Grammar):
-    rules = _make_string_rules(')')
-class MatchGrammar4(Grammar):
-    rules = _make_string_rules(']')
-class MatchGrammar5(Grammar):
-    rules = _make_string_rules('}')
-class MatchGrammar6(Grammar):
-    rules = _make_string_rules('>')
-
-class QuotedGrammar1(Grammar):
-    rules = _make_string_rules(')')
-class QuotedGrammar2(Grammar):
-    rules = _make_string_rules('}')
-class QuotedGrammar3(Grammar):
-    rules = _make_string_rules('>')
-class QuotedGrammar4(Grammar):
-    rules = _make_string_rules(']')
+class QuotedGrammar1(Grammar): rules = _make_string_rules(')')
+class QuotedGrammar2(Grammar): rules = _make_string_rules('}')
+class QuotedGrammar3(Grammar): rules = _make_string_rules('>')
+class QuotedGrammar4(Grammar): rules = _make_string_rules(']')

 class PerlGrammar(Grammar):
    rules = [
-        RegionRule(r'heredoc', r"<<(?P<heredoc>[a-zA-Z_][a-zA-Z0-9_]+)", None, ';\n', StringGrammar, r'^%(heredoc)s$'),
-        RegionRule(r'heredoc', r'<< *"(?P<heredoc>[a-zA-Z0-9_]+)" *;', StringGrammar, r'^%(heredoc)s$'),
-        RegionRule(r'heredoc', r"<< *'(?P<heredoc>[a-zA-Z0-9_]+)' *;", DataGrammar, r'^%(heredoc)s$'),
-        RegionRule(r'evaldoc', r"<< *`(?P<heredoc>[a-zA-Z0-9_]+)` *;", StringGrammar, r'^%(heredoc)s$'),
+        RegionRule(r'heredoc', r"<<(?P<heredoc>" + word1 + ")", None,
+                   ';\n', StringGrammar, r'^%(heredoc)s$'),
+        RegionRule(r'heredoc', r'<< *"(?P<heredoc>[^"]+)"', None,
+                   ';\n', StringGrammar, r'^%(heredoc)s$'),
+        RegionRule(r'heredoc', r"<< *'(?P<heredoc>[^']+)'", None,
+                   ";\n", DataGrammar, r'^%(heredoc)s$'),
+        RegionRule(r'evaldoc', r"<< *`(?P<heredoc>[^`]+)`", None,
+                   ";\n", StringGrammar, r'^%(heredoc)s$'),

        RegionRule(r'endblock', r"^__END__|__DATA__ *$", DataGrammar, r''),
-        RegionRule(r'pod', r'^=[a-zA-Z0-9_]+', PodGrammar, r'^=cut'),
+        RegionRule(r'pod', '^=' + word1, PodGrammar, r'^=cut'),
+
+        OverridePatternRule(r'comment', r'#@@:(?P<token>' + pname + '):(?P<mode>' + pname + ') *$'),
+
+        PatternMatchRule('x', r'(sub)( +)(' + word2 + r')( *)(\()( *)([\[\]\\@$%&*;]+)( *)(\))',
+                         'perl_keyword', 'spaces', 'sub', 'spaces', 'delimiter',
+                         'spaces', 'prototype', 'spaces', 'delimiter'),

-        OverridePatternRule(r'comment', r'#@@:(?P<token>[.a-zA-Z0-9_]+):(?P<mode>[.a-zA-Z0-9_]+) *$'),
-        PatternGroupRule(r'prototype', r'delimiter', r'\(', r'prototype', r'[\[\]\\@$%&*;]+', r'delimiter', '\)'),
        PatternRule(r'comment', r'#.*$'),
        RegionRule(r'perl_string', r'"', StringGrammar, r'"'),
        RegionRule(r'perl_string', r"'", StrictStringGrammar, r"'"),
        RegionRule(r'evalstring', r"`", EvalGrammar, r"`"),
        PatternRule(r'number', r'0?\.[0-9]+|[0-9]+(?:\.[0-9]+)?'),
        PatternRule(r'perl_keyword', r"(?<!->)(?:STDIN|STDERR|STDOUT|continue|do|else|elsif|eval|foreach|for|if|last|my|next|our|package|require|return|sub|undef|unless|until|use|while)(?![a-zA-Z0-9_])"),
-        PatternRule(r'hash_key', r'(?<={)[A-Za-z0-9_]+(?=})'),
-        PatternRule(r'perl_method', r'(?<=->)[A-Za-z_][A-Za-z0-9_]*'),
-        PatternRule(r'hash_key', r'[A-Za-z0-9_]+(?= *=>)'),
-        PatternRule(r'length', r"\$#[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"),
-        PatternRule(r'cast', r'[\$\@\%\^\&](?= *{)'),
-        PatternRule(r'scalar', r"\$[\[\]<>ab/'\"_@\?#\$!%^|&*()](?![A-Za-z0-9_])"),
+        PatternRule(r'hash_key', r'(?<={)' + wchr2 + '+(?=})'),
+        PatternRule(r'perl_method', r'(?<=->)' + word1),
+        PatternRule(r'hash_key', wchr2 + r'+(?= *=>)'),
+        PatternRule(r'length', r"\$#" + word2),
+        PatternRule(r'cast', r'[\$\@\%\&\*](?= *{)'),
+        PatternRule(r'scalar', r"\$[\[\]<>ab/'\"_@\?#\$!%^|&*()](?!" + wchr2 + ")"),
        PatternRule(r'array', r"@_"),
-        PatternRule(r'perl_function', r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*(?=-> *\()"),
-        PatternRule(r'scalar', r"\$\$*[A-Za-z0-9_](?:[A-Za-z0-9_]|::)*"),
-        PatternRule(r'array', r"@\$*[A-Za-z_](?:[A-Za-z0-9_]|::)*"),
-        PatternRule(r'perl_hash', r"%\$*[A-Za-z_](?:[A-Za-z0-9_]|::)*"),
-        PatternRule(r'deref', r"[@%\$&\*](?={)"),
+        PatternRule(r'perl_function', r"\$\$*" + word2 + "(?=-> *\()"),
+        PatternRule(r'scalar', r"\$\$*" + word2),
+        PatternRule(r'array', r"@\$*" + word2),
+        PatternRule(r'perl_hash', r"%\$*" + word2),

        # match regexes; paired delimiters
        RegionRule(r'match', r'm *(?P<delim>\()', MatchGrammar3, r'\)[a-z]*'),
@ -144,12 +144,12 @@ class PerlGrammar(Grammar):
        RegionRule(r'translate', r'(?:y|tr) *(?P<delim>[^ a-zA-Z0-9_])', TranslateGrammarX, r'%(delim)s', TranslateGrammarX, r'%(delim)s[a-z]*'),

        # some more basic stuff
-        PatternRule(r'package', r"(?<=package )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*"),
-        PatternRule(r'sub', r"(?<=sub )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*"),
-        PatternRule(r'use', r"(?<=use )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*"),
-        PatternRule(r'require', r"(?<=require )(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*"),
-        PatternRule(r'perl_label', r'[a-zA-Z_][a-zA-Z0-9_]*:(?!:)'),
-        PatternRule(r'perl_function', r"&\$*(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*"),
+        PatternRule(r'package', r"(?<=package )" + word2),
+        PatternRule(r'sub', r"(?<=sub )" + word2),
+        PatternRule(r'use', r"(?<=use )" + word2),
+        PatternRule(r'require', r"(?<=require )" + word2),
+        PatternRule(r'perl_label', word1 + ':(?!:)'),
+        PatternRule(r'perl_function', r"&\$*" + word2),
        PatternRule(r'perl_builtin', r"(?<!->)&?(?:write|warn|wantarray|waitpid|wait|vec|values|utime|use|untie|unshift|unpack|unlink|undef|umask|ucfirst|uc|truncate|times|time|tied|tie|telldir|tell|syswrite|system|sysseek|sysread|sysopen|syscall|symlink|substr|sub|study|stat|srand|sqrt|sprintf|split|splice|sort|socketpair|socket|sleep|sin|shutdown|shmwrite|shmread|shmget|shmctl|shift|setsockopt|setservent|setpwent|setprotoent|setpriority|setpgrp|setnetent|sethostent|setgrent|send|semop|semget|semctl|select|seekdir|seek|scalar|rmdir|rindex|rewinddir|reverse|return|reset|require|rename|ref|redo|recv|readpipe|readlink|readline|readdir|read|rand|quotemeta|push|prototype|printf|print|pos|pop|pipe|package|pack|our|ord|opendir|open|oct|no|next|my|msgsnd|msgrcv|msgget|msgctl|mkdir|map|lstat|log|lock|localtime|local|listen|link|length|lcfirst|lc|last|kill|keys|join|ioctl|int|index|import|hex|grep|goto|gmtime|glob|getsockopt|getsockname|getservent|getservbyport|getservbyname|getpwuid|getpwnam|getpwent|getprotoent|getprotobynumber|getprotobyname|getpriority|getppid|getpgrp|getpeername|getnetent|getnetbyname|getnetbyaddr|getlogin|gethostent|gethostbyname|gethostbyaddr|getgrnam|getgrgid|getgrent|getc|formline|format|fork|flock|fileno|fcntl|exp|exit|exists|exec|eval|eof|endservent|endpwent|endprotoent|endnetent|endhostent|endgrent|each|dump|do|die|delete|defined|dbmopen|dbmclose|crypt|cos|continue|connect|closedir|close|chroot|chr|chown|chop|chomp|chmod|chdir|caller|bless|binmode|bind|atan2|alarm|accept|abs)(?![a-zA-Z0-9_])"),

        # quote operator: qq(), qx() and qr() usually interpolate
@ -169,17 +169,16 @@ class PerlGrammar(Grammar):
        RegionRule(r'quoted', r'qw?#', NoHash, r'#'),
        RegionRule(r'quoted', r'qw? *(?P<delim>[^ a-zA-Z0-9#])', Grammar, r'%(delim)s'),

-        PatternRule(r'perl_function', r"(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*(?= *\()"),
-        PatternRule(r'perl_namespace', r"(?:[a-zA-Z_][a-zA-Z_0-9]*\:\:)+(?:[a-zA-Z_][a-zA-Z_0-9]*)?"),
-        PatternRule(r'perl_class', r"(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*(?=->)"),
+        PatternRule(r'perl_function', word2 + "(?= *\()"),
+        PatternRule(r'perl_class', word2 + "(?=->)"),

        # some basic stuff
        PatternRule(r'delimiter', r"::|->|=>|(?<!:):(?!=:)|[,;=\?(){}\[\]\(\)]"),
-        PatternRule(r'noperator', r"-[rwxoRWXOezsfdlpSbctugkTBMAC]"),
+        PatternRule(r'noperator', r"-[rwxoRWXOezsfdlpSbctugkTBMAC](?!" + wchr2 + ")"),
        PatternRule(r'operator', r"\+=|-=|\*=|/=|//=|%=|&=\|\^=|>>=|<<=|\*\*=|\\"),
        PatternRule(r'operator', r"\+\+|\+|<=>|<>|<<|<=|<|-|>>|>=|>|\*\*|\*|&&|&|\|\||\||/|\^|==|//|~|=~|!~|!=|%|!|\.|x(?![a-zA-Z_])"),
        PatternRule(r'noperator', r"(?:xor|or|not|ne|lt|le|gt|ge|eq|cmp|and)(?![a-zA-Z_])"),
-        PatternRule(r'bareword', r'(?:[a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*'),
+        PatternRule(r'bareword', word2),

        PatternRule(r'spaces', r' +'),
        PatternRule(r"eol", r"\n$"),
@ -682,7 +681,6 @@ class Perl(mode.Fundamental):
        'hash_key':       ('green', 'default', 'bold'),
        'perl_method':    ('cyan', 'default', 'bold'),
        'perl_function':  ('cyan', 'default', 'bold'),
-        'perl_namespace': ('cyan', 'default', 'bold'),
        'perl_builtin':   ('magenta', 'default', 'bold'),
        'perl_label':     ('cyan', 'default', 'bold'),
        'package':        ('cyan', 'default', 'bold'),
--- a/mode/rst.py
+++ b/mode/rst.py
@ -1,5 +1,5 @@
 import color, method, mode
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule, OverrideRegionRule
+from lex import Grammar, PatternRule, RegionRule, OverrideRegionRule
 from method import Method
 from mode.text import TextInsertSpace
 from mode.python import PythonGrammar
--- a/mode/sh.py
+++ b/mode/sh.py
@ -1,74 +1,83 @@
 import commands
 import color, mode, tab
-from lex import Grammar, PatternRule, RegionRule, PatternGroupRule, OverridePatternRule
+from lex import Grammar, PatternRule, RegionRule, PatternMatchRule, OverridePatternRule
 from method import Method

-class StringGrammar1(Grammar):
-    rules = [
+char  = '[a-zA-Z0-9_]'
+word  = char + '+'
+pname = '[.a-zA-Z0-9_]+'
+
+class StringGrammar1(Grammar): pass
+class StringGrammar2(Grammar): pass
+class HereGrammar(Grammar): pass
+class EvalGrammar(Grammar): pass
+class NevalGrammar(Grammar): pass
+class StanzaGrammar(Grammar): pass
+class CaseGrammar(Grammar): pass
+class TestGrammar(Grammar): pass
+class ShGrammar(Grammar): pass
+
+StringGrammar1.rules = [
    PatternRule(r'data', r'[^\']+'),
 ]
-class StringGrammar2(Grammar):
-    rules = [
+
+StringGrammar2.rules = [
    PatternRule(r'escaped', r'\\.'),
-        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
-        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
+    PatternRule(r'variable', r"\${(?:" + word + "|\?\$)}"),
+    PatternRule(r"variable", r"\$" + word),
    PatternRule(r'variable', r"\$(?=\()"),
    PatternRule(r'data', r'[^\\$"]+'),
 ]

-class HereGrammar(Grammar):
-    rules = [
+HereGrammar.rules = [
    PatternRule(r'escaped', r'\\.'),
-        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
-        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
+    PatternRule(r'variable', r"\${(?:" + word + "|\?\$)}"),
+    PatternRule(r"variable", r"\$" + word),
    PatternRule(r'variable', r"\$(?=\()"),
    PatternRule(r'data', r'[^\\$]+'),
 ]        

-class EvalGrammar(Grammar):
-    rules = [
+EvalGrammar.rules = [
    RegionRule(r'string', "'", StringGrammar1, "'"),
    RegionRule(r'string', '"', StringGrammar2, '"'),
    PatternRule(r'escaped', r'\\.'),
-        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
-        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
+    PatternRule(r'variable', r"\${(?:" + word + "|\?\$)}"),
+    PatternRule(r"variable", r"\$" + word),
    PatternRule(r'variable', r"\$(?=\()"),
    PatternRule(r'data', r'[^\\`$]+'),
 ]
-class NevalGrammar(Grammar):
-    rules = [
+
+NevalGrammar.rules = [
    RegionRule(r'string', "'", Grammar, "'"),
    RegionRule(r'string', '"', StringGrammar2, '"'),
    PatternRule(r'escaped', r'\\.'),
-        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
-        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
+    PatternRule(r'variable', r"\${(?:" + word + "|\?\$)}"),
+    PatternRule(r"variable", r"\$" + word),
    PatternRule(r'variable', r"\$(?=\()"),
    PatternRule(r'data', r'[^\\)$]+'),
 ]

-class StanzaGrammar(Grammar):
-    rules = [
+StanzaGrammar.rules = [
    PatternRule(r'spaces', r' +'),
    PatternRule(r'start_cont', r'.+\\'),
    PatternRule(r'eol', r'\n'),
 ]
-class CaseGrammar(Grammar):
-    rules = [
+
+CaseGrammar.rules = [
    PatternRule(r'comment', r'#.*$'),
    PatternRule(r'spaces', r' +'),
-        RegionRule(r'stanza', r'.+\\\n$', StanzaGrammar, r'.+\)', Grammar, r';;'),
-        RegionRule(r'stanza', r'.+?\)', Grammar, r';;'),
+    RegionRule(r'stanza', r'.+\\\n$', StanzaGrammar, r'.+\)', ShGrammar, r';;'),
+    RegionRule(r'stanza', r'.+?\)', ShGrammar, r';;'),
    PatternRule(r'eol', r'\n'),
 ]

-class TestGrammar(Grammar):
-    rules = [
+TestGrammar.rules = [
    PatternRule(r'spaces', r' +'),
    PatternRule(r'sh_builtin', r"(?<![-a-zA-Z0-9_])(?:source|alias|bg|bind|break|builtin|cd|command|compgen|complete|declare|dirs|disown|echo|enable|eval|exec|exit|export|fc|fg|getops|hash|help|history|jobs|kill|let|local|logout|popd|printf|pushd|pwd|readonly|read|return|set|shift|shopt|suspend|test|times|trap|type|ulimit|umask|unalias|unset|wait)(?![a-zA-Z0-9_=/])"),
    PatternRule(r'sh_reserved', r"(?:done|do|elif|else|esac|fi|for|function|if|in|select|then|until|while|time)(?![a-zA-Z0-9_=/])"),
    PatternRule(r'binop', r'==|=|!='),
-        PatternRule(r'binop', r'-(?:nt|ot|ef|eq|ne|lt|gt|le|ge)(?![a-zA-Z0-9_])'),
-        PatternRule(r'unop', r'-[a-zA-Z](?![a-zA-Z0-9_])'),
+    PatternRule(r'binop', r'-(?:nt|ot|ef|eq|ne|lt|gt|le|ge)(?!' + char + ')'),
+    PatternRule(r'unop', r'-[a-zA-Z](?!' + char + ')'),
    PatternRule(r'continuation', r'\\\n$'),
    PatternRule(r'redirect', r'<|>'),
    PatternRule(r'delimiter', r";;|[();{}|&><]"),
@ -77,25 +86,27 @@ class TestGrammar(Grammar):
    RegionRule(r'test3', r'\[', None, r'\]'),
    RegionRule(r'eval', r'`', EvalGrammar, r'`'),
    RegionRule(r'neval', r'\$\(', NevalGrammar, r'\)'),
-        PatternRule(r'variable', r"(?:^|(?<= ))[a-zA-Z_][a-zA-Z0-9_]*(?==)"),
-        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
-        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
+    PatternRule(r'variable', r"(?:^|(?<= ))" + word + "(?==)"),
+    PatternRule(r'variable', r"\${(?:" + word + "|\?\$)}"),
+    PatternRule(r"variable", r"\$" + word),
    PatternRule(r'variable', r"\$(?=\()"),
    RegionRule(r'string', "'", StringGrammar1, "'"),
    RegionRule(r'string', '"', StringGrammar2, '"'),
-        PatternRule(r'bareword', r'[-a-zA-Z0-9_.]+'),
+    PatternRule(r'sh_bareword', r'[-a-zA-Z0-9_.]+'),
 ]

-class ShGrammar(Grammar):
-    rules = [
-        PatternGroupRule(r'vardecl', r'spaces', r'^ +', r'variable', r'[a-zA-Z_][a-zA-Z0-9_]*', r'delimiter', r'='),
-        PatternGroupRule(r'vardecl', r'sh_builtin', r'(?:alias|export)', r'spaces', r' +', r'variable', r'[a-zA-Z_][a-zA-Z0-9_]*', r'delimiter', r'='),
-        PatternRule(r'variable', r"^[a-zA-Z_][a-zA-Z0-9_]*(?==)"),
+ShGrammar.rules = [
+    PatternMatchRule('x', '( *)(' + word + ')(=)',
+                     'spaces', 'variable', 'delimiter'),
+    PatternMatchRule('x', '(alias|export)( +)(' + word + ')(=)',
+                     'sh_builtin', 'spaces', 'variable', 'delimiter'),
+    PatternMatchRule('x', '(unset)( +)(' + word + ')',
+                     'sh_builtin', 'spaces', 'variable'),

    PatternRule(r'spaces', r' +'),
-        RegionRule(r'heredoc', r"<<[<\\]?(?P<heredoc>[a-zA-Z_][a-zA-Z0-9_]*)", None, "\n", HereGrammar, r'^%(heredoc)s$'),
-        RegionRule(r'heredoc', r"<<-(?P<heredoc>[a-zA-Z_][a-zA-Z0-9_]*)", None, "\n", HereGrammar, r'^ *%(heredoc)s$'),
-        PatternRule(r'sh_function', r'[a-zA-Z_][a-zA-Z0-9_]*(?= *\(\))'),
+    RegionRule(r'heredoc', r"<<[<\\]?(?P<heredoc>" + word + ")", None, "\n", HereGrammar, r'^%(heredoc)s$'),
+    RegionRule(r'heredoc', r"<<-(?P<heredoc>" + word + ")", None, "\n", HereGrammar, r'^ *%(heredoc)s$'),
+    PatternRule(r'sh_function', word + r'(?= *\(\))'),
    PatternRule(r'sh_reserved', r"(?:done|do|elif|else|esac|fi|for|function|if|in|select|then|until|while|time)(?![a-zA-Z0-9_=/])"),
    RegionRule(r'case', r'case', None, 'in', CaseGrammar, r'esac'),
    PatternRule(r'sh_builtin', r"(?<![-a-zA-Z0-9_])(?:source|alias|bg|bind|break|builtin|cd|command|compgen|complete|declare|dirs|disown|echo|enable|eval|exec|exit|export|fc|fg|getops|hash|help|history|jobs|kill|let|local|logout|popd|printf|pushd|pwd|readonly|read|return|set|shift|shopt|suspend|times|trap|type|ulimit|umask|unalias|unset|wait)(?![a-zA-Z0-9_=/])(?![-a-zA-Z0-9_])"),
@ -106,25 +117,18 @@ class ShGrammar(Grammar):
    PatternRule(r'delimiter', r";;|[();{}|&><:=/]"),
    RegionRule(r'eval', r'`', EvalGrammar, r'`'),
    RegionRule(r'neval', r'\$\(', NevalGrammar, r'\)'),
-        PatternRule(r'variable', r"^[a-zA-Z_][a-zA-Z0-9_]*(?==)"),
-        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
-        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
-        PatternRule(r'variable', r"\$(?=\()"),
+    PatternRule(r'variable', r"\${(?:" + word + "|\?\$)}"),
+    PatternRule(r"variable", r"\$" + word),
+    #PatternRule(r'variable', r"\$(?=\()"),
    RegionRule(r'string', "'", StringGrammar1, "'"),
    RegionRule(r'string', '"', StringGrammar2, '"'),
-        OverridePatternRule(r'comment', r'#@@:(?P<token>[.a-zA-Z0-9_]+):(?P<mode>[.a-zA-Z0-9_]+) *$'),
+    OverridePatternRule(r'comment', r'#@@:(?P<token>' + pname + '):(?P<mode>' + pname + ') *$'),
    PatternRule(r'comment', r'#.*$'),
-        PatternRule(r'bareword', r'(?:[-a-zA-Z0-9_.]|\\.)+'),
+    PatternRule(r'sh_bareword', r'(?:[-a-zA-Z0-9_.]|\\.)+'),
    PatternRule(r'continuation', r'\\\n$'),
    PatternRule(r'eol', r'\n$'),
 ]

-# hacks to get some circular grammar refs
-rule = CaseGrammar.rules[2]
-rule.pairs[1] = (ShGrammar, rule.pairs[1][1])
-rule = CaseGrammar.rules[3]
-rule.pairs[0] = (ShGrammar, rule.pairs[0][1])
-
 class ShTabber(tab.StackTabber):
    def is_base(self, y):
        if y == 0:
--- a/mode/xml.py
+++ b/mode/xml.py
@ -1,41 +1,6 @@
 import color, method, mode
 from lex import Grammar, Rule, PatternRule, RegionRule

-class PatternGroupRule(PatternRule):
-    def __init__(self, name, *args):
-        assert args and len(args) % 2 == 0
-        i = 0
-        pairs = []
-        while i < len(args):
-            tokname, pattern = args[i], args[i+1]
-            pairs.append((tokname, re.compile(pattern, self.reflags)))
-            i += 2
-        Rule.__init__(self, name)
-        self.pairs = tuple(pairs)
-    def match(self, lexer, parent):
-        (x, y) = (lexer.x, lexer.y)
-        matches = []
-        for (tokname, tokre) in self.pairs:
-            if y >= len(lexer.lines):
-                return []
-            line = self.get_line(lexer, y)
-            m    = tokre.match(line, x)
-            if m:
-                x += len(m.group(0))
-                if x >= len(line):
-                    x = 0
-                    y += 1
-                matches.append((tokname, m))
-            else:
-                return []
-        assert len(matches) == len(self.pairs)
-        return matches
-    def lex(self, lexer, parent, matches):
-        if matches:
-            for (tokname, m) in matches:
-                yield self.make_token(lexer, m.group(0), tokname, parent, m.groupdict())
-        raise StopIteration
-
 class StringGrammar1(Grammar):
    rules = [
        PatternRule(r'data', r'[^"&]+'),