mode fixes and awk

--HG--
branch : pmacs2

application.py

@@ -129,7 +129,7 @@ class Application(object):
                  'python', 'replace', 'rst', 'scheme', 'search', 'sh', 'sql',
                  'tt', 'text', 'text2', 'which', 'xml', 'cheetah', 'colortext',
                  'latex', 'insertmini', 'conf', 'haskell', 'erlang',
-                 'iperl', 'iperlmini', 'ipython', 'ipythonmini',
+                 'iperl', 'iperlmini', 'ipython', 'ipythonmini', 'awk',
                  'bds', #XYZ
                  'shell', 'shellmini', 'fstab'
         )

code_examples/outline_classic11.awk.txt

@@ -0,0 +1,84 @@
+# filename: outline_classic11.awk
+#   author: Eric Pement - pemente@northpark.edu
+#     date: 22 March 2001
+#  version: 1.1
+#
+#  purpose: GNU awk script to modify files created and saved in GNU Emacs
+#           "outline-mode" into classic indented, outline format. E.g.,
+#
+#  INPUT FILE        OUTPUT FILE
+#  ==============    ===================
+#  * Line 1        | A. Line 1
+#  ** Line 2       |   1. Line 2
+#  *** Line 3      |     a. Line 3
+#  *** Line 4      |     b. Line 4
+#  **** Line 5     |       (1) Line 5
+#  ***** Line 6    |         (a) Line 6
+#  ***** Line 7    |         (b) Line 7
+#  ** Line 8       |   2. Line 8
+#  * Line 9        | B. Line 9
+#  ** Line 10      |   1. Line 10
+#
+# NEW! variable "num" determines the amount of increasing indentation.
+# Default is 2 (indent by 2, 4, 6, 8... spaces), but this can be
+# controlled by using the -v switch from the command line. E.g.,
+#
+#       awk -v num=4 -f outline_classic11.awk yourfile.txt
+#
+# Note: this script expects a maximum of five asterisks (*) on a line.
+# Lines of plain text (no asterisks) in source file are NOT indented.
+BEGIN {
+  if (num == "") num = 2            # if num is not defined, set it to 2
+  split("ABCDEFGHIJKLMNOPQRSTUVWXYZ",Lev1,"")
+  split("abcdefghijklmnopqrstuvwxyz",Lev3,"")
+  split("abcdefghijklmnopqrstuvwxyz",Lev5,"")
+}
+
+/^\*/ {
+
+  this_len = match($0,/\*([^*]|$)/);  # get number of stars in 1st field
+  array[this_len]++;                  # increment index of current leaf
+
+  if ( this_len - last_len > 1 ) {    # check for invalid outline levels
+    if (FILENAME == "-" ) myfile = "(piped from standard input)"
+    else                  myfile = FILENAME
+
+    error_message = "\a\a" \
+"************************************************\n" \
+"  WARNING! The input file has an invalid number \n" \
+"  of asterisks on line " NR ", below.         \n\n" \
+"  The previous outline level had " last_len " asterisks,  \n" \
+"  but the current/next level has " this_len " asterisks!\n\n" \
+"  You have inadvertently skipped one level of   \n" \
+"  indentation. Processing halted so you can fix \n" \
+"  the input file, \x22"   myfile   "\x22.       \n" \
+"************************************************\n" \
+">>>\n" \
+"Error on Line #" NR " :" ;
+
+    print error_message, $0 > "/dev/stderr" ;
+    exit 1;
+  }
+
+  if ( this_len < last_len ) {    # if we have moved up a branch...
+     for (i = this_len + 1; i <= last_len; i++)
+        array[i] = 0;             # .. reset the leaves below us
+  }
+
+  for (j=1; j <= this_len; j++){  # build up the prefix string
+     if      (j == 1)   prefix = Lev1[array[j]] "."
+     else if (j == 2)   prefix = array[j] "."
+     else if (j == 3)   prefix = Lev3[array[j]] "."
+     else if (j == 4)   prefix = "(" array[j] ")"
+     else if (j == 5)   prefix = "(" Lev5[array[j]] ")"
+  }
+
+  indent_level = (this_len - 1) * num ;
+  indentation  = sprintf("%+" indent_level "s", "") ;
+
+  sub(/^\*+/, indentation prefix) ;
+  last_len = this_len ;
+  prefix = "" ;
+}
+{ print }
+# --- end of script ---

code_examples/wfault.sh

@@ -77,7 +77,7 @@ fi
 
 if 0; then
     foo
-    else
+else
     bar
 fi
 

mode/awk.py

@@ -0,0 +1,27 @@
+import commands
+import color, mode, tab
+from lex import Grammar, PatternRule, RegionRule
+from mode.python import StringGrammar2
+
+class AwkGrammar(Grammar):
+    rules = [
+        PatternRule(r'comment', '#.*\n$'),
+        PatternRule(r'awk_global_var', r'(?:TEXTDOMAIN|SUBSEP|RLENGTH|RSTART|RT|RS|PROCINFO|ORS|OFS|OFMT|NR|NF|LINT|IGNORECASE|FS|FNR|FILENAME|FIELDWIDTHS|ERRNO|ENVIRON|CONVFMT|BINMODE|ARGV|ARGIND|ARGC)(?![a-zA-Z0-9_])'),
+        PatternRule(r'delimiter', r'(?:[\{\}()\[\]?:;,\$]|=(?!=)|\+=|-=|\*=|/=|\%=|\^=)'),
+        PatternRule(r'keyword', r'(?:if|else|while|do|for|break|continue|delete|exit|close|getline|nextfile|next|printf|print|system|fflush|atan2|cos|exp|int|log|rand|sin|sqrt|srand|asorti|asort|gensub|gsub|index|length|match|split|sprintf|strtonum|substr|sub|tolower|toupper|mktime|strftime|systime|and|compl|lshift|or|xor|rshift|bindtextdomain|dcgettext|dcngettext|function|extension)(?![a-zA-Z0-9_])'),
+        PatternRule(r'unop', r'!(?![=~])|--|\+\+'),
+        PatternRule(r'binop', r'(?:&&|\|\||<=|>=|!=|!~|==|\^|%|[-~/+*<>])'),
+        RegionRule(r'string', r'"', StringGrammar2, r'"'),
+        PatternRule(r'function', r'[a-zA-Z_][a-zA-Z0-9_]*(?=\()'),
+        PatternRule(r'identifier', r'[a-zA-Z_][a-zA-Z0-9_]*'),
+    ]
+
+class Awk(mode.Fundamental):
+    modename   = 'awk'
+    extensions = ['.awk']
+    grammar    = AwkGrammar
+    colors     = {
+        'awk_global_var': ('cyan', 'default', 'bold'),
+    }
+
+install = Awk.install

mode/c.py

@@ -2,23 +2,30 @@ import os, re
 from subprocess import Popen, PIPE, STDOUT
 import color, default, method, method.shell, mode, tab
 from lex import Grammar, PatternRule, RegionRule, PatternGroupRule, OverridePatternRule
-from mode.python import StringGrammar
+from mode.python import StringGrammar2
+
+class CommentGrammar(Grammar):
+    rules = [
+        PatternRule(r'data', r'(?:[^\*]|\*(?!/))+'),
+    ]
 
 # this might not be complete...
 # see http://gcc.gnu.org/onlinedocs/gcc-2.95.3/cpp_3.html#SEC44
 class MacroGrammar(Grammar):
     rules = [
-        RegionRule(r'comment', r'/\*', Grammar, r'\*/'),
+        PatternRule(r'spaces', r' +'),
+
+        RegionRule(r'comment', r'/\*', CommentGrammar, r'\*/'),
         PatternRule(r'comment', r'//.*$'),
 
         PatternRule('name', r'(?:(?<=#define )) *[a-zA-Z_][a-zA-Z0-9_]*'),
-        PatternRule(r"unop", r"\+=|-=|\*=|/=|//=|%=|&=|\^=|>>=|<<=|\*\*=|\|="),
+        PatternRule(r"unop", r"!(?!=)|\+=|-=|\*=|/=|//=|%=|&=\|\^=|>>=|<<=|\*\*="),
         PatternRule(r'binop', r"\+|<>|<<|<=|<|-|>>|>=|>|\*\*|&|\*|\||/|\^|==|//|~|!=|%"),
         PatternRule(r"delimiter", r"->|\.|\(|\)|\[|\]|{|}|@|,|:|`|;|=|\?"),
         PatternRule(r"identifier", r"[a-zA-Z_][a-zA-Z0-9_]*"),
         PatternRule(r"integer", r"-?(?:0(?![x0-9])|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
         PatternRule(r"float", r"-?(?:[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+)"),
-        RegionRule(r'string', '"', StringGrammar, '"'),
+        RegionRule(r'string', '"', StringGrammar2, '"'),
         PatternRule(r'char', r"'.'|'\\.'|'\\[0-7]{3}'"),
         PatternRule(r"continued", r"\\\n$"),
     ]
@@ -39,16 +46,15 @@ class CGrammar(Grammar):
 
         PatternRule(r'builtin', r"(?:NULL|TRUE|FALSE)"),
         PatternRule(r'label', r'[a-zA-Z_][a-zA-Z0-9_]*(?=:)'),
-        PatternRule(r'identifier', r"[a-zA-Z_][a-zA-Z0-9_]*(?! *[\(:])"),
 
         RegionRule(r'macro', r'# *(?:assert|cpu|define|elif|else|endif|error|ident|ifdef|ifndef|if|import|include_next|line|machine|pragma|pragma_once|system|unassert|undef|warning)(?!=[a-zA-Z0-9_])', MacroGrammar, r'\n$'),
 
-        RegionRule(r'comment', r'/\*', Grammar, r'\*/'),
+        RegionRule(r'comment', r'/\*', CommentGrammar, r'\*/'),
         PatternRule(r'comment', r'//.*$'),
 
-        RegionRule(r'string', '"', StringGrammar, '"'),
+        RegionRule(r'string', '"', StringGrammar2, '"'),
 
-        PatternRule(r"unop", r"!|\+=|-=|\*=|/=|//=|%=|&=\|\^=|>>=|<<=|\*\*="),
+        PatternRule(r"unop", r"!(?!=)|\+=|-=|\*=|/=|//=|%=|&=\|\^=|>>=|<<=|\*\*="),
         PatternRule(r'binop', r"\+|<>|<<|<=|<|-|>>|>=|>|\*\*|&|\*|\||/|\^|==|//|~|!=|%"),
         PatternRule(r"integer", r"(?:0(?![x0-9])|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
         PatternRule(r"float", r"[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"),
@@ -59,13 +65,8 @@ class CGrammar(Grammar):
 
         PatternGroupRule(r'includegrp', r'macro.start', r'# *include', r'spaces',
                          r' +', r'header', r'< *[-A-Za-z/0-9_.]+ *>|" *[-A-Za-z/0-9_.]+ *"'),
-        #PatternGroupRule(r'ifndefgrp', r'macro.start', r'# *ifndef', r'spaces',
-        #                 r' +', r'header', r'< *[-A-Za-z/0-9_.]+ *>|" *[-A-Za-z/0-9_.]+ *"'),
 
-        #PatternRule(r'include', r'#include(?!=[a-zA-Z0-9_])'),
+        PatternRule(r'identifier', r"[a-zA-Z_][a-zA-Z0-9_]*"),
-        #PatternRule(r'header', r'<[-A-Za-z/0-9_\.]+>|"[-A-Za-z/0-9_\.]+"'),
-        #PatternRule(r'enumname', r'(?<=enum ) *[a-zA-Z_][a-zA-Z0-9_]*'),
-        #PatternRule(r'structname', r'(?<=struct ) *[a-zA-Z_][a-zA-Z0-9_]*'),
 
         OverridePatternRule(r'comment', r'/\* *@@:(?P<token>[.a-zA-Z0-9_]+):(?P<mode>[.a-zA-Z0-9_]+) *\*/$'),
         OverridePatternRule(r'comment', r'// *@@:(?P<token>[.a-zA-Z0-9_]+):(?P<mode>[.a-zA-Z0-9_]+) *$'),

mode/conf.py

@@ -1,13 +1,13 @@
 import color, mode
 from lex import Grammar, PatternRule, RegionRule
-from mode.sh import StringGrammar
+from mode.python import StringGrammar1, StringGrammar2
 
 class ConfGrammar(Grammar):
     rules = [
         PatternRule(r'comment', r'#.*$'),
         PatternRule(r'comment', r'//.*$'),
-        RegionRule(r'string', r"'", Grammar, r"'"),
+        RegionRule(r'string', r"'", StringGrammar1, r"'"),
-        RegionRule(r'string', r'"', StringGrammar, r'"'),
+        RegionRule(r'string', r'"', StringGrammar2, r'"'),
     ]
 
 class Conf(mode.Fundamental):

mode/console.py

@@ -1,11 +1,11 @@
 import color, mode
 from lex import Grammar, PatternRule, RegionRule
-from mode.python import StringGrammar, PythonGrammar
+from mode.python import StringGrammar1, StringGrammar2, PythonGrammar
 
 class ConsoleGrammar(Grammar):
     rules = [
-        RegionRule(r'string', r'"', StringGrammar, r'"'),
+        RegionRule(r'string', r'"', StringGrammar2, r'"'),
-        RegionRule(r'string', r"'", StringGrammar, r"'"),
+        RegionRule(r'string', r"'", StringGrammar1, r"'"),
         RegionRule(r'console_input', r'^(?:>>>|-->)', PythonGrammar, '\n$'),
         PatternRule(r'console_mesg', r'^[A-Za-z].*$'),
         PatternRule(r'console_reserved', r'True|False|None'),

mode/fstab.py

@@ -1,6 +1,5 @@
 import color, mode
 from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
-from mode.sh import StringGrammar
 
 class FstabGrammar(Grammar):
     rules = [
@@ -14,7 +13,6 @@ class FstabGrammar(Grammar):
                          r'fstab_pass', r' *(?:[^# \n\t]+|\\.)+',
                          r'fstab_error', r'[^\n]*',
                          r'fstab_eol', r'\n')
-        
     ]
 
 class Fstab(mode.Fundamental):

mode/haskell.py

@@ -3,10 +3,13 @@ import color, completer, default, mode, method, regex, tab
 from point import Point
 from lex import Grammar, PatternRule, RegionRule, OverridePatternRule, NocasePatternRule
 
+#class CommentGrammar(Grammar):
+#    rules = [
+#        RegionRule(r'comment', r'{-', None, r'-}'),
+#    ]
 class CommentGrammar(Grammar):
-    rules = [
+    pass
-        RegionRule(r'comment', r'{-', mode.lisp.StringGrammar, r'-}'),
+CommentGrammar.rules.append(RegionRule(r'comment', r'{-', None, r'-}'))
-    ]
 
 class StringGrammar(Grammar):
     rules = [

mode/ipython.py

@@ -1,11 +1,11 @@
 import color, mode
 from lex import Grammar, PatternRule, RegionRule
-from mode.python import StringGrammar, PythonGrammar
+from mode.python import StringGrammar1, StringGrammar2, PythonGrammar
 
 class IpythonGrammar(Grammar):
     rules = [
-        RegionRule(r'string', r'"', StringGrammar, r'"'),
+        RegionRule(r'string', r'"', StringGrammar2, r'"'),
-        RegionRule(r'string', r"'", StringGrammar, r"'"),
+        RegionRule(r'string', r"'", StringGrammar1, r"'"),
         RegionRule(r'ipython_input', r'^(?:>>>|\.\.>)', PythonGrammar, '\n$'),
         PatternRule(r'ipython_reserved', r'undef'),
     ]

mode/java.py

@@ -1,10 +1,11 @@
 import color, mode, tab
 from lex import Grammar, PatternRule, RegionRule
-from mode.python import StringGrammar
+from mode.python import StringGrammar2
 from mode.c import CTabber
 
 class JavaGrammar(Grammar):
     rules = [
+        PatternRule(r"spaces", r" +"),
         PatternRule(r'import', r'(?<=import ) *[a-zA-Z0-9_.*]+'),
         PatternRule(r'package', r'(?<=package ) *[a-zA-Z0-9_.*]+'),
 
@@ -24,7 +25,7 @@ class JavaGrammar(Grammar):
 
         PatternRule(r"java_integer", r"(?:0(?![x0-9])|[1-9][0-9]*|0[0-7]+|0[xX][0-9a-fA-F]+)[lL]?"),
         PatternRule(r"java_float", r"[0-9]+\.[0-9]*|\.[0-9]+|(?:[0-9]|[0-9]+\.[0-9]*|\.[0-9]+)[eE][\+-]?[0-9]+"),
-        RegionRule(r'string', '"', StringGrammar, '"'),
+        RegionRule(r'string', '"', StringGrammar2, '"'),
         PatternRule(r'java_char', r"'.'|'\\.'|'\\[0-7]{3}'"),
         PatternRule(r"eol", r"\n$"),
     ]

mode/lisp.py

@@ -2,14 +2,14 @@ import commands, os.path, sets, string, sys, traceback
 import color, completer, default, mode, method, regex, tab
 from point import Point
 from lex import Grammar, PatternRule, RegionRule, OverridePatternRule
-from mode.python import StringGrammar
+from mode.python import StringGrammar2
 from method import CommentRegion, UncommentRegion
 
 class LispGrammar(Grammar):
     rules = [
         PatternRule(r'comment', r';.*$'),
         PatternRule(r'delimiter', r'[()]'),
-        RegionRule(r'string', r'"', StringGrammar, r'"'),
+        RegionRule(r'string', r'"', StringGrammar2, r'"'),
         PatternRule(r'spaces', r' +'),
         PatternRule(r'eol', r'\n'),
     ]

mode/make.py

@@ -2,24 +2,34 @@ import color, mode
 from lex import Grammar, PatternRule, RegionRule, PatternGroupRule
 from mode.sh import ShGrammar, Sh
 
-class StringGrammar(Grammar):
+class StringGrammar1(Grammar):
     rules = [
         PatternRule(r'escaped', r'\\.'),
         PatternRule(r'escaped', r'\$\$'),
         PatternRule(r'variable', r"\$\([a-zA-Z0-9_]+\)"),
+        PatternRule(r'data', r"[^\\']+"),
+    ]
+class StringGrammar2(Grammar):
+    rules = [
+        PatternRule(r'escaped', r'\\.'),
+        PatternRule(r'escaped', r'\$\$'),
+        PatternRule(r'variable', r"\$\([a-zA-Z0-9_]+\)"),
+        PatternRule(r'data', r'[^\\"]+'),
     ]
 
 class MakeGrammar(Grammar):
     rules = [
         PatternRule(r'comment', r'#.*\n$'),
+        PatternRule(r'spaces', r' +$'),
         PatternRule(r'continuation', r'\\\n$'),
         PatternRule(r'targets', r'^[^:= ]+(?= *:)'),
         RegionRule(r'assignment', r'^[^:= ]+ *=', ShGrammar, r'\n'),
         RegionRule(r'actions', r'^    ', ShGrammar, r'\n'),
         PatternRule(r'make_variable', r'\$\([a-zA-Z0-9_]+\)'),
         PatternRule(r'make_variable', r'\${[a-zA-Z0-9_]+}'),
-        RegionRule(r'make_string', r'"', StringGrammar, r'"'),
+        RegionRule(r'make_string', r'"', StringGrammar2, r'"'),
-        RegionRule(r'make_string', r"'", StringGrammar, r"'"),
+        RegionRule(r'make_string', r"'", StringGrammar1, r"'"),
+        PatternRule(r'eol', r'\n$'),
     ]
 
 class Make(mode.Fundamental):

mode/python.py

@@ -3,12 +3,33 @@ import color, completer, context, default, mode, method, regex, tab
 from point import Point
 from lex import Grammar, PatternRule, RegionRule, OverridePatternRule
 
-class StringGrammar(Grammar):
+class StringGrammar1(Grammar):
     rules = [
         PatternRule(r'octal', r'\\[0-7]{3}'),
         PatternRule(r'hex', r'\\x[0-9a-fA-F]{2}'),
         PatternRule(r'escaped', r'\\.'),
-        PatternRule(r'data', r'[^\\"\']+'),
+        PatternRule(r'data', r"[^\\']+"),
+    ]
+class StringGrammar2(Grammar):
+    rules = [
+        PatternRule(r'octal', r'\\[0-7]{3}'),
+        PatternRule(r'hex', r'\\x[0-9a-fA-F]{2}'),
+        PatternRule(r'escaped', r'\\.'),
+        PatternRule(r'data', r'[^\\"]+'),
+    ]
+class StringGrammar3(Grammar):
+    rules = [
+        PatternRule(r'octal', r'\\[0-7]{3}'),
+        PatternRule(r'hex', r'\\x[0-9a-fA-F]{2}'),
+        PatternRule(r'escaped', r'\\.'),
+        PatternRule(r'data', r"(?:[^\\']|'(?!')|''(?!'))+"),
+    ]
+class StringGrammar4(Grammar):
+    rules = [
+        PatternRule(r'octal', r'\\[0-7]{3}'),
+        PatternRule(r'hex', r'\\x[0-9a-fA-F]{2}'),
+        PatternRule(r'escaped', r'\\.'),
+        PatternRule(r'data', r'(?:[^\\"]|"(?!")|""(?!"))+'),
     ]
 
 class PythonGrammar(Grammar):
@@ -24,14 +45,14 @@ class PythonGrammar(Grammar):
         PatternRule(r'private_identifier', r'__[a-zA-Z0-9_]*'),
         PatternRule(r'hidden_identifier', r'_[a-zA-Z0-9_]*'),
 
-        RegionRule(r'rawstring', r'r"""', StringGrammar, r'"""'),
+        RegionRule(r'rawstring', r'r"""', StringGrammar4, r'"""'),
-        RegionRule(r'rawstring', r"r'''", StringGrammar, r"'''"),
+        RegionRule(r'rawstring', r"r'''", StringGrammar3, r"'''"),
-        RegionRule(r'rawstring', r'r"', StringGrammar, r'"'),
+        RegionRule(r'rawstring', r'r"', StringGrammar2, r'"'),
-        RegionRule(r'rawstring', r"r'", StringGrammar, r"'"),
+        RegionRule(r'rawstring', r"r'", StringGrammar1, r"'"),
-        RegionRule(r'string', r'u?"""', StringGrammar, r'"""'),
+        RegionRule(r'string', r'u?"""', StringGrammar4, r'"""'),
-        RegionRule(r'string', r"u?'''", StringGrammar, r"'''"),
+        RegionRule(r'string', r"u?'''", StringGrammar3, r"'''"),
-        RegionRule(r'string', r'u?"', StringGrammar, r'"'),
+        RegionRule(r'string', r'u?"', StringGrammar2, r'"'),
-        RegionRule(r'string', r"u?'", StringGrammar, r"'"),
+        RegionRule(r'string', r"u?'", StringGrammar1, r"'"),
 
         PatternRule(r'identifier', r'[a-zA-Z_][a-zA-Z0-9_]*'),
         PatternRule(r'delimiter', r'\(|\)|\[|\]|{|}|@|,|:|\.|`|=|;|\+=|-=|\*=|/=|//=|%=|&=|\|=|\^=|>>=|<<=|\*\*='),

mode/scheme.py

@@ -8,7 +8,7 @@ class SchemeGrammar(Grammar):
     rules = [
         PatternRule(r'comment', r';.*$'),
         PatternRule(r'delimiter', r'[()]'),
-        RegionRule(r'string', r'"', mode.lisp.StringGrammar, r'"'),
+        RegionRule(r'string', r'"', mode.lisp.StringGrammar2, r'"'),
         PatternRule(r'spaces', r' +'),
         PatternRule(r'eol', r'\n'),
         PatternRule(r'abbrev', r"'|`|,\@|,"),

mode/sh.py

@@ -3,79 +3,111 @@ import color, mode, tab
 from lex import Grammar, PatternRule, RegionRule, OverridePatternRule
 from method import Method
 
-class StringGrammar(Grammar):
+class StringGrammar1(Grammar):
+    rules = [
+        PatternRule(r'data', r'[^\']+'),
+    ]
+class StringGrammar2(Grammar):
     rules = [
         PatternRule(r'escaped', r'\\.'),
         PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
         PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
         PatternRule(r'variable', r"\$(?=\()"),
+        PatternRule(r'data', r'[^\\$"]+'),
     ]
 
+class HereGrammar(Grammar):
+    rules = [
+        PatternRule(r'escaped', r'\\.'),
+        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
+        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
+        PatternRule(r'variable', r"\$(?=\()"),
+        PatternRule(r'data', r'[^\\$]+'),
+    ]        
+
 class EvalGrammar(Grammar):
     rules = [
-        RegionRule(r'string', "'", Grammar, "'"),
+        RegionRule(r'string', "'", StringGrammar1, "'"),
-        RegionRule(r'string', '"', StringGrammar, '"'),
+        RegionRule(r'string', '"', StringGrammar2, '"'),
         PatternRule(r'escaped', r'\\.'),
         PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
         PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
         PatternRule(r'variable', r"\$(?=\()"),
+        PatternRule(r'data', r'[^\\`$]+'),
+    ]
+class NevalGrammar(Grammar):
+    rules = [
+        RegionRule(r'string', "'", Grammar, "'"),
+        RegionRule(r'string', '"', StringGrammar2, '"'),
+        PatternRule(r'escaped', r'\\.'),
+        PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
+        PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
+        PatternRule(r'variable', r"\$(?=\()"),
+        PatternRule(r'data', r'[^\\)$]+'),
     ]
 
 class StanzaGrammar(Grammar):
     rules = [
-        PatternRule(r'ws', r' +'),
+        PatternRule(r'spaces', r' +'),
         PatternRule(r'start_cont', r'.+\\'),
+        PatternRule(r'eol', r'\n'),
     ]
 class CaseGrammar(Grammar):
     rules = [
         PatternRule(r'comment', r'#.*$'),
-        PatternRule(r'ws', r' +'),
+        PatternRule(r'spaces', r' +'),
         RegionRule(r'stanza', r'.+\\\n$', StanzaGrammar, r'.+\)', Grammar, r';;'),
         RegionRule(r'stanza', r'.+\)', Grammar, r';;'),
+        PatternRule(r'eol', r'\n'),
     ]
 
 class TestGrammar(Grammar):
     rules = [
+        PatternRule(r'spaces', r' +'),
         PatternRule(r'sh_builtin', r"(?<![-a-zA-Z0-9_])(?:source|alias|bg|bind|break|builtin|cd|command|compgen|complete|declare|dirs|disown|echo|enable|eval|exec|exit|export|fc|fg|getops|hash|help|history|jobs|kill|let|local|logout|popd|printf|pushd|pwd|readonly|read|return|set|shift|shopt|suspend|test|times|trap|type|ulimit|umask|unalias|unset|wait)(?![a-zA-Z0-9_=/])"),
         PatternRule(r'sh_reserved', r"(?:done|do|elif|else|esac|fi|for|function|if|in|select|then|until|while|time)(?![a-zA-Z0-9_=/])"),
-        #PatternRule(r'binop', r'==|=|!=|<|>'),
         PatternRule(r'binop', r'==|=|!='),
         PatternRule(r'binop', r'-(?:nt|ot|ef|eq|ne|lt|gt|le|ge)(?![a-zA-Z0-9_])'),
         PatternRule(r'unop', r'-[a-zA-Z](?![a-zA-Z0-9_])'),
         PatternRule(r'continuation', r'\\\n$'),
         PatternRule(r'redirect', r'<|>'),
         PatternRule(r'delimiter', r";;|[();{}|&><]"),
-        RegionRule(r'eval', r'`', StringGrammar, r'`'),
+        RegionRule(r'test', r'test', None, r';|\n'),
-        RegionRule(r'neval', r'\$\(', StringGrammar, r'\)'),
+        RegionRule(r'test2', r'\[\[', None, r'\]\]'),
+        RegionRule(r'test3', r'\[', None, r'\]'),
+        RegionRule(r'eval', r'`', EvalGrammar, r'`'),
+        RegionRule(r'neval', r'\$\(', NevalGrammar, r'\)'),
         PatternRule(r'variable', r"(?:^|(?<= ))[a-zA-Z_][a-zA-Z0-9_]*(?==)"),
         PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
         PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
         PatternRule(r'variable', r"\$(?=\()"),
-        RegionRule(r'string', "'", Grammar, "'"),
+        RegionRule(r'string', "'", StringGrammar1, "'"),
-        RegionRule(r'string', '"', StringGrammar, '"'),
+        RegionRule(r'string', '"', StringGrammar2, '"'),
         PatternRule(r'bareword', r'[-a-zA-Z0-9_.]+'),
     ]
 
 class ShGrammar(Grammar):
     rules = [
-        RegionRule(r'heredoc', r"<<[<\\]?(?P<heredoc>[a-zA-Z_][a-zA-Z0-9_]*)", None, "\n", StringGrammar, r'^%(heredoc)s$'),
+        PatternRule(r'spaces', r' +'),
-        RegionRule(r'heredoc', r"<<-(?P<heredoc>[a-zA-Z_][a-zA-Z0-9_]*)", None, "\n", StringGrammar, r'^ *%(heredoc)s$'),
+        RegionRule(r'heredoc', r"<<[<\\]?(?P<heredoc>[a-zA-Z_][a-zA-Z0-9_]*)", None, "\n", HereGrammar, r'^%(heredoc)s$'),
+        RegionRule(r'heredoc', r"<<-(?P<heredoc>[a-zA-Z_][a-zA-Z0-9_]*)", None, "\n", HereGrammar, r'^ *%(heredoc)s$'),
         PatternRule(r'sh_function', r'[a-zA-Z_][a-zA-Z0-9_]*(?= *\(\))'),
         PatternRule(r'sh_reserved', r"(?:done|do|elif|else|esac|fi|for|function|if|in|select|then|until|while|time)(?![a-zA-Z0-9_=/])"),
         RegionRule(r'case', r'case', None, 'in', CaseGrammar, r'esac'),
         PatternRule(r'sh_builtin', r"(?<![-a-zA-Z0-9_])(?:source|alias|bg|bind|break|builtin|cd|command|compgen|complete|declare|dirs|disown|echo|enable|eval|exec|exit|export|fc|fg|getops|hash|help|history|jobs|kill|let|local|logout|popd|printf|pushd|pwd|readonly|read|return|set|shift|shopt|suspend|times|trap|type|ulimit|umask|unalias|unset|wait)(?![a-zA-Z0-9_=/])(?![-a-zA-Z0-9_])"),
         RegionRule(r'test', r'test', TestGrammar, r';|\n'),
-        RegionRule(r'test2', r'\[', TestGrammar, r'\]'),
+        RegionRule(r'test2', r'\[\[', TestGrammar, r'\]\]'),
+        RegionRule(r'test3', r'\[', TestGrammar, r'\]'),
         PatternRule(r'redirect', r'<|>'),
-        PatternRule(r'delimiter', r";;|[();{}|&><]"),
+        PatternRule(r'delimiter', r";;|[();{}|&><:=/]"),
         RegionRule(r'eval', r'`', EvalGrammar, r'`'),
-        RegionRule(r'neval', r'\$\(', EvalGrammar, r'\)'),
+        RegionRule(r'neval', r'\$\(', NevalGrammar, r'\)'),
-        PatternRule(r'variable', r"(?:^|(?<= ))[a-zA-Z_][a-zA-Z0-9_]*(?==)"),
+        PatternRule(r'variable', r"^[a-zA-Z_][a-zA-Z0-9_]*(?==)"),
         PatternRule(r'variable', r"\${(?:[a-zA-Z0-9_]+|\?\$)}"),
         PatternRule(r"variable", r"\$[a-zA-Z0-9_]*"),
         PatternRule(r'variable', r"\$(?=\()"),
-        RegionRule(r'string', "'", Grammar, "'"),
+        RegionRule(r'string', "'", StringGrammar1, "'"),
-        RegionRule(r'string', '"', StringGrammar, '"'),
+        RegionRule(r'string', '"', StringGrammar2, '"'),
         OverridePatternRule(r'comment', r'#@@:(?P<token>[.a-zA-Z0-9_]+):(?P<mode>[.a-zA-Z0-9_]+) *$'),
         PatternRule(r'comment', r'#.*$'),
         PatternRule(r'bareword', r'(?:[-a-zA-Z0-9_.]|\\.)+'),

mode/shellmini.py

@@ -1,7 +1,6 @@
 import code, os, re, string, StringIO, sys, traceback
 import buffer, color, completer, lex, method, mode, window
 from lex import Grammar, PatternRule, RegionRule
-from mode.sh import StringGrammar
 from point import Point
 from method import Method
 from subprocess import Popen, PIPE, STDOUT

mode/sql.py

@@ -1,12 +1,14 @@
 import mode, tab
 from lex import Grammar, PatternRule, NocasePatternRule, RegionRule, NocaseRegionRule
-from mode.python import StringGrammar
+from mode.python import StringGrammar1, StringGrammar2
 
 class PlPgSqlGrammar(Grammar):
     rules = [
+        PatternRule(r'spaces', r' +'),
+
         PatternRule(r'comment', r'--.*\n$'),
         RegionRule(r'comment', '/\*', Grammar, '\*/'),
-        PatternRule(r'delimiter', r':=|[():;,\.\[\]]'),
+        PatternRule(r'delimiter', r':=|[():;,\.\[\]~]'),
     
         NocasePatternRule(r'attribute', r'(?:check|exists|unique|not null|default|primary key|minvalue|foreign key|references)(?![A-Za-z0-9_])'),
         NocasePatternRule(r'sql_keyword', r'(?:declare|begin|end if|end loop|end|raise notice|return|if|while|for|else)(?![A-Za-z0-9_])'),
@@ -15,8 +17,8 @@ class PlPgSqlGrammar(Grammar):
         NocasePatternRule(r'pseudokeyword', r'(?:returns|language|right join|left join|inner join|outer join|join|where|null|true|false|into|values|as|from|order by|next|asc|desc|limit|distinct|cascade|alias for|using|on)(?![A-Za-z0-9_])'),
         NocasePatternRule(r'sql_type', r'(?:void|row|serial|varchar|float|integer|int|text|timestamptz|timestamp|datetz|date|timetz|time|boolean|bool|record|row)(?![A-Za-z0-9_])'),
         PatternRule(r'sql_builtin', r'(?:nextval|current_timestamp|current_time|current_date)(?![A-Za-z0-9_])'),
-        RegionRule(r'string', "''", StringGrammar, "''"),
+        RegionRule(r'string', "''", StringGrammar1, "''"),
-        RegionRule(r'sql_quoted', '"', StringGrammar, '"'),
+        RegionRule(r'sql_quoted', '"', StringGrammar2, '"'),
         PatternRule(r'sql_variable', r'\$[1-9][0-9]*'),
         PatternRule(r'sql_bareword', r'[A-Za-z0-9_]+'),
         PatternRule(r'empty', r'^ *\n$'),
@@ -25,9 +27,11 @@ class PlPgSqlGrammar(Grammar):
 
 class FunctionGrammar(Grammar):
     rules = [
+        PatternRule(r'spaces', r' +'),
+
         PatternRule(r'comment', r'--.*\n$'),
         RegionRule(r'comment', '/\*', Grammar, '\*/'),
-        PatternRule(r'delimiter', r':=|[():;,\.\$\[\]]'),
+        PatternRule(r'delimiter', r':=|[():;,\.\$\[\]~]'),
         
         PatternRule(r'name', r'[a-zA-Z_][a-zA-Z0-9_]*(?=\()'),
         NocasePatternRule(r'sql_keyword', r'(?:as|returns|language)'),
@@ -43,9 +47,11 @@ class FunctionGrammar(Grammar):
 
 class SqlGrammar(Grammar):
     rules = [
+        PatternRule(r'spaces', r' +'),
+
         PatternRule(r'comment', r'--.*\n$'),
         RegionRule(r'comment', '/\*', Grammar, '\*/'),
-        PatternRule(r'delimiter', r':=|[():;,\.\$\[\]]'),
+        PatternRule(r'delimiter', r':=|[():;,\.\$\[\]~]'),
     
         NocaseRegionRule(r'function', r'create function', FunctionGrammar, r';'),
         NocaseRegionRule(r'function', r'create or replace function', FunctionGrammar, r';'),
@@ -56,8 +62,8 @@ class SqlGrammar(Grammar):
         NocasePatternRule(r'pseudokeyword', r'(?:returns|language|right join|left join|inner join|outer join|join|where|null|true|false|into|values|as|from|order by|asc|desc|limit|distinct|cascade|using|on)(?![A-Za-z0-9_])'),
         NocasePatternRule(r'sql_type', r'(?:void|row|serial|varchar|float|integer|int|text|timestamptz|timestamp|datetz|date|timetz|time|boolean|bool)(?![A-Za-z0-9_])'),
         PatternRule(r'sql_builtin', r'(?:nextval|current_timestamp|current_time|current_date)(?![A-Za-z0-9_])'),
-        RegionRule(r'string', "'", StringGrammar, "'"),
+        RegionRule(r'string', "'", StringGrammar1, "'"),
-        RegionRule(r'sql_quoted', '"', StringGrammar, '"'),
+        RegionRule(r'sql_quoted', '"', StringGrammar2, '"'),
         PatternRule(r'sql_bareword', r'[A-Za-z0-9_]+'),
         PatternRule(r'empty', r'^ *\n$'),
         PatternRule(r'eol', r'\n'),