--HG--

branch : pmacs2

IDEAS

@@ -1,3 +1,19 @@
+2008/05/23:
+
+Buffers should run in their own threads and/or processes and communicate with
+the main program through a series of locks and/or IPC calls. This will allow
+two different useful features: (1) user navigation while reparsing, (2) buffers
+representing pipes/sockets (e.g. shells, chat clients, tail -f, etc.)
+
+solving (2) with asyncore or other non-blocking IO would work, but (1) almost
+certainly requires seperate processes or threads.
+
+2008/05/23:
+
+Semantic tab-completion via iperl/ipython buffers... after syntax check, the
+file will be read into an interactive buffer, which can then be hooked into in
+order to support tab completion!
+
 2008/05/11:
 
 perl/python "open-module-by-name" command (auto-completing?).

application.py

@@ -2,7 +2,6 @@
 import curses, curses.ascii, getpass, os, re, string, sys, termios, time
 import traceback
 from subprocess import Popen, PIPE, STDOUT
-#from collections import defaultdict
 
 import buffer, bufferlist, color, completer, keyinput, method, minibuffer, mode
 import util, window
@@ -129,7 +128,7 @@ class Application(object):
                  'python', 'replace', 'rst', 'scheme', 'search', 'sh', 'sql',
                  'tt', 'text', 'text2', 'which', 'xml', 'cheetah', 'colortext',
                  'latex', 'insertmini', 'conf', 'haskell', 'erlang',
-                 'iperl', 'iperlmini',
+                 'iperl', 'iperlmini', 'ipython', 'ipythonmini'
         )
         for name in names:
             exec("import mode.%s; mode.%s.install(self)" % (name, name))

buffer.py

@@ -1,6 +1,7 @@
 import codecs, datetime, grp, md5, os, pwd, re, sets, shutil, stat, string
 import aes, dirutil, regex, highlight, lex
 from point import Point
+from subprocess import Popen, PIPE, STDOUT
 
 # undo/redo stack constants
 ACT_NORM    = 0
@@ -398,7 +399,6 @@ class ConsoleBuffer(Buffer):
     def readonly(self):
         return True
 
-# console is another singleton
 iperl = None
 class IperlBuffer(Buffer):
     btype    = 'iperl'
@@ -423,6 +423,76 @@ class IperlBuffer(Buffer):
     def readonly(self):
         return True
 
+def get_ipython_name(parent):
+    if hasattr(parent, 'path'):
+        return '*IPython:%s*' % parent.name()
+    else:
+        return '*IPython*'
+
+class IpythonBuffer(Buffer):
+    btype    = 'ipython'
+    modename = 'ipython'
+    readre   = re.compile('^([A-Z]+):(.*)\n$')
+    _name = '*IPython*'
+    def __init__(self, parent):
+        if hasattr(parent, 'path'):
+            self.parent = parent
+        else:
+            self.parent = None
+        Buffer.__init__(self)
+        f = open('/dev/null', 'w')
+        self.pipe = Popen(self.get_cmd(), stdin=PIPE, stdout=PIPE, stderr=f)
+        self.prompt = '***'
+        self.clear()
+        self.pipe_read()
+        self._name = get_ipython_name(parent)
+    def name(self):
+        return self._name
+    def get_cmd(self):
+        if self.parent:
+            return ('epython', '-p', self.parent.path)
+        else:
+            return ('epython',)
+    def pipe_readline(self):
+        line = self.pipe.stdout.readline()
+        m = self.readre.match(line)
+        if m:
+            return (m.group(1), m.group(2))
+        else:
+            return (None, line.rstrip())
+    def pipe_read(self):
+        lines = []
+        while True:
+            (type_, value) = self.pipe_readline()
+            if type_ == 'PROMPT':
+                self.prompt = value.strip() + ' '
+                break
+            value.rstrip()
+            if value:
+                lines.append(value)
+        if lines:
+            output = '\n'.join(lines) + '\n'
+            p = self.get_buffer_end()
+            self.insert_string(p, output, force=True)
+    def pipe_write(self, s):
+        self.pipe.stdin.write("%s\n" % s)
+        self.pipe.stdin.flush()
+    def pipe_read_completions(self):
+        (typ_, value) = self.pipe_readline()
+        assert typ_ == 'COMPLETIONS', '%r %r' % (typ_, value)
+        candidates = [x for x in value.split('|') if x]
+        return candidates
+
+    def clear(self):
+        self.set_data('', force=True)
+    def changed(self):
+        return False
+    def close(self):
+        global ipython
+        ipython = None
+    def readonly(self):
+        return True
+
 class BinaryDataException(Exception):
     pass
 

code_examples/Reporting2.pm

@@ -8,7 +8,6 @@ sub foo {
         unless 9 && 3;
 }
 
-#@@:heredoc:sql
 my $s = <<EOT;
 select '<foo attr="3">bar bar</foo>';
 drop table foog;

mode/consolemini.py

@@ -9,6 +9,8 @@ LIMIT = 79
 
 class ConsoleExec(method.Method):
     def _execute(self, w, **vargs):
+        if w.application.completion_window_is_open():
+            w.application.close_completion_buffer()
         s = w.buffer.make_string()
         w.mode.history[-1] = s
         w.mode.history.append('')
@@ -86,6 +88,8 @@ class ConsoleExec(method.Method):
 class ConsoleCancel(method.Method):
     def execute(self, w, **vargs):
         w.application.close_mini_buffer()
+        if w.application.completion_window_is_open():
+            w.application.close_completion_buffer()
 class ConsoleClear(method.Method):
     def execute(self, w, **vargs):
         a = w.application
@@ -154,25 +158,29 @@ class ConsoleTab(method.Method):
             else:
                 break
 
+        if not names:
+            return
+
         obj  = None
         g    = globals()
         i    = 0
-        name = None
+        name = names[0]
-        while i < len(names):
+        if len(names) > 1:
-            name = names[i]
+            while i < len(names):
-            if obj is None:
+                name = names[i]
-                if name in w.mode.locals:
+                if obj is None:
-                    obj = w.mode.locals[name]
+                    if name in w.mode.locals:
-                elif name in w.mode.globals:
+                        obj = w.mode.locals[name]
-                    obj = w.mode.globals[name]
+                    elif name in w.mode.globals:
+                        obj = w.mode.globals[name]
+                    else:
+                        break
                 else:
-                    break
+                    if hasattr(obj, name):
-            else:
+                        obj = getattr(obj, name)
-                if hasattr(obj, name):
+                    else:
-                    obj = getattr(obj, name)
+                        break
-                else:
+                i += 1
-                    break
-            i += 1
 
         if i == len(names) - 1:
             if obj is not None:

mode/iperl.py

@@ -16,5 +16,8 @@ class Iperl(mode.Fundamental):
         'iperl_input.start': ('red', 'default', 'bold'),
         'iperl_reserved':    ('magenta', 'default', 'bold'),
     }
+    def __init__(self, w):
+        mode.Fundamental.__init__(self, w)
+        self.add_bindings('iperl-start', ('M-e',))
 
 install = Iperl.install

mode/iperlmini.py

@@ -1,7 +1,6 @@
 import code, re, string, StringIO, sys, traceback
 import buffer, color, completer, lex, method, mode, mode.mini, mode.consolemini, window
 from subprocess import Popen, PIPE, STDOUT
-#from lex import Grammar, PatternRule
 from mode.perl import PerlGrammar
 from point import Point
 
@@ -10,6 +9,8 @@ LIMIT = 79
 
 class IperlExec(method.Method):
     def _execute(self, w, **vargs):
+        if w.application.completion_window_is_open():
+            w.application.close_completion_buffer()
         s = w.buffer.make_string()
         w.mode.history[-1] = s
         w.mode.history.append('')
@@ -17,16 +18,14 @@ class IperlExec(method.Method):
         w.mode.hindex = len(w.mode.history) - 1
 
         a = w.application
-        if not a.has_buffer_name('*IPerl*'):
+        b = w.mode._get_iperl()
-            raise Exception, "No iperl found!"
-        b = a.bufferlist.get_buffer_by_name('*IPerl*')
         if a.window().buffer is not b:
             a.switch_buffer(b)
         p = a.get_mini_buffer_prompt()
         b.insert_string(b.get_buffer_end(), p + s + '\n', force=True)
 
-        w.mode.pipe.stdin.write("ENTER:%s\n" % s)
+        b.pipe.stdin.write("ENTER:%s\n" % s)
-        w.mode.pipe.stdin.flush()
+        b.pipe.stdin.flush()
         output = w.mode._read()
         b.insert_string(b.get_buffer_end(), output, force=True)
 
@@ -34,6 +33,7 @@ class IperlTab(method.Method):
     def execute(self, w, **vargs):
         a = w.application
         s = w.buffer.make_string()
+        b = w.mode._get_iperl()
 
         x2 = w.logical_cursor().x
         if not s or s[:x2].isspace():
@@ -47,8 +47,8 @@ class IperlTab(method.Method):
             x1 -= 1
         word = line[x1:x2]
 
-        w.mode.pipe.stdin.write("COMPLETE2:%d:%d:%s\n" % (x1, x2, s))
+        b.pipe.stdin.write("COMPLETE:%d:%d:%s\n" % (x1, x2, s))
-        w.mode.pipe.stdin.flush()
+        b.pipe.stdin.flush()
         (typ_, value) = w.mode._readline()
         assert typ_ == 'COMPLETIONS', '%r %r' % (typ_, value)
 
@@ -90,18 +90,21 @@ class IperlMini(mode.Fundamental):
                 IperlPageUp, IperlPageDown, IperlGotoBeginning, IperlGotoEnd]
     readre = re.compile('^([A-Z]+):(.*)\n$')
     def _readline(self):
-        line = self.pipe.stdout.readline()
+        b = self._get_iperl()
+        line = b.pipe.stdout.readline()
         m = self.readre.match(line)
         if m:
             return (m.group(1), m.group(2))
         else:
             return ('RAW', line.rstrip())
     def _read(self):
+        b = self._get_iperl()
         output = []
         while True:
             (type_, value) = self._readline()
             if type_ == 'PROMPT':
-                self.window.application.set_mini_buffer_prompt(value + ' ')
+                b.prompt = value.strip() + ' '
+                self.window.application.set_mini_buffer_prompt(b.prompt)
                 break
             value.rstrip()
             if value:
@@ -111,16 +114,25 @@ class IperlMini(mode.Fundamental):
         else:
             return ''
 
+    def _get_iperl(self):
+        a = self.window.application
+        if not a.has_buffer_name('*IPerl*'):
+            raise Exception, "No iperl found!"
+        b = a.bufferlist.get_buffer_by_name('*IPerl*')
+        return b
+
     def __init__(self, w):
         mode.Fundamental.__init__(self, w)
         self.history = ['']
         self.hindex  = 0
-        cmd = ('iperl', '-p')
+        b = self._get_iperl()
-        f = open('/dev/null', 'w')
+        if hasattr(b, 'pipe'):
-        self.pipe = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=f)
+            self.window.application.set_mini_buffer_prompt(b.prompt)
-        (type_, value) = self._readline()
+        else:
-        assert type_ == 'PROMPT', type_
+            cmd = ('iperl', '-p')
-        w.application.set_mini_buffer_prompt('%s ' % value.strip())
+            f = open('/dev/null', 'w')
+            b.pipe = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=f)
+            self._read()
         self.add_bindings('iperl-exec', ('RETURN',))
         self.add_bindings('console-clear', ('C-l',))
         self.add_bindings('console-cancel', ('C-]',))

mode/ipython.py

@@ -0,0 +1,23 @@
+import color, mode
+from lex import Grammar, PatternRule, RegionRule
+from mode.python import StringGrammar, PythonGrammar
+
+class IpythonGrammar(Grammar):
+    rules = [
+        RegionRule(r'string', r'"', StringGrammar, r'"'),
+        RegionRule(r'string', r"'", StringGrammar, r"'"),
+        RegionRule(r'ipython_input', r'^(?:>>>|\.\.>)', PythonGrammar, '\n$'),
+        PatternRule(r'ipython_reserved', r'undef'),
+    ]
+class Ipython(mode.Fundamental):
+    modename = 'IPython'
+    grammar = IpythonGrammar()
+    colors = {
+        'ipython_input.start': ('red', 'default', 'bold'),
+        'ipython_reserved':    ('magenta', 'default', 'bold'),
+    }
+    def __init__(self, w):
+        mode.Fundamental.__init__(self, w)
+        self.add_bindings('ipython-start', ('M-e',))
+
+install = Ipython.install

mode/ipythonmini.py

@@ -0,0 +1,131 @@
+import code, re, string, StringIO, sys
+import buffer, color, completer, lex, method, mode, mode.mini, mode.consolemini, window
+from subprocess import Popen, PIPE, STDOUT
+from mode.python import PythonGrammar
+from point import Point
+
+class IpythonExec(method.Method):
+    def _execute(self, w, **vargs):
+        if w.application.completion_window_is_open():
+            w.application.close_completion_buffer()
+        s = w.buffer.make_string()
+        w.mode.history[-1] = s
+        w.mode.history.append('')
+        w.buffer.set_data('')
+        w.mode.hindex = len(w.mode.history) - 1
+
+        b = w.mode.get_ipython()
+        a = w.application
+        if a.window().buffer is not b:
+            a.switch_buffer(b)
+        p = a.get_mini_buffer_prompt()
+        b.insert_string(b.get_buffer_end(), p + s + '\n', force=True)
+
+        b.pipe_write("ENTER:%s" % s)
+        w.mode.read_sync()
+
+class IpythonTab(method.Method):
+    def execute(self, w, **vargs):
+        a = w.application
+        s = w.buffer.make_string()
+
+        x2 = w.logical_cursor().x
+        if not s or s[:x2].isspace():
+            w.insert_string_at_cursor(' ' * w.mode.tabwidth)
+            return
+
+        r = re.compile('^[a-zA-Z0-9_.]$')
+        line = s
+        x1 = x2
+        while x1 > 0 and r.match(s[x1 - 1]):
+            x1 -= 1
+        word = line[x1:x2]
+
+        b = w.mode.get_ipython()
+        b.pipe_write("COMPLETE:%s" % word)
+        candidates = b.pipe_read_completions()
+        w.mode.read_sync()
+
+        if candidates:
+            s = completer.find_common_string(candidates)
+            w.buffer.delete(Point(x1, 0), Point(x2, 0), force=True)
+            w.insert_string_at_cursor(s)
+            mode.mini.use_completion_window(a, s, candidates)
+        elif a.completion_window_is_open():
+            a.close_completion_buffer()
+
+class IpythonStart(method.Method):
+    '''Evaluate python expressions (for advanced use and debugging only)'''
+    def execute(self, w, **vargs):
+        a = w.application
+        if not a.has_buffer_name('*IPython*'):
+            b = buffer.IpythonBuffer(None)
+            a.add_buffer(b)
+            window.Window(b, a)
+        b = a.bufferlist.get_buffer_by_name('*IPython*')
+        self.main_buffer = b
+        if a.window().buffer is not b:
+            a.switch_buffer(b)
+        f = lambda x: None
+        w.application.open_mini_buffer('*** ', f, self, None, 'ipythonmini')
+
+class IpythonPathStart(method.Method):
+    '''xyz'''
+    def execute(self, w, **vargs):
+        a = w.application
+        if w.buffer.btype == 'ipython':
+            b = w.buffer
+        else:
+            name = buffer.get_ipython_name(w.buffer)
+            if not a.has_buffer_name(name):
+                b = buffer.IpythonBuffer(w.buffer)
+                a.add_buffer(b)
+                window.Window(b, a)
+            else:
+                b = a.get_buffer_by_name(name)
+        self.main_buffer = b
+        if a.window().buffer is not b:
+            a.switch_buffer(b)
+        f = lambda x: None
+        w.application.open_mini_buffer('*** ', f, self, None, 'ipythonmini')
+
+class IpythonPageUp(mode.consolemini.ConsolePageUp):
+    subbuf = '*IPython*'
+class IpythonPageDown(mode.consolemini.ConsolePageDown):
+    subbuf = '*IPython*'
+class IpythonGotoBeginning(mode.consolemini.ConsoleGotoBeginning):
+    subbuf = '*IPython*'
+class IpythonGotoEnd(mode.consolemini.ConsoleGotoEnd):
+    subbuf = '*IPython*'
+
+class IpythonMini(mode.Fundamental):
+    modename = 'IpythonMini'
+    actions  = [IpythonExec, IpythonTab, IpythonStart, IpythonPathStart,
+                IpythonPageUp, IpythonPageDown, IpythonGotoBeginning, IpythonGotoEnd]
+    def get_ipython(self):
+        return self.window.buffer.method.main_buffer
+    def read_sync(self):
+        b = self.get_ipython()
+        b.pipe_read()
+        self.window.application.set_mini_buffer_prompt(b.prompt)
+    def __init__(self, w):
+        mode.Fundamental.__init__(self, w)
+        self.history = ['']
+        self.hindex  = 0
+        b = self.window.buffer.method.main_buffer
+        assert hasattr(b, 'pipe')
+        self.window.application.set_mini_buffer_prompt(b.prompt)
+        self.add_bindings('ipython-exec', ('RETURN',))
+        self.add_bindings('console-clear', ('C-l',))
+        self.add_bindings('console-cancel', ('C-]',))
+        self.add_bindings('console-history-prev', ('C-p', 'UP'))
+        self.add_bindings('console-history-next', ('C-n', 'DOWN'))
+        self.add_bindings('ipython-tab', ('TAB',))
+        self.add_bindings('ipython-page-up', ('M-v',))
+        self.add_bindings('ipython-page-down', ('C-v',))
+        self.add_bindings('ipython-goto-beginning', ('M-<',))
+        self.add_bindings('ipython-goto-end', ('M->',))
+        for c in string.letters + string.digits + string.punctuation:
+            self.add_binding('insert-string-%s' % c, c)
+
+install = IpythonMini.install

tools/epython

@@ -0,0 +1,121 @@
+#!/usr/bin/python
+import code
+import optparse
+import os
+import sys
+import StringIO
+import traceback
+
+class EPython(object):
+    def __init__(self, globals_, locals_):
+        self.lines   = []
+        self.globals = globals_
+        self.locals  = locals_
+
+    def reset(self):
+        self.lines = []
+
+    def prompt(self):
+        if self.lines:
+            return "PROMPT:..>"
+        else:
+            return "PROMPT:>>>"
+
+    def handle(self, line):
+        if line.startswith('ENTER:'):
+            self.push(line[6:-1])
+        elif line.startswith('COMPLETE:'):
+            self.complete(line[9:-1])
+        else:
+            print "ERROR:invalid input"
+
+    def complete(self, s):
+        candidates = []
+        names      = s.split('.')
+        obj        = None
+        i          = 0
+        name       = names[0]
+        if len(names) > 1:
+            while i < len(names):
+                name = names[i]
+                if obj is None:
+                    if name in self.locals:
+                        obj = self.locals[name]
+                    elif name in self.globals:
+                        obj = self.globals[name]
+                    else:
+                        break
+                else:
+                    if hasattr(obj, name):
+                        obj = getattr(obj, name)
+                    else:
+                        break
+                i += 1
+
+        if i == len(names) - 1:
+            base = '.'.join(names[:-1])
+            if base:
+                base += '.'
+            if obj is not None:
+                newnames = dir(obj)
+            else:
+                newnames = set()
+                newnames.update(dir(__builtins__))
+                newnames.update(self.locals)
+                newnames.update(self.globals)
+            candidates = [base + x for x in newnames if x.startswith(name)]
+        print "COMPLETIONS:%s" % ('|'.join(candidates))
+
+    def push(self, s):
+        self.lines.append(s)
+        s2 = '\n'.join(self.lines)
+        try:
+            code_obj = code.compile_command(s2)
+            if code_obj is None:
+                return
+            try:
+                exec code_obj in self.globals, self.locals
+            except Exception, e:
+                print str(e) + "\n" + traceback.format_exc()
+        except (SyntaxError, OverflowError, ValueError), e:
+            print str(e) + "\n" + traceback.format_exc()
+        self.reset()
+
+    def main(self):
+        while True:
+            print '\n' + self.prompt()
+            sys.stdout.flush()
+            line = sys.stdin.readline()
+            if not line:
+                break
+            self.handle(line)
+
+if __name__ == "__main__":
+    stanzas = []
+    def add_eval(option, opt, value, parser):
+        stanzas.append(('eval', value))
+    def add_path(option, opt, value, parser):
+        stanzas.append(('path', value))
+    parser = optparse.OptionParser()
+    parser.add_option('-e', '--eval', type='string', action='callback', callback=add_eval)
+    parser.add_option('-p', '--path', type='string', action='callback', callback=add_path)
+    parser.parse_args()
+    del parser, add_path, add_eval
+
+    __EP = EPython(globals_=globals(), locals_=locals())
+    sys.path.insert(0, os.getcwd())
+    for (type_, value) in stanzas:
+        #break
+        if type_ == 'eval':
+            exec value in __EP.globals, __EP.locals
+            __EP.push(value)
+        elif type_ == 'path':
+            f = open(value, 'r')
+            exec f in __EP.globals, __EP.locals
+            f.close()
+        else:
+            raise Exception, 'how can this happen?'
+    del stanzas
+
+    __EP.reset()
+    __EP.main()

tools/iperl

@@ -302,7 +302,7 @@ sub main {
         # display the prompt and read some input
         my $line;
         if($pipe) {
-            print "PROMPT:$prompt\n";
+            print "\nPROMPT:$prompt\n";
             $line = <STDIN>;
         } else {
             $line = $term->readline("$prompt ");
@@ -314,14 +314,7 @@ sub main {
         if($pipe) {
             if($line =~ m/^ENTER:(.*)$/) {
                 $line = $1;
-            } elsif($line =~ m/^COMPLETE:(.*)$/) {
+            } elsif($line =~ m/^COMPLETE:(\d+):(\d+):(.*)$/) {
-                my $line = $1;
-                while($line =~ m/\G(?<=[^a-zA-Z0-9_])[a-zA-Z0-9_]/g) {}
-                my $x = pos($line) ? pos($line) : 0;
-                my $word = substr($line, $x);
-                draw_completions(complete($line, $word, $x));
-                next;
-            } elsif($line =~ m/^COMPLETE2:(\d+):(\d+):(.*)$/) {
                 my $x = $2;
                 my $line = $3;
                 my $word = substr($line, $1, $x - $1);