pmacs3/buffer.py

import codecs, datetime, grp, md5, os, pwd, re, sets, shutil, stat, string
import fcntl, select, pty, threading
import aes, dirutil, regex, highlight, lex, term
from point import Point
from subprocess import Popen, PIPE, STDOUT
from keyinput import MAP

# undo/redo stack constants
ACT_NONE    = -1
ACT_NORM    = 0
ACT_UNDO    = 1
ACT_REDO    = 2
STACK_LIMIT = 1024

class ReadOnlyError(Exception):
    pass

# used for undo/redo stacks when text will need to be added back
class AddMove(object):
    def __init__(self, buffer, p, lines):
        self.buffer = buffer
        self.p = p
        self.lines = lines
        self.undo_id = buffer.undo_id
    def restore(self, act=ACT_UNDO):
        assert act == ACT_UNDO or act == ACT_REDO
        self.buffer.insert_lines(self.p, self.lines, act)
    def getpos(self):
        return self.p

# used for undo/redo stacks when text will need to be removed
class DelMove(object):
    def __init__(self, buffer, p1, p2):
        self.buffer = buffer
        self.p1     = p1
        self.p2     = p2
        self.undo_id = buffer.undo_id
    def restore(self, act):
        assert act == ACT_UNDO or act == ACT_REDO
        self.buffer.delete(self.p1, self.p2, act)
    def getpos(self):
        return self.p1

# abstract class
class Buffer(object):
    btype   = 'generic'
    mac_re  = re.compile('\r(?!\n)')
    unix_re = re.compile('(?<!\r)\n')
    win_re  = re.compile('\r\n')
    def __init__(self, stack_limit=STACK_LIMIT):
        self.lines       = [""]
        self.windows     = []
        self.undo_id     = 1
        self.undo_stack  = []
        self.redo_stack  = []
        self.stack_limit = stack_limit
        self.nl          = '\n'
        self.modified    = False
        self.highlights  = {}
        self.indentlvl   = 4
        self.writetabs   = False

    def _detect_nl_type(self, data):
        mac_c  = len(self.mac_re.findall(data))
        unix_c = len(self.unix_re.findall(data))
        win_c  = len(self.win_re.findall(data))
        if (unix_c and mac_c) or (unix_c and win_c) or (mac_c and win_c):
            # warn the user?
            #raise Exception, 'inconsistent line endings %r' % \
            #    (data, [unix_c, mac_c, win_c])
            pass

        if unix_c >= win_c and unix_c >= mac_c:
            return '\n'
        elif mac_c >= win_c:
            return '\r'
        else:
            return '\r\n'

    # basic file operation stuff
    def _open_file_r(self, path):
        path = os.path.realpath(path)
        if not os.path.isfile(path):
            raise Exception, "Path '%s' does not exist" % (path)
        if not os.access(path, os.R_OK):
            raise Exception, "Path '%s' cannot be read" % (path)
        f = open(path, 'r')
        return f
    def _open_file_w(self, path):
        if os.path.isfile(path):
            raise Exception, "Path '%s' already exists" % (path)
        d = os.path.dirname(path)
        if not os.access(d, os.R_OK):
            raise Exception, "Dir '%s' cannot be read" % (path)
        if not os.access(d, os.W_OK):
            raise Exception, "Dir '%s' cannot be written" % (path)
        f = open(path, 'w')
        return f
    def _temp_path(self, path):
        (dirname, basename) = os.path.split(path)
        return os.path.join(dirname, ".__%s__pmacs" % (basename))

    # undo/redo stack
    def _stack_trim(self, stack):
        if self.stack_limit:
            while len(stack) > self.stack_limit:
                stack.pop(0)
    def add_to_stack(self, move, act):
        if act == ACT_NONE:
            pass
        elif act == ACT_NORM:
            self.redo_stack = []
            self.undo_stack.append(move)
            self._stack_trim(self.undo_stack)
        elif act == ACT_UNDO:
            self.redo_stack.append(move)
            self._stack_trim(self.redo_stack)
        elif act == ACT_REDO:
            self.undo_stack.append(move)
            self._stack_trim(self.undo_stack)
        else:
            raise Exception, "Invalid act: %d" % (act)
    def undo(self):
        if len(self.undo_stack):
            undo_id = self.undo_stack[-1].undo_id
            pos = None
            while self.undo_stack and self.undo_stack[-1].undo_id == undo_id:
                move = self.undo_stack.pop(-1)
                move.restore(ACT_UNDO)
                pos = move.getpos()
            return pos
        else:
            raise Exception, "Nothing to Undo!"
    def redo(self):
        if len(self.redo_stack):
            undo_id = self.redo_stack[-1].undo_id
            pos = None
            while self.redo_stack and self.redo_stack[-1].undo_id == undo_id:
                move = self.redo_stack.pop(-1)
                move.restore(ACT_REDO)
                pos = move.getpos()
            return pos
        else:
            raise Exception, "Nothing to Redo!"

    # window-buffer communication
    def add_window(self, w):
        if w not in self.windows:
            self.windows.append(w)
            modename = w.mode.name()
            if modename not in self.highlights and w.mode.lexer is not None:
                self.highlights[modename] = highlight.Highlighter(w.mode.lexer)
                self.highlights[modename].highlight(self.lines)

    def remove_window(self, w):
        if w in self.windows:
            self.windows.remove(w)
            modename = w.mode.name()
            if modename in self.highlights:
                for w2 in self.windows:
                    if w2.mode.name() == modename:
                        return
                del self.highlights[modename]
    def _region_add(self, p1, p2, lines, act):
        move = DelMove(self, p1, p2)
        self.add_to_stack(move, act)
        for name in self.highlights:
            self.highlights[name].relex_add(self.lines, p1.y, p1.x, lines)
        for w in self.windows:
            w.region_added(p1, lines)
    def _region_del(self, p1, p2, lines, act):
        move = AddMove(self, p1, lines)
        self.add_to_stack(move, act)
        for name in self.highlights:
            self.highlights[name].relex_del(self.lines, p1.y, p1.x, p2.y, p2.x)
        for w in self.windows:
            w.region_removed(p1, p2)

    # internal validation
    def _validate_point(self, p):
        self._validate_xy(p.x, p.y)
    def _validate_xy(self, x, y):
        assert y >= 0 and y < len(self.lines), \
            "xy1: %d >= 0 and %d < %d" % (y, y, len(self.lines))
        assert x >= 0 and x <= len(self.lines[y]), \
            "xy2: %d >= 0 and %d <= %d" % (x, x, len(self.lines[y]))
    def _validate_y(self, y):
        assert y >= 0 and y < len(self.lines), \
            "y: %d >= 0 and %d < %d" % (y, y, len(self.lines))

    # deal with the actual logical document string
    def num_chars(self):
        n = 0
        for line in self.lines[:-1]:
            n += len(line) + 1
        n += len(self.lines[-1])
        return n
    def num_lines(self):
        return len(self.lines)
    def make_string(self):
        if self.writetabs:
            lines = []
            for line in self.lines:
                i = 0
                while i < len(line) and line[i] == ' ':
                    i += 1
                j, k = i // self.indentlvl, i % self.indentlvl
                lines.append(('\t' * j) + (' ' * k) + line[i:])
            return self.nl.join(lines)
        else:
            return self.nl.join(self.lines)

    # methods to be overridden by subclasses
    def name(self):
        return "Generic"
    def close(self):
        pass
    def open(self):
        pass
    def changed(self):
        return self.modified
    def reload(self):
        raise Exception, "%s reload: Unimplemented" % (self.name())
    def save_as(self, path, force=False):
        # check to see if the path exists, and if we're prepared to overwrite it
        # if yes to both, get its mode so we can preserve the path's permissions
        mode = None
        if os.path.exists(path):
            if force:
                mode = os.stat(self.path)[0]
            else:
                raise Exception, "oh no! %r already exists" % path

        # create the string that we're going to write into the file
        data = self.write_filter(self.make_string())

        # create a safe temporary path to write to, and write out data to it
        temp_path = self._temp_path()
        f2 = self._open_file_w(temp_path)
        f2.write(data)
        f2.close()

        # move the temporary file to the actual path; maybe change permissions
        shutil.move(temp_path, path)
        if mode:
            os.chmod(path, mode)

        # the file has not been modified now
        self.modified = False
    def readonly(self):
        return False
    def read_filter(self, data):
        return data
    def write_filter(self, data):
        return data

    # point retrieval
    def get_buffer_start(self):
        return Point(0, 0)
    def get_buffer_end(self):
        return Point(len(self.lines[-1]), len(self.lines) - 1)

    # data retrieval
    def get_sublines(self, p1, p2):
        self._validate_point(p1)
        self._validate_point(p2)
        assert p1 <= p2, "p1.x (%d) > p2.x (%d)" % (p1.x, p2.x)
        lines = []
        x = p1.x
        for i in range(p1.y, p2.y):
            lines.append(self.lines[i][x:])
            x = 0
        lines.append(self.lines[p2.y][x:p2.x])
        return lines
    def get_substring(self, p1, p2):
        lines = self.get_sublines(p1, p2)
        return '\n'.join(lines)

    # buffer set
    def set_lines(self, lines, force=False):
        if not force and self.readonly():
            raise Exception, "set_data: buffer is readonly"
        start = self.get_buffer_start()
        end   = self.get_buffer_end()
        self.delete(start, end, force=force)
        self.insert_lines(start, lines, force=force)
        self.modified = True
    def set_data(self, data, force=False):
        lines = data.split('\n')
        self.set_lines(lines, force)

    # insertion into buffer
    def insert_lines(self, p, lines, act=ACT_NORM, force=False):
        llen = len(lines)
        assert llen > 0
        if not force and self.readonly():
            raise ReadOnlyError("buffer is read-only")
        p2 = p.vadd(len(lines[-1]), llen - 1)
        if llen > 1:
            self.lines.insert(p.y + 1, [])
            self.lines[p.y + 1] = lines[-1] + self.lines[p.y][p.x:]
            self.lines[p.y] = self.lines[p.y][:p.x] + lines[0]
            for i in range(1, llen - 1):
                self.lines.insert(p.y + i, lines[i])
        else:
            self.lines[p.y] = self.lines[p.y][:p.x] + lines[-1] + self.lines[p.y][p.x:]
        self._region_add(p, p2, lines, act)
        self.modified = True
    def insert_string(self, p, s, act=ACT_NORM, force=False):
        lines = s.split("\n")
        self.insert_lines(p, lines, act, force)
        
    # deletion from buffer
    def delete(self, p1, p2, act=ACT_NORM, force=False):
        """delete characters from p1 up to p2 from the buffer"""
        if not force and self.readonly():
            raise ReadOnlyError("buffer is read-only")
        self._validate_point(p1)
        self._validate_point(p2)
        if p1 == p2:
            return
        assert p1 < p2, "p1 %r > p2 %r" % (p1, p2)
        lines = self.get_sublines(p1, p2)
        line1 = self.lines[p1.y]
        line2 = self.lines[p2.y]
        self.lines[p1.y:p2.y+1] = ["%s%s" % (line1[:p1.x], line2[p2.x:])]
        self._region_del(p1, p2, lines, act)
        self.modified = True
    def delete_char(self, p, act=ACT_NORM, force=False):
        if p.x == len(self.lines[p.y]):
            p2 = Point(0, p.y + 1)
        else:
            p2 = Point(p.x + 1, p.y)
        self.delete(p, p2, act=act, force=force)
    def overwrite_char(self, p, c, act=ACT_NORM, force=False):
        self.delete_char(p, act=act, force=force)
        self.insert_string(p, c, act=act, force=force)
    def delete_line(self, y, act=ACT_NORM, force=False):
        line = self.lines[y]
        p1 = Point(0, y)
        if y < len(self.lines) - 1:
            p2 = Point(0, y + 1)
        else:
            p2 = Point(len(self.lines[-1]), y)
        self.delete(p1, p2, act, force)

    # random
    def count_leading_whitespace(self, y):
        line = self.lines[y]
        m = regex.leading_whitespace.match(line)
        if m:
            return m.end()
        else:
            # should not happen
            raise Exception, "iiiijjjj"

# scratch is a singleton
scratch = None
class ScratchBuffer(Buffer):
    btype = 'scratch'
    def __new__(cls, *args, **kwargs):
        global scratch
        if scratch is None:
            scratch = object.__new__(ScratchBuffer, *args, **kwargs)
        return scratch
    def name(self):
        return "*Scratch*"
    def close(self):
        global scratch
        scratch = None

class DataBuffer(Buffer):
    btype = 'data'
    def __init__(self, name, data):
        Buffer.__init__(self)
        self._name = name
        self.lines = data.split("\n")
    def name(self):
        return self._name
    def close(self):
        pass
    def readonly(self):
        return True

# console is another singleton
console = None
class ConsoleBuffer(Buffer):
    btype    = 'console'
    modename = 'console'
    def __new__(cls, *args, **kwargs):
        global console
        if console is None:
            b = object.__new__(ConsoleBuffer, *args, **kwargs)
            console = b
        return console
    def __init__(self):
        Buffer.__init__(self)
        self.clear()
    def clear(self):
        lines = ['Python Console\n',
                 "Evaluate python expressions in the editor's context (self)\n",
                 'Press Control-] to exit\n']
        console.set_data(''.join(lines), force=True)
    def name(self):
        return '*Console*'
    def changed(self):
        return False
    def close(self):
        global console
        console = None
    def readonly(self):
        return True

class InterpreterPipeError(Exception):
    pass

class InterpreterBuffer(Buffer):
    _basename = 'Interpreter'
    def create_name(cls, parent):
        if hasattr(parent, 'path'):
            return '*%s:%s*' % (cls._basename, parent.name())
        else:
            return '*%s*' % cls._basename
    create_name = classmethod(create_name)
    btype    = 'interpreter'
    readre   = re.compile('^([A-Z]+):(.*)\n$')
    def __init__(self, parent, app):
        self.application = app
        if parent and hasattr(parent, 'path'):
            self.parent = parent
        else:
            self.parent = None
        Buffer.__init__(self)
        cmd = self.get_cmd()
        env = dict(os.environ)
        env.update(self.get_env())
        f = open('/dev/null', 'w')
        self.pipe = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=f, env=env)
        self.prompt = '***'
        self.clear()
        self.pipe_read()
        self._name = self.create_name(parent)
    def name(self):
        return self._name
    def get_env(self):
        return {}
    def get_cmd(self):
        raise Exception, 'unimplemented'
    def pipe_readline(self):
        if self.pipe.poll() is not None:
            raise InterpreterPipeError('broken pipe')
        line = self.pipe.stdout.readline()
        m = self.readre.match(line)
        if m:
            return (m.group(1), m.group(2))
        else:
            return (None, line.rstrip())
    def pipe_read(self):
        lines = []
        while True:
            (type_, value) = self.pipe_readline()
            if type_ == 'PROMPT':
                self.prompt = value.strip() + ' '
                break
            value.rstrip()
            if value:
                lines.append(value)
        if lines:
            output = '\n'.join(lines) + '\n'
            p = self.get_buffer_end()
            self.insert_string(p, output, force=True)
    def pipe_write(self, s):
        self.pipe.stdin.write("%s\n" % s)
        self.pipe.stdin.flush()
    def completions(self, word):
        self.pipe_write("COMPLETE:%s" % word)
        candidates = self.pipe_read_completions()
        self.pipe_read()
        return candidates
    def pipe_read_completions(self):
        (typ_, value) = self.pipe_readline()
        assert typ_ == 'COMPLETIONS', '%r %r' % (typ_, value)
        candidates = [x for x in value.split('|') if x]
        return candidates
    def clear(self):
        self.set_data('', force=True)
    def changed(self):
        return False
    def readonly(self):
        return True

class IperlBuffer(InterpreterBuffer):
    _basename = 'IPerl'
    btype     = 'iperl'
    modename  = 'iperl'
    def create_name(cls, parent):
        if parent and hasattr(parent, 'path'):
            if parent.path.endswith('.pm'):
                return '*%s:%s*' % (cls._basename, parent.name())
            else:
                raise Exception, "not a perl module"
        else:
            return '*%s*' % cls._basename
    create_name = classmethod(create_name)
    def get_cmd(self):
        if self.parent:
            return ('iperl', '-p', '-r', self.parent.path)
        else:
            return ('iperl', '-p')
    def get_env(self):
        return {'PERL5LIB': self.application.config.get('perl.lib', '.')}
    def readline_completions(self, x1, x2, line):
        self.pipe.stdin.write("READLINE:%d:%d:%s\n" % (x1, x2, line))
        self.pipe.stdin.flush()
        (typ_, value) = self.pipe_readline()
        assert typ_ == 'COMPLETIONS', '%r %r' % (typ_, value)
        candidates = [x for x in value.split('|') if x]
        self.pipe_read()
        return candidates

class IpythonBuffer(InterpreterBuffer):
    _basename = 'IPython'
    btype     = 'ipython'
    modename  = 'ipython'
    def get_cmd(self):
        if self.parent:
            return ('epython', '-p', '-r', self.parent.path)
        else:
            return ('epython', '-p')
    def get_env(self):
        return {'PYTHONPATH': self.application.config.get('python.lib', '.')}

class BinaryDataException(Exception):
    pass

class PipeBuffer(Buffer):
    btype = 'pipe'
    terms = {
        'dumb':  term.Dumb,
        'xterm': term.XTerm,
    }
    modename = 'colortext'
    def __init__(self, cmd, args, name=None, term='dumb'):
        Buffer.__init__(self)
        self.cmd = cmd
        if name:
            self._name = name
        else:
            self._name = '*Pipe*'

        self.term     = self.terms[term]()
        self._pid, self._pty = pty.fork()
        if self._pid == 0:
            os.execve(cmd, [cmd] + args, {'TERM': self.term.name})

        self._lock    = threading.Lock()
        self._towrite = ''
        self._done    = False
        self._set_nonblock(self._pty)
        self._thread = threading.Thread(target=self.pipe_read)
        self._thread.setDaemon(True)
        self._thread.start()

    def _set_nonblock(self, fd):
        flags = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NDELAY)

    def _filter_output(self, output):
        output2 = []
        i = 0
        escaped = []
        for c in output:
            if escaped:
                escaped.append(c)
                if c == 'm':
                    escaped = []
            elif c == '\x1b':
                escaped.append(c)
            elif c == '\n':
                output2.append(c)
                i = 0
            elif c == '\t':
                j = i % 8
                output2.append(' ' * (8 - j))
                i += 8 - j
            elif c == '\a':
                pass
            elif c == '\b':
                if i > 0:
                    output2.pop(-1)
                    i -= 1
            else:
                output2.append(c)
                i += 1
        return ''.join(output2)

    def pipe_read(self):
        fd = self._pty
        while not self._done:
            if self._towrite:
                ifd, ofd, efd = select.select([fd], [fd], [fd], 0.1)
            else:
                ifd, ofd, efd = select.select([fd], [], [fd], 0.1)
            if ifd:
                data = os.read(ifd[0], 1024)
                end = self.get_buffer_end()
                data = self.term.filter(data)
                self.insert_string(end, data, force=True, act=ACT_NONE)
            if ofd:
                self._lock.acquire()
                n = os.write(ofd[0], self._towrite)
                self._towrite = self._towrite[n:]
                self._lock.release()
            if efd:
                raise Exception, "exception is ready: %s" % repr(efd)
        os.close(fd)

    def pipe_write(self, s):
        self._lock.acquire()
        self._towrite += s
        self._lock.release()

    def insert_string(self, p, s, act=ACT_NORM, force=False):
        lines = s.split("\n")
        self.insert_lines(p, lines, act, force)

    def insert_lines(self, p, lines, act=ACT_NORM, force=False):
        llen = len(lines)
        assert llen > 0
        if not force and self.readonly():
            raise ReadOnlyError("buffer is read-only")
        p2 = p.vadd(len(lines[-1]), llen - 1)
        if llen > 1:
            self.lines.insert(p.y + 1, [])
            self.lines[p.y + 1] = lines[-1] + self.lines[p.y][p.x:]
            self.lines[p.y] = self.lines[p.y][:p.x] + lines[0]
            for i in range(1, llen - 1):
                self.lines.insert(p.y + i, lines[i])
        else:
            self.lines[p.y] = self.lines[p.y][:p.x] + lines[-1] + self.lines[p.y][p.x:]
        self._region_add(p, p2, lines, act)
        self.modified = True
        
    def name(self): return self._name
    def changed(self): return False
    def readonly(self): return True
    def undo(self, move, act): raise Exception, "invalid"
    def redo(self, move, act): raise Exception, "invalid"
    def reload(self): raise Exception, "invalid"
    def close(self): raise Exception, "invalid"

class FileBuffer(Buffer):
    btype = 'file'
    def __init__(self, path, name=None):
        '''fb = FileBuffer(path)'''
        Buffer.__init__(self)
        self.path = os.path.realpath(path)
        self.checksum = None
        self.bytemark = ''
        if name is None:
            self._name = os.path.basename(self.path)
        else:
            self._name = name
        if os.path.exists(self.path) and not os.access(self.path, os.W_OK):
            self._readonly = True
        else:
            self._readonly = False
    def readonly(self):
        return self._readonly

    def _open_file_r(self, path=None):
        if path is None:
            path = self.path
        path = os.path.realpath(path)
        self.path = path
        if not os.path.isfile(path):
            raise Exception, "Path '%s' does not exist" % (path)
        if not os.access(path, os.R_OK):
            raise Exception, "Path '%s' cannot be read" % (path)
        f = open(path, 'r')
        return f
    def _open_file_w(self, path=None):
        if path is None:
            path = self.path
        if os.path.isfile(path):
            raise Exception, "Path '%s' already exists" % (path)
        d = os.path.dirname(path)
        if not os.access(d, os.R_OK):
            raise Exception, "Dir '%s' cannot be read" % (path)
        if not os.access(d, os.W_OK):
            raise Exception, "Dir '%s' cannot be written" % (path)
        f = open(path, 'w')
        return f
    def _temp_path(self, path=None):
        if path is None:
            path = self.path
        (dirname, basename) = os.path.split(path)
        return os.path.join(dirname, ".__%s__pmacs" % (basename))

    # methods for dealing with the underlying resource, etc.
    def name(self):
        return self._name
    def path_exists(self):
        return os.path.exists(self.path)
    def store_checksum(self, data):
        self.checksum = md5.new(data)
    def read(self):
        if self.path_exists():
            f = self._open_file_r()
            data = f.read()
            if '\t' in data:
                self.writetabs = True
            
            f.close()
            self.store_checksum(data)
        else:
            data = ''

        if data.startswith('\xEF\xBB\xBF'):
            # utf-8
            self.bytemark = data[:3]
            data          = data[3:]

        self.nl = self._detect_nl_type(data)
        data = self.read_filter(data)
        data = data.replace("\t", "    ")
        for i in range(0, min(len(data), 128)):
            if data[i] not in string.printable:
                raise BinaryDataException, "binary files are not supported"
        #FIXME: this is horrible...but maybe not as horrible as using tabs??
        return data
    def open(self):
        data = self.read()
        self.lines = data.split(self.nl)
    def reload(self):
        data = self.read()
        self.set_data(data)
    def changed_on_disk(self):
        assert self.checksum is not None
        f = open(self.path)
        data = f.read()
        f.close()
        m = md5.new(data)
        return self.checksum.digest() != m.digest()
    def save(self, force=False):
        if self.readonly():
            raise ReadOnlyError("can't save read-only file")

        if self.checksum is not None and force is False:
            # the file already existed and we took a checksum so make sure it's
            # still the same right now
            if not self.path_exists():
                raise Exception, "oh no! %r disappeared!" % self.path
            if self.changed_on_disk():
                raise Exception, "oh no! %r has changed on-disk!" % self.path

        temp_path = self._temp_path()
        data = self.make_string()
        if self.windows[0].mode.savetabs:
            data = data.replace("    ", "\t")
            
        data = self.write_filter(data)

        f2 = self._open_file_w(temp_path)
        f2.write(self.bytemark + data)
        f2.close()

        if self.path_exists():
            mode = os.stat(self.path)[0]
            os.chmod(temp_path, mode)

        shutil.move(temp_path, self.path)
        self.store_checksum(data)
        self.modified = False
    def save_as(self, path):
        self.path = path
        self.save()

class AesBuffer(FileBuffer):
    btype = 'aesfile'
    def __init__(self, path, password, name=None):
        '''fb = FileBuffer(path)'''
        FileBuffer.__init__(self, path, name)
        self.password = password
    def read_filter(self, data):
        return aes.decrypt_data(data, self.password)
    def write_filter(self, data):
        return aes.encrypt_data(data, self.password)

class Binary32Buffer(FileBuffer):
    btype     = 'bin32file'
    grouppad  = 2
    groupsize = 8
    numgroups = 2
    bytepad   = 1
    data      = None
    wordsize  = 4
    def __init__(self, path, name=None):
        '''fb = FileBuffer(path)'''
        FileBuffer.__init__(self, path, name)
    def _detect_nl_type(self, data):
        return '\n'
    def cursorx_to_datax(self, cy, cx):
        bytespace = 2 + self.bytepad
        groupspace = bytespace * self.groupsize - self.bytepad + self.grouppad

        groupmod = (cx + self.grouppad) % groupspace
        if groupmod < self.grouppad:
            return None

        groupdiv = (cx + 2) // groupspace
        if groupdiv >= self.numgroups:
            return None

        bytemod = (cx + self.bytepad - groupdiv) % bytespace
        if bytemod == 0:
            return None
        
        bytediv = ((cx + self.bytepad) % groupspace) // bytespace
        ix = self.groupsize * groupdiv  + bytediv
        if ix < len(self.rawdata[cy]):
            return ix
        else:
            return None

    def datax_to_cursorx(self, ix):
        groupsize = (((2 + self.bytepad) * self.groupsize) + self.grouppad)
        maxsize = groupsize * self.numgroups - self.grouppad
        if ix < maxsize:
            return (ix // self.groupsize) * (self.grouppad - 1) + ix * (2 + self.bytepad)
        else:
            return None
    def datax_to_cursory(self, ix):
        return ix // (self.groupsize * self.numgroups)

    def get_address(self, cy, ix):
        return (cy * self.numgroups * self.groupsize) + ix

    def overwrite_char(self, p, c, act=ACT_NORM, force=False):
        ix = self.cursorx_to_datax(p.y, p.x)
        if ix is None:
            return
        Buffer.overwrite_char(self, p, c, act, force)
        cx = self.datax_to_cursorx(ix)
        c = chr(int(self.lines[p.y][cx:cx + 2], 16))
        rawline = self.rawdata[p.y]
        self.rawdata[p.y] = rawline[0:ix] + c + rawline[ix + 1:]

    def read_filter(self, data):
        bytepad      = ' ' * self.bytepad
        grouppad     = ' ' * self.grouppad
        self.rawdata = []
        lines        = []
        i            = 0
        while i < len(data):
            self.rawdata.append(data[i:i + self.numgroups * self.groupsize])
            j = 0
            groups = []
            while j < self.numgroups * self.groupsize and i + j < len(data):
                bytes = []
                for c in data[i + j:i + j + self.groupsize]:
                    bytes.append(string.hexdigits[ord(c) / 16] + string.hexdigits[ord(c) % 16])
                groups.append(bytepad.join(bytes))
                j += self.groupsize
            lines.append(grouppad.join(groups))
            i += self.numgroups * self.groupsize
        if not self.rawdata:
            self.rawdata = ['']
        return '\n'.join(lines)
    def write_filter(self, data):
        return ''.join(self.rawdata)

class DirBuffer(Buffer):
    btype = 'dir'
    def __init__(self, path, name=None):
        Buffer.__init__(self)
        self.path = os.path.realpath(path)
    def changed(self):
        return False
    def readonly(self):
        return True
    def name(self):
        return self.path
    def path_exists(self):
        return os.path.exists(self.path)

    def _get_names(self):
        if not self.path_exists():
            raise Exception, "directory %r does not exists" % self.path
        names = os.listdir(self.path)
        if self.path != '/':
            names.insert(0, '..')
        names.insert(0, '.')
        return names
    def _make_path(self, name):
        return os.path.join(self.path, name)
    def _get_lines(self):
        names = self._get_names()

        fieldlines = []
        maxlens = [0] * 5
        for name in names:
            path = self._make_path(name)
            fields = dirutil.path_fields(path, name)
            for i in range(0, 5):
                try:
                    maxlens[i] = max(maxlens[i], len(fields[i]))
                except:
                    raise Exception, '%d %r' % (i, fields[i])
            fieldlines.append(fields)
        
        fieldlines.sort(cmp=dirutil.path_sort)
        fmt = '%%%ds %%-%ds %%-%ds %%%ds %%%ds %%s' % tuple(maxlens)

        lines = []
        for fields in fieldlines:
            s = fmt % fields
            lines.append(s)
        return lines
    def open(self):
        self.lines = self._get_lines()
    def reload(self):
        lines = self._get_lines()
        self.set_lines(lines, force=True)
    def save(self, force=False):
        raise Exception, "can't save a directory buffer"
    def save_as(self, path):
        raise Exception, "can't save a directory buffer"

class PathListBuffer(DirBuffer):
    btype = 'pathlist'
    def __init__(self, name, paths):
        Buffer.__init__(self)
        self.paths = list(paths)
        self.path  = os.getcwd()
        self._name = name
    def path_exists(self):
        raise Exception
    def _get_names(self):
        cwd = os.getcwd()
        return [x.replace(cwd, '.', 1) for x in self.paths]
    def _make_path(self, name):
        if name.startswith('.'):
            return name.replace('.', os.getcwd(), 1)
        else:
            return name
    def name(self):
        return self._name

# NOTE: this highlighter will not reprocess the data given. it is intended to
#       be used with read-only buffers like DataBuffer and ColorBuffer
class ColorHighlighter(highlight.Highlighter):
    color_re = re.compile(r'\[([a-zA-Z0-9\*:]+)\]')
    color_map = {
        'B': 'black',
        'r': 'red',
        'g': 'green',
        'y': 'yellow',
        'b': 'blue',
        'm': 'magenta',
        'c': 'cyan',
        'w': 'white',
        'd': 'default',
        '*': 'bold',
    }
    def __init__(self):
        self.tokens = []
    def append_token(self, y, x, s, color):
        s2 = s.replace('\\[', '[')
        s2 = s2.replace('\\]', ']')
        s2 = s2.replace('\\\\', '\\')
        t = lex.Token('color_data', None, y, x, s2, color)
        self.tokens[y].append(t)
        return len(s) - len(s2)
    def delete_token(self, y, i):
        pass
    def relex(self, lines, y1, x1, y2, x2, token=None):
        pass
    def relex_del(self, lines, y1, x1, y2, x2):
        pass
    def highlight(self, lines):
        if self.tokens:
            return
        self.tokens = [[] for l in lines]
        #self.tokens = [None] * len(lines)
        for y in range(0, len(lines)):
            self.tokens[y] = []
            line = lines[y]
            c = ['default', 'default']
            i = 0
            offset = 0
            while i < len(line):
                m = self.color_re.search(line, i)
                if m:
                    (j, k) = (m.start(), m.end())
                    if j > i:
                        offset += self.append_token(y, i - offset, line[i:j], c)
                    fields = m.group(1).split(':')
                    c = [self.color_map.get(x, x) for x in fields]
                    offset += k - j
                    i = k
                else:
                    offset += self.append_token(y, i - offset, line[i:], c)
                    break

class ColorDataBuffer(DataBuffer):
    btype    = 'colordata'
    modename = 'colortext'
    color_re = re.compile(r'\[([a-z:]+)\]')
    def _highlight(self, data):
        data2 = ColorHighlighter.color_re.sub('', data)
        data2 = data2.replace('\\[', '[')
        data2 = data2.replace('\\]', ']')
        data2 = data2.replace('\\\\', '\\')
        Buffer.set_data(self, data2, force=True)
        lines = data.split(self.nl)
        self.highlights = {'Colortext': ColorHighlighter()}
        self.highlights['Colortext'].highlight(lines)
        self.modified = False
    def __init__(self, name, data):
        DataBuffer.__init__(self, name, '')
        self._highlight(data)
    def set_data(self, data, force=True):
        self._highlight(data)
    
ABOUT_DATA = '''
[r:d:*]===============================================================================

[y:d:*]************    ********** ******        ****** ****    ********     *********
[y:d:*]**************  ******************     *************   **********   ***********
[y:d:*]*******   ***** ******  *****  ****   ****    ******  ****    **** *****    ***
[y:d:*]  ***       ***   ***    ***    ***  ****      ****  ****           *******
[y:d:*]  ***       ***   ***    ***    ***  ****      ****  ****               ******
[y:d:*]  *****   ***** *****   *****   ****  ****    ******  ****    **** ****    ****
[y:d:*]  ************  *****   *****   ****   *************   **********   **********
[y:d:*]  **********    *****   *****   ****     ****** ****    ********     ********
[y:d:*]  ***
[y:d:*]  ***         [c:d:*]pmacs[d:d:*] is a python-based text editor by [c:d:*]Erik Osheim[d:d:*], [b:d:*](c) 2005-2008
[y:d:*]  ***         [c:d:*]pmacs[d:d:*] is available to you under the [c:d:*]GNU General Public License v2
[y:d:*]*****
[y:d:*]*****         [d:d:*]to view available commands use [c:d:*]C-c M-h [b:d:*](show-bindings-buffer)
[y:d:*]*****         [d:d:*]open files with [c:d:*]C-x C-f[d:d:*]; save with [c:d:*]C-x C-s[d:d:*]; quit with [c:d:*]C-x C-c

[r:d:*]===============================================================================
'''
class AboutBuffer(ColorDataBuffer):
    def __init__(self):
        ColorDataBuffer.__init__(self, '*About*', ABOUT_DATA)

class ShellBuffer(Buffer):
    btype    = 'shell'
    modename = 'shell'
    def __init__(self):
        Buffer.__init__(self)
        self.clear()
    def clear(self):
        lines = ['=== Shell Console\n',
                 "=== Bash your brains out!\n"]
        self.set_data(''.join(lines), force=True)
    def name(self):
        return '*Shell*'
    def changed(self):
        return False
    def readonly(self):
        return True