diff --git a/buffer.py b/buffer.py index a70523a..690e772 100644 --- a/buffer.py +++ b/buffer.py @@ -15,6 +15,7 @@ class Buffer(object): assert nl in ('\n', '\r', '\r\n'), "Invalid line ending" self.nl = nl self.modified = False + pass def num_chars(self): n = 0 diff --git a/mode_python.py b/mode_python.py index a66afee..5123eef 100644 --- a/mode_python.py +++ b/mode_python.py @@ -1,262 +1,130 @@ -import commands, os.path, sets, string, sys - +import commands, os.path, sets, string import color, completer, default, mode2, lex2, method, regex, tab2 import ctag_python - from point2 import Point -from lex2 import Grammar, ConstantRule, PatternRule, RegionRule, DualRegionRule +from lex2 import Grammar, PatternRule, RegionRule class StringGrammar(Grammar): rules = [ - PatternRule( - name=r'octal', - pattern=r'\\[0-7]{3}', - ), - PatternRule( - name=r'escaped', - pattern=r'\\.', - ), - #PatternRule( - # name=r'format', - # pattern=r'%(?:\([a-zA-Z_]+\))?[-# +]*(?:[0-9]+|\*)?\.?(?:[0-9]+|\*)?[hlL]?[a-zA-Z%]', - #), + PatternRule(name=r'octal', pattern=r'\\[0-7]{3}'), + PatternRule(name=r'escaped', pattern=r'\\.'), ] class PythonGrammar(Grammar): rules = [ - PatternRule( - name=r'functiondef', - pattern=r'(?<=def )[a-zA-Z_][a-zA-Z0-9_]*', - ), - PatternRule( - name=r'classdef', - pattern=r'(?<=class )[a-zA-Z_][a-zA-Z0-9_]*', - ), - PatternRule( - name=r'reserved', - pattern=r'(?:True|None|False|Exception|self)(?![a-zA-Z0-9_])', - ), - PatternRule( - name=r'keyword', - pattern=r'(?:yield|while|try|return|raise|print|pass|or|not|lambda|is|in|import|if|global|from|for|finally|exec|except|else|elif|del|def|continue|class|break|assert|as|and)(?![a-zA-Z0-9_])', - ), - PatternRule( - name=r"builtin", - pattern=r'(?>=|<<=|\*\*=', - ), - PatternRule( - name=r"operator", - pattern=r"\+|<>|<<|<=|<|-|>>|>=|>|\*\*|&|\*|\||/|\^|==|//|~|!=|%", - ), - - PatternRule( - name=r"integer", - pattern=r"(?"""|\'\'\')', - grammar=Grammar(), - end=r'%(tag)s', - ), - RegionRule( - name=r'tq_string', - start=r'(?P"""|\'\'\')', - grammar=Grammar(), - end=r'%(tag)s', - ), - RegionRule( - name=r'string', - start=r'(?P"|\')', - grammar=StringGrammar(), - end=r'%(tag)s', - ), - - PatternRule( - name=r'comment', - pattern=r'#.*$', - ), - PatternRule( - name=r'continuation', - pattern=r'\\$', - ), + PatternRule(name=r'functionname', pattern=r'(?<=def )[a-zA-Z_][a-zA-Z0-9_]*'), + PatternRule(name=r'classname', pattern=r'(?<=class )[a-zA-Z_][a-zA-Z0-9_]*'), + PatternRule(name=r'reserved', pattern=r'(?:True|None|False|Exception|self)(?![a-zA-Z0-9_])'), + PatternRule(name=r'keyword', pattern=r'(?:yield|while|try|return|raise|print|pass|or|not|lambda|is|in|import|if|global|from|for|finally|exec|except|else|elif|del|def|continue|class|break|assert|as|and)(?![a-zA-Z0-9_])'), + PatternRule(name=r"builtin", pattern=r'(?>=|<<=|\*\*='), + PatternRule(name=r"operator", pattern=r"\+|<>|<<|<=|<|-|>>|>=|>|\*\*|&|\*|\||/|\^|==|//|~|!=|%"), + PatternRule(name=r"integer", pattern=r"(?"""|\'\'\')', grammar=Grammar(), end=r'%(tag)s'), + RegionRule(name=r'string', start=r'(?P"""|\'\'\')', grammar=StringGrammar(), end=r'%(tag)s'), + RegionRule(name=r'string', start=r'(?P"|\')', grammar=StringGrammar(), end=r'%(tag)s'), + PatternRule(name=r'comment', pattern=r'#.*$'), + PatternRule(name=r'continuation', pattern=r'\\$'), ] -class PythonTabber(tab2.Tabber): +class PythonTabber(tab2.StackTabber): + unanchored_names = ('null', 'string', 'docstring', 'comment') + endlevel_names = ('pass', 'return', 'yield', 'raise', 'break', 'continue') + startlevel_names = ('if', 'try', 'class', 'def', 'for', 'while', 'try') + def __init__(self, m): + tab2.StackTabber.__init__(self, m) + self.base_level = 0 + def is_base(self, y): if y == 0: return True - highlighter = self.mode.window.buffer.highlights[self.mode.name()] - if not highlighter.tokens[y]: + tokens = self.get_tokens(y) + if not tokens: return False - t = highlighter.tokens[y][0] - return t.name == 'keyword' and t.string == 'def' - - def __init__(self, m): - tab2.Tabber.__init__(self, m) - - - - def stack_append(self, item): - self.tab_stack.append(item) - def stack_pop(self): - self.tab_stack.pop(-1) - - def base_indentation_level(self, y): - return y == 0 - - def calculate_tabs(self, start=0, goal=None): - lines = self.mode.window.buffer.lines - tokens = self.mode.highlighter.tokens - buffer = self.mode.window.buffer - - if self.levels is None: - self.levels = [None] * (len(lines)) - - self.index = 0 - self.y = start - self.base = 0 - self.tab_stack = [] - - # we want to process every logical line in the file - while self.y < len(lines): - line = lines[self.y] - start_index = self.index - - start_point = point.Point(0, self.y) - start_offset = buffer.get_point_offset(start_point) - end_point = point.Point(len(line), self.y) - end_offset = buffer.get_point_offset(end_point) - - # we want to find all the tokens on the line we are currently processing - while self.index < len(tokens): - token = tokens[self.index] - if token.end > end_offset: - break - self.index += 1 - - self.handle_line(line, - start_offset, start_index, - end_offset, self.index) - - self.levels[self.y] = self.line_depth - self.y += 1 - if goal is not None and self.y > goal: - break - - def get_line_depth(self): - if len(self.tab_stack) > 0: - return self.tab_stack[-1][1] + elif tokens[0].name not in self.unanchored_names: + return True else: - return self.base + return False - def handle_line(self, line, start_offset, start_index, end_offset, end_index): - self.line_depth = self.get_line_depth() - tokens = self.mode.highlighter.tokens + def get_level(self, y): + self._calc_level(y) + return self.lines.get(y) - if start_index >= len(tokens): - return - if regex.whitespace.match(line): - return + def _calc_level(self, y): + target = y + while not self.is_base(y) and y > 0: + y -= 1 - if len(self.tab_stack) == 0 and tokens[start_index].start >= start_offset: - self.base = util.count_leading_whitespace(line) - - for i in range(start_index, end_index): - token = tokens[i] - s = token.string - if s in self.start_tags: - if i < end_index - 1: - i = tokens[i+1].start - start_offset - elif len(self.tab_stack) > 0: - i = self.tab_stack[-1][1] + 4 + self.markers = [] + while y <= target: + self.popped = False + tokens = self.get_tokens(y) + currlvl = self.get_curr_level() + if y < target and tokens: + if self.token_is_whitespace(y, 0): + l = len(tokens[0].string) else: - i = self.base + 4 - self.stack_append((s, i)) - elif s in self.close_tags: - assert len(self.tab_stack), "Unbalanced closing tag" - assert self.tab_stack[-1][0] == self.close_tags[s], "Unmatched closing tag" - self.stack_pop() - if i == start_index: - self.line_depth = self.get_line_depth() + l = 0 + while currlvl > l: + self._pop() + currlvl = self.get_curr_level() + self.popped = True + for i in range(0, len(tokens)): + currlvl = self._handle_token(currlvl, y, i) + self.lines[y] = currlvl + self.record[y] = tuple(self.markers) + y += 1 - if tokens[start_index].start < start_offset: - self.line_depth = -1 - - prebase = self.base - s = tokens[start_index].string - e = tokens[end_index-1].string - - if s == "except" or s == "elif" or s == "else": - if self.y > 0 and self.line_depth == self.levels[self.y - 1]: - self.line_depth = max(0, self.line_depth - 4) - elif (s == "return" or s == "raise" or s == "yield" or s == "break" or - s == "pass" or s == 'continue'): - self.base = max(0, self.base - 4) - - if e == "\\": - if len(self.tab_stack) and self.tab_stack[-1][0] == "\\": + def _handle_other_token(self, currlvl, y, i): + token = self.get_token(y, i) + fqname = token.fqname() + if fqname == 'string.start': + self._opt_append('string', None) + elif fqname == 'string.end': + self._opt_pop('string') + elif fqname == 'docstring.start': + self._opt_append('docstring', None) + elif fqname == 'docstring.end': + self._opt_pop('docstring') + elif fqname == 'delimiter': + if token.string == ':' and self.markers and self.markers[-1].name in ('[', '{'): + # we are in a list range [:] or dictionary key/value {:} + pass + elif self.is_rightmost_token(y, i): + # we are at the end of a block pass else: - self.stack_append(("\\", prebase + 4)) - return - elif e == ":": - self.base += 4 - elif len(self.tab_stack) and self.tab_stack[-1][0] == "\\": - self.stack_pop() - - def get_indentation_level(self, y): - if self.levels is not None and self.levels[y] is not None: - result = self.levels[y] - else: - i = max(0, y - 1) - while i > 0: - if self.base_indentation_level(i): - break - i -= 1 - self.calculate_tabs(i, y) - result = self.levels[y] - if result == -1: - return None - return result + # we are doing a one-liner + self._pop() + elif fqname == 'keyword': + if token.string in self.endlevel_names: + self._pop() + elif token.string in self.startlevel_names and self.is_leftmost_token(y, i): + self._append(token.string, currlvl + 4) + elif token.string in ('elif', 'else') and self.is_leftmost_token(y, i): + if not self.popped: + self._pop_until('if', 'elif') + currlvl = self.get_curr_level() + self._append(token.string, currlvl + 4) + elif token.string == 'except' and self.is_leftmost_token(y, i): + if not self.popped: + self._pop_until('try') + currlvl = self.get_curr_level() + self._append(token.string, currlvl + 4) + elif token.string == 'finally' and self.is_leftmost_token(y, i): + if not self.popped: + self._pop_until('try', 'except') + currlvl = self.get_curr_level() + self._append(token.string, currlvl + 4) + return currlvl class Python(mode2.Fundamental): tabbercls = PythonTabber @@ -283,8 +151,8 @@ class Python(mode2.Fundamental): 'keyword': color.build('cyan', 'default'), 'reserved': color.build('cyan', 'default'), 'builtin': color.build('cyan', 'default'), - 'functiondef': color.build('blue', 'default'), - 'classdef': color.build('green', 'default'), + 'functionname': color.build('blue', 'default'), + 'classname': color.build('green', 'default'), 'string.start': color.build('green', 'default'), 'string.null': color.build('green', 'default'), @@ -297,10 +165,6 @@ class Python(mode2.Fundamental): 'float': color.build('default', 'default'), 'imaginary': color.build('default', 'default'), - 'tq_string.start': color.build('green', 'default'), - 'tq_string.null': color.build('green', 'default'), - 'tq_string.end': color.build('green', 'default'), - 'docstring.start': color.build('green', 'default'), 'docstring.null': color.build('green', 'default'), 'docstring.end': color.build('green', 'default'), diff --git a/tab2.py b/tab2.py index 9cdb96c..99af1d7 100644 --- a/tab2.py +++ b/tab2.py @@ -9,24 +9,33 @@ class Marker: return '' % (self.name, self.level) class Tabber: + wsre = regex.whitespace + wst = 'null' def __init__(self, m): self.mode = m self.lines = {} + def get_highlighter(self): + return self.mode.window.buffer.highlights[self.mode.name()] + def get_tokens(self, y): + return self.mode.window.buffer.highlights[self.mode.name()].tokens[y] + def get_token(self, y, i): + return self.mode.window.buffer.highlights[self.mode.name()].tokens[y][i] + def token_is_whitespace(self, y, i): + token = self.get_token(y, i) + return token.name == self.wst and self.wsre.match(token.string) def get_next_left_token(self, y, i): tokens = self.get_tokens(y) assert i >= 0 and i < len(tokens) for j in range(1, i): - m = regex.whitespace.match(tokens[i - j].string) - if not m: + if not self.token_is_whitespace(y, i - j): return tokens[i - j] return None def get_next_right_token(self, y, i): tokens = self.get_tokens(y) assert i >= 0 and i < len(tokens) for j in range(i + 1, len(tokens)): - m = regex.whitespace.match(tokens[j].string) - if not m: + if not self.token_is_whitespace(y, j): return tokens[j] return None def is_leftmost_token(self, y, i): @@ -36,6 +45,20 @@ class Tabber: def is_only_token(self, y, i): return self.is_leftmost_token(y, i) and self.is_rightmost_token(y, i) + def get_leftmost_token(self, y): + tokens = self.get_tokens(y) + for i in range(0, len(tokens)): + if not self.token_is_whitespace(y, i): + return tokens[i] + return None + def get_rightmost_token(self, y): + tokens = self.get_tokens(y) + i = len(tokens) - 1 + for j in range(0, len(tokens)): + if not self.token_is_whitespace(y, i - j): + return tokens[i - j] + return None + def region_added(self, p, newlines): self.lines = {} def region_removed(self, p1, p2): @@ -52,12 +75,6 @@ class Tabber: def _calc_level(self, y): pass -def get_level(markers): - if markers: - return markers[-1].level - else: - return 0 - class StackTabber(Tabber): def __init__(self, m): self.mode = m @@ -70,10 +87,6 @@ class StackTabber(Tabber): return self.markers[-1].level else: return 0 - def get_tokens(self, y): - return self.mode.window.buffer.highlights[self.mode.name()].tokens[y] - def get_token(self, y, i): - return self.mode.window.buffer.highlights[self.mode.name()].tokens[y][i] def region_added(self, p, newlines): self.lines = {} @@ -87,9 +100,6 @@ class StackTabber(Tabber): def is_base(self, y): return y == 0 def _calc_level(self, y): - highlighter = self.mode.window.buffer.highlights[self.mode.name()] - target = y - # first we need to step back to find the last place where we have tab # stops figured out, or a suitable place to start while not self.is_base(y) and y > 0: @@ -97,13 +107,14 @@ class StackTabber(Tabber): # ok now, let's do this shit self.markers = [] - currlvl = 0 + currlvl = 0 + target = y while y <= target: currlvl = self.get_curr_level() tokens = self.get_tokens(y) for i in range(0, len(tokens)): currlvl = self._handle_token(currlvl, y, i) - self.lines[y] = currlvl + self.lines[y] = currlvl self.record[y] = tuple(self.markers) y += 1 @@ -150,6 +161,13 @@ class StackTabber(Tabber): self.markers.append(Marker(name, level)) def _pop(self): self.markers.pop(-1) + def _pop_until(self, *names): + while self.markers: + if self.markers[-1].name in names: + self.markers.pop(-1) + return + else: + self.markers.pop(-1) def _opt_append(self, name, level): if self.markers and self.markers[-1].name == name: pass