From 02b2aafb77a04115ab8acf28856538362a3db566 Mon Sep 17 00:00:00 2001 From: moculus Date: Sat, 5 May 2007 17:22:19 +0000 Subject: [PATCH] believed to work --HG-- branch : pmacs2 --- highlight2.py | 254 ++++++++++++++++++++++++++++++++++++-------------- test3.py | 23 ++++- 2 files changed, 208 insertions(+), 69 deletions(-) diff --git a/highlight2.py b/highlight2.py index 6622424..7073d73 100644 --- a/highlight2.py +++ b/highlight2.py @@ -1,4 +1,5 @@ import sys +import lex2 color_list = [] color_list.extend(['\033[3%dm' % x for x in range(0, 8)]) @@ -22,7 +23,9 @@ class Highlighter: def dump(self, fmt='(%3s, %2s) | %s'): print fmt % ('y', 'x', 'string') - for group in self.tokens: + for i in range(0, len(self.tokens)): + group = self.tokens[i] + print 'LINE %d' % i for token in group: print fmt % (token.y, token.x, token.string) @@ -52,6 +55,83 @@ class Highlighter: for token in self.lexer: self.tokens[token.y].append(token) + # relexing + # ====================== + def relex(self, lines, y1, x1, y2, x2): + # start the relexing process + self.lexer.lex(lines, y1, 0) + + # this keeps track of the current y coordinate, the current token index + # on line[y], and the current "new token", respectively. + y = y1 + i = 0 + getnext = True + new_token = None + + while True: + # if we have overstepped our bounds, then exit! + if y >= len(lines): + break + + # if we need another new_token, then try to get it. + if getnext: + try: + new_token = self.lexer.next() + getnext = False + except StopIteration: + # ok, so this means that ALL the rest of the tokens didn't + # show up, because we're done. so delete them and exit + for j in range(y, len(lines)): + del self.tokens[j][i:] + i = 0 + break + + # if our next token is one a future line, we need to just get rid of + # all our old tokens until we get there + while new_token.y > y: + del self.tokens[y][i:] + i = 0 + y += 1 + + # ok, so see if we have current tokens on this line; if so get it + if i < len(self.tokens[y]): + old_token = self.tokens[y][i] + assert old_token.y == y, "%d == %d" % (old_token.y, y) + else: + old_token = None + + if old_token is None: + # since we don't have a previous token at this location, just + # insert the new one + self.tokens[y].insert(i, new_token) + i += 1 + getnext = True + elif old_token == new_token: + # if they match, then leave the old one alone + i += 1 + getnext = True + if new_token.y >= y2 and new_token.end_x() >= x2: + # in this case, we can (probably) assume that the rest of + # the lines will lex the same way + break + elif old_token.x < new_token.end_x(): + # ok, so we haven't gotten to this new token yet. obviously + # this token never showed up in the new lexing, so delete it. + del self.tokens[y][i] + elif old_token.x >= new_token.end_x(): + # ok, this token is further out, so just insert the new token + # ahead of it, move our counter out and continue + self.tokens[y].insert(i, new_token) + i += 1 + getnext = True + else: + # this should never happen + raise Exception, "this isn't happening" + + + + # deletion + # ====================== def update_del(self, lines, y1, x1, y2, x2): assert y1 >= 0 assert y1 <= y2 @@ -60,15 +140,25 @@ class Highlighter: xdelta = x2 - x1 ydelta = y2 - y1 - newtokens = [[] for x in range(0, len(self.tokens) - ydelta)] + # construct a new token data structure; it will have one list for + # every y index in lines. also, fill in tokens before the change + newtokens = [[] for x in lines] for y in range(0, y1): newtokens[y] = self.tokens[y] - for y in range(y1, len(lines)): + # so for each line we currently have, we need to process every token, + # transferring them from the old structure to the new, and modifying + # them appropriately to take the change into account. + for y in range(y1, len(self.tokens)): while self.tokens[y]: + # so remove the token from the old structure, and figure out + # where it stands in relation to the deletion token = self.tokens[y].pop(0) tx1 = token.x tx2 = token.x + len(token.string) + + # the notation "*|*| " refers to what the text spans, i.e.: + # before|during|after if (y, tx2) <= (y1, x1): # *| | newtokens[y].append(token) @@ -79,89 +169,117 @@ class Highlighter: token.x -= xdelta newtokens[token.y].append(token) elif (y, tx1) < (y1, x1): - token2 = token.copy() if (y, tx2) <= (y2, x2): # *|*| - s = token2.string[:x1 - tx1] + token.string = token.string[:x1 - tx1] else: # *|*|* - s = token2.string[:x1 - tx1] + token2.string[x2 - tx1:] - token2.string = s - newtokens[y].append(token2) + token.string = token.string[:x1 - tx1] + token.string[x2 - tx1:] + newtokens[y].append(token) elif (y, tx1) < (y2, x2): if (y, tx2) <= (y2, x2): # |*| pass else: # |*|* - token2 = token.copy() - token2.x = x1 - token2.y = token2.y - ydelta - token2.string = token2.string[x2 - tx1:] - newtokens[token2.y].append(token2) + token.x = x1 + token.y -= ydelta + token.string = token.string[x2 - tx1:] + newtokens[token.y].append(token) + else: + raise Exception, "this should never happen: %r" % token + # ok, now that we have built a correct new structure, store a reference + # to it instead. self.tokens = newtokens def relex_del(self, lines, y1, x1, y2, x2): + # first let's update our existing tokens to fix their offsets, etc. self.update_del(lines, y1, x1, y2, x2) - self.lexer.lex(lines, y1, 0) - y = y1 - i = 0 - getnext = True + # then let's do some relexing + self.relex(lines, y1, x1, y2, x2) - while True: - if y >= len(lines): - break + # addition + # ====================== + def update_add(self, lines, y1, x1, newlines): + assert y1 >= 0 + assert len(newlines) > 0 - if getnext: - try: - new_token = self.lexer.next() - getnext = False - except StopIteration: - for j in range(y, len(lines)): - print 'DELETE END ROW %d[%d:]: %r' % (j, i, [x.string for x in self.tokens[j][i:]]) - del self.tokens[j][i:] - i = 0 - break + y2 = y1 + len(newlines) - 1 + if y2 == y1: + x2 = x1 + len(newlines[0]) + else: + x2 = len(newlines[-1]) - # if our next token is one a future line, we need to just get rid of - # all our old tokens until we get there - while new_token.y > y: - print 'DELETE MID ROW %d[%d:]: %r' % (y, i, [x.string for x in self.tokens[y][i:]]) - del self.tokens[y][i:] - i = 0 - y += 1 + xdelta = x2 - x1 + ydelta = y2 - y1 - if i < len(self.tokens[y]): - old_token = self.tokens[y][i] - assert old_token.y == y + # construct a new token data structure, with the right number of lines + newtokens = [] + for i in range(0, len(self.tokens) + ydelta): + newtokens.append([]) + + # copy the tokens that show up before the changed line + for y in range(0, y1): + newtokens[y] = self.tokens[y] + + # process the tokens that show up on the changed line + post_change_list = [] + for t in self.tokens[y1]: + tx1 = t.x + tx2 = t.x + len(t.string) + ty = t.y + ts = t.string + if tx2 <= x1: + # '*| ' before the insertion + newtokens[y1].append(t) + elif tx1 >= x1: + # ' |*' after the insertion + t.x += xdelta + t.y = y2 + post_change_list.append(t) else: - old_token = None + # '*|*' around the insertion + t1 = t.copy() + t1.string = t.string[:x1 - tx1] + newtokens[y1].append(t1) - if old_token is None: - print 'DEFAULT INSERT %d[%d]: %r' % (y, i, new_token.string) - self.tokens[y].insert(i, new_token) - i += 1 - getnext = True - continue - elif old_token == new_token: - print 'MATCH %d[%d]: %r == %r' % (y, i, old_token.string, new_token.string) - i += 1 - getnext = True - if new_token.y >= y2 and new_token.end_x() >= x2: - break - else: - continue - elif old_token.x < new_token.end_x(): - print 'DELETE BEFORE %d[%d]: %r' % (y, i, old_token.string) - del self.tokens[y][i] - continue - elif old_token.x >= new_token.end_x(): - print 'INSERT %d[%d]: %r' % (y, i, new_token.string) - self.tokens[y].insert(i, new_token) - i += 1 - getnext = True - continue - else: - raise Exception, "what what?" - + t2 = t.copy() + t2.string = t.string[x1 - tx1:] + t2.x = x2 + t2.y = y2 + post_change_list.append(t2) + + # add in the new data + newtokens[y1].append(lex2.Token('new', '', y1, x1, newlines[0])) + for i in range(1, len(newlines)): + yi = y1 + i + newtokens[yi].append(lex2.Token('new', '', yi, 0, newlines[i])) + + # add the post-change tokens back + for t in post_change_list: + newtokens[y2].append(t) + + # for each subsequent line, fix it's tokens' y coordinates + for y in range(y1 + 1, len(self.tokens)): + for t in self.tokens[y]: + t.y += ydelta + newtokens[t.y].append(t) + + # ok, now that we have built a correct new structure, store a reference + # to it instead. + self.tokens = newtokens + + def relex_add(self, lines, y1, x1, newlines): + # first let's update our existing tokens to fix their offsets, etc. + self.update_add(lines, y1, x1, newlines) + + # create some extra info that we need + y2 = y1 + len(newlines) - 1 + if y2 == y1: + x2 = x1 + len(newlines[0]) + else: + x2 = len(newlines[-1]) + + # now let's start the relexing process + self.relex(lines, y1, x1, y2, x2) diff --git a/test3.py b/test3.py index b21656a..8dca865 100644 --- a/test3.py +++ b/test3.py @@ -166,7 +166,7 @@ for path in args: h.dump() else: h.display(token_colors[opts.grammar]) - else: + elif False: (y1, x1) = (5, 9) (y2, x2) = (7, 14) #(y2, x2) = (82, 2) @@ -182,3 +182,24 @@ for path in args: h.dump() else: h.display(token_colors[opts.grammar]) + else: + #newlines = ['one two three'] + newlines = ['one two three', 'cat', 'dog', 'del self.foo[3]', 'oops'] + (y1, x1) = (5, 9) + + if len(newlines) > 1: + lines.insert(y1 + 1, newlines[-1] + lines[y1][x1:]) + lines[y1] = lines[y1][:x1] + newlines[0] + for i in range(1, len(newlines) - 1): + newline = newlines[i] + lines.insert(y1 + i, newline) + else: + lines[y1] = lines[y1][:x1] + newlines[0] + lines[y1][x1:] + + h.relex_add(lines, y1, x1, newlines) + #h.update_add(lines, y1, x1, newlines) + #h.highlight(lines) + if opts.dump: + h.dump() + else: + h.display(token_colors[opts.grammar])