initial unicode support

--HG--
branch : pmacs2

BUGS

@@ -1,5 +1,16 @@
 === OUSTANDING BUGS ===
 
+2009/06/09: UNICODE ISSUES
+    1. some double-width characters don't work right (see japanese.txt)
+
+    2. need input methods for unicode characters
+
+    3. regexes need to be made unicode-aware
+
+    4. make ad-hoc strings in application u'' strings
+
+    5. handling terminal output type (utf-8, latin-1, ascii)
+
 2009/06/09:
     1. many of the dir-mode methods need better names, to make it clear they
     aren't general purpose.

application.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import curses, curses.ascii, getpass, os, re, string, sys, termios, time
+import locale
 import math
 import traceback
 from subprocess import Popen, PIPE, STDOUT
@@ -1046,6 +1047,8 @@ if __name__ == "__main__":
         'aes':  open_aes_file,
     }
 
+    locale.setlocale(locale.LC_ALL, '')
+
     # preprocess args
     argv = list(sys.argv[1:])
     goto_line = None

buffer/__init__.py

@@ -631,11 +631,12 @@ class FileBuffer(Buffer):
 
         self.nl = self._detect_nl_type(data)
         data = self.read_filter(data)
+        try:
+            data = data.decode('utf-8')
             data = data.replace("\t", "    ")
-        for i in range(0, min(len(data), 128)):
-            if data[i] not in string.printable:
-                raise BinaryDataException("binary files are not supported")
             return data
+        except UnicodeDecodeError:
+            raise BinaryDataException("binary files are not supported")
     def open(self):
         data = self.read()
         self.lines = data.split(self.nl)

code_examples/georgian.txt

@@ -0,0 +1,3 @@
+ვეპხის ტყაოსანი შოთა რუსთაველი
+
+ღმერთსი შემვედრე, ნუთუ კვლა დამხსნას სოფლისა შრომასა, ცეცხლს, წყალსა და მიწასა, ჰაერთა თანა მრომასა; მომცნეს ფრთენი და აღვფრინდე, მივჰხვდე მას ჩემსა ნდომასა, დღისით და ღამით ვჰხედვიდე მზისა ელვათა კრთომაასა. 

code_examples/greek.txt

@@ -0,0 +1,3 @@
+Τη γλώσσα μου έδωσαν ελληνική
+το σπίτι φτωχικό στις αμμουδιές του Ομήρου.
+Μονάχη έγνοια η γλώσσα μου στις αμμουδιές του Ομήρου.

code_examples/hebrew.txt

@@ -0,0 +1 @@
+זה כיף סתם לשמוע איך תנצח קרפד עץ טוב בגן

code_examples/japanese.txt

@@ -0,0 +1,4 @@
+いろはにほへど　ちりぬるを
+わがよたれぞ　つねならむ
+うゐのおくやま　けふこえて
+あさきゆめみじ　ゑひもせず

code_examples/runes.txt

@@ -0,0 +1,3 @@
+ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ
+ᛋᚳᛖᚪᛚ᛫ᚦᛖᚪᚻ᛫ᛗᚪᚾᚾᚪ᛫ᚷᛖᚻᚹᛦᛚᚳ᛫ᛗᛁᚳᛚᚢᚾ᛫ᚻᛦᛏ᛫ᛞᚫᛚᚪᚾ
+ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬

code_examples/russian.txt

@@ -0,0 +1,5 @@
+абвгдеёжзийклмно 
+прстуфхцчшчьыъэюя 
+АБВГДЕЁЖЗИЙ 
+КЛМНОПРСТУФ 
+ХЦЧШЩЬЫЪЭЮЯ

render.py

@@ -35,7 +35,8 @@ class RenderString(object):
                 s = '%-*s' % (x2 - x, self.string)
             else:
                 s = self.string
-            cwin.addstr(self.y + y, self.x + x, s, self.attrs)
+            s2 = s.encode('utf-8')
+            cwin.addstr(self.y + y, self.x + x, s2, self.attrs)
         except Exception, e:
             raise
             #v = (self.y, y, self.x, x, self.string, self.attrs, str(e))