web2py.gluon.languages

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 This file is part of the web2py Web Framework 6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 """ 9 10 import os 11 import re 12 import cgi 13 import portalocker 14 import logging 15 import marshal 16 import copy_reg 17 from fileutils import listdir 18 import settings 19 from cfs import getcfs 20 21 __all__ = ['translator', 'findT', 'update_all_languages'] 22 23 is_gae = settings.global_settings.web2py_runtime_gae 24 25 # pattern to find T(blah blah blah) expressions 26 27 PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\ 28 + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\ 29 + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\ 30 + r'(?:"(?:[^"\\]|\\.)*"))' 31 32 regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL) 33 34 # patter for a valid accept_language 35 36 regex_language = \ 37 re.compile('^[a-zA-Z]{2}(\-[a-zA-Z]{2})?(\-[a-zA-Z]+)?$') 38 39

40 -def read_dict_aux(filename):

41 fp = open(filename, 'r') 42 portalocker.lock(fp, portalocker.LOCK_SH) 43 lang_text = fp.read().replace('\r\n', '\n') 44 portalocker.unlock(fp) 45 fp.close() 46 if not lang_text.strip(): 47 return {} 48 try: 49 return eval(lang_text) 50 except: 51 logging.error('Syntax error in %s' % filename) 52 return {}

53

54 -def read_dict(filename):

55 return getcfs('language:%s'%filename,filename, 56 lambda filename=filename:read_dict_aux(filename))

57

58 -def utf8_repr(s):

59 r''' # note that we use raw strings to avoid having to use double back slashes below 60 61 utf8_repr() works same as repr() when processing ascii string 62 >>> utf8_repr('abc') == utf8_repr("abc") == repr('abc') == repr("abc") == "'abc'" 63 True 64 >>> utf8_repr('a"b"c') == repr('a"b"c') == '\'a"b"c\'' 65 True 66 >>> utf8_repr("a'b'c") == repr("a'b'c") == '"a\'b\'c"' 67 True 68 >>> utf8_repr('a\'b"c') == repr('a\'b"c') == utf8_repr("a'b\"c") == repr("a'b\"c") == '\'a\\\'b"c\'' 69 True 70 >>> utf8_repr('a\r\nb') == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n 71 True 72 73 Unlike repr(), utf8_repr() remains utf8 content when processing utf8 string 74 >>> utf8_repr('中文字') == utf8_repr("中文字") == "'中文字'" != repr('中文字') 75 True 76 >>> utf8_repr('中"文"字') == "'中\"文\"字'" != repr('中"文"字') 77 True 78 >>> utf8_repr("中'文'字") == '"中\'文\'字"' != repr("中'文'字") 79 True 80 >>> utf8_repr('中\'文"字') == utf8_repr("中'文\"字") == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字") 81 True 82 >>> utf8_repr('中\r\n文') == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n 83 True 84 ''' 85 if (s.find("'") >= 0) and (s.find('"') < 0): # only single quote exists 86 s = ''.join(['"', s, '"']) # s = ''.join(['"', s.replace('"','\\"'), '"']) 87 else: 88 s = ''.join(["'", s.replace("'","\\'"), "'"]) 89 return s.replace("\n","\\n").replace("\r","\\r")

90 91

92 -def write_dict(filename, contents):

93 try: 94 fp = open(filename, 'w') 95 except IOError: 96 if not is_gae: 97 logging.warning('Unable to write to file %s' % filename) 98 return 99 portalocker.lock(fp, portalocker.LOCK_EX) 100 fp.write('# coding: utf8\n{\n') 101 for key in sorted(contents): 102 fp.write('%s: %s,\n' % (utf8_repr(key), utf8_repr(contents[key]))) 103 fp.write('}\n') 104 portalocker.unlock(fp) 105 fp.close()

106 107

108 -class lazyT(object):

109 110 """ 111 never to be called explicitly, returned by translator.__call__ 112 """ 113 114 m = None 115 s = None 116 T = None 117

118 - def __init__( 119 self, 120 message, 121 symbols = {}, 122 T = None, 123 ):

124 self.m = message 125 self.s = symbols 126 self.T = T

127

128 - def __repr__(self):

129 return "<lazyT %s>" % (repr(str(self.m)), )

130

131 - def __str__(self):

132 return self.T.translate(self.m, self.s)

133

134 - def __eq__(self, other):

135 return self.T.translate(self.m, self.s) == other

136

137 - def __ne__(self, other):

138 return self.T.translate(self.m, self.s) != other

139

140 - def __add__(self, other):

141 return '%s%s' % (self, other)

142

143 - def __radd__(self, other):

144 return '%s%s' % (other, self)

145

146 - def __cmp__(self,other):

147 return cmp(str(self),str(other))

148

149 - def __hash__(self):

150 return hash(str(self))

151

152 - def __getattr__(self, name):

153 return getattr(str(self),name)

154

155 - def __getitem__(self, i):

156 return str(self)[i]

157

158 - def __getslice__(self, i, j):

159 return str(self)[i:j]

160

161 - def __iter__(self):

162 for c in str(self): yield c

163

164 - def __len__(self):

165 return len(str(self))

166

167 - def xml(self):

168 return cgi.escape(str(self))

169

170 - def encode(self, *a, **b):

171 return str(self).encode(*a, **b)

172

173 - def decode(self, *a, **b):

174 return str(self).decode(*a, **b)

175

176 - def read(self):

177 return str(self)

178

179 - def __mod__(self, symbols):

180 return self.T.translate(self.m, symbols)

181 182

183 -class translator(object):

184 185 """ 186 this class is instantiated by gluon.compileapp.build_environment 187 as the T object 188 189 :: 190 191 T.force(None) # turns off translation 192 T.force('fr, it') # forces web2py to translate using fr.py or it.py 193 194 T(\"Hello World\") # translates \"Hello World\" using the selected file 195 196 notice 1: there is no need to force since, by default, T uses 197 accept_language to determine a translation file. 198 199 notice 2: en and en-en are considered different languages! 200 """ 201

202 - def __init__(self, request):

203 self.request = request 204 self.folder = request.folder 205 self.current_languages = ['en'] 206 self.accepted_language = None 207 self.language_file = None 208 self.http_accept_language = request.env.http_accept_language 209 self.requested_languages = self.force(self.http_accept_language) 210 self.lazy = True 211 self.otherTs = {}

212

213 - def get_possible_languages(self):

214 possible_languages = [lang for lang in self.current_languages] 215 file_ending = re.compile("\.py$") 216 for langfile in os.listdir(os.path.join(self.folder,'languages')): 217 if file_ending.search(langfile): 218 possible_languages.append(file_ending.sub('',langfile)) 219 return possible_languages

220

221 - def set_current_languages(self, *languages):

222 if len(languages) == 1 and isinstance(languages[0], (tuple, list)): 223 languages = languages[0] 224 self.current_languages = languages 225 self.force(self.http_accept_language)

226

227 - def force(self, *languages):

228 if not languages or languages[0] is None: 229 languages = [] 230 if len(languages) == 1 and isinstance(languages[0], (str, unicode)): 231 languages = languages[0] 232 if languages: 233 if isinstance(languages, (str, unicode)): 234 accept_languages = languages.split(';') 235 languages = [] 236 [languages.extend(al.split(',')) for al in accept_languages] 237 languages = [item.strip().lower() for item in languages \ 238 if regex_language.match(item.strip())] 239 240 for language in languages: 241 if language in self.current_languages: 242 self.accepted_language = language 243 break 244 filename = os.path.join(self.folder, 'languages/', language + '.py') 245 if os.path.exists(filename): 246 self.accepted_language = language 247 self.language_file = filename 248 self.t = read_dict(filename) 249 return languages 250 self.language_file = None 251 self.t = {} # ## no language by default 252 return languages

253

254 - def __call__(self, message, symbols={}, language=None, lazy=None):

255 if lazy is None: 256 lazy = self.lazy 257 if not language: 258 if lazy: 259 return lazyT(message, symbols, self) 260 else: 261 return self.translate(message, symbols) 262 else: 263 try: 264 otherT = self.otherTs[language] 265 except KeyError: 266 otherT = self.otherTs[language] = translator(self.request) 267 otherT.force(language) 268 return otherT(message, symbols, lazy=lazy)

269

270 - def translate(self, message, symbols):

271 """ 272 user ## to add a comment into a translation string 273 the comment can be useful do discriminate different possible 274 translations for the same string (for example different locations) 275 276 T(' hello world ') -> ' hello world ' 277 T(' hello world ## token') -> 'hello world' 278 T('hello ## world ## token') -> 'hello ## world' 279 280 the ## notation is ignored in multiline strings and strings that 281 start with ##. this is to allow markmin syntax to be translated 282 """ 283 #for some reason languages.py gets executed before gaehandler.py 284 # is able to set web2py_runtime_gae, so re-check here 285 is_gae = settings.global_settings.web2py_runtime_gae 286 if not message.startswith('#') and not '\n' in message: 287 tokens = message.rsplit('##', 1) 288 else: 289 # this allows markmin syntax in translations 290 tokens = [message] 291 if len(tokens) == 2: 292 tokens[0] = tokens[0].strip() 293 message = tokens[0] + '##' + tokens[1].strip() 294 mt = self.t.get(message, None) 295 if mt is None: 296 self.t[message] = mt = tokens[0] 297 if self.language_file and not is_gae: 298 write_dict(self.language_file, self.t) 299 if symbols or symbols == 0: 300 return mt % symbols 301 return mt

302 303

304 -def findT(path, language='en-us'):

305 """ 306 must be run by the admin app 307 """ 308 filename = os.path.join(path, 'languages', '%s.py' % language) 309 sentences = read_dict(filename) 310 mp = os.path.join(path, 'models') 311 cp = os.path.join(path, 'controllers') 312 vp = os.path.join(path, 'views') 313 for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\ 314 + listdir(vp, '.+\.html', 0): 315 fp = open(file, 'r') 316 portalocker.lock(fp, portalocker.LOCK_SH) 317 data = fp.read() 318 portalocker.unlock(fp) 319 fp.close() 320 items = regex_translate.findall(data) 321 for item in items: 322 try: 323 message = eval(item) 324 if not message.startswith('#') and not '\n' in message: 325 tokens = message.rsplit('##', 1) 326 else: 327 # this allows markmin syntax in translations 328 tokens = [message] 329 if len(tokens) == 2: 330 message = tokens[0].strip() + '##' + tokens[1].strip() 331 if message and not message in sentences: 332 sentences[message] = message 333 except: 334 pass 335 write_dict(filename, sentences)

336 337 ### important to allow safe session.flash=T(....)

338 -def lazyT_unpickle(data):

339 return marshal.loads(data)

340 -def lazyT_pickle(data):

341 return lazyT_unpickle, (marshal.dumps(str(data)),)

342 copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle) 343

344 -def update_all_languages(application_path):

345 path = os.path.join(application_path, 'languages/') 346 for language in listdir(path, '^\w+(\-\w+)?\.py$'): 347 findT(application_path, language[:-3])

348 349 350 if __name__ == '__main__': 351 import doctest 352 doctest.testmod() 353

Source Code for Module web2py.gluon.languages