Package translate :: Package storage :: Module base
[hide private]
[frames] | no frames]

Source Code for Module translate.storage.base

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  # 
  4  # Copyright 2006-2009 Zuza Software Foundation 
  5  # 
  6  # This file is part of the Translate Toolkit. 
  7  # 
  8  # This program is free software; you can redistribute it and/or modify 
  9  # it under the terms of the GNU General Public License as published by 
 10  # the Free Software Foundation; either version 2 of the License, or 
 11  # (at your option) any later version. 
 12  # 
 13  # This program is distributed in the hope that it will be useful, 
 14  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  # GNU General Public License for more details. 
 17  # 
 18  # You should have received a copy of the GNU General Public License 
 19  # along with this program; if not, see <http://www.gnu.org/licenses/>. 
 20   
 21  """Base classes for storage interfaces. 
 22   
 23  @organization: Zuza Software Foundation 
 24  @copyright: 2006-2009 Zuza Software Foundation 
 25  @license: U{GPL <http://www.fsf.org/licensing/licenses/gpl.html>} 
 26  """ 
 27   
 28  try: 
 29      import cPickle as pickle 
 30  except: 
 31      import pickle 
 32  from exceptions import NotImplementedError 
 33  import translate.i18n 
 34  from translate.storage.placeables import StringElem, general, parse as rich_parse 
 35  from translate.misc.typecheck import accepts, Self, IsOneOf 
 36  from translate.misc.multistring import multistring 
 37   
38 -def force_override(method, baseclass):
39 """Forces derived classes to override method.""" 40 41 if type(method.im_self) == type(baseclass): 42 # then this is a classmethod and im_self is the actual class 43 actualclass = method.im_self 44 else: 45 actualclass = method.im_class 46 if actualclass != baseclass: 47 raise NotImplementedError( 48 "%s does not reimplement %s as required by %s" % \ 49 (actualclass.__name__, method.__name__, baseclass.__name__) 50 )
51 52
53 -class ParseError(Exception):
54 - def __init__(self, inner_exc):
55 self.inner_exc = inner_exc
56
57 - def __str__(self):
58 return repr(self.inner_exc)
59 60
61 -class TranslationUnit(object):
62 """Base class for translation units. 63 64 Our concept of a I{translation unit} is influenced heavily by XLIFF: 65 U{http://www.oasis-open.org/committees/xliff/documents/xliff-specification.htm} 66 67 As such most of the method- and variable names borrows from XLIFF terminology. 68 69 A translation unit consists of the following: 70 - A I{source} string. This is the original translatable text. 71 - A I{target} string. This is the translation of the I{source}. 72 - Zero or more I{notes} on the unit. Notes would typically be some 73 comments from a translator on the unit, or some comments originating from 74 the source code. 75 - Zero or more I{locations}. Locations indicate where in the original 76 source code this unit came from. 77 - Zero or more I{errors}. Some tools (eg. L{pofilter <filters.pofilter>}) can run checks on 78 translations and produce error messages. 79 80 @group Source: *source* 81 @group Target: *target* 82 @group Notes: *note* 83 @group Locations: *location* 84 @group Errors: *error* 85 """ 86 87 rich_parsers = [] 88 """A list of functions to use for parsing a string into a rich string tree.""" 89
90 - def __init__(self, source):
91 """Constructs a TranslationUnit containing the given source string.""" 92 self.notes = "" 93 self._store = None 94 self.source = source 95 self._target = None 96 self._rich_source = None 97 self._rich_target = None
98
99 - def __eq__(self, other):
100 """Compares two TranslationUnits. 101 102 @type other: L{TranslationUnit} 103 @param other: Another L{TranslationUnit} 104 @rtype: Boolean 105 @return: Returns True if the supplied TranslationUnit equals this unit. 106 """ 107 return self.source == other.source and self.target == other.target
108
109 - def rich_to_multistring(cls, elem_list):
110 """Convert a "rich" string tree to a C{multistring}: 111 112 >>> from translate.storage.placeables.interfaces import X 113 >>> rich = [StringElem(['foo', X(id='xxx', sub=[' ']), 'bar'])] 114 >>> TranslationUnit.rich_to_multistring(rich) 115 multistring(u'foo bar') 116 """ 117 return multistring([unicode(elem) for elem in elem_list])
118 rich_to_multistring = classmethod(rich_to_multistring) 119
120 - def multistring_to_rich(cls, mulstring):
121 """Convert a multistring to a list of "rich" string trees: 122 123 >>> target = multistring([u'foo', u'bar', u'baz']) 124 >>> TranslationUnit.multistring_to_rich(target) 125 [<StringElem([<StringElem([u'foo'])>])>, 126 <StringElem([<StringElem([u'bar'])>])>, 127 <StringElem([<StringElem([u'baz'])>])>] 128 """ 129 if isinstance(mulstring, multistring): 130 return [rich_parse(s, cls.rich_parsers) for s in mulstring.strings] 131 return [rich_parse(mulstring, cls.rich_parsers)]
132
133 - def setsource(self, source):
134 """Sets the source string to the given value.""" 135 self._rich_source = None 136 self._source = source
137 source = property(lambda self: self._source, setsource) 138
139 - def settarget(self, target):
140 """Sets the target string to the given value.""" 141 self._rich_target = None 142 self._target = target
143 target = property(lambda self: self._target, settarget) 144
145 - def _get_rich_source(self):
146 if self._rich_source is None: 147 self._rich_source = self.multistring_to_rich(self.source) 148 return self._rich_source
149 - def _set_rich_source(self, value):
150 if not hasattr(value, '__iter__'): 151 raise ValueError('value must be iterable') 152 if len(value) < 1: 153 raise ValueError('value must have at least one element.') 154 if not isinstance(value[0], StringElem): 155 raise ValueError('value[0] must be of type StringElem.') 156 self._rich_source = list(value) 157 self.source = self.rich_to_multistring(value)
158 rich_source = property(_get_rich_source, _set_rich_source) 159 """ @see: rich_to_multistring 160 @see: multistring_to_rich""" 161
162 - def _get_rich_target(self):
163 if self._rich_target is None: 164 self._rich_target = self.multistring_to_rich(self.target) 165 return self._rich_target
166 - def _set_rich_target(self, value):
167 if not hasattr(value, '__iter__'): 168 raise ValueError('value must be iterable') 169 if len(value) < 1: 170 raise ValueError('value must have at least one element.') 171 if not isinstance(value[0], StringElem): 172 raise ValueError('value[0] must be of type StringElem.') 173 self._rich_target = list(value) 174 self.target = self.rich_to_multistring(value)
175 rich_target = property(_get_rich_target, _set_rich_target) 176 """ @see: rich_to_multistring 177 @see: multistring_to_rich""" 178
179 - def gettargetlen(self):
180 """Returns the length of the target string. 181 182 @note: Plural forms might be combined. 183 @rtype: Integer 184 """ 185 length = len(self.target or "") 186 strings = getattr(self.target, "strings", []) 187 if strings: 188 length += sum([len(pluralform) for pluralform in strings[1:]]) 189 return length
190
191 - def getid(self):
192 """A unique identifier for this unit. 193 194 @rtype: string 195 @return: an identifier for this unit that is unique in the store 196 197 Derived classes should override this in a way that guarantees a unique 198 identifier for each unit in the store. 199 """ 200 return self.source
201
202 - def getlocations(self):
203 """A list of source code locations. 204 205 @note: Shouldn't be implemented if the format doesn't support it. 206 @rtype: List 207 """ 208 return []
209
210 - def addlocation(self, location):
211 """Add one location to the list of locations. 212 213 @note: Shouldn't be implemented if the format doesn't support it. 214 """ 215 pass
216
217 - def addlocations(self, location):
218 """Add a location or a list of locations. 219 220 @note: Most classes shouldn't need to implement this, 221 but should rather implement L{addlocation()}. 222 @warning: This method might be removed in future. 223 """ 224 if isinstance(location, list): 225 for item in location: 226 self.addlocation(item) 227 else: 228 self.addlocation(location)
229
230 - def getcontext(self):
231 """Get the message context.""" 232 return ""
233
234 - def getnotes(self, origin=None):
235 """Returns all notes about this unit. 236 237 It will probably be freeform text or something reasonable that can be 238 synthesised by the format. 239 It should not include location comments (see L{getlocations()}). 240 """ 241 return getattr(self, "notes", "")
242
243 - def addnote(self, text, origin=None):
244 """Adds a note (comment). 245 246 @type text: string 247 @param text: Usually just a sentence or two. 248 @type origin: string 249 @param origin: Specifies who/where the comment comes from. 250 Origin can be one of the following text strings: 251 - 'translator' 252 - 'developer', 'programmer', 'source code' (synonyms) 253 """ 254 if getattr(self, "notes", None): 255 self.notes += '\n'+text 256 else: 257 self.notes = text
258
259 - def removenotes(self):
260 """Remove all the translator's notes.""" 261 self.notes = u''
262
263 - def adderror(self, errorname, errortext):
264 """Adds an error message to this unit. 265 266 @type errorname: string 267 @param errorname: A single word to id the error. 268 @type errortext: string 269 @param errortext: The text describing the error. 270 """ 271 pass
272
273 - def geterrors(self):
274 """Get all error messages. 275 276 @rtype: Dictionary 277 """ 278 return {}
279
280 - def markreviewneeded(self, needsreview=True, explanation=None):
281 """Marks the unit to indicate whether it needs review. 282 283 @keyword needsreview: Defaults to True. 284 @keyword explanation: Adds an optional explanation as a note. 285 """ 286 pass
287
288 - def istranslated(self):
289 """Indicates whether this unit is translated. 290 291 This should be used rather than deducing it from .target, 292 to ensure that other classes can implement more functionality 293 (as XLIFF does). 294 """ 295 return bool(self.target) and not self.isfuzzy()
296
297 - def istranslatable(self):
298 """Indicates whether this unit can be translated. 299 300 This should be used to distinguish real units for translation from 301 header, obsolete, binary or other blank units. 302 """ 303 return True
304
305 - def isfuzzy(self):
306 """Indicates whether this unit is fuzzy.""" 307 return False
308
309 - def markfuzzy(self, value=True):
310 """Marks the unit as fuzzy or not.""" 311 pass
312
313 - def isheader(self):
314 """Indicates whether this unit is a header.""" 315 return False
316
317 - def isreview(self):
318 """Indicates whether this unit needs review.""" 319 return False
320
321 - def isblank(self):
322 """Used to see if this unit has no source or target string. 323 324 @note: This is probably used more to find translatable units, 325 and we might want to move in that direction rather and get rid of this. 326 """ 327 return not (self.source or self.target)
328
329 - def hasplural(self):
330 """Tells whether or not this specific unit has plural strings.""" 331 #TODO: Reconsider 332 return False
333
334 - def getsourcelanguage(self):
335 return getattr(self._store, "sourcelanguage", "en")
336
337 - def gettargetlanguage(self):
338 return getattr(self._store, "targetlanguage", None)
339
340 - def merge(self, otherunit, overwrite=False, comments=True):
341 """Do basic format agnostic merging.""" 342 if not self.target or overwrite: 343 self.rich_target = otherunit.rich_target
344
345 - def unit_iter(self):
346 """Iterator that only returns this unit.""" 347 yield self
348
349 - def getunits(self):
350 """This unit in a list.""" 351 return [self]
352
353 - def buildfromunit(cls, unit):
354 """Build a native unit from a foreign unit, preserving as much 355 information as possible.""" 356 if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy): 357 return unit.copy() 358 newunit = cls(unit.source) 359 newunit.target = unit.target 360 newunit.markfuzzy(unit.isfuzzy()) 361 locations = unit.getlocations() 362 if locations: 363 newunit.addlocations(locations) 364 notes = unit.getnotes() 365 if notes: 366 newunit.addnote(notes) 367 return newunit
368 buildfromunit = classmethod(buildfromunit) 369 370 xid = property(lambda self: None, lambda self, value: None) 371 rid = property(lambda self: None, lambda self, value: None)
372 373
374 -class TranslationStore(object):
375 """Base class for stores for multiple translation units of type UnitClass.""" 376 377 UnitClass = TranslationUnit 378 """The class of units that will be instantiated and used by this class""" 379 Name = "Base translation store" 380 """The human usable name of this store type""" 381 Mimetypes = None 382 """A list of MIME types associated with this store type""" 383 Extensions = None 384 """A list of file extentions associated with this store type""" 385 _binary = False 386 """Indicates whether a file should be accessed as a binary file.""" 387 suggestions_in_format = False 388 """Indicates if format can store suggestions and alternative translation for a unit""" 389
390 - def __init__(self, unitclass=None):
391 """Constructs a blank TranslationStore.""" 392 self.units = [] 393 self.sourcelanguage = None 394 self.targetlanguage = None 395 if unitclass: 396 self.UnitClass = unitclass 397 super(TranslationStore, self).__init__()
398
399 - def getsourcelanguage(self):
400 """Gets the target language for this store""" 401 return self.sourcelanguage
402
403 - def setsourcelanguage(self, sourcelanguage):
404 """Sets the source language for this store""" 405 self.sourcelanguage = sourcelanguage
406
407 - def gettargetlanguage(self):
408 """Gets the target language for this store""" 409 return self.targetlanguage
410
411 - def settargetlanguage(self, targetlanguage):
412 """Sets the target language for this store""" 413 self.targetlanguage = targetlanguage
414
415 - def unit_iter(self):
416 """Iterator over all the units in this store.""" 417 for unit in self.units: 418 yield unit
419
420 - def getunits(self):
421 """Return a list of all units in this store.""" 422 return [unit for unit in self.unit_iter()]
423
424 - def addunit(self, unit):
425 """Appends the given unit to the object's list of units. 426 427 This method should always be used rather than trying to modify the 428 list manually. 429 430 @type unit: L{TranslationUnit} 431 @param unit: The unit that will be added. 432 """ 433 unit._store = self 434 self.units.append(unit)
435
436 - def addsourceunit(self, source):
437 """Adds and returns a new unit with the given source string. 438 439 @rtype: L{TranslationUnit} 440 """ 441 unit = self.UnitClass(source) 442 self.addunit(unit) 443 return unit
444
445 - def findunit(self, source):
446 """Finds the unit with the given source string. 447 448 @rtype: L{TranslationUnit} or None 449 """ 450 if len(getattr(self, "sourceindex", [])): 451 if source in self.sourceindex: 452 return self.sourceindex[source][0] 453 else: 454 for unit in self.units: 455 if unit.source == source: 456 return unit 457 return None
458 459
460 - def findunits(self, source):
461 """Finds the unit with the given source string. 462 463 @rtype: L{TranslationUnit} or None 464 """ 465 if len(getattr(self, "sourceindex", [])): 466 if source in self.sourceindex: 467 return self.sourceindex[source] 468 else: 469 #FIXME: maybe we should generate index here instead since 470 #we'll scan all units anyway 471 result = [] 472 for unit in self.units: 473 if unit.source == source: 474 result.append(unit) 475 return result 476 return None
477
478 - def translate(self, source):
479 """Returns the translated string for a given source string. 480 481 @rtype: String or None 482 """ 483 unit = self.findunit(source) 484 if unit and unit.target: 485 return unit.target 486 else: 487 return None
488
489 - def remove_unit_from_index(self, unit):
490 """Remove a unit from source and locaton indexes""" 491 def remove_unit(source): 492 if source in self.sourceindex: 493 try: 494 self.sourceindex[source].remove(unit) 495 if len(self.sourceindex[source]) == 0: 496 del(self.sourceindex[source]) 497 except ValueError: 498 pass
499 500 if unit.hasplural(): 501 for source in unit.source.strings: 502 remove_unit(source) 503 else: 504 remove_unit(unit.source) 505 506 for location in unit.getlocations(): 507 if location in self.locationindex and self.locationindex[location] is not None \ 508 and self.locationindex[location] == unit: 509 del(self.locationindex[location])
510 511
512 - def add_unit_to_index(self, unit):
513 """Add a unit to source and location idexes""" 514 def insert_unit(source): 515 if not source in self.sourceindex: 516 self.sourceindex[source] = [unit] 517 else: 518 self.sourceindex[source].append(unit)
519 520 if unit.hasplural(): 521 for source in unit.source.strings: 522 insert_unit(source) 523 else: 524 insert_unit(unit.source) 525 526 for location in unit.getlocations(): 527 if location in self.locationindex: 528 # if sources aren't unique, don't use them 529 #FIXME: maybe better store a list of units like sourceindex 530 self.locationindex[location] = None 531 else: 532 self.locationindex[location] = unit 533
534 - def makeindex(self):
535 """Indexes the items in this store. At least .sourceindex should be usefull.""" 536 self.locationindex = {} 537 self.sourceindex = {} 538 for unit in self.units: 539 # Do we need to test if unit.source exists? 540 self.add_unit_to_index(unit)
541
542 - def require_index(self):
543 """make sure source index exists""" 544 if not hasattr(self, "sourceindex"): 545 self.makeindex()
546 547
548 - def __getstate__(self):
549 odict = self.__dict__.copy() 550 odict['fileobj'] = None 551 return odict
552
553 - def __setstate__(self, dict):
554 self.__dict__.update(dict) 555 if getattr(self, "filename", False): 556 self.fileobj = open(self.filename)
557
558 - def __str__(self):
559 """Converts to a string representation that can be parsed back using L{parsestring()}.""" 560 # We can't pickle fileobj if it is there, so let's hide it for a while. 561 fileobj = getattr(self, "fileobj", None) 562 self.fileobj = None 563 dump = pickle.dumps(self) 564 self.fileobj = fileobj 565 return dump
566
567 - def isempty(self):
568 """Returns True if the object doesn't contain any translation units.""" 569 if len(self.units) == 0: 570 return True 571 for unit in self.units: 572 if unit.istranslatable(): 573 return False 574 return True
575
576 - def _assignname(self):
577 """Tries to work out what the name of the filesystem file is and 578 assigns it to .filename.""" 579 fileobj = getattr(self, "fileobj", None) 580 if fileobj: 581 filename = getattr(fileobj, "name", getattr(fileobj, "filename", None)) 582 if filename: 583 self.filename = filename
584
585 - def parsestring(cls, storestring):
586 """Converts the string representation back to an object.""" 587 newstore = cls() 588 if storestring: 589 newstore.parse(storestring) 590 return newstore
591 parsestring = classmethod(parsestring) 592
593 - def parse(self, data):
594 """parser to process the given source string""" 595 self.units = pickle.loads(data).units
596
597 - def savefile(self, storefile):
598 """Writes the string representation to the given file (or filename).""" 599 if isinstance(storefile, basestring): 600 mode = 'w' 601 if self._binary: 602 mode = 'wb' 603 storefile = open(storefile, mode) 604 self.fileobj = storefile 605 self._assignname() 606 storestring = str(self) 607 storefile.write(storestring) 608 storefile.close()
609
610 - def save(self):
611 """Save to the file that data was originally read from, if available.""" 612 fileobj = getattr(self, "fileobj", None) 613 mode = 'w' 614 if self._binary: 615 mode = 'wb' 616 if not fileobj: 617 filename = getattr(self, "filename", None) 618 if filename: 619 fileobj = file(filename, mode) 620 else: 621 fileobj.close() 622 filename = getattr(fileobj, "name", getattr(fileobj, "filename", None)) 623 if not filename: 624 raise ValueError("No file or filename to save to") 625 fileobj = fileobj.__class__(filename, mode) 626 self.savefile(fileobj)
627
628 - def parsefile(cls, storefile):
629 """Reads the given file (or opens the given filename) and parses back to an object.""" 630 mode = 'r' 631 if cls._binary: 632 mode = 'rb' 633 if isinstance(storefile, basestring): 634 storefile = open(storefile, mode) 635 mode = getattr(storefile, "mode", mode) 636 #For some reason GzipFile returns 1, so we have to test for that here 637 if mode == 1 or "r" in mode: 638 storestring = storefile.read() 639 storefile.close() 640 else: 641 storestring = "" 642 newstore = cls.parsestring(storestring) 643 newstore.fileobj = storefile 644 newstore._assignname() 645 return newstore
646 parsefile = classmethod(parsefile) 647