1 "table definitions"
2 import os
3 import sys
4 import csv
5 import codecs
6 import locale
7 import unicodedata
8 import weakref
9 from array import array
10 from bisect import bisect_left, bisect_right
11 from decimal import Decimal
12 from shutil import copyfileobj
13 import dbf
14 from dbf import _io as io
15 from dbf.dates import Date, DateTime, Time
16 from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode, DoNotIndex
17
18 input_decoding = locale.getdefaultlocale()[1]
19 default_codepage = 'cp1252'
20 return_ascii = False
21 temp_dir = os.environ.get("DBF_TEMP") or os.environ.get("TEMP") or ""
22
23 version_map = {
24 '\x02' : 'FoxBASE',
25 '\x03' : 'dBase III Plus',
26 '\x04' : 'dBase IV',
27 '\x05' : 'dBase V',
28 '\x30' : 'Visual FoxPro',
29 '\x31' : 'Visual FoxPro (auto increment field)',
30 '\x43' : 'dBase IV SQL',
31 '\x7b' : 'dBase IV w/memos',
32 '\x83' : 'dBase III Plus w/memos',
33 '\x8b' : 'dBase IV w/memos',
34 '\x8e' : 'dBase IV w/SQL table',
35 '\xf5' : 'FoxPro w/memos'}
36
37 code_pages = {
38 '\x00' : ('ascii', "plain ol' ascii"),
39 '\x01' : ('cp437', 'U.S. MS-DOS'),
40 '\x02' : ('cp850', 'International MS-DOS'),
41 '\x03' : ('cp1252', 'Windows ANSI'),
42 '\x04' : ('mac_roman', 'Standard Macintosh'),
43 '\x08' : ('cp865', 'Danish OEM'),
44 '\x09' : ('cp437', 'Dutch OEM'),
45 '\x0A' : ('cp850', 'Dutch OEM (secondary)'),
46 '\x0B' : ('cp437', 'Finnish OEM'),
47 '\x0D' : ('cp437', 'French OEM'),
48 '\x0E' : ('cp850', 'French OEM (secondary)'),
49 '\x0F' : ('cp437', 'German OEM'),
50 '\x10' : ('cp850', 'German OEM (secondary)'),
51 '\x11' : ('cp437', 'Italian OEM'),
52 '\x12' : ('cp850', 'Italian OEM (secondary)'),
53 '\x13' : ('cp932', 'Japanese Shift-JIS'),
54 '\x14' : ('cp850', 'Spanish OEM (secondary)'),
55 '\x15' : ('cp437', 'Swedish OEM'),
56 '\x16' : ('cp850', 'Swedish OEM (secondary)'),
57 '\x17' : ('cp865', 'Norwegian OEM'),
58 '\x18' : ('cp437', 'Spanish OEM'),
59 '\x19' : ('cp437', 'English OEM (Britain)'),
60 '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'),
61 '\x1B' : ('cp437', 'English OEM (U.S.)'),
62 '\x1C' : ('cp863', 'French OEM (Canada)'),
63 '\x1D' : ('cp850', 'French OEM (secondary)'),
64 '\x1F' : ('cp852', 'Czech OEM'),
65 '\x22' : ('cp852', 'Hungarian OEM'),
66 '\x23' : ('cp852', 'Polish OEM'),
67 '\x24' : ('cp860', 'Portugese OEM'),
68 '\x25' : ('cp850', 'Potugese OEM (secondary)'),
69 '\x26' : ('cp866', 'Russian OEM'),
70 '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'),
71 '\x40' : ('cp852', 'Romanian OEM'),
72 '\x4D' : ('cp936', 'Chinese GBK (PRC)'),
73 '\x4E' : ('cp949', 'Korean (ANSI/OEM)'),
74 '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'),
75 '\x50' : ('cp874', 'Thai (ANSI/OEM)'),
76 '\x57' : ('cp1252', 'ANSI'),
77 '\x58' : ('cp1252', 'Western European ANSI'),
78 '\x59' : ('cp1252', 'Spanish ANSI'),
79 '\x64' : ('cp852', 'Eastern European MS-DOS'),
80 '\x65' : ('cp866', 'Russian MS-DOS'),
81 '\x66' : ('cp865', 'Nordic MS-DOS'),
82 '\x67' : ('cp861', 'Icelandic MS-DOS'),
83 '\x68' : (None, 'Kamenicky (Czech) MS-DOS'),
84 '\x69' : (None, 'Mazovia (Polish) MS-DOS'),
85 '\x6a' : ('cp737', 'Greek MS-DOS (437G)'),
86 '\x6b' : ('cp857', 'Turkish MS-DOS'),
87 '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'),
88 '\x79' : ('cp949', 'Korean Windows'),
89 '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'),
90 '\x7b' : ('cp932', 'Japanese Windows'),
91 '\x7c' : ('cp874', 'Thai Windows'),
92 '\x7d' : ('cp1255', 'Hebrew Windows'),
93 '\x7e' : ('cp1256', 'Arabic Windows'),
94 '\xc8' : ('cp1250', 'Eastern European Windows'),
95 '\xc9' : ('cp1251', 'Russian Windows'),
96 '\xca' : ('cp1254', 'Turkish Windows'),
97 '\xcb' : ('cp1253', 'Greek Windows'),
98 '\x96' : ('mac_cyrillic', 'Russian Macintosh'),
99 '\x97' : ('mac_latin2', 'Macintosh EE'),
100 '\x98' : ('mac_greek', 'Greek Macintosh') }
101
102 if sys.version_info[:2] < (2, 6):
105 "Emulate PyProperty_Type() in Objects/descrobject.c"
106
107 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
108 self.fget = fget
109 self.fset = fset
110 self.fdel = fdel
111 self.__doc__ = doc or fget.__doc__
113 self.fget = func
114 if not self.__doc__:
115 self.__doc__ = fget.__doc__
116 - def __get__(self, obj, objtype=None):
117 if obj is None:
118 return self
119 if self.fget is None:
120 raise AttributeError, "unreadable attribute"
121 return self.fget(obj)
123 if self.fset is None:
124 raise AttributeError, "can't set attribute"
125 self.fset(obj, value)
127 if self.fdel is None:
128 raise AttributeError, "can't delete attribute"
129 self.fdel(obj)
131 self.fset = func
132 return self
134 self.fdel = func
135 return self
136
138 """Provides routines to extract and save data within the fields of a dbf record."""
139 __slots__ = ['_recnum', '_layout', '_data', '_dirty', '__weakref__']
141 """calls appropriate routine to fetch value stored in field from array
142 @param record_data: the data portion of the record
143 @type record_data: array of characters
144 @param fielddef: description of the field definition
145 @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags'
146 @returns: python data stored in field"""
147
148 field_type = fielddef['type']
149 classtype = yo._layout.fieldtypes[field_type]['Class']
150 retrieve = yo._layout.fieldtypes[field_type]['Retrieve']
151 if classtype is not None:
152 datum = retrieve(record_data, fielddef, yo._layout.memo, classtype)
153 else:
154 datum = retrieve(record_data, fielddef, yo._layout.memo)
155 if field_type in yo._layout.character_fields:
156 datum = yo._layout.decoder(datum)[0]
157 if yo._layout.return_ascii:
158 try:
159 datum = yo._layout.output_encoder(datum)[0]
160 except UnicodeEncodeError:
161 datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore')
162 return datum
164 "calls appropriate routine to convert value to ascii bytes, and save it in record"
165 field_type = fielddef['type']
166 update = yo._layout.fieldtypes[field_type]['Update']
167 if field_type in yo._layout.character_fields:
168 if not isinstance(value, unicode):
169 if yo._layout.input_decoder is None:
170 raise NonUnicode("String not in unicode format, no default encoding specified")
171 value = yo._layout.input_decoder(value)[0]
172 value = yo._layout.encoder(value)[0]
173 bytes = array('c', update(value, fielddef, yo._layout.memo))
174 size = fielddef['length']
175 if len(bytes) > size:
176 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size))
177 blank = array('c', ' ' * size)
178 start = fielddef['start']
179 end = start + size
180 blank[:len(bytes)] = bytes[:]
181 yo._data[start:end] = blank[:]
182 yo._dirty = True
197 results = []
198 if not specs:
199 specs = yo._layout.index
200 specs = _normalize_tuples(tuples=specs, length=2, filler=[_nop])
201 for field, func in specs:
202 results.append(func(yo[field]))
203 return tuple(results)
204
210 if name[0:2] == '__' and name[-2:] == '__':
211 raise AttributeError, 'Method %s is not implemented.' % name
212 elif name == 'record_number':
213 return yo._recnum
214 elif name == 'delete_flag':
215 return yo._data[0] != ' '
216 elif not name in yo._layout.fields:
217 raise FieldMissing(name)
218 try:
219 fielddef = yo._layout[name]
220 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef)
221 return value
222 except DbfError, error:
223 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message)
224 raise
241 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
280 if type(name) == str:
281 yo.__setattr__(name, value)
282 elif type(name) in (int, long):
283 yo.__setattr__(yo._layout.fields[name], value)
284 elif type(name) == slice:
285 sequence = []
286 for field in yo._layout.fields[name]:
287 sequence.append(field)
288 if len(sequence) != len(value):
289 raise DbfError("length of slices not equal")
290 for field, val in zip(sequence, value):
291 yo[field] = val
292 else:
293 raise TypeError("%s is not a field name" % name)
295 result = []
296 for seq, field in enumerate(yo.field_names):
297 result.append("%3d - %-10s: %s" % (seq, field, yo[field]))
298 return '\n'.join(result)
300 return yo._data.tostring()
302 "creates a blank record data chunk"
303 layout = yo._layout
304 ondisk = layout.ondisk
305 layout.ondisk = False
306 yo._data = array('c', ' ' * layout.header.record_length)
307 layout.memofields = []
308 for field in layout.fields:
309 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']())
310 if layout[field]['type'] in layout.memotypes:
311 layout.memofields.append(field)
312 layout.blankrecord = yo._data[:]
313 layout.ondisk = ondisk
315 "marks record as deleted"
316 yo._data[0] = '*'
317 yo._dirty = True
318 return yo
319 @property
324 "saves a dictionary into a record's fields\nkeys with no matching field will raise a FieldMissing exception unless drop_missing = True"
325 old_data = yo._data[:]
326 try:
327 for key in dictionary:
328 if not key in yo.field_names:
329 if drop:
330 continue
331 raise FieldMissing(key)
332 yo.__setattr__(key, dictionary[key])
333 except:
334 yo._data[:] = old_data
335 raise
336 return yo
337 @property
339 "marked for deletion?"
340 return yo._data[0] == '*'
349 @property
351 "physical record number"
352 return yo._recnum
353 @property
355 table = yo._layout.table()
356 if table is None:
357 raise DbfError("table is no longer available")
358 return table
360 for dbfindex in yo._layout.table()._indexen:
361 dbfindex(yo)
363 "blanks record"
364 if keep_fields is None:
365 keep_fields = []
366 keep = {}
367 for field in keep_fields:
368 keep[field] = yo[field]
369 if yo._layout.blankrecord == None:
370 yo._createBlankRecord()
371 yo._data[:] = yo._layout.blankrecord[:]
372 for field in keep_fields:
373 yo[field] = keep[field]
374 yo._dirty = True
375 return yo
377 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty."
378 keys = yo._layout.fields
379 if blank:
380 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys]
381 else:
382 values = [yo[field] for field in keys]
383 return dict(zip(keys, values))
385 "marks record as active"
386 yo._data[0] = ' '
387 yo._dirty = True
388 return yo
398 """Provides access to memo fields as dictionaries
399 must override _init, _get_memo, and _put_memo to
400 store memo contents to disk"""
402 "initialize disk file usage"
404 "retrieve memo contents from disk"
406 "store memo contents to disk"
408 ""
409 yo.meta = meta
410 yo.memory = {}
411 yo.nextmemo = 1
412 yo._init()
413 yo.meta.newmemofile = False
415 "gets the memo in block"
416 if yo.meta.ignorememos or not block:
417 return ''
418 if yo.meta.ondisk:
419 return yo._get_memo(block)
420 else:
421 return yo.memory[block]
423 "stores data in memo file, returns block number"
424 if yo.meta.ignorememos or data == '':
425 return 0
426 if yo.meta.inmemory:
427 thismemo = yo.nextmemo
428 yo.nextmemo += 1
429 yo.memory[thismemo] = data
430 else:
431 thismemo = yo._put_memo(data)
432 return thismemo
435 "dBase III specific"
436 yo.meta.memo_size= 512
437 yo.record_header_length = 2
438 if yo.meta.ondisk and not yo.meta.ignorememos:
439 if yo.meta.newmemofile:
440 yo.meta.mfd = open(yo.meta.memoname, 'w+b')
441 yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508)
442 else:
443 try:
444 yo.meta.mfd = open(yo.meta.memoname, 'r+b')
445 yo.meta.mfd.seek(0)
446 yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4))
447 except:
448 raise DbfError("memo file appears to be corrupt")
450 block = int(block)
451 yo.meta.mfd.seek(block * yo.meta.memo_size)
452 eom = -1
453 data = ''
454 while eom == -1:
455 newdata = yo.meta.mfd.read(yo.meta.memo_size)
456 if not newdata:
457 return data
458 data += newdata
459 eom = data.find('\x1a\x1a')
460 return data[:eom].rstrip()
462 data = data.rstrip()
463 length = len(data) + yo.record_header_length
464 blocks = length // yo.meta.memo_size
465 if length % yo.meta.memo_size:
466 blocks += 1
467 thismemo = yo.nextmemo
468 yo.nextmemo = thismemo + blocks
469 yo.meta.mfd.seek(0)
470 yo.meta.mfd.write(io.packLongInt(yo.nextmemo))
471 yo.meta.mfd.seek(thismemo * yo.meta.memo_size)
472 yo.meta.mfd.write(data)
473 yo.meta.mfd.write('\x1a\x1a')
474 double_check = yo._get_memo(thismemo)
475 if len(double_check) != len(data):
476 uhoh = open('dbf_memo_dump.err','wb')
477 uhoh.write('thismemo: %d' % thismemo)
478 uhoh.write('nextmemo: %d' % yo.nextmemo)
479 uhoh.write('saved: %d bytes' % len(data))
480 uhoh.write(data)
481 uhoh.write('retrieved: %d bytes' % len(double_check))
482 uhoh.write(double_check)
483 uhoh.close()
484 raise DbfError("unknown error: memo not saved")
485 return thismemo
488 "Visual Foxpro 6 specific"
489 if yo.meta.ondisk and not yo.meta.ignorememos:
490 yo.record_header_length = 8
491 if yo.meta.newmemofile:
492 if yo.meta.memo_size == 0:
493 yo.meta.memo_size = 1
494 elif 1 < yo.meta.memo_size < 33:
495 yo.meta.memo_size *= 512
496 yo.meta.mfd = open(yo.meta.memoname, 'w+b')
497 nextmemo = 512 // yo.meta.memo_size
498 if nextmemo * yo.meta.memo_size < 512:
499 nextmemo += 1
500 yo.nextmemo = nextmemo
501 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \
502 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504)
503 else:
504 try:
505 yo.meta.mfd = open(yo.meta.memoname, 'r+b')
506 yo.meta.mfd.seek(0)
507 header = yo.meta.mfd.read(512)
508 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True)
509 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True)
510 except:
511 raise DbfError("memo file appears to be corrupt")
513 yo.meta.mfd.seek(block * yo.meta.memo_size)
514 header = yo.meta.mfd.read(8)
515 length = io.unpackLongInt(header[4:], bigendian=True)
516 return yo.meta.mfd.read(length)
518 data = data.rstrip()
519 yo.meta.mfd.seek(0)
520 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True)
521 yo.meta.mfd.seek(0)
522 length = len(data) + yo.record_header_length
523 blocks = length // yo.meta.memo_size
524 if length % yo.meta.memo_size:
525 blocks += 1
526 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True))
527 yo.meta.mfd.seek(thismemo*yo.meta.memo_size)
528 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data)
529 return thismemo
530
532 """Provides a framework for dbf style tables."""
533 _version = 'basic memory table'
534 _versionabbv = 'dbf'
535 _fieldtypes = {
536 'D' : { 'Type':'Date', 'Init':io.addDate, 'Blank':Date.today, 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Class':None},
537 'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool, 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Class':None},
538 'M' : { 'Type':'Memo', 'Init':io.addMemo, 'Blank':str, 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Class':None} }
539 _memoext = ''
540 _memotypes = tuple('M', )
541 _memoClass = _DbfMemo
542 _yesMemoMask = ''
543 _noMemoMask = ''
544 _fixed_fields = ('M','D','L')
545 _variable_fields = tuple()
546 _character_fields = tuple('M', )
547 _decimal_fields = tuple()
548 _numeric_fields = tuple()
549 _currency_fields = tuple()
550 _dbfTableHeader = array('c', '\x00' * 32)
551 _dbfTableHeader[0] = '\x00'
552 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
553 _dbfTableHeader[10] = '\x01'
554 _dbfTableHeader[29] = '\x00'
555 _dbfTableHeader = _dbfTableHeader.tostring()
556 _dbfTableHeaderExtra = ''
557 _supported_tables = []
558 _read_only = False
559 _meta_only = False
560 _use_deleted = True
561 backup = False
563 "implements the weakref structure for DbfLists"
567 yo._lists = set([s for s in yo._lists if s() is not None])
568 return (s() for s in yo._lists if s() is not None)
570 yo._lists = set([s for s in yo._lists if s() is not None])
571 return len(yo._lists)
572 - def add(yo, new_list):
573 yo._lists.add(weakref.ref(new_list))
574 yo._lists = set([s for s in yo._lists if s() is not None])
576 "implements the weakref structure for seperate indexes"
580 yo._indexen = set([s for s in yo._indexen if s() is not None])
581 return (s() for s in yo._indexen if s() is not None)
583 yo._indexen = set([s for s in yo._indexen if s() is not None])
584 return len(yo._indexen)
585 - def add(yo, new_list):
586 yo._indexen.add(weakref.ref(new_list))
587 yo._indexen = set([s for s in yo._indexen if s() is not None])
602 if len(data) != 32:
603 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data))
604 yo._data = array('c', data + '\x0d')
606 "get/set code page of table"
607 if cp is None:
608 return yo._data[29]
609 else:
610 cp, sd, ld = _codepage_lookup(cp)
611 yo._data[29] = cp
612 return cp
613 @property
619 @data.setter
621 if len(bytes) < 32:
622 raise DbfError("length for data of %d is less than 32" % len(bytes))
623 yo._data[:] = array('c', bytes)
624 @property
626 "extra dbf info (located after headers, before data records)"
627 fieldblock = yo._data[32:]
628 for i in range(len(fieldblock)//32+1):
629 cr = i * 32
630 if fieldblock[cr] == '\x0d':
631 break
632 else:
633 raise DbfError("corrupt field structure")
634 cr += 33
635 return yo._data[cr:].tostring()
636 @extra.setter
638 fieldblock = yo._data[32:]
639 for i in range(len(fieldblock)//32+1):
640 cr = i * 32
641 if fieldblock[cr] == '\x0d':
642 break
643 else:
644 raise DbfError("corrupt field structure")
645 cr += 33
646 yo._data[cr:] = array('c', data)
647 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
648 @property
650 "number of fields (read-only)"
651 fieldblock = yo._data[32:]
652 for i in range(len(fieldblock)//32+1):
653 cr = i * 32
654 if fieldblock[cr] == '\x0d':
655 break
656 else:
657 raise DbfError("corrupt field structure")
658 return len(fieldblock[:cr]) // 32
659 @property
661 "field block structure"
662 fieldblock = yo._data[32:]
663 for i in range(len(fieldblock)//32+1):
664 cr = i * 32
665 if fieldblock[cr] == '\x0d':
666 break
667 else:
668 raise DbfError("corrupt field structure")
669 return fieldblock[:cr].tostring()
670 @fields.setter
672 fieldblock = yo._data[32:]
673 for i in range(len(fieldblock)//32+1):
674 cr = i * 32
675 if fieldblock[cr] == '\x0d':
676 break
677 else:
678 raise DbfError("corrupt field structure")
679 cr += 32
680 fieldlen = len(block)
681 if fieldlen % 32 != 0:
682 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen)
683 yo._data[32:cr] = array('c', block)
684 yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))
685 fieldlen = fieldlen // 32
686 recordlen = 1
687 for i in range(fieldlen):
688 recordlen += ord(block[i*32+16])
689 yo._data[10:12] = array('c', io.packShortInt(recordlen))
690 @property
692 "number of records (maximum 16,777,215)"
693 return io.unpackLongInt(yo._data[4:8].tostring())
694 @record_count.setter
697 @property
699 "length of a record (read_only) (max of 65,535)"
700 return io.unpackShortInt(yo._data[10:12].tostring())
701 @property
703 "starting position of first record in file (must be within first 64K)"
704 return io.unpackShortInt(yo._data[8:10].tostring())
705 @start.setter
708 @property
710 "date of last table modification (read-only)"
711 return io.unpackDate(yo._data[1:4].tostring())
712 @property
714 "dbf version"
715 return yo._data[0]
716 @version.setter
720 "implements the weakref table for records"
722 yo._meta = meta
723 yo._weakref_list = [weakref.ref(lambda x: None)] * count
725 maybe = yo._weakref_list[index]()
726 if maybe is None:
727 if index < 0:
728 index += yo._meta.header.record_count
729 size = yo._meta.header.record_length
730 location = index * size + yo._meta.header.start
731 yo._meta.dfd.seek(location)
732 if yo._meta.dfd.tell() != location:
733 raise ValueError("unable to seek to offset %d in file" % location)
734 bytes = yo._meta.dfd.read(size)
735 if not bytes:
736 raise ValueError("unable to read record data from %s at location %d" % (yo._meta.filename, location))
737 maybe = _DbfRecord(recnum=index, layout=yo._meta, kamikaze=bytes, _fromdisk=True)
738 yo._weakref_list[index] = weakref.ref(maybe)
739 return maybe
741 yo._weakref_list.append(weakref.ref(record))
743 yo._weakref_list[:] = []
745 return yo._weakref_list.pop()
747 "returns records using current index"
749 yo._table = table
750 yo._index = -1
751 yo._more_records = True
755 while yo._more_records:
756 yo._index += 1
757 if yo._index >= len(yo._table):
758 yo._more_records = False
759 continue
760 record = yo._table[yo._index]
761 if not yo._table.use_deleted and record.has_been_deleted:
762 continue
763 return record
764 else:
765 raise StopIteration
767 "constructs fieldblock for disk table"
768 fieldblock = array('c', '')
769 memo = False
770 yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask))
771 for field in yo._meta.fields:
772 if yo._meta.fields.count(field) > 1:
773 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)")
774 fielddef = array('c', '\x00' * 32)
775 fielddef[:11] = array('c', io.packStr(field))
776 fielddef[11] = yo._meta[field]['type']
777 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start']))
778 fielddef[16] = chr(yo._meta[field]['length'])
779 fielddef[17] = chr(yo._meta[field]['decimals'])
780 fielddef[18] = chr(yo._meta[field]['flags'])
781 fieldblock.extend(fielddef)
782 if yo._meta[field]['type'] in yo._meta.memotypes:
783 memo = True
784 yo._meta.header.fields = fieldblock.tostring()
785 if memo:
786 yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask))
787 if yo._meta.memo is None:
788 yo._meta.memo = yo._memoClass(yo._meta)
790 "dBase III specific"
791 if yo._meta.header.version == '\x83':
792 try:
793 yo._meta.memo = yo._memoClass(yo._meta)
794 except:
795 yo._meta.dfd.close()
796 yo._meta.dfd = None
797 raise
798 if not yo._meta.ignorememos:
799 for field in yo._meta.fields:
800 if yo._meta[field]['type'] in yo._memotypes:
801 if yo._meta.header.version != '\x83':
802 yo._meta.dfd.close()
803 yo._meta.dfd = None
804 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
805 elif not os.path.exists(yo._meta.memoname):
806 yo._meta.dfd.close()
807 yo._meta.dfd = None
808 raise DbfError("Table structure corrupt: memo fields exist without memo file")
809 break
811 "builds the FieldList of names, types, and descriptions from the disk file"
812 yo._meta.fields[:] = []
813 offset = 1
814 fieldsdef = yo._meta.header.fields
815 if len(fieldsdef) % 32 != 0:
816 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
817 if len(fieldsdef) // 32 != yo.field_count:
818 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
819 for i in range(yo.field_count):
820 fieldblock = fieldsdef[i*32:(i+1)*32]
821 name = io.unpackStr(fieldblock[:11])
822 type = fieldblock[11]
823 if not type in yo._meta.fieldtypes:
824 raise DbfError("Unknown field type: %s" % type)
825 start = offset
826 length = ord(fieldblock[16])
827 offset += length
828 end = start + length
829 decimals = ord(fieldblock[17])
830 flags = ord(fieldblock[18])
831 if name in yo._meta.fields:
832 raise DbfError('Duplicate field name found: %s' % name)
833 yo._meta.fields.append(name)
834 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
836 "Returns field information Name Type(Length[,Decimals])"
837 name = yo._meta.fields[i]
838 type = yo._meta[name]['type']
839 length = yo._meta[name]['length']
840 decimals = yo._meta[name]['decimals']
841 if type in yo._decimal_fields:
842 description = "%s %s(%d,%d)" % (name, type, length, decimals)
843 elif type in yo._fixed_fields:
844 description = "%s %s" % (name, type)
845 else:
846 description = "%s %s(%d)" % (name, type, length)
847 return description
849 "loads the records from disk to memory"
850 if yo._meta_only:
851 raise DbfError("%s has been closed, records are unavailable" % yo.filename)
852 dfd = yo._meta.dfd
853 header = yo._meta.header
854 dfd.seek(header.start)
855 allrecords = dfd.read()
856 dfd.seek(0)
857 length = header.record_length
858 for i in range(header.record_count):
859 record_data = allrecords[length*i:length*i+length]
860 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True))
861 dfd.seek(0)
863 if specs is None:
864 specs = yo.field_names
865 elif isinstance(specs, str):
866 specs = specs.split(sep)
867 else:
868 specs = list(specs)
869 specs = [s.strip() for s in specs]
870 return specs
872 "synchronizes the disk file with current data"
873 if yo._meta.inmemory:
874 return
875 fd = yo._meta.dfd
876 fd.seek(0)
877 fd.write(yo._meta.header.data)
878 if not headeronly:
879 for record in yo._table:
880 record._update_disk()
881 fd.flush()
882 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length)
883 if 'db3' in yo._versionabbv:
884 fd.seek(0, os.SEEK_END)
885 fd.write('\x1a')
886 fd.flush()
887 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length + 1)
888
896 if name in ('_table'):
897 if yo._meta.ondisk:
898 yo._table = yo._Table(len(yo), yo._meta)
899 else:
900 yo._table = []
901 yo._loadtable()
902 return object.__getattribute__(yo, name)
904 if type(value) == int:
905 if not -yo._meta.header.record_count <= value < yo._meta.header.record_count:
906 raise IndexError("Record %d is not in table." % value)
907 return yo._table[value]
908 elif type(value) == slice:
909 sequence = List(desc='%s --> %s' % (yo.filename, value), field_names=yo.field_names)
910 yo._dbflists.add(sequence)
911 for index in range(len(yo))[value]:
912 record = yo._table[index]
913 if yo.use_deleted is True or not record.has_been_deleted:
914 sequence.append(record)
915 return sequence
916 else:
917 raise TypeError('type <%s> not valid for indexing' % type(value))
918 - def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False,
919 read_only=False, keep_memos=False, meta_only=False, codepage=None,
920 numbers='default', strings=str, currency=Decimal):
921 """open/create dbf file
922 filename should include path if needed
923 field_specs can be either a ;-delimited string or a list of strings
924 memo_size is always 512 for db3 memos
925 ignore_memos is useful if the memo file is missing or corrupt
926 read_only will load records into memory, then close the disk file
927 keep_memos will also load any memo fields into memory
928 meta_only will ignore all records, keeping only basic table information
929 codepage will override whatever is set in the table itself"""
930 if filename[0] == filename[-1] == ':':
931 if field_specs is None:
932 raise DbfError("field list must be specified for memory tables")
933 elif type(yo) is DbfTable:
934 raise DbfError("only memory tables supported")
935 yo._dbflists = yo._DbfLists()
936 yo._indexen = yo._Indexen()
937 yo._meta = meta = yo._MetaData()
938 for datatypes, classtype in (
939 (yo._character_fields, strings),
940 (yo._numeric_fields, numbers),
941 (yo._currency_fields, currency),
942 ):
943 for datatype in datatypes:
944 yo._fieldtypes[datatype]['Class'] = classtype
945 meta.numbers = numbers
946 meta.strings = strings
947 meta.currency = currency
948 meta.table = weakref.ref(yo)
949 meta.filename = filename
950 meta.fields = []
951 meta.fieldtypes = yo._fieldtypes
952 meta.fixed_fields = yo._fixed_fields
953 meta.variable_fields = yo._variable_fields
954 meta.character_fields = yo._character_fields
955 meta.decimal_fields = yo._decimal_fields
956 meta.numeric_fields = yo._numeric_fields
957 meta.memotypes = yo._memotypes
958 meta.ignorememos = ignore_memos
959 meta.memo_size = memo_size
960 meta.input_decoder = codecs.getdecoder(input_decoding)
961 meta.output_encoder = codecs.getencoder(input_decoding)
962 meta.return_ascii = return_ascii
963 meta.header = header = yo._TableHeader(yo._dbfTableHeader)
964 header.extra = yo._dbfTableHeaderExtra
965 header.data
966 if filename[0] == filename[-1] == ':':
967 yo._table = []
968 meta.ondisk = False
969 meta.inmemory = True
970 meta.memoname = filename
971 else:
972 base, ext = os.path.splitext(filename)
973 if ext == '':
974 meta.filename = base + '.dbf'
975 meta.memoname = base + yo._memoext
976 meta.ondisk = True
977 meta.inmemory = False
978 if field_specs:
979 if meta.ondisk:
980 meta.dfd = open(meta.filename, 'w+b')
981 meta.newmemofile = True
982 yo.add_fields(field_specs)
983 header.codepage(codepage or default_codepage)
984 cp, sd, ld = _codepage_lookup(meta.header.codepage())
985 meta.decoder = codecs.getdecoder(sd)
986 meta.encoder = codecs.getencoder(sd)
987 return
988 try:
989 dfd = meta.dfd = open(meta.filename, 'r+b')
990 except IOError, e:
991 raise DbfError(str(e))
992 dfd.seek(0)
993 meta.header = header = yo._TableHeader(dfd.read(32))
994 if not header.version in yo._supported_tables:
995 dfd.close()
996 dfd = None
997 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version)))
998 cp, sd, ld = _codepage_lookup(meta.header.codepage())
999 yo._meta.decoder = codecs.getdecoder(sd)
1000 yo._meta.encoder = codecs.getencoder(sd)
1001 fieldblock = dfd.read(header.start - 32)
1002 for i in range(len(fieldblock)//32+1):
1003 fieldend = i * 32
1004 if fieldblock[fieldend] == '\x0d':
1005 break
1006 else:
1007 raise DbfError("corrupt field structure in header")
1008 if len(fieldblock[:fieldend]) % 32 != 0:
1009 raise DbfError("corrupt field structure in header")
1010 header.fields = fieldblock[:fieldend]
1011 header.extra = fieldblock[fieldend+1:]
1012 yo._initializeFields()
1013 yo._checkMemoIntegrity()
1014 meta.current = -1
1015 if len(yo) > 0:
1016 meta.current = 0
1017 dfd.seek(0)
1018 if meta_only:
1019 yo.close(keep_table=False, keep_memos=False)
1020 elif read_only:
1021 yo.close(keep_table=True, keep_memos=keep_memos)
1022 if codepage is not None:
1023 cp, sd, ld = _codepage_lookup(codepage)
1024 yo._meta.decoder = codecs.getdecoder(sd)
1025 yo._meta.encoder = codecs.getencoder(sd)
1026
1034 if yo._read_only:
1035 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename
1036 elif yo._meta_only:
1037 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename
1038 else:
1039 return __name__ + ".Table('%s')" % yo._meta.filename
1041 if yo._read_only:
1042 status = "read-only"
1043 elif yo._meta_only:
1044 status = "meta-only"
1045 else:
1046 status = "read/write"
1047 str = """
1048 Table: %s
1049 Type: %s
1050 Codepage: %s
1051 Status: %s
1052 Last updated: %s
1053 Record count: %d
1054 Field count: %d
1055 Record length: %d """ % (yo.filename, version_map.get(yo._meta.header.version,
1056 'unknown - ' + hex(ord(yo._meta.header.version))), yo.codepage, status,
1057 yo.last_update, len(yo), yo.field_count, yo.record_length)
1058 str += "\n --Fields--\n"
1059 for i in range(len(yo._meta.fields)):
1060 str += "%11d) %s\n" % (i, yo._fieldLayout(i))
1061 return str
1062 @property
1064 return "%s (%s)" % code_pages[yo._meta.header.codepage()]
1065 @codepage.setter
1066 - def codepage(yo, cp):
1067 cp = code_pages[yo._meta.header.codepage(cp)][0]
1068 yo._meta.decoder = codecs.getdecoder(cp)
1069 yo._meta.encoder = codecs.getencoder(cp)
1070 yo._update_disk(headeronly=True)
1071 @property
1073 "the number of fields in the table"
1074 return yo._meta.header.field_count
1075 @property
1077 "a list of the fields in the table"
1078 return yo._meta.fields[:]
1079 @property
1081 "table's file name, including path (if specified on open)"
1082 return yo._meta.filename
1083 @property
1085 "date of last update"
1086 return yo._meta.header.update
1087 @property
1089 "table's memo name (if path included in filename on open)"
1090 return yo._meta.memoname
1091 @property
1093 "number of bytes in a record"
1094 return yo._meta.header.record_length
1095 @property
1097 "index number of the current record"
1098 return yo._meta.current
1099 @property
1103 @property
1105 "process or ignore deleted records"
1106 return yo._use_deleted
1107 @use_deleted.setter
1110 @property
1112 "returns the dbf type of the table"
1113 return yo._version
1115 """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]]
1116 backup table is created with _backup appended to name
1117 then modifies current structure"""
1118 all_records = [record for record in yo]
1119 if yo:
1120 yo.create_backup()
1121 yo._meta.blankrecord = None
1122 meta = yo._meta
1123 offset = meta.header.record_length
1124 fields = yo._list_fields(field_specs, sep=';')
1125 for field in fields:
1126 try:
1127 name, format = field.split()
1128 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum():
1129 raise DbfError("%s invalid: field names must start with a letter, and can only contain letters, digits, and _" % name)
1130 name = name.lower()
1131 if name in meta.fields:
1132 raise DbfError("Field '%s' already exists" % name)
1133 field_type = format[0].upper()
1134 if len(name) > 10:
1135 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name)))
1136 if not field_type in meta.fieldtypes.keys():
1137 raise DbfError("Unknown field type: %s" % field_type)
1138 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format)
1139 except ValueError:
1140 raise DbfError("invalid field specifier: %s (multiple fields should be separated with ';'" % field)
1141 start = offset
1142 end = offset + length
1143 offset = end
1144 meta.fields.append(name)
1145 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0}
1146 if meta[name]['type'] in yo._memotypes and meta.memo is None:
1147 meta.memo = yo._memoClass(meta)
1148 for record in yo:
1149 record[name] = meta.fieldtypes[field_type]['Blank']()
1150 yo._buildHeaderFields()
1151 yo._update_disk()
1152 - def append(yo, kamikaze='', drop=False, multiple=1):
1153 "adds <multiple> blank records, and fills fields with dict/tuple values if present"
1154 if not yo.field_count:
1155 raise DbfError("No fields defined, cannot append")
1156 empty_table = len(yo) == 0
1157 dictdata = False
1158 tupledata = False
1159 if not isinstance(kamikaze, _DbfRecord):
1160 if isinstance(kamikaze, dict):
1161 dictdata = kamikaze
1162 kamikaze = ''
1163 elif isinstance(kamikaze, tuple):
1164 tupledata = kamikaze
1165 kamikaze = ''
1166 newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze)
1167 yo._table.append(newrecord)
1168 yo._meta.header.record_count += 1
1169 try:
1170 if dictdata:
1171 newrecord.gather_fields(dictdata, drop=drop)
1172 elif tupledata:
1173 for index, item in enumerate(tupledata):
1174 newrecord[index] = item
1175 elif kamikaze == str:
1176 for field in yo._meta.memofields:
1177 newrecord[field] = ''
1178 elif kamikaze:
1179 for field in yo._meta.memofields:
1180 newrecord[field] = kamikaze[field]
1181 newrecord.write_record()
1182 except Exception:
1183 yo._table.pop()
1184 yo._meta.header.record_count = yo._meta.header.record_count - 1
1185 yo._update_disk()
1186 raise
1187 multiple -= 1
1188 if multiple:
1189 data = newrecord._data
1190 single = yo._meta.header.record_count
1191 total = single + multiple
1192 while single < total:
1193 multi_record = _DbfRecord(single, yo._meta, kamikaze=data)
1194 yo._table.append(multi_record)
1195 for field in yo._meta.memofields:
1196 multi_record[field] = newrecord[field]
1197 single += 1
1198 multi_record.write_record()
1199 yo._meta.header.record_count = total
1200 yo._meta.current = yo._meta.header.record_count - 1
1201 newrecord = multi_record
1202 yo._update_disk(headeronly=True)
1203 if empty_table:
1204 yo._meta.current = 0
1205 return newrecord
1206 - def bof(yo, _move=False):
1221 - def bottom(yo, get_record=False):
1222 """sets record pointer to bottom of table
1223 if get_record, seeks to and returns last (non-deleted) record
1224 DbfError if table is empty
1225 Bof if all records deleted and use_deleted is False"""
1226 yo._meta.current = yo._meta.header.record_count
1227 if get_record:
1228 try:
1229 return yo.prev()
1230 except Bof:
1231 yo._meta.current = yo._meta.header.record_count
1232 raise Eof()
1233 - def close(yo, keep_table=False, keep_memos=False):
1234 """closes disk files
1235 ensures table data is available if keep_table
1236 ensures memo data is available if keep_memos"""
1237 yo._meta.inmemory = True
1238 if keep_table:
1239 replacement_table = []
1240 for record in yo._table:
1241 replacement_table.append(record)
1242 yo._table = replacement_table
1243 else:
1244 if yo._meta.ondisk:
1245 yo._meta_only = True
1246 if yo._meta.mfd is not None:
1247 if not keep_memos:
1248 yo._meta.ignorememos = True
1249 else:
1250 memo_fields = []
1251 for field in yo.field_names:
1252 if yo.is_memotype(field):
1253 memo_fields.append(field)
1254 for record in yo:
1255 for field in memo_fields:
1256 record[field] = record[field]
1257 yo._meta.mfd.close()
1258 yo._meta.mfd = None
1259 if yo._meta.ondisk:
1260 yo._meta.dfd.close()
1261 yo._meta.dfd = None
1262 if keep_table:
1263 yo._read_only = True
1264 yo._meta.ondisk = False
1266 "creates a backup table -- ignored if memory table"
1267 if yo.filename[0] == yo.filename[-1] == ':':
1268 return
1269 if new_name is None:
1270 upper = yo.filename.isupper()
1271 name, ext = os.path.splitext(os.path.split(yo.filename)[1])
1272 extra = '_BACKUP' if upper else '_backup'
1273 new_name = os.path.join(temp_dir, name + extra + ext)
1274 else:
1275 overwrite = True
1276 if overwrite or not yo.backup:
1277 bkup = open(new_name, 'wb')
1278 try:
1279 yo._meta.dfd.seek(0)
1280 copyfileobj(yo._meta.dfd, bkup)
1281 yo.backup = new_name
1282 finally:
1283 bkup.close()
1287 "returns current logical record, or its index"
1288 if yo._meta.current < 0:
1289 raise Bof()
1290 elif yo._meta.current >= yo._meta.header.record_count:
1291 raise Eof()
1292 if index:
1293 return yo._meta.current
1294 return yo._table[yo._meta.current]
1296 """removes field(s) from the table
1297 creates backup files with _backup appended to the file name,
1298 then modifies current structure"""
1299 doomed = yo._list_fields(doomed)
1300 for victim in doomed:
1301 if victim not in yo._meta.fields:
1302 raise DbfError("field %s not in table -- delete aborted" % victim)
1303 all_records = [record for record in yo]
1304 yo.create_backup()
1305 for victim in doomed:
1306 yo._meta.fields.pop(yo._meta.fields.index(victim))
1307 start = yo._meta[victim]['start']
1308 end = yo._meta[victim]['end']
1309 for record in yo:
1310 record._data = record._data[:start] + record._data[end:]
1311 for field in yo._meta.fields:
1312 if yo._meta[field]['start'] == end:
1313 end = yo._meta[field]['end']
1314 yo._meta[field]['start'] = start
1315 yo._meta[field]['end'] = start + yo._meta[field]['length']
1316 start = yo._meta[field]['end']
1317 yo._buildHeaderFields()
1318 yo._update_disk()
1319 - def eof(yo, _move=False):
1334 - def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
1335 """writes the table using CSV or tab-delimited format, using the filename
1336 given if specified, otherwise the table name"""
1337 if filename is not None:
1338 path, filename = os.path.split(filename)
1339 else:
1340 path, filename = os.path.split(yo.filename)
1341 filename = os.path.join(path, filename)
1342 field_specs = yo._list_fields(field_specs)
1343 if records is None:
1344 records = yo
1345 format = format.lower()
1346 if format not in ('csv', 'tab', 'fixed'):
1347 raise DbfError("export format: csv, tab, or fixed -- not %s" % format)
1348 if format == 'fixed':
1349 format = 'txt'
1350 base, ext = os.path.splitext(filename)
1351 if ext.lower() in ('', '.dbf'):
1352 filename = base + "." + format[:3]
1353 fd = open(filename, 'w')
1354 try:
1355 if format == 'csv':
1356 csvfile = csv.writer(fd, dialect='dbf')
1357 if header:
1358 csvfile.writerow(field_specs)
1359 for record in records:
1360 fields = []
1361 for fieldname in field_specs:
1362 fields.append(record[fieldname])
1363 csvfile.writerow(fields)
1364 elif format == 'tab':
1365 if header:
1366 fd.write('\t'.join(field_specs) + '\n')
1367 for record in records:
1368 fields = []
1369 for fieldname in field_specs:
1370 fields.append(str(record[fieldname]))
1371 fd.write('\t'.join(fields) + '\n')
1372 else:
1373 header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w')
1374 header.write("%-15s Size\n" % "Field Name")
1375 header.write("%-15s ----\n" % ("-" * 15))
1376 sizes = []
1377 for field in field_specs:
1378 size = yo.size(field)[0]
1379 sizes.append(size)
1380 header.write("%-15s %3d\n" % (field, size))
1381 header.write('\nTotal Records in file: %d\n' % len(records))
1382 header.close()
1383 for record in records:
1384 fields = []
1385 for i, field_name in enumerate(field_specs):
1386 fields.append("%-*s" % (sizes[i], record[field_name]))
1387 fd.write(''.join(fields) + '\n')
1388 finally:
1389 fd.close()
1390 fd = None
1391 return len(records)
1393 "returns record at physical_index[recno]"
1394 return yo._table[recno]
1395 - def goto(yo, criteria):
1396 """changes the record pointer to the first matching (non-deleted) record
1397 criteria should be either a tuple of tuple(value, field, func) triples,
1398 or an integer to go to"""
1399 if isinstance(criteria, int):
1400 if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count:
1401 raise IndexError("Record %d does not exist" % criteria)
1402 if criteria < 0:
1403 criteria += yo._meta.header.record_count
1404 yo._meta.current = criteria
1405 return yo.current()
1406 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop])
1407 specs = tuple([(field, func) for value, field, func in criteria])
1408 match = tuple([value for value, field, func in criteria])
1409 current = yo.current(index=True)
1410 matchlen = len(match)
1411 while not yo.Eof():
1412 record = yo.current()
1413 results = record(*specs)
1414 if results == match:
1415 return record
1416 return yo.goto(current)
1418 "returns True if name is a variable-length field type"
1419 return yo._meta[name]['type'] in yo._decimal_fields
1421 "returns True if name is a memo type field"
1422 return yo._meta[name]['type'] in yo._memotypes
1423 - def new(yo, filename, field_specs=None, codepage=None):
1437 "set record pointer to next (non-deleted) record, and return it"
1438 if yo.eof(_move=True):
1439 raise Eof()
1440 return yo.current()
1442 meta = yo._meta
1443 meta.inmemory = False
1444 meta.ondisk = True
1445 yo._read_only = False
1446 yo._meta_only = False
1447 if '_table' in dir(yo):
1448 del yo._table
1449 dfd = meta.dfd = open(meta.filename, 'r+b')
1450 dfd.seek(0)
1451 meta.header = header = yo._TableHeader(dfd.read(32))
1452 if not header.version in yo._supported_tables:
1453 dfd.close()
1454 dfd = None
1455 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version)))
1456 cp, sd, ld = _codepage_lookup(meta.header.codepage())
1457 meta.decoder = codecs.getdecoder(sd)
1458 meta.encoder = codecs.getencoder(sd)
1459 fieldblock = dfd.read(header.start - 32)
1460 for i in range(len(fieldblock)//32+1):
1461 fieldend = i * 32
1462 if fieldblock[fieldend] == '\x0d':
1463 break
1464 else:
1465 raise DbfError("corrupt field structure in header")
1466 if len(fieldblock[:fieldend]) % 32 != 0:
1467 raise DbfError("corrupt field structure in header")
1468 header.fields = fieldblock[:fieldend]
1469 header.extra = fieldblock[fieldend+1:]
1470 yo._initializeFields()
1471 yo._checkMemoIntegrity()
1472 meta.current = -1
1473 if len(yo) > 0:
1474 meta.current = 0
1475 dfd.seek(0)
1476
1477 - def pack(yo, _pack=True):
1478 "physically removes all deleted records"
1479 for dbfindex in yo._indexen:
1480 dbfindex.clear()
1481 newtable = []
1482 index = 0
1483 offset = 0
1484 for record in yo._table:
1485 found = False
1486 if record.has_been_deleted and _pack:
1487 for dbflist in yo._dbflists:
1488 if dbflist._purge(record, record.record_number - offset, 1):
1489 found = True
1490 record._recnum = -1
1491 else:
1492 record._recnum = index
1493 newtable.append(record)
1494 index += 1
1495 if found:
1496 offset += 1
1497 found = False
1498 yo._table.clear()
1499 for record in newtable:
1500 yo._table.append(record)
1501 yo._meta.header.record_count = index
1502 yo._current = -1
1503 yo._update_disk()
1504 yo.reindex()
1506 "set record pointer to previous (non-deleted) record, and return it"
1507 if yo.bof(_move=True):
1508 raise Bof
1509 return yo.current()
1510 - def query(yo, sql_command=None, python=None):
1511 "uses exec to perform queries on the table"
1512 if sql_command:
1513 return sql(yo, sql_command)
1514 elif python is None:
1515 raise DbfError("query: python parameter must be specified")
1516 possible = List(desc="%s --> %s" % (yo.filename, python), field_names=yo.field_names)
1517 yo._dbflists.add(possible)
1518 query_result = {}
1519 select = 'query_result["keep"] = %s' % python
1520 g = {}
1521 use_deleted = yo.use_deleted
1522 for record in yo:
1523 query_result['keep'] = False
1524 g['query_result'] = query_result
1525 exec select in g, record
1526 if query_result['keep']:
1527 possible.append(record)
1528 record.write_record()
1529 return possible
1531 for dbfindex in yo._indexen:
1532 dbfindex.reindex()
1534 "renames an existing field"
1535 if yo:
1536 yo.create_backup()
1537 if not oldname in yo._meta.fields:
1538 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname)
1539 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum():
1540 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits")
1541 newname = newname.lower()
1542 if newname in yo._meta.fields:
1543 raise DbfError("field --%s-- already exists" % newname)
1544 if len(newname) > 10:
1545 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname)))
1546 yo._meta[newname] = yo._meta[oldname]
1547 yo._meta.fields[yo._meta.fields.index(oldname)] = newname
1548 yo._buildHeaderFields()
1549 yo._update_disk(headeronly=True)
1551 """resizes field (C only at this time)
1552 creates backup file, then modifies current structure"""
1553 if not 0 < new_size < 256:
1554 raise DbfError("new_size must be between 1 and 255 (use delete_fields to remove a field)")
1555 doomed = yo._list_fields(doomed)
1556 for victim in doomed:
1557 if victim not in yo._meta.fields:
1558 raise DbfError("field %s not in table -- resize aborted" % victim)
1559 all_records = [record for record in yo]
1560 yo.create_backup()
1561 for victim in doomed:
1562 start = yo._meta[victim]['start']
1563 end = yo._meta[victim]['end']
1564 eff_end = min(yo._meta[victim]['length'], new_size)
1565 yo._meta[victim]['length'] = new_size
1566 yo._meta[victim]['end'] = start + new_size
1567 blank = array('c', ' ' * new_size)
1568 for record in yo:
1569 new_data = blank[:]
1570 new_data[:eff_end] = record._data[start:start+eff_end]
1571 record._data = record._data[:start] + new_data + record._data[end:]
1572 for field in yo._meta.fields:
1573 if yo._meta[field]['start'] == end:
1574 end = yo._meta[field]['end']
1575 yo._meta[field]['start'] = start + new_size
1576 yo._meta[field]['end'] = start + new_size + yo._meta[field]['length']
1577 start = yo._meta[field]['end']
1578 yo._buildHeaderFields()
1579 yo._update_disk()
1580 - def size(yo, field):
1581 "returns size of field as a tuple of (length, decimals)"
1582 if field in yo:
1583 return (yo._meta[field]['length'], yo._meta[field]['decimals'])
1584 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1586 """return list of fields suitable for creating same table layout
1587 @param fields: list of fields or None for all fields"""
1588 field_specs = []
1589 fields = yo._list_fields(fields)
1590 try:
1591 for name in fields:
1592 field_specs.append(yo._fieldLayout(yo.field_names.index(name)))
1593 except ValueError:
1594 raise DbfError("field --%s-- does not exist" % name)
1595 return field_specs
1596 - def top(yo, get_record=False):
1597 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record
1598 DbfError if table is empty
1599 Eof if all records are deleted and use_deleted is False"""
1600 yo._meta.current = -1
1601 if get_record:
1602 try:
1603 return yo.next()
1604 except Eof:
1605 yo._meta.current = -1
1606 raise Bof()
1607 - def type(yo, field):
1608 "returns type of field"
1609 if field in yo:
1610 return yo._meta[field]['type']
1611 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1612 - def zap(yo, areyousure=False):
1613 """removes all records from table -- this cannot be undone!
1614 areyousure must be True, else error is raised"""
1615 if areyousure:
1616 if yo._meta.inmemory:
1617 yo._table = []
1618 else:
1619 yo._table.clear()
1620 yo._meta.header.record_count = 0
1621 yo._current = -1
1622 yo._update_disk()
1623 else:
1624 raise DbfError("You must say you are sure to wipe the table")
1626 """Provides an interface for working with dBase III tables."""
1627 _version = 'dBase III Plus'
1628 _versionabbv = 'db3'
1629 _fieldtypes = {
1630 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None},
1631 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None},
1632 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None},
1633 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None},
1634 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric, 'Class':None} }
1635 _memoext = '.dbt'
1636 _memotypes = ('M',)
1637 _memoClass = _Db3Memo
1638 _yesMemoMask = '\x80'
1639 _noMemoMask = '\x7f'
1640 _fixed_fields = ('D','L','M')
1641 _variable_fields = ('C','N')
1642 _character_fields = ('C','M')
1643 _decimal_fields = ('N',)
1644 _numeric_fields = ('N',)
1645 _currency_fields = tuple()
1646 _dbfTableHeader = array('c', '\x00' * 32)
1647 _dbfTableHeader[0] = '\x03'
1648 _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
1649 _dbfTableHeader[10] = '\x01'
1650 _dbfTableHeader[29] = '\x03'
1651 _dbfTableHeader = _dbfTableHeader.tostring()
1652 _dbfTableHeaderExtra = ''
1653 _supported_tables = ['\x03', '\x83']
1654 _read_only = False
1655 _meta_only = False
1656 _use_deleted = True
1658 "dBase III specific"
1659 if yo._meta.header.version == '\x83':
1660 try:
1661 yo._meta.memo = yo._memoClass(yo._meta)
1662 except:
1663 yo._meta.dfd.close()
1664 yo._meta.dfd = None
1665 raise
1666 if not yo._meta.ignorememos:
1667 for field in yo._meta.fields:
1668 if yo._meta[field]['type'] in yo._memotypes:
1669 if yo._meta.header.version != '\x83':
1670 yo._meta.dfd.close()
1671 yo._meta.dfd = None
1672 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
1673 elif not os.path.exists(yo._meta.memoname):
1674 yo._meta.dfd.close()
1675 yo._meta.dfd = None
1676 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1677 break
1679 "builds the FieldList of names, types, and descriptions"
1680 yo._meta.fields[:] = []
1681 offset = 1
1682 fieldsdef = yo._meta.header.fields
1683 if len(fieldsdef) % 32 != 0:
1684 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1685 if len(fieldsdef) // 32 != yo.field_count:
1686 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1687 for i in range(yo.field_count):
1688 fieldblock = fieldsdef[i*32:(i+1)*32]
1689 name = io.unpackStr(fieldblock[:11])
1690 type = fieldblock[11]
1691 if not type in yo._meta.fieldtypes:
1692 raise DbfError("Unknown field type: %s" % type)
1693 start = offset
1694 length = ord(fieldblock[16])
1695 offset += length
1696 end = start + length
1697 decimals = ord(fieldblock[17])
1698 flags = ord(fieldblock[18])
1699 yo._meta.fields.append(name)
1700 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1702 'Provides an interface for working with FoxPro 2 tables'
1703 _version = 'Foxpro'
1704 _versionabbv = 'fp'
1705 _fieldtypes = {
1706 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None},
1707 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None},
1708 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None},
1709 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None},
1710 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None},
1711 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1712 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None},
1713 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None},
1714 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} }
1715 _memoext = '.fpt'
1716 _memotypes = ('G','M','P')
1717 _memoClass = _VfpMemo
1718 _yesMemoMask = '\xf5'
1719 _noMemoMask = '\x03'
1720 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1721 _variable_fields = ('C','F','N')
1722 _character_fields = ('C','M')
1723 _decimal_fields = ('F','N')
1724 _numeric_fields = ('F','N')
1725 _currency_fields = tuple()
1726 _supported_tables = ('\x03', '\xf5')
1727 _dbfTableHeader = array('c', '\x00' * 32)
1728 _dbfTableHeader[0] = '\x30'
1729 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1730 _dbfTableHeader[10] = '\x01'
1731 _dbfTableHeader[29] = '\x03'
1732 _dbfTableHeader = _dbfTableHeader.tostring()
1733 _dbfTableHeaderExtra = '\x00' * 263
1734 _use_deleted = True
1736 if os.path.exists(yo._meta.memoname):
1737 try:
1738 yo._meta.memo = yo._memoClass(yo._meta)
1739 except:
1740 yo._meta.dfd.close()
1741 yo._meta.dfd = None
1742 raise
1743 if not yo._meta.ignorememos:
1744 for field in yo._meta.fields:
1745 if yo._meta[field]['type'] in yo._memotypes:
1746 if not os.path.exists(yo._meta.memoname):
1747 yo._meta.dfd.close()
1748 yo._meta.dfd = None
1749 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1750 break
1752 "builds the FieldList of names, types, and descriptions"
1753 yo._meta.fields[:] = []
1754 offset = 1
1755 fieldsdef = yo._meta.header.fields
1756 if len(fieldsdef) % 32 != 0:
1757 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
1758 if len(fieldsdef) // 32 != yo.field_count:
1759 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
1760 for i in range(yo.field_count):
1761 fieldblock = fieldsdef[i*32:(i+1)*32]
1762 name = io.unpackStr(fieldblock[:11])
1763 type = fieldblock[11]
1764 if not type in yo._meta.fieldtypes:
1765 raise DbfError("Unknown field type: %s" % type)
1766 elif type == '0':
1767 return
1768 start = offset
1769 length = ord(fieldblock[16])
1770 offset += length
1771 end = start + length
1772 decimals = ord(fieldblock[17])
1773 flags = ord(fieldblock[18])
1774 yo._meta.fields.append(name)
1775 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1776
1778 'Provides an interface for working with Visual FoxPro 6 tables'
1779 _version = 'Visual Foxpro v6'
1780 _versionabbv = 'vfp'
1781 _fieldtypes = {
1782 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None},
1783 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency, 'Class':None},
1784 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble, 'Class':None},
1785 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None},
1786 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None},
1787 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger, 'Class':None},
1788 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None},
1789 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None},
1790 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime, 'Class':None},
1791 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1792 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1793 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None},
1794 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} }
1795 _memoext = '.fpt'
1796 _memotypes = ('G','M','P')
1797 _memoClass = _VfpMemo
1798 _yesMemoMask = '\x30'
1799 _noMemoMask = '\x30'
1800 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
1801 _variable_fields = ('C','F','N')
1802 _character_fields = ('C','M')
1803 _decimal_fields = ('F','N')
1804 _numeric_fields = ('B','F','I','N','Y')
1805 _currency_fields = ('Y',)
1806 _supported_tables = ('\x30',)
1807 _dbfTableHeader = array('c', '\x00' * 32)
1808 _dbfTableHeader[0] = '\x30'
1809 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
1810 _dbfTableHeader[10] = '\x01'
1811 _dbfTableHeader[29] = '\x03'
1812 _dbfTableHeader = _dbfTableHeader.tostring()
1813 _dbfTableHeaderExtra = '\x00' * 263
1814 _use_deleted = True
1816 if os.path.exists(yo._meta.memoname):
1817 try:
1818 yo._meta.memo = yo._memoClass(yo._meta)
1819 except:
1820 yo._meta.dfd.close()
1821 yo._meta.dfd = None
1822 raise
1823 if not yo._meta.ignorememos:
1824 for field in yo._meta.fields:
1825 if yo._meta[field]['type'] in yo._memotypes:
1826 if not os.path.exists(yo._meta.memoname):
1827 yo._meta.dfd.close()
1828 yo._meta.dfd = None
1829 raise DbfError("Table structure corrupt: memo fields exist without memo file")
1830 break
1832 "builds the FieldList of names, types, and descriptions"
1833 yo._meta.fields[:] = []
1834 offset = 1
1835 fieldsdef = yo._meta.header.fields
1836 for i in range(yo.field_count):
1837 fieldblock = fieldsdef[i*32:(i+1)*32]
1838 name = io.unpackStr(fieldblock[:11])
1839 type = fieldblock[11]
1840 if not type in yo._meta.fieldtypes:
1841 raise DbfError("Unknown field type: %s" % type)
1842 elif type == '0':
1843 return
1844 start = io.unpackLongInt(fieldblock[12:16])
1845 length = ord(fieldblock[16])
1846 offset += length
1847 end = start + length
1848 decimals = ord(fieldblock[17])
1849 flags = ord(fieldblock[18])
1850 yo._meta.fields.append(name)
1851 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1852 -class List(object):
1853 "list of Dbf records, with set-like behavior"
1854 _desc = ''
1855 - def __init__(yo, new_records=None, desc=None, key=None, field_names=None):
1856 yo.field_names = field_names
1857 yo._list = []
1858 yo._set = set()
1859 if key is not None:
1860 yo.key = key
1861 if key.__doc__ is None:
1862 key.__doc__ = 'unknown'
1863 key = yo.key
1864 yo._current = -1
1865 if isinstance(new_records, yo.__class__) and key is new_records.key:
1866 yo._list = new_records._list[:]
1867 yo._set = new_records._set.copy()
1868 yo._current = 0
1869 elif new_records is not None:
1870 for record in new_records:
1871 value = key(record)
1872 item = (record.record_table, record.record_number, value)
1873 if value not in yo._set:
1874 yo._set.add(value)
1875 yo._list.append(item)
1876 yo._current = 0
1877 if desc is not None:
1878 yo._desc = desc
1880 key = yo.key
1881 if isinstance(other, (DbfTable, list)):
1882 other = yo.__class__(other, key=key)
1883 if isinstance(other, yo.__class__):
1884 result = yo.__class__()
1885 result._set = yo._set.copy()
1886 result._list[:] = yo._list[:]
1887 result.key = yo.key
1888 if key is other.key:
1889 for item in other._list:
1890 if item[2] not in result._set:
1891 result._set.add(item[2])
1892 result._list.append(item)
1893 else:
1894 for rec in other:
1895 value = key(rec)
1896 if value not in result._set:
1897 result._set.add(value)
1898 result._list.append((rec.record_table, rec.record_number, value))
1899 result._current = 0 if result else -1
1900 return result
1901 return NotImplemented
1903 if isinstance(record, tuple):
1904 item = record
1905 else:
1906 item = yo.key(record)
1907 return item in yo._set
1909 if isinstance(key, int):
1910 item = yo._list.pop[key]
1911 yo._set.remove(item[2])
1912 elif isinstance(key, slice):
1913 yo._set.difference_update([item[2] for item in yo._list[key]])
1914 yo._list.__delitem__(key)
1915 else:
1916 raise TypeError
1918 if isinstance(key, int):
1919 count = len(yo._list)
1920 if not -count <= key < count:
1921 raise IndexError("Record %d is not in list." % key)
1922 return yo._get_record(*yo._list[key])
1923 elif isinstance(key, slice):
1924 result = yo.__class__()
1925 result._list[:] = yo._list[key]
1926 result._set = set(result._list)
1927 result.key = yo.key
1928 result._current = 0 if result else -1
1929 return result
1930 else:
1931 raise TypeError('indices must be integers')
1933 return (table.get_record(recno) for table, recno, value in yo._list)
1935 return len(yo._list)
1941 if yo._desc:
1942 return "%s(key=%s - %s - %d records)" % (yo.__class__, yo.key.__doc__, yo._desc, len(yo._list))
1943 else:
1944 return "%s(key=%s - %d records)" % (yo.__class__, yo.key.__doc__, len(yo._list))
1946 key = yo.key
1947 if isinstance(other, (DbfTable, list)):
1948 other = yo.__class__(other, key=key)
1949 if isinstance(other, yo.__class__):
1950 result = yo.__class__()
1951 result._list[:] = other._list[:]
1952 result._set = other._set.copy()
1953 result.key = key
1954 lost = set()
1955 if key is other.key:
1956 for item in yo._list:
1957 if item[2] in result._list:
1958 result._set.remove(item[2])
1959 lost.add(item)
1960 else:
1961 for rec in other:
1962 value = key(rec)
1963 if value in result._set:
1964 result._set.remove(value)
1965 lost.add((rec.record_table, rec.record_number, value))
1966 result._list = [item for item in result._list if item not in lost]
1967 result._current = 0 if result else -1
1968 return result
1969 return NotImplemented
1971 key = yo.key
1972 if isinstance(other, (DbfTable, list)):
1973 other = yo.__class__(other, key=key)
1974 if isinstance(other, yo.__class__):
1975 result = yo.__class__()
1976 result._list[:] = yo._list[:]
1977 result._set = yo._set.copy()
1978 result.key = key
1979 lost = set()
1980 if key is other.key:
1981 for item in other._list:
1982 if item[2] in result._set:
1983 result._set.remove(item[2])
1984 lost.add(item[2])
1985 else:
1986 for rec in other:
1987 value = key(rec)
1988 if value in result._set:
1989 result._set.remove(value)
1990 lost.add(value)
1991 result._list = [item for item in result._list if item[2] not in lost]
1992 result._current = 0 if result else -1
1993 return result
1994 return NotImplemented
1996 if item[2] not in yo._set:
1997 yo._set.add(item[2])
1998 yo._list.append(item)
1999 - def _get_record(yo, table=None, rec_no=None, value=None):
2000 if table is rec_no is None:
2001 table, rec_no, value = yo._list[yo._current]
2002 return table.get_record(rec_no)
2003 - def _purge(yo, record, old_record_number, offset):
2004 partial = record.record_table, old_record_number
2005 records = sorted(yo._list, key=lambda item: (item[0], item[1]))
2006 for item in records:
2007 if partial == item[:2]:
2008 found = True
2009 break
2010 elif partial[0] is item[0] and partial[1] < item[1]:
2011 found = False
2012 break
2013 else:
2014 found = False
2015 if found:
2016 yo._list.pop(yo._list.index(item))
2017 yo._set.remove(item[2])
2018 start = records.index(item) + found
2019 for item in records[start:]:
2020 if item[0] is not partial[0]:
2021 break
2022 i = yo._list.index(item)
2023 yo._set.remove(item[2])
2024 item = item[0], (item[1] - offset), item[2]
2025 yo._list[i] = item
2026 yo._set.add(item[2])
2027 return found
2034 if yo._list:
2035 yo._current = len(yo._list) - 1
2036 return yo._get_record()
2037 raise DbfError("dbf.List is empty")
2039 yo._list = []
2040 yo._set = set()
2041 yo._current = -1
2043 if yo._current < 0:
2044 raise Bof()
2045 elif yo._current == len(yo._list):
2046 raise Eof()
2047 return yo._get_record()
2048 - def extend(yo, new_records):
2064 - def goto(yo, index_number):
2065 if yo._list:
2066 if 0 <= index_number <= len(yo._list):
2067 yo._current = index_number
2068 return yo._get_record()
2069 raise DbfError("index %d not in dbf.List of %d records" % (index_number, len(yo._list)))
2070 raise DbfError("dbf.List is empty")
2071 - def index(yo, sort=None, reverse=False):
2072 "sort= ((field_name, func), (field_name, func),) | 'ORIGINAL'"
2073 if sort is None:
2074 results = []
2075 for field, func in yo._meta.index:
2076 results.append("%s(%s)" % (func.__name__, field))
2077 return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed])
2078 yo._meta.index_reversed = reverse
2079 if sort == 'ORIGINAL':
2080 yo._index = range(yo._meta.header.record_count)
2081 yo._meta.index = 'ORIGINAL'
2082 if reverse:
2083 yo._index.reverse()
2084 return
2085 new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop])
2086 yo._meta.index = tuple(new_sort)
2087 yo._meta.orderresults = [''] * len(yo)
2088 for record in yo:
2089 yo._meta.orderresults[record.record_number] = record()
2090 yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
2091 - def index(yo, record, start=None, stop=None):
2103 - def key(yo, record):
2107 if yo._current < len(yo._list):
2108 yo._current += 1
2109 if yo._current < len(yo._list):
2110 return yo._get_record()
2111 raise Eof()
2112 - def pop(yo, index=None):
2113 if index is None:
2114 table, recno, value = yo._list.pop()
2115 else:
2116 table, recno, value = yo._list.pop(index)
2117 yo._set.remove(value)
2118 return yo._get_record(table, recno, value)
2120 if yo._current >= 0:
2121 yo._current -= 1
2122 if yo._current > -1:
2123 return yo._get_record()
2124 raise Bof()
2132 if yo._list:
2133 yo._current = 0
2134 return yo._get_record()
2135 raise DbfError("dbf.List is empty")
2136 - def sort(yo, key=None, reverse=False):
2140
2152 "returns records using this index"
2154 yo.table = table
2155 yo.records = records
2156 yo.index = 0
2168 - def __init__(yo, table, key, field_names=None):
2169 yo._table = table
2170 yo._values = []
2171 yo._rec_by_val = []
2172 yo._records = {}
2173 yo.__doc__ = key.__doc__ or 'unknown'
2174 yo.key = key
2175 yo.field_names = field_names or table.field_names
2176 for record in table:
2177 value = key(record)
2178 if value is DoNotIndex:
2179 continue
2180 rec_num = record.record_number
2181 if not isinstance(value, tuple):
2182 value = (value, )
2183 vindex = bisect_right(yo._values, value)
2184 yo._values.insert(vindex, value)
2185 yo._rec_by_val.insert(vindex, rec_num)
2186 yo._records[rec_num] = value
2187 table._indexen.add(yo)
2189 rec_num = record.record_number
2190 if rec_num in yo._records:
2191 value = yo._records[rec_num]
2192 vindex = bisect_left(yo._values, value)
2193 yo._values.pop(vindex)
2194 yo._rec_by_val.pop(vindex)
2195 value = yo.key(record)
2196 if value is DoNotIndex:
2197 return
2198 if not isinstance(value, tuple):
2199 value = (value, )
2200 vindex = bisect_right(yo._values, value)
2201 yo._values.insert(vindex, value)
2202 yo._rec_by_val.insert(vindex, rec_num)
2203 yo._records[rec_num] = value
2205 if isinstance(match, _DbfRecord):
2206 if match.record_table is yo._table:
2207 return match.record_number in yo._records
2208 match = yo.key(match)
2209 elif not isinstance(match, tuple):
2210 match = (match, )
2211 return yo.find(match) != -1
2213 if isinstance(key, int):
2214 count = len(yo._values)
2215 if not -count <= key < count:
2216 raise IndexError("Record %d is not in list." % key)
2217 rec_num = yo._rec_by_val[key]
2218 return yo._table.get_record(rec_num)
2219 elif isinstance(key, slice):
2220 result = List(field_names=yo._table.field_names)
2221 yo._table._dbflists.add(result)
2222 start, stop, step = key.start, key.stop, key.step
2223 if start is None: start = 0
2224 if stop is None: stop = len(yo._rec_by_val)
2225 if step is None: step = 1
2226 for loc in range(start, stop, step):
2227 record = yo._table.get_record(yo._rec_by_val[loc])
2228 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2229 result._current = 0 if result else -1
2230 return result
2231 elif isinstance (key, (str, unicode, tuple, _DbfRecord)):
2232 if isinstance(key, _DbfRecord):
2233 key = yo.key(key)
2234 elif not isinstance(key, tuple):
2235 key = (key, )
2236 loc = yo.find(key)
2237 if loc == -1:
2238 raise KeyError(key)
2239 return yo._table.get_record(yo._rec_by_val[loc])
2240 else:
2241 raise TypeError('indices must be integers, match objects must by strings or tuples')
2245 yo._table.close()
2246 yo._values[:] = []
2247 yo._rec_by_val[:] = []
2248 yo._records.clear()
2249 return False
2253 return len(yo._records)
2255 target = target[:len(match)]
2256 if isinstance(match[-1], (str, unicode)):
2257 target = list(target)
2258 target[-1] = target[-1][:len(match[-1])]
2259 target = tuple(target)
2260 return target == match
2262 value = yo._records.get(rec_num)
2263 if value is not None:
2264 vindex = bisect_left(yo._values, value)
2265 del yo._records[rec_num]
2266 yo._values.pop(vindex)
2267 yo._rec_by_val.pop(vindex)
2268 - def _search(yo, match, lo=0, hi=None):
2269 if hi is None:
2270 hi = len(yo._values)
2271 return bisect_left(yo._values, match, lo, hi)
2273 "removes all entries from index"
2274 yo._values[:] = []
2275 yo._rec_by_val[:] = []
2276 yo._records.clear()
2279 - def find(yo, match, partial=False):
2280 "returns numeric index of (partial) match, or -1"
2281 if isinstance(match, _DbfRecord):
2282 if match.record_number in yo._records:
2283 return yo._values.index(yo._records[match.record_number])
2284 else:
2285 return -1
2286 if not isinstance(match, tuple):
2287 match = (match, )
2288 loc = yo._search(match)
2289 while loc < len(yo._values) and yo._values[loc] == match:
2290 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
2291 loc += 1
2292 continue
2293 return loc
2294 if partial:
2295 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
2296 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
2297 loc += 1
2298 continue
2299 return loc
2300 return -1
2302 "returns numeric index of either (partial) match, or position of where match would be"
2303 if isinstance(match, _DbfRecord):
2304 if match.record_number in yo._records:
2305 return yo._values.index(yo._records[match.record_number])
2306 else:
2307 match = yo.key(match)
2308 if not isinstance(match, tuple):
2309 match = (match, )
2310 loc = yo._search(match)
2311 return loc
2312 - def index(yo, match, partial=False):
2313 "returns numeric index of (partial) match, or raises ValueError"
2314 loc = yo.find(match, partial)
2315 if loc == -1:
2316 if isinstance(match, _DbfRecord):
2317 raise ValueError("table <%s> record [%d] not in index <%s>" % (yo._table.filename, match.record_number, yo.__doc__))
2318 else:
2319 raise ValueError("match criteria <%s> not in index" % (match, ))
2320 return loc
2322 "reindexes all records"
2323 for record in yo._table:
2324 yo(record)
2325 - def query(yo, sql_command=None, python=None):
2326 """recognized sql commands are SELECT, UPDATE, REPLACE, INSERT, DELETE, and RECALL"""
2327 if sql_command:
2328 return sql(yo, sql_command)
2329 elif python is None:
2330 raise DbfError("query: python parameter must be specified")
2331 possible = List(desc="%s --> %s" % (yo._table.filename, python), field_names=yo._table.field_names)
2332 yo._table._dbflists.add(possible)
2333 query_result = {}
2334 select = 'query_result["keep"] = %s' % python
2335 g = {}
2336 for record in yo:
2337 query_result['keep'] = False
2338 g['query_result'] = query_result
2339 exec select in g, record
2340 if query_result['keep']:
2341 possible.append(record)
2342 record.write_record()
2343 return possible
2344 - def search(yo, match, partial=False):
2345 "returns dbf.List of all (partially) matching records"
2346 result = List(field_names=yo._table.field_names)
2347 yo._table._dbflists.add(result)
2348 if not isinstance(match, tuple):
2349 match = (match, )
2350 loc = yo._search(match)
2351 if loc == len(yo._values):
2352 return result
2353 while loc < len(yo._values) and yo._values[loc] == match:
2354 record = yo._table.get_record(yo._rec_by_val[loc])
2355 if not yo._table.use_deleted and record.has_been_deleted:
2356 loc += 1
2357 continue
2358 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2359 loc += 1
2360 if partial:
2361 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
2362 record = yo._table.get_record(yo._rec_by_val[loc])
2363 if not yo._table.use_deleted and record.has_been_deleted:
2364 loc += 1
2365 continue
2366 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
2367 loc += 1
2368 return result
2369
2370 csv.register_dialect('dbf', DbfCsv)
2371
2372 -def sql_select(records, chosen_fields, condition, field_names):
2373 if chosen_fields != '*':
2374 field_names = chosen_fields.replace(' ','').split(',')
2375 result = condition(records)
2376 result.modified = 0, 'record' + ('','s')[len(result)>1]
2377 result.field_names = field_names
2378 return result
2379
2380 -def sql_update(records, command, condition, field_names):
2381 possible = condition(records)
2382 modified = sql_cmd(command, field_names)(possible)
2383 possible.modified = modified, 'record' + ('','s')[modified>1]
2384 return possible
2385
2386 -def sql_delete(records, dead_fields, condition, field_names):
2387 deleted = condition(records)
2388 deleted.modified = len(deleted), 'record' + ('','s')[len(deleted)>1]
2389 deleted.field_names = field_names
2390 if dead_fields == '*':
2391 for record in deleted:
2392 record.delete_record()
2393 record.write_record()
2394 else:
2395 keep = [f for f in field_names if f not in dead_fields.replace(' ','').split(',')]
2396 for record in deleted:
2397 record.reset_record(keep_fields=keep)
2398 record.write_record()
2399 return deleted
2400
2401 -def sql_recall(records, all_fields, condition, field_names):
2422
2423 -def sql_add(records, new_fields, condition, field_names):
2433
2434 -def sql_drop(records, dead_fields, condition, field_names):
2444
2445 -def sql_pack(records, command, condition, field_names):
2455
2456 -def sql_resize(records, fieldname_newsize, condition, field_names):
2457 tables = set()
2458 possible = condition(records)
2459 for record in possible:
2460 tables.add(record.record_table)
2461 fieldname, newsize = fieldname_newsize.split()
2462 newsize = int(newsize)
2463 for table in tables:
2464 table.resize_field(fieldname, newsize)
2465 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1]
2466 possible.field_names = field_names
2467 return possible
2468
2469 sql_functions = {
2470 'select' : sql_select,
2471 'update' : sql_update,
2472 'replace': sql_update,
2473 'insert' : None,
2474 'delete' : sql_delete,
2475 'recall' : sql_recall,
2476 'add' : sql_add,
2477 'drop' : sql_drop,
2478 'count' : None,
2479 'pack' : sql_pack,
2480 'resize' : sql_resize,
2481 }
2484 "creates a function matching the sql criteria"
2485 function = """def func(records):
2486 \"\"\"%s\"\"\"
2487 matched = List(field_names=records[0].field_names)
2488 for rec in records:
2489 %s
2490
2491 if %s:
2492 matched.append(rec)
2493 return matched"""
2494 fields = []
2495 for field in records[0].field_names:
2496 if field in criteria:
2497 fields.append(field)
2498 fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields])
2499 g = dbf.sql_user_functions.copy()
2500 g['List'] = List
2501 function %= (criteria, fields, criteria)
2502
2503 exec function in g
2504 return g['func']
2505
2506 -def sql_cmd(command, field_names):
2507 "creates a function matching to apply command to each record in records"
2508 function = """def func(records):
2509 \"\"\"%s\"\"\"
2510 changed = 0
2511 for rec in records:
2512 %s
2513
2514 %s
2515
2516 %s
2517 changed += rec.write_record()
2518 return changed"""
2519 fields = []
2520 for field in field_names:
2521 if field in command:
2522 fields.append(field)
2523 pre_fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields])
2524 post_fields = '\n '.join(['rec.%s = %s' % (field, field) for field in fields])
2525 g = dbf.sql_user_functions.copy()
2526 if '=' not in command and ' with ' in command.lower():
2527 offset = command.lower().index(' with ')
2528 command = command[:offset] + ' = ' + command[offset+6:]
2529 function %= (command, pre_fields, command, post_fields)
2530
2531 exec function in g
2532 return g['func']
2533
2534 -def sql(records, command):
2535 """recognized sql commands are SELECT, UPDATE | REPLACE, DELETE, RECALL, ADD, DROP"""
2536 sql_command = command
2537 if ' where ' in command:
2538 command, condition = command.split(' where ', 1)
2539 condition = sql_criteria(records, condition)
2540 else:
2541 def condition(records):
2542 return records[:]
2543 name, command = command.split(' ', 1)
2544 command = command.strip()
2545 name = name.lower()
2546 field_names = records[0].field_names
2547 if sql_functions.get(name) is None:
2548 raise DbfError('unknown SQL command: %s' % name.upper())
2549 result = sql_functions[name](records, command, condition, field_names)
2550 tables = set()
2551 for record in result:
2552 tables.add(record.record_table)
2553 for list_table in tables:
2554 list_table._dbflists.add(result)
2555 return result
2557 "returns parameter unchanged"
2558 return value
2560 "ensures each tuple is the same length, using filler[-missing] for the gaps"
2561 final = []
2562 for t in tuples:
2563 if len(t) < length:
2564 final.append( tuple([item for item in t] + filler[len(t)-length:]) )
2565 else:
2566 final.append(t)
2567 return tuple(final)
2569 if cp not in code_pages:
2570 for code_page in sorted(code_pages.keys()):
2571 sd, ld = code_pages[code_page]
2572 if cp == sd or cp == ld:
2573 if sd is None:
2574 raise DbfError("Unsupported codepage: %s" % ld)
2575 cp = code_page
2576 break
2577 else:
2578 raise DbfError("Unsupported codepage: %s" % cp)
2579 sd, ld = code_pages[cp]
2580 return cp, sd, ld
2581 -def ascii(new_setting=None):
2588 -def codepage(cp=None):
2589 "get/set default codepage for any new tables"
2590 global default_codepage
2591 cp, sd, ld = _codepage_lookup(cp or default_codepage)
2592 default_codepage = sd
2593 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2601 version = 'dBase IV w/memos (non-functional)'
2602 _versionabbv = 'db4'
2603 _fieldtypes = {
2604 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
2605 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
2606 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
2607 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
2608 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
2609 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
2610 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
2611 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
2612 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
2613 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2614 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2615 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
2616 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
2617 _memoext = '.dbt'
2618 _memotypes = ('G','M','P')
2619 _memoClass = _VfpMemo
2620 _yesMemoMask = '\x8b'
2621 _noMemoMask = '\x04'
2622 _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
2623 _variable_fields = ('C','F','N')
2624 _character_fields = ('C','M')
2625 _decimal_fields = ('F','N')
2626 _numeric_fields = ('B','F','I','N','Y')
2627 _currency_fields = ('Y',)
2628 _supported_tables = ('\x04', '\x8b')
2629 _dbfTableHeader = ['\x00'] * 32
2630 _dbfTableHeader[0] = '\x8b'
2631 _dbfTableHeader[10] = '\x01'
2632 _dbfTableHeader[29] = '\x03'
2633 _dbfTableHeader = ''.join(_dbfTableHeader)
2634 _dbfTableHeaderExtra = ''
2635 _use_deleted = True
2637 "dBase III specific"
2638 if yo._meta.header.version == '\x8b':
2639 try:
2640 yo._meta.memo = yo._memoClass(yo._meta)
2641 except:
2642 yo._meta.dfd.close()
2643 yo._meta.dfd = None
2644 raise
2645 if not yo._meta.ignorememos:
2646 for field in yo._meta.fields:
2647 if yo._meta[field]['type'] in yo._memotypes:
2648 if yo._meta.header.version != '\x8b':
2649 yo._meta.dfd.close()
2650 yo._meta.dfd = None
2651 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos")
2652 elif not os.path.exists(yo._meta.memoname):
2653 yo._meta.dfd.close()
2654 yo._meta.dfd = None
2655 raise DbfError("Table structure corrupt: memo fields exist without memo file")
2656 break
2657