1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 """
24 interface for differrent indexing engines for the translate toolkit
25
26 """
27
28 __revision__ = "$Id: __init__.py 8503 2008-09-27 08:58:30Z dwaynebailey $"
29
30 import CommonIndexer
31 import os
32 import shutil
33
34 """ TODO for indexing engines:
35 * get rid of jToolkit.glock dependency
36 * add partial matching at the beginning of a term
37 * do a proper cleanup - e.g.: the pylucene lockfiles remain in /tmp/
38 * do unittests for PyLucene v1.x
39 """
40
42 """get a list of the available supported indexing engines
43
44 search through the translate.search.indexer package for modules derived from
45 the CommonIndexer class
46 """
47 result = []
48
49 indexer_dir = os.path.dirname(os.path.abspath(__file__))
50
51
52 all_files = os.listdir(indexer_dir)
53 all_files.sort()
54 for mod_file in all_files:
55 if mod_file == __file__:
56
57 continue
58 mod_path = os.path.join(indexer_dir, mod_file)
59 if (not mod_path.endswith(".py")) or (not os.path.isfile(mod_path)) \
60 or (not os.access(mod_path, os.R_OK)):
61
62 continue
63
64 mod_name = mod_file[:-3]
65
66 try:
67 module = __import__(mod_name, globals(), {})
68 except ImportError:
69
70 continue
71
72 if not (hasattr(module, "is_available") and \
73 callable(module.is_available) and \
74 module.is_available()):
75 continue
76 for item in dir(module):
77 try:
78 element = getattr(module, item)
79 except TypeError:
80
81 continue
82 try:
83
84 if issubclass(element, CommonIndexer.CommonDatabase) \
85 and not element is CommonIndexer.CommonDatabase:
86
87
88 result.append(element)
89 except TypeError:
90
91 continue
92 return result
93
95 """sort a given list of indexer classes according to the given order
96
97 the list of preferred indexers are strings that should match the filenames
98 (without suppix ".py") of the respective modules (e.g.: XapianIndexer or
99 PyLuceneIndexer)
100
101 @param indexer_classes: the list of all available indexer classes
102 @type indexer_classes: list of CommonIndexer.CommonDatabase objects
103 @param pref_order: list of preferred indexer names
104 @type pref_order: str
105 @return: sorted list of indexer classes
106 @rtype: list of CommonIndexer.CommonDatabase objects
107 """
108
109 get_indexer_name = lambda indexer_class: \
110 os.path.basename(indexer_class.__module__).split(".")[-1]
111
112 avail_indexers = indexer_classes[:]
113 result = []
114
115 for choice in pref_order:
116
117 matches = [ indexer for indexer in avail_indexers
118 if get_indexer_name(indexer) == choice ]
119
120 for match_item in matches:
121 result.append(match_item)
122 avail_indexers.remove(match_item)
123
124 return result + avail_indexers
125
126
127
128 _AVAILABLE_INDEXERS = _get_available_indexers()
129
130
131
132 HAVE_INDEXER = bool(_AVAILABLE_INDEXERS)
133
134
136 """return an appropriate indexer for the given directory
137
138 If the directory already exists, then we check, if one of the available
139 indexers knows how to handle it. Otherwise we return the first available
140 indexer.
141
142 @raise IndexError: there is no indexing engine available
143 @raise ValueError: the database location already exists, but we did not find
144 a suitable indexing engine for it
145 @raise OSError: any error that could occour while creating or opening the
146 database
147
148 @param basedir: the parent directory of (possible) different indexing
149 databases
150 @type basedir: string
151 @return: the class of the most appropriate indexer
152 @rtype: subclass of L{CommonIndexer.CommonDatabase}
153 """
154 if not _AVAILABLE_INDEXERS:
155 raise IndexError("Indexer: no indexing engines are available")
156 if preference is None:
157 preference = []
158
159 preferred_indexers = _sort_indexers_by_preference(_AVAILABLE_INDEXERS,
160 preference)
161 if os.path.exists(basedir):
162 for index_class in preferred_indexers:
163 try:
164
165
166
167 return index_class(basedir, create_allowed=False)
168 except (ValueError, OSError):
169
170 continue
171
172
173 shutil.rmtree(basedir, ignore_errors=True)
174 print "Deleting invalid indexing directory '%s'" % basedir
175
176
177 return preferred_indexers[0](basedir)
178
179
180 if __name__ == "__main__":
181
182 for ONE_INDEX in _AVAILABLE_INDEXERS:
183 print ONE_INDEX
184