1
2 """
3
4 @author: Fabio Erculiani <[email protected]>
5 @contact: [email protected]
6 @copyright: Fabio Erculiani
7 @license: GPL-2
8
9 B{Entropy Framework object disk serializer module}.
10
11 This module contains Entropy Python object serialization functions and
12 disk dumpers.
13
14 Serialized objects are stored to disk with proper permissions by default
15 into path given by entropy.const's etpConst['dumpstoragedir'].
16
17 Permissions are set using entropy.const's const_setup_perms and
18 const_setup_file functions.
19
20 Objects are serialized using Python's cPickle/pickle modules, thus
21 they must be "pickable". Please read Python Library reference for
22 more information.
23
24 """
25
26 import sys
27 import os
28 import time
29
30 from entropy.const import etpConst, const_setup_file, const_is_python3, \
31 const_mkstemp
32
33 COMPAT_PICKLE_PROTOCOL = 0
34
35 if const_is_python3():
36 import pickle
37 else:
38 try:
39 import cPickle as pickle
40 except ImportError:
41 import pickle
42
43 pickle.HIGHEST_PROTOCOL = COMPAT_PICKLE_PROTOCOL
44 pickle.DEFAULT_PROTOCOL = COMPAT_PICKLE_PROTOCOL
45
46 D_EXT = etpConst['cachedumpext']
47 D_DIR = etpConst['dumpstoragedir']
48 E_GID = etpConst['entropygid']
49 if E_GID == None:
50 E_GID = 0
51
52
53 -def dumpobj(name, my_object, complete_path = False, ignore_exceptions = True,
54 dump_dir = None, custom_permissions = None):
55 """
56 Dump pickable object to file
57
58 @param name: name of the object
59 @type name: string
60 @param my_object: object to dump
61 @type my_object: any Python "pickable" object
62 @keyword complete_path: consider "name" argument as
63 a complete path (this overrides the default dump
64 path given by etpConst['dumpstoragedir'])
65 @type complete_path: bool
66 @keyword ignore_exceptions: ignore any possible exception
67 (EOFError, IOError, OSError,)
68 @type ignore_exceptions: bool
69 @keyword dump_dir: alternative dump directory
70 @type dump_dir: string
71 @keyword custom_permissions: give custom permission bits
72 @type custom_permissions: octal
73 @return: None
74 @rtype: None
75 @raise EOFError: could be caused by pickle.dump, ignored if
76 ignore_exceptions is True
77 @raise IOError: could be caused by pickle.dump, ignored if
78 ignore_exceptions is True
79 @raise OSError: could be caused by pickle.dump, ignored if
80 ignore_exceptions is True
81 """
82 if dump_dir is None:
83 dump_dir = D_DIR
84 if custom_permissions is None:
85 custom_permissions = 0o664
86
87 while True:
88 tmp_fd, tmp_dmpfile = None, None
89 try:
90 if complete_path:
91 dmpfile = name
92 c_dump_dir = os.path.dirname(name)
93 else:
94 _dmp_path = os.path.join(dump_dir, name)
95 dmpfile = _dmp_path+D_EXT
96 c_dump_dir = os.path.dirname(_dmp_path)
97
98 my_dump_dir = c_dump_dir
99 d_paths = []
100 while not os.path.isdir(my_dump_dir):
101 d_paths.append(my_dump_dir)
102 my_dump_dir = os.path.dirname(my_dump_dir)
103 if d_paths:
104 d_paths = sorted(d_paths)
105 for d_path in d_paths:
106 os.mkdir(d_path)
107 const_setup_file(d_path, E_GID, 0o775)
108
109 dmp_name = os.path.basename(dmpfile)
110 tmp_fd, tmp_dmpfile = const_mkstemp(
111 dir=c_dump_dir, prefix=dmp_name)
112
113
114
115
116 with open(tmp_dmpfile, "wb") as dmp_f:
117 if const_is_python3():
118 pickle.dump(my_object, dmp_f,
119 protocol = COMPAT_PICKLE_PROTOCOL, fix_imports = True)
120 else:
121 pickle.dump(my_object, dmp_f)
122
123 const_setup_file(tmp_dmpfile, E_GID, custom_permissions)
124 os.rename(tmp_dmpfile, dmpfile)
125
126 except RuntimeError:
127 try:
128 os.remove(dmpfile)
129 except OSError:
130 pass
131 except (EOFError, IOError, OSError):
132 if not ignore_exceptions:
133 raise
134 finally:
135 if tmp_fd is not None:
136 try:
137 os.close(tmp_fd)
138 except (IOError, OSError):
139 pass
140 if tmp_dmpfile is not None:
141 try:
142 os.remove(tmp_dmpfile)
143 except (IOError, OSError):
144 pass
145 break
146
148 """
149 Serialize object to ser_f (file)
150
151 @param myobj: Python object to serialize
152 @type myobj: any Python picklable object
153 @param ser_f: file object to write to
154 @type ser_f: file object
155 @keyword do_seek: move file cursor back to the beginning
156 of ser_f
157 @type do_seek: bool
158 @return: file object where data has been written
159 @rtype: file object
160 @raise RuntimeError: caused by pickle.dump in case of
161 system errors
162 @raise EOFError: caused by pickle.dump in case of
163 race conditions on multi-processing or multi-threading
164 @raise IOError: caused by pickle.dump in case of
165 race conditions on multi-processing or multi-threading
166 @raise pickle.PicklingError: when object cannot be recreated
167 """
168 if const_is_python3():
169 pickle.dump(myobj, ser_f, protocol = COMPAT_PICKLE_PROTOCOL,
170 fix_imports = True)
171 else:
172 pickle.dump(myobj, ser_f)
173 ser_f.flush()
174 if do_seek:
175 ser_f.seek(0)
176 return ser_f
177
179 """
180 Unserialize file to object (file)
181
182 @param serial_f: file object which data will be read from
183 @type serial_f: file object
184 @return: rebuilt object
185 @rtype: any Python pickable object
186 @raise pickle.UnpicklingError: when object cannot be recreated
187 """
188 if const_is_python3():
189 return pickle.load(serial_f, fix_imports = True,
190 encoding = etpConst['conf_raw_encoding'])
191 else:
192 return pickle.load(serial_f)
193
195 """
196 Unserialize pickle string to object
197
198 @param mystring: data stream in string form to reconstruct
199 @type mystring: string
200 @return: reconstructed object
201 @rtype: any Python pickable object
202 @raise pickle.UnpicklingError: when object cannot be recreated
203 """
204 if const_is_python3():
205 return pickle.loads(mystring, fix_imports = True,
206 encoding = etpConst['conf_raw_encoding'])
207 else:
208 return pickle.loads(mystring)
209
211 """
212 Serialize object to string
213
214 @param myobj: object to serialize
215 @type myobj: any Python picklable object
216 @return: serialized string
217 @rtype: string
218 @raise pickle.PicklingError: when object cannot be recreated
219 """
220 if const_is_python3():
221 return pickle.dumps(myobj, protocol = COMPAT_PICKLE_PROTOCOL,
222 fix_imports = True, encoding = etpConst['conf_raw_encoding'])
223 else:
224 return pickle.dumps(myobj)
225
226 -def loadobj(name, complete_path = False, dump_dir = None, aging_days = None):
227 """
228 Load object from a file
229 @param name: name of the object to load
230 @type name: string
231 @keyword complete_path: determine whether name argument
232 is a complete disk path to serialized object
233 @type complete_path: bool
234 @keyword dump_dir: alternative dump directory
235 @type dump_dir: string
236 @keyword aging_days: if int, consider the cached file invalid
237 if older than aging_days.
238 @type aging_days: int
239 @return: object or None
240 @rtype: any Python pickable object or None
241 """
242 if dump_dir is None:
243 dump_dir = D_DIR
244
245 while True:
246 if complete_path:
247 dmpfile = name
248 else:
249 dump_path = os.path.join(dump_dir, name)
250 dmpfile = dump_path + D_EXT
251
252 if aging_days is not None:
253 cur_t = time.time()
254 try:
255 mtime = os.path.getmtime(dmpfile)
256 except (IOError, OSError):
257 mtime = 0.0
258 if abs(cur_t - mtime) > (aging_days * 86400):
259
260
261
262
263
264
265
266 return None
267
268 try:
269 with open(dmpfile, "rb") as dmp_f:
270 obj = None
271 try:
272 if const_is_python3():
273 obj = pickle.load(dmp_f, fix_imports = True,
274 encoding = etpConst['conf_raw_encoding'])
275 else:
276 obj = pickle.load(dmp_f)
277 except (ValueError, EOFError, IOError,
278 OSError, pickle.UnpicklingError, TypeError,
279 AttributeError, ImportError, SystemError,):
280 pass
281 return obj
282 except (IOError, OSError,):
283 pass
284 break
285
287 """
288 Get dumped object mtime
289
290 @param name: object name
291 @type name: string
292 @keyword dump_dir: alternative dump directory
293 @type dump_dir: string
294 @return: mtime of the file containing the serialized object or 0
295 if not found
296 @rtype: int
297 """
298 if dump_dir is None:
299 dump_dir = D_DIR
300 mtime = 0
301 dump_path = os.path.join(dump_dir, name+D_EXT)
302 try:
303 mtime = os.path.getmtime(dump_path)
304 except (IOError, OSError):
305 mtime = 0
306 return int(mtime)
307
309 """
310 Remove cached object referenced by its object name
311
312 @param name: object name
313 @type name: string
314 @keyword dump_dir: alternative dump directory
315 @type dump_dir: string
316 @return: bool representing whether object has been
317 removed or not
318 @rtype: bool
319 @raise OSError: in case of troubles with os.remove()
320 """
321 if dump_dir is None:
322 dump_dir = D_DIR
323 filepath = dump_dir + os.path.sep + name + D_EXT
324 try:
325 os.remove(filepath)
326 return True
327 except (OSError, IOError) as err:
328 if err.errno not in (errno.ENOENT, errno.ENOTDIR):
329 raise
330 return False
331