Package entropy :: Module dump

Source Code for Module entropy.dump

  1  # -*- coding: utf-8 -*- 
  2  """ 
  3   
  4      @author: Fabio Erculiani <[email protected]> 
  5      @contact: [email protected] 
  6      @copyright: Fabio Erculiani 
  7      @license: GPL-2 
  8   
  9      B{Entropy Framework object disk serializer module}. 
 10   
 11      This module contains Entropy Python object serialization functions and 
 12      disk dumpers. 
 13   
 14      Serialized objects are stored to disk with proper permissions by default 
 15      into path given by entropy.const's etpConst['dumpstoragedir']. 
 16   
 17      Permissions are set using entropy.const's const_setup_perms and 
 18      const_setup_file functions. 
 19   
 20      Objects are serialized using Python's cPickle/pickle modules, thus 
 21      they must be "pickable". Please read Python Library reference for 
 22      more information. 
 23   
 24  """ 
 25   
 26  import sys 
 27  import os 
 28  import time 
 29   
 30  from entropy.const import etpConst, const_setup_file, const_is_python3, \ 
 31      const_mkstemp 
 32  # Always use MAX pickle protocol to <=2, to allow Python 2 and 3 support 
 33  COMPAT_PICKLE_PROTOCOL = 0 
 34   
 35  if const_is_python3(): 
 36      import pickle 
 37  else: 
 38      try: 
 39          import cPickle as pickle 
 40      except ImportError: 
 41          import pickle 
 42   
 43  pickle.HIGHEST_PROTOCOL = COMPAT_PICKLE_PROTOCOL 
 44  pickle.DEFAULT_PROTOCOL = COMPAT_PICKLE_PROTOCOL 
 45   
 46  D_EXT = etpConst['cachedumpext'] 
 47  D_DIR = etpConst['dumpstoragedir'] 
 48  E_GID = etpConst['entropygid'] 
 49  if E_GID == None: 
 50      E_GID = 0 
 51   
 52   
53 -def dumpobj(name, my_object, complete_path = False, ignore_exceptions = True, 54 dump_dir = None, custom_permissions = None):
55 """ 56 Dump pickable object to file 57 58 @param name: name of the object 59 @type name: string 60 @param my_object: object to dump 61 @type my_object: any Python "pickable" object 62 @keyword complete_path: consider "name" argument as 63 a complete path (this overrides the default dump 64 path given by etpConst['dumpstoragedir']) 65 @type complete_path: bool 66 @keyword ignore_exceptions: ignore any possible exception 67 (EOFError, IOError, OSError,) 68 @type ignore_exceptions: bool 69 @keyword dump_dir: alternative dump directory 70 @type dump_dir: string 71 @keyword custom_permissions: give custom permission bits 72 @type custom_permissions: octal 73 @return: None 74 @rtype: None 75 @raise EOFError: could be caused by pickle.dump, ignored if 76 ignore_exceptions is True 77 @raise IOError: could be caused by pickle.dump, ignored if 78 ignore_exceptions is True 79 @raise OSError: could be caused by pickle.dump, ignored if 80 ignore_exceptions is True 81 """ 82 if dump_dir is None: 83 dump_dir = D_DIR 84 if custom_permissions is None: 85 custom_permissions = 0o664 86 87 while True: # trap ctrl+C 88 tmp_fd, tmp_dmpfile = None, None 89 try: 90 if complete_path: 91 dmpfile = name 92 c_dump_dir = os.path.dirname(name) 93 else: 94 _dmp_path = os.path.join(dump_dir, name) 95 dmpfile = _dmp_path+D_EXT 96 c_dump_dir = os.path.dirname(_dmp_path) 97 98 my_dump_dir = c_dump_dir 99 d_paths = [] 100 while not os.path.isdir(my_dump_dir): 101 d_paths.append(my_dump_dir) 102 my_dump_dir = os.path.dirname(my_dump_dir) 103 if d_paths: 104 d_paths = sorted(d_paths) 105 for d_path in d_paths: 106 os.mkdir(d_path) 107 const_setup_file(d_path, E_GID, 0o775) 108 109 dmp_name = os.path.basename(dmpfile) 110 tmp_fd, tmp_dmpfile = const_mkstemp( 111 dir=c_dump_dir, prefix=dmp_name) 112 # WARNING: it has been observed that using 113 # os.fdopen() below in multi-threaded scenarios 114 # is causing EBADF. There is probably a race 115 # condition down in the stack. 116 with open(tmp_dmpfile, "wb") as dmp_f: 117 if const_is_python3(): 118 pickle.dump(my_object, dmp_f, 119 protocol = COMPAT_PICKLE_PROTOCOL, fix_imports = True) 120 else: 121 pickle.dump(my_object, dmp_f) 122 123 const_setup_file(tmp_dmpfile, E_GID, custom_permissions) 124 os.rename(tmp_dmpfile, dmpfile) 125 126 except RuntimeError: 127 try: 128 os.remove(dmpfile) 129 except OSError: 130 pass 131 except (EOFError, IOError, OSError): 132 if not ignore_exceptions: 133 raise 134 finally: 135 if tmp_fd is not None: 136 try: 137 os.close(tmp_fd) 138 except (IOError, OSError): 139 pass 140 if tmp_dmpfile is not None: 141 try: 142 os.remove(tmp_dmpfile) 143 except (IOError, OSError): 144 pass 145 break
146
147 -def serialize(myobj, ser_f, do_seek = True):
148 """ 149 Serialize object to ser_f (file) 150 151 @param myobj: Python object to serialize 152 @type myobj: any Python picklable object 153 @param ser_f: file object to write to 154 @type ser_f: file object 155 @keyword do_seek: move file cursor back to the beginning 156 of ser_f 157 @type do_seek: bool 158 @return: file object where data has been written 159 @rtype: file object 160 @raise RuntimeError: caused by pickle.dump in case of 161 system errors 162 @raise EOFError: caused by pickle.dump in case of 163 race conditions on multi-processing or multi-threading 164 @raise IOError: caused by pickle.dump in case of 165 race conditions on multi-processing or multi-threading 166 @raise pickle.PicklingError: when object cannot be recreated 167 """ 168 if const_is_python3(): 169 pickle.dump(myobj, ser_f, protocol = COMPAT_PICKLE_PROTOCOL, 170 fix_imports = True) 171 else: 172 pickle.dump(myobj, ser_f) 173 ser_f.flush() 174 if do_seek: 175 ser_f.seek(0) 176 return ser_f
177
178 -def unserialize(serial_f):
179 """ 180 Unserialize file to object (file) 181 182 @param serial_f: file object which data will be read from 183 @type serial_f: file object 184 @return: rebuilt object 185 @rtype: any Python pickable object 186 @raise pickle.UnpicklingError: when object cannot be recreated 187 """ 188 if const_is_python3(): 189 return pickle.load(serial_f, fix_imports = True, 190 encoding = etpConst['conf_raw_encoding']) 191 else: 192 return pickle.load(serial_f)
193
194 -def unserialize_string(mystring):
195 """ 196 Unserialize pickle string to object 197 198 @param mystring: data stream in string form to reconstruct 199 @type mystring: string 200 @return: reconstructed object 201 @rtype: any Python pickable object 202 @raise pickle.UnpicklingError: when object cannot be recreated 203 """ 204 if const_is_python3(): 205 return pickle.loads(mystring, fix_imports = True, 206 encoding = etpConst['conf_raw_encoding']) 207 else: 208 return pickle.loads(mystring)
209
210 -def serialize_string(myobj):
211 """ 212 Serialize object to string 213 214 @param myobj: object to serialize 215 @type myobj: any Python picklable object 216 @return: serialized string 217 @rtype: string 218 @raise pickle.PicklingError: when object cannot be recreated 219 """ 220 if const_is_python3(): 221 return pickle.dumps(myobj, protocol = COMPAT_PICKLE_PROTOCOL, 222 fix_imports = True, encoding = etpConst['conf_raw_encoding']) 223 else: 224 return pickle.dumps(myobj)
225
226 -def loadobj(name, complete_path = False, dump_dir = None, aging_days = None):
227 """ 228 Load object from a file 229 @param name: name of the object to load 230 @type name: string 231 @keyword complete_path: determine whether name argument 232 is a complete disk path to serialized object 233 @type complete_path: bool 234 @keyword dump_dir: alternative dump directory 235 @type dump_dir: string 236 @keyword aging_days: if int, consider the cached file invalid 237 if older than aging_days. 238 @type aging_days: int 239 @return: object or None 240 @rtype: any Python pickable object or None 241 """ 242 if dump_dir is None: 243 dump_dir = D_DIR 244 245 while True: 246 if complete_path: 247 dmpfile = name 248 else: 249 dump_path = os.path.join(dump_dir, name) 250 dmpfile = dump_path + D_EXT 251 252 if aging_days is not None: 253 cur_t = time.time() 254 try: 255 mtime = os.path.getmtime(dmpfile) 256 except (IOError, OSError): 257 mtime = 0.0 258 if abs(cur_t - mtime) > (aging_days * 86400): 259 # do not unlink since other consumers might 260 # have different aging settings. 261 #try: 262 # os.remove(dmpfile) 263 #except (OSError, IOError): 264 # # did my best 265 # pass 266 return None 267 268 try: 269 with open(dmpfile, "rb") as dmp_f: 270 obj = None 271 try: 272 if const_is_python3(): 273 obj = pickle.load(dmp_f, fix_imports = True, 274 encoding = etpConst['conf_raw_encoding']) 275 else: 276 obj = pickle.load(dmp_f) 277 except (ValueError, EOFError, IOError, 278 OSError, pickle.UnpicklingError, TypeError, 279 AttributeError, ImportError, SystemError,): 280 pass 281 return obj 282 except (IOError, OSError,): 283 pass 284 break
285
286 -def getobjmtime(name, dump_dir = None):
287 """ 288 Get dumped object mtime 289 290 @param name: object name 291 @type name: string 292 @keyword dump_dir: alternative dump directory 293 @type dump_dir: string 294 @return: mtime of the file containing the serialized object or 0 295 if not found 296 @rtype: int 297 """ 298 if dump_dir is None: 299 dump_dir = D_DIR 300 mtime = 0 301 dump_path = os.path.join(dump_dir, name+D_EXT) 302 try: 303 mtime = os.path.getmtime(dump_path) 304 except (IOError, OSError): 305 mtime = 0 306 return int(mtime)
307
308 -def removeobj(name, dump_dir = None):
309 """ 310 Remove cached object referenced by its object name 311 312 @param name: object name 313 @type name: string 314 @keyword dump_dir: alternative dump directory 315 @type dump_dir: string 316 @return: bool representing whether object has been 317 removed or not 318 @rtype: bool 319 @raise OSError: in case of troubles with os.remove() 320 """ 321 if dump_dir is None: 322 dump_dir = D_DIR 323 filepath = dump_dir + os.path.sep + name + D_EXT 324 try: 325 os.remove(filepath) 326 return True 327 except (OSError, IOError) as err: 328 if err.errno not in (errno.ENOENT, errno.ENOTDIR): 329 raise 330 return False
331