Package entropy :: Package client :: Package interfaces :: Package package :: Module _content

Source Code for Module entropy.client.interfaces.package._content

  1  # -*- coding: utf-8 -*- 
  2  """ 
  3   
  4      @author: Fabio Erculiani <[email protected]> 
  5      @contact: [email protected] 
  6      @copyright: Fabio Erculiani 
  7      @license: GPL-2 
  8   
  9      B{Entropy Package Manager Client Package Interface}. 
 10   
 11  """ 
 12  import codecs 
 13  import errno 
 14  import sys 
 15  import os 
 16   
 17  from entropy.const import etpConst, const_mkstemp 
 18   
 19  import entropy.tools 
 20   
 21   
22 -class FileContentReader(object):
23
24 - def __init__(self, path, enc=None):
25 if enc is None: 26 self._enc = etpConst['conf_encoding'] 27 else: 28 self._enc = enc 29 self._cpath = path 30 self._file = None 31 self._eof = False
32
33 - def _open_f(self):
34 # opening the file in universal newline mode 35 # fixes the readline() issues wrt 36 # truncated lines. 37 if isinstance(self._cpath, int): 38 self._file = entropy.tools.codecs_fdopen( 39 self._cpath, "rU", self._enc) 40 else: 41 self._file = codecs.open( 42 self._cpath, "rU", self._enc)
43
44 - def __iter__(self):
45 # reset object status, this makes possible 46 # to reuse the iterator more than once 47 # restarting from the beginning. It is really 48 # important for scenarios where transactions 49 # have to be rolled back and replayed. 50 self.close() 51 self._open_f() 52 # reset EOF status on each new iteration 53 self._eof = False 54 return self
55
56 - def __enter__(self):
57 return self
58
59 - def __exit__(self, exc_type, exc_val, trace_obj):
60 self.close()
61
62 - def __next__(self):
63 return self.next()
64
65 - def next(self):
66 if self._eof: 67 raise StopIteration() 68 if self._file is None: 69 self._open_f() 70 71 line = self._file.readline() 72 if not line: 73 self.close() 74 self._eof = True 75 raise StopIteration() 76 77 # non-deterministic BUG with 78 # ca-certificates and Python crappy 79 # API causes readline() to return 80 # partial lines when non ASCII cruft 81 # is on the line. This is probably a 82 # Python bug. 83 # Example of partial readline(): 84 # 0|obj|/usr/share/ca-certificates/mozilla/NetLock_Arany_=Class_Gold=_F\xc3\x85 85 # and the next call: 86 # \xc2\x91tan\xc3\x83\xc2\xbas\xc3\x83\xc2\xadtv\xc3\x83\xc2\xa1ny.crt\n 87 # Try to workaround it by reading ahead 88 # if line does not end with \n 89 # HOWEVER: opening the file in 90 # Universal Newline mode fixes it. 91 # But let's keep the check for QA. 92 # 2012-08-14: is has been observed that 93 # Universal Newline mode is not enough 94 # to avoid this issue. 95 while not line.endswith("\n"): 96 part_line = self._file.readline() 97 line += part_line 98 sys.stderr.write( 99 "FileContentReader, broken readline()" 100 ", executing fixup code\n") 101 sys.stderr.write("%s\n" % (repr(part_line),)) 102 # break infinite loops 103 # and let it crash 104 if not part_line: # EOF 105 break 106 107 _package_id, _ftype, _path = line[:-1].split("|", 2) 108 # must be legal or die! 109 _package_id = int(_package_id) 110 return _package_id, _path, _ftype
111
112 - def close(self):
113 if self._file is not None: 114 self._file.close() 115 self._file = None
116 117
118 -class FileContentWriter(object):
119 120 TMP_SUFFIX = "__filter_tmp" 121
122 - def __init__(self, path, enc=None):
123 self._file = None 124 if enc is None: 125 self._enc = etpConst['conf_encoding'] 126 else: 127 self._enc = enc 128 self._cpath = path 129 # callers expect that file is created 130 # on open object instantiation, don't 131 # remove this or things like os.rename() 132 # will fail 133 self._open_f()
134
135 - def _open_f(self):
136 if isinstance(self._cpath, int): 137 self._file = entropy.tools.codecs_fdopen( 138 self._cpath, "w", self._enc) 139 else: 140 self._file = codecs.open( 141 self._cpath, "w", self._enc)
142
143 - def __enter__(self):
144 return self
145
146 - def __exit__(self, exc_type, exc_val, trace_obj):
147 self.close()
148
149 - def write(self, package_id, path, ftype):
150 """ 151 Write an entry to file. 152 """ 153 if self._file is None: 154 self._open_f() 155 156 if package_id is not None: 157 self._file.write(str(package_id)) 158 else: 159 self._file.write("0") 160 self._file.write("|") 161 self._file.write(ftype) 162 self._file.write("|") 163 self._file.write(path) 164 self._file.write("\n")
165
166 - def close(self):
167 if self._file is not None: 168 self._file.flush() 169 self._file.close() 170 self._file = None
171 172
173 -class FileContentSafetyWriter(object):
174
175 - def __init__(self, path, enc=None):
176 self._file = None 177 if enc is None: 178 self._enc = etpConst['conf_encoding'] 179 else: 180 self._enc = enc 181 self._cpath = path 182 # callers expect that file is created 183 # on open object instantiation, don't 184 # remove this or things like os.rename() 185 # will fail 186 self._open_f()
187
188 - def _open_f(self):
189 if isinstance(self._cpath, int): 190 self._file = entropy.tools.codecs_fdopen( 191 self._cpath, "w", self._enc) 192 else: 193 self._file = codecs.open( 194 self._cpath, "w", self._enc)
195
196 - def __enter__(self):
197 return self
198
199 - def __exit__(self, exc_type, exc_val, trace_obj):
200 self.close()
201
202 - def write(self, path, sha256, mtime):
203 """ 204 Write an entry to file. 205 """ 206 if self._file is None: 207 self._open_f() 208 209 self._file.write("%f" % (mtime,)) 210 self._file.write("|") 211 self._file.write(sha256) 212 self._file.write("|") 213 self._file.write(path) 214 self._file.write("\n")
215
216 - def close(self):
217 if self._file is not None: 218 self._file.flush() 219 self._file.close() 220 self._file = None
221 222
223 -class FileContentSafetyReader(object):
224
225 - def __init__(self, path, enc=None):
226 if enc is None: 227 self._enc = etpConst['conf_encoding'] 228 else: 229 self._enc = enc 230 self._cpath = path 231 self._file = None 232 self._eof = False
233
234 - def _open_f(self):
235 # opening the file in universal newline mode 236 # fixes the readline() issues wrt 237 # truncated lines. 238 if isinstance(self._cpath, int): 239 self._file = entropy.tools.codecs_fdopen( 240 self._cpath, "rU", self._enc) 241 else: 242 self._file = codecs.open( 243 self._cpath, "rU", self._enc)
244
245 - def __iter__(self):
246 # reset object status, this makes possible 247 # to reuse the iterator more than once 248 # restarting from the beginning. It is really 249 # important for scenarios where transactions 250 # have to be rolled back and replayed. 251 self.close() 252 self._open_f() 253 # reset EOF status on each new iteration 254 self._eof = False 255 return self
256
257 - def __enter__(self):
258 return self
259
260 - def __exit__(self, exc_type, exc_val, trace_obj):
261 self.close()
262
263 - def __next__(self):
264 return self.next()
265
266 - def next(self):
267 if self._eof: 268 raise StopIteration() 269 if self._file is None: 270 self._open_f() 271 272 line = self._file.readline() 273 if not line: 274 self.close() 275 self._eof = True 276 raise StopIteration() 277 278 # non-deterministic BUG with 279 # ca-certificates and Python crappy 280 # API causes readline() to return 281 # partial lines when non ASCII cruft 282 # is on the line. This is probably a 283 # Python bug. 284 # Example of partial readline(): 285 # 0|obj|/usr/share/ca-certificates/mozilla/NetLock_Arany_=Class_Gold=_F\xc3\x85 286 # and the next call: 287 # \xc2\x91tan\xc3\x83\xc2\xbas\xc3\x83\xc2\xadtv\xc3\x83\xc2\xa1ny.crt\n 288 # Try to workaround it by reading ahead 289 # if line does not end with \n 290 # HOWEVER: opening the file in 291 # Universal Newline mode fixes it. 292 # But let's keep the check for QA. 293 # 2012-08-14: is has been observed that 294 # Universal Newline mode is not enough 295 # to avoid this issue. 296 while not line.endswith("\n"): 297 part_line = self._file.readline() 298 line += part_line 299 sys.stderr.write( 300 "FileContentReader, broken readline()" 301 ", executing fixup code\n") 302 sys.stderr.write("%s\n" % (repr(part_line),)) 303 # break infinite loops 304 # and let it crash 305 if not part_line: # EOF 306 break 307 308 _mtime, _sha256, _path = line[:-1].split("|", 2) 309 # must be legal or die! 310 _mtime = float(_mtime) 311 return _path, _sha256, _mtime
312
313 - def close(self):
314 if self._file is not None: 315 self._file.close() 316 self._file = None
317 318
319 -def generate_content_safety_file(content_safety):
320 """ 321 Generate a file containing the "content_safety" metadata, 322 reading by content_safety list or iterator. Each item 323 of "content_safety" must contain (path, sha256, mtime). 324 Each item shall be written to file, one per line, 325 in the following form: "<mtime>|<sha256>|<path>". 326 The order of the element in "content_safety" will be kept. 327 """ 328 tmp_dir = os.path.join( 329 etpConst['entropyunpackdir'], 330 "__generate_content_safety_file_f") 331 try: 332 os.makedirs(tmp_dir, 0o755) 333 except OSError as err: 334 if err.errno != errno.EEXIST: 335 raise 336 337 tmp_fd, tmp_path = None, None 338 generated = False 339 try: 340 tmp_fd, tmp_path = const_mkstemp( 341 prefix="PackageContentSafety", 342 dir=tmp_dir) 343 with FileContentSafetyWriter(tmp_fd) as tmp_f: 344 for path, sha256, mtime in content_safety: 345 tmp_f.write(path, sha256, mtime) 346 347 generated = True 348 return tmp_path 349 finally: 350 if tmp_fd is not None: 351 try: 352 os.close(tmp_fd) 353 except OSError: 354 pass 355 if tmp_path is not None and not generated: 356 try: 357 os.remove(tmp_path) 358 except (OSError, IOError): 359 pass
360 361
362 -def generate_content_file(content, package_id = None, 363 filter_splitdebug = False, 364 splitdebug = None, 365 splitdebug_dirs = None):
366 """ 367 Generate a file containing the "content" metadata, 368 reading by content list or iterator. Each item 369 of "content" must contain (path, ftype). 370 Each item shall be written to file, one per line, 371 in the following form: "[<package_id>|]<ftype>|<path>". 372 The order of the element in "content" will be kept. 373 """ 374 tmp_dir = os.path.join( 375 etpConst['entropyunpackdir'], 376 "__generate_content_file_f") 377 try: 378 os.makedirs(tmp_dir, 0o755) 379 except OSError as err: 380 if err.errno != errno.EEXIST: 381 raise 382 383 tmp_fd, tmp_path = None, None 384 generated = False 385 try: 386 tmp_fd, tmp_path = const_mkstemp( 387 prefix="PackageContent", 388 dir=tmp_dir) 389 with FileContentWriter(tmp_fd) as tmp_f: 390 for path, ftype in content: 391 if filter_splitdebug and not splitdebug: 392 # if filter_splitdebug is enabled, this 393 # code filters out all the paths starting 394 # with splitdebug_dirs, if splitdebug is 395 # disabled for package. 396 _skip = False 397 for split_dir in splitdebug_dirs: 398 if path.startswith(split_dir): 399 _skip = True 400 break 401 if _skip: 402 continue 403 tmp_f.write(package_id, path, ftype) 404 405 generated = True 406 return tmp_path 407 finally: 408 if tmp_fd is not None: 409 try: 410 os.close(tmp_fd) 411 except OSError: 412 pass 413 if tmp_path is not None and not generated: 414 try: 415 os.remove(tmp_path) 416 except (OSError, IOError): 417 pass
418 419
420 -def merge_content_file(content_file, sorted_content, 421 cmp_func):
422 """ 423 Given a sorted content_file content and a sorted list of 424 content (sorted_content), apply the "merge" step of a merge 425 sort algorithm. In other words, add the sorted_content to 426 content_file keeping content_file content ordered. 427 It is of couse O(n+m) where n = lines in content_file and 428 m = sorted_content length. 429 """ 430 tmp_content_file = content_file + FileContentWriter.TMP_SUFFIX 431 432 sorted_ptr = 0 433 _sorted_path = None 434 _sorted_ftype = None 435 _package_id = 0 # will be filled 436 try: 437 with FileContentWriter(tmp_content_file) as tmp_w: 438 with FileContentReader(content_file) as tmp_r: 439 for _package_id, _path, _ftype in tmp_r: 440 441 while True: 442 443 try: 444 _sorted_path, _sorted_ftype = \ 445 sorted_content[sorted_ptr] 446 except IndexError: 447 _sorted_path = None 448 _sorted_ftype = None 449 450 if _sorted_path is None: 451 tmp_w.write(_package_id, _path, _ftype) 452 break 453 454 cmp_outcome = cmp_func(_path, _sorted_path) 455 if cmp_outcome < 0: 456 tmp_w.write(_package_id, _path, _ftype) 457 break 458 459 # always privilege _ftype over _sorted_ftype 460 # _sorted_ftype might be invalid 461 tmp_w.write( 462 _package_id, _sorted_path, _ftype) 463 sorted_ptr += 1 464 if cmp_outcome == 0: 465 # write only one 466 break 467 468 # add the remainder 469 if sorted_ptr < len(sorted_content): 470 _sorted_rem = sorted_content[sorted_ptr:] 471 for _sorted_path, _sorted_ftype in _sorted_rem: 472 tmp_w.write( 473 _package_id, _sorted_path, _sorted_ftype) 474 475 os.rename(tmp_content_file, content_file) 476 finally: 477 try: 478 os.remove(tmp_content_file) 479 except OSError as err: 480 if err.errno != errno.ENOENT: 481 raise
482 483
484 -def filter_content_file(content_file, filter_func):
485 """ 486 This method rewrites the content of content_file by applying 487 a filter to the path elements. 488 """ 489 tmp_content_file = content_file + FileContentWriter.TMP_SUFFIX 490 try: 491 with FileContentWriter(tmp_content_file) as tmp_w: 492 with FileContentReader(content_file) as tmp_r: 493 for _package_id, _path, _ftype in tmp_r: 494 if filter_func(_path): 495 tmp_w.write(_package_id, _path, _ftype) 496 os.rename(tmp_content_file, content_file) 497 finally: 498 try: 499 os.remove(tmp_content_file) 500 except OSError as err: 501 if err.errno != errno.ENOENT: 502 raise
503