Package entropy :: Package spm :: Package plugins :: Package interfaces :: Package portage_plugin :: Module xpak

Source Code for Module entropy.spm.plugins.interfaces.portage_plugin.xpak

  1  # -*- coding: utf-8 -*- 
  2  # Copyright 2001-2004 Gentoo Foundation 
  3  # Distributed under the terms of the GNU General Public License v2 
  4  # $Id: xpak.py 5595 2007-01-12 08:08:03Z antarus $ 
  5   
  6   
  7  # The format for a tbz2/xpak: 
  8  # 
  9  #  tbz2: tar.bz2 + xpak + (xpak_offset) + "STOP" 
 10  #  xpak: "XPAKPACK" + (index_len) + (data_len) + index + data + "XPAKSTOP" 
 11  # index: (pathname_len) + pathname + (data_offset) + (data_len) 
 12  #        index entries are concatenated end-to-end. 
 13  #  data: concatenated data chunks, end-to-end. 
 14  # 
 15  # [tarball]XPAKPACKIIIIDDDD[index][data]XPAKSTOPOOOOSTOP 
 16  # 
 17  # (integer) == encodeint(integer)  ===> 4 characters (big-endian copy) 
 18  # '+' means concatenate the fields ===> All chunks are strings 
 19   
 20  import sys 
 21  import os, shutil, errno 
 22  from stat import ST_SIZE, ST_MTIME, ST_CTIME 
 23  from entropy.const import const_convert_to_rawstring, \ 
 24      const_convert_to_unicode, const_is_python3 
 25   
 26  if const_is_python3(): 
 27      XPAKPACK = b"XPAKPACK" 
 28      XPAKSTOP = b"XPAKSTOP" 
 29      STOP = b"STOP" 
 30  else: 
 31      XPAKPACK = "XPAKPACK" 
 32      XPAKSTOP = "XPAKSTOP" 
 33      STOP = "STOP" 
 34   
35 -def _addtolist(mylist, curdir, _nested = False):
36 """(list, dir) --- Takes an array(list) and appends all files from dir down 37 the directory tree. Returns nothing. list is modified.""" 38 39 if not _nested: 40 curdir = os.path.normpath(curdir) 41 for x in os.listdir(curdir): 42 x_path = os.path.join(curdir, x) 43 if os.path.isdir(x_path): 44 _addtolist(mylist, x_path, _nested = True) 45 elif x_path not in mylist: 46 mylist.append(x_path) 47 48 if not _nested: 49 for idx in range(len(mylist)): 50 mylist[idx] = mylist[idx][len(curdir)+1:]
51
52 -def encodeint(myint):
53 """Takes a 4 byte integer and converts it into a string of 4 characters. 54 Returns the characters in a string.""" 55 part1 = chr((myint >> 24 ) & 0x000000ff) 56 part2 = chr((myint >> 16 ) & 0x000000ff) 57 part3 = chr((myint >> 8 ) & 0x000000ff) 58 part4 = chr(myint & 0x000000ff) 59 if const_is_python3(): 60 return bytes(part1 + part2 + part3 + part4, 'raw_unicode_escape') 61 else: 62 return part1 + part2 + part3 + part4
63
64 -def decodeint(mystring):
65 """Takes a 4 byte string and converts it into a 4 byte integer. 66 Returns an integer.""" 67 myint = 0 68 if const_is_python3(): 69 myint = myint+mystring[3] 70 myint = myint+(mystring[2] << 8) 71 myint = myint+(mystring[1] << 16) 72 myint = myint+(mystring[0] << 24) 73 else: 74 myint = myint+ord(mystring[3]) 75 myint = myint+(ord(mystring[2]) << 8) 76 myint = myint+(ord(mystring[1]) << 16) 77 myint = myint+(ord(mystring[0]) << 24) 78 return myint
79
80 -def xpak(rootdir, outfile=None):
81 """(rootdir,outfile) -- creates an xpak segment of the directory 'rootdir' 82 and under the name 'outfile' if it is specified. Otherwise it returns the 83 xpak segment.""" 84 mylist = [] 85 _addtolist(mylist, const_convert_to_rawstring(rootdir)) 86 87 mylist.sort() 88 mydata = {} 89 for x in mylist: 90 x_path = os.path.join(const_convert_to_rawstring(rootdir), x) 91 with open(x_path, "rb") as a: 92 mydata[x] = a.read() 93 94 xpak_segment = xpak_mem(mydata) 95 if outfile: 96 outf = open(outfile, "wb") 97 outf.write(xpak_segment) 98 outf.close() 99 else: 100 return xpak_segment
101
102 -def xpak_mem(mydata):
103 """Create an xpack segement from a map object.""" 104 if const_is_python3(): 105 indexglob = b"" 106 dataglob = b"" 107 else: 108 indexglob = "" 109 dataglob = "" 110 indexpos = 0 111 datapos = 0 112 113 for x, newglob in mydata.items(): 114 mydatasize = len(newglob) 115 int_enc_dir = encodeint(len(x)) + x 116 indexglob += int_enc_dir + encodeint(datapos) + encodeint(mydatasize) 117 indexpos = indexpos + 4 + len(x) + 4 + 4 118 dataglob = dataglob + newglob 119 datapos += mydatasize 120 return XPAKPACK \ 121 + encodeint(len(indexglob)) \ 122 + encodeint(len(dataglob)) \ 123 + indexglob \ 124 + dataglob \ 125 + XPAKSTOP
126
127 -def xsplit(infile):
128 """(infile) -- Splits the infile into two files. 129 'infile.index' contains the index segment. 130 'infile.dat' contails the data segment.""" 131 myfile = open(infile, "rb") 132 mydat = myfile.read() 133 myfile.close() 134 135 splits = xsplit_mem(mydat) 136 if not splits: 137 return False 138 139 myfile = open(infile+".index", "wb") 140 myfile.write(splits[0]) 141 myfile.close() 142 myfile = open(infile+".dat", "wb") 143 myfile.write(splits[1]) 144 myfile.close() 145 return True
146
147 -def xsplit_mem(mydat):
148 if mydat[0:8] != XPAKPACK: 149 return None 150 if mydat[-8:] != XPAKSTOP: 151 return None 152 indexsize = decodeint(mydat[8:12]) 153 #datasize = decodeint(mydat[12:16]) not used 154 return (mydat[16:indexsize+16], mydat[indexsize+16:-8])
155
156 -def getindex(infile):
157 """(infile) -- grabs the index segment from the infile and returns it.""" 158 myfile = open(infile, "rb") 159 myheader = myfile.read(16) 160 if myheader[0:8] != XPAKPACK: 161 myfile.close() 162 return 163 indexsize = decodeint(myheader[8:12]) 164 myindex = myfile.read(indexsize) 165 myfile.close() 166 return myindex
167
168 -def getboth(infile):
169 """(infile) -- grabs the index and data segments from the infile. 170 Returns an array [indexSegment,dataSegment]""" 171 myfile = open(infile, "rb") 172 myheader = myfile.read(16) 173 if myheader[0:8] != XPAKPACK: 174 myfile.close() 175 return 176 indexsize = decodeint(myheader[8:12]) 177 datasize = decodeint(myheader[12:16]) 178 myindex = myfile.read(indexsize) 179 mydata = myfile.read(datasize) 180 myfile.close() 181 return myindex, mydata
182
183 -def listindex(myindex):
184 """Print to the terminal the filenames listed in the indexglob passed in.""" 185 for x in getindex_mem(myindex): 186 print(x)
187
188 -def getindex_mem(myindex):
189 """Returns the filenames listed in the indexglob passed in.""" 190 myindexlen = len(myindex) 191 startpos = 0 192 myret = [] 193 while ((startpos+8) < myindexlen): 194 mytestlen = decodeint(myindex[startpos:startpos+4]) 195 myret = myret + [myindex[startpos+4:startpos+4+mytestlen]] 196 startpos = startpos + mytestlen + 12 197 return myret
198
199 -def searchindex(myindex, myitem):
200 """(index,item) -- Finds the offset and length of the file 'item' in the 201 datasegment via the index 'index' provided.""" 202 mylen = len(myitem) 203 myindexlen = len(myindex) 204 startpos = 0 205 while ((startpos+8) < myindexlen): 206 mytestlen = decodeint(myindex[startpos:startpos+4]) 207 if mytestlen == mylen: 208 if myitem == myindex[startpos+4:startpos+4+mytestlen]: 209 #found 210 datapos = decodeint(myindex[startpos+4+mytestlen:startpos+8+mytestlen]) 211 datalen = decodeint(myindex[startpos+8+mytestlen:startpos+12+mytestlen]) 212 return datapos, datalen 213 startpos = startpos+mytestlen+12
214
215 -def getitem(myid, myitem):
216 myindex = myid[0] 217 mydata = myid[1] 218 myloc = searchindex(myindex, myitem) 219 if not myloc: 220 return None 221 return mydata[myloc[0]:myloc[0]+myloc[1]]
222
223 -def xpand(myid, mydest):
224 myindex = myid[0] 225 mydata = myid[1] 226 227 myindexlen = len(myindex) 228 startpos = 0 229 while ((startpos+8) < myindexlen): 230 namelen = decodeint(myindex[startpos:startpos+4]) 231 datapos = decodeint(myindex[startpos+4+namelen:startpos+8+namelen]) 232 datalen = decodeint(myindex[startpos+8+namelen:startpos+12+namelen]) 233 myname = myindex[startpos+4:startpos+4+namelen] 234 myname = os.path.join(mydest, const_convert_to_unicode(myname)) 235 dirname = os.path.dirname(myname) 236 if not os.path.exists(dirname): 237 os.makedirs(dirname) 238 with open(myname, "wb") as mydat: 239 mydat.write(mydata[datapos:datapos+datalen]) 240 startpos = startpos+namelen+12
241 242
243 -class tbz2:
244 - def __init__(self, myfile):
245 self.file = myfile 246 self.filestat = None 247 self.index = "" 248 self.infosize = 0 249 self.xpaksize = 0 250 self.indexsize = None 251 self.datasize = None 252 self.indexpos = None 253 self.datapos = None 254 self.scan()
255
256 - def decompose(self, datadir, cleanup=1):
257 """Alias for unpackinfo() --- Complement to recompose() but optionally 258 deletes the destination directory. Extracts the xpak from the tbz2 into 259 the directory provided. Raises IOError if scan() fails. 260 Returns result of upackinfo().""" 261 if not self.scan(): 262 raise IOError 263 if cleanup: 264 self.cleanup(datadir) 265 if not os.path.exists(datadir): 266 os.makedirs(datadir) 267 return self.unpackinfo(datadir)
268
269 - def compose(self, datadir, cleanup = 0):
270 """Alias for recompose().""" 271 return self.recompose(datadir, cleanup)
272
273 - def recompose(self, datadir, cleanup = 0):
274 """Creates an xpak segment from the datadir provided, truncates the tbz2 275 to the end of regular data if an xpak segment already exists, and adds 276 the new segment to the file with terminating info.""" 277 xpdata = xpak(datadir) 278 self.recompose_mem(xpdata) 279 if cleanup: 280 self.cleanup(datadir)
281
282 - def recompose_mem(self, xpdata):
283 self.scan() 284 with open(self.file, "ab+") as myfile: 285 myfile.seek(-self.xpaksize, os.SEEK_END) 286 myfile.truncate() 287 myfile.write(xpdata+encodeint(len(xpdata))+STOP) 288 return 1
289
290 - def cleanup(self, datadir):
291 datadir_split = os.path.split(datadir) 292 if len(datadir_split) >= 2 and len(datadir_split[1]) > 0: 293 # This is potentially dangerous, 294 # thus the above sanity check. 295 try: 296 shutil.rmtree(datadir) 297 except OSError as oe: 298 if oe.errno == errno.ENOENT: 299 pass 300 else: 301 raise oe
302
303 - def scan(self):
304 """Scans the tbz2 to locate the xpak segment and setup internal values. 305 This function is called by relevant functions already.""" 306 try: 307 mystat = os.stat(self.file) 308 if self.filestat: 309 changed = 0 310 for x in [ST_SIZE, ST_MTIME, ST_CTIME]: 311 if mystat[x] != self.filestat[x]: 312 changed = 1 313 if not changed: 314 return 1 315 self.filestat = mystat 316 a = open(self.file, "rb") 317 a.seek(-16, os.SEEK_END) 318 trailer = a.read() 319 self.infosize = 0 320 self.xpaksize = 0 321 if trailer[-4:] != STOP: 322 a.close() 323 return 0 324 if trailer[0:8] != XPAKSTOP: 325 a.close() 326 return 0 327 self.infosize = decodeint(trailer[8:12]) 328 self.xpaksize = self.infosize+8 329 a.seek(-(self.xpaksize), os.SEEK_END) 330 header = a.read(16) 331 if header[0:8] != XPAKPACK: 332 a.close() 333 return 0 334 self.indexsize = decodeint(header[8:12]) 335 self.datasize = decodeint(header[12:16]) 336 self.indexpos = a.tell() 337 self.index = a.read(self.indexsize) 338 self.datapos = a.tell() 339 a.close() 340 return 2 341 except SystemExit: 342 raise 343 except: 344 return 0
345
346 - def filelist(self):
347 """Return an array of each file listed in the index.""" 348 if not self.scan(): 349 return None 350 return getindex_mem(self.index)
351
352 - def getfile(self, myfile, mydefault=None):
353 """Finds 'myfile' in the data segment and returns it.""" 354 if not self.scan(): 355 return None 356 myresult = searchindex(self.index, myfile) 357 if not myresult: 358 return mydefault 359 a = open(self.file, "rb") 360 a.seek(self.datapos + myresult[0], 0) 361 myreturn = a.read(myresult[1]) 362 a.close() 363 return myreturn
364
365 - def getelements(self, myfile):
366 """A split/array representation of tbz2.getfile()""" 367 mydat = self.getfile(myfile) 368 if not mydat: 369 return [] 370 return mydat.split()
371
372 - def unpackinfo(self, mydest):
373 """Unpacks all the files from the dataSegment into 'mydest'.""" 374 if not self.scan(): 375 return 0 376 377 a = open(self.file, "rb") 378 startpos = 0 379 380 while ((startpos+8) < self.indexsize): 381 namelen = decodeint(self.index[startpos:startpos+4]) 382 datapos = decodeint(self.index[startpos+4+namelen:startpos+8+namelen]) 383 datalen = decodeint(self.index[startpos+8+namelen:startpos+12+namelen]) 384 myname = self.index[startpos+4:startpos+4+namelen] 385 myname = os.path.join(mydest, myname) 386 dirname = os.path.dirname(myname) 387 if not os.path.exists(dirname): 388 os.makedirs(dirname) 389 a.seek(self.datapos + datapos) 390 with open(myname, "wb") as mydat: 391 mydat.write(a.read(datalen)) 392 startpos = startpos + namelen + 12 393 394 a.close() 395 return 1
396
397 - def get_data(self):
398 """Returns all the files from the dataSegment as a map object.""" 399 if not self.scan(): 400 return 0 401 a = open(self.file, "rb") 402 mydata = {} 403 startpos = 0 404 while ((startpos+8) < self.indexsize): 405 namelen = decodeint(self.index[startpos:startpos+4]) 406 datapos = decodeint(self.index[startpos+4+namelen:startpos+8+namelen]) 407 datalen = decodeint(self.index[startpos+8+namelen:startpos+12+namelen]) 408 myname = self.index[startpos+4:startpos+4+namelen] 409 a.seek(self.datapos+datapos) 410 mydata[myname] = a.read(datalen) 411 startpos = startpos+namelen+12 412 a.close() 413 return mydata
414
415 - def getboth(self):
416 """Returns an array [indexSegment,dataSegment]""" 417 if not self.scan(): 418 return None 419 420 a = open(self.file, "rb") 421 a.seek(self.datapos) 422 mydata = a.read(self.datasize) 423 a.close() 424 425 return self.index, mydata
426