Back to index

enigmail  1.4.3
Classes | Functions | Variables
optimizejars Namespace Reference

Classes

class  MyStruct
class  BinaryBlob

Functions

def format_struct
def size_of
def assert_true
def optimizejar
def optimize
def deoptimize
def main

Variables

list local_file_header
list cdir_entry
list cdir_end
dictionary type_mapping = { "uint32":"I", "uint16":"H"}
int ENDSIG = 0x06054b50
tuple jar_regex = re.compile("\\.jar?$")

Function Documentation

def optimizejars.assert_true (   cond,
  msg 
)

Definition at line 145 of file optimizejars.py.

00145 
00146 def assert_true(cond, msg):
00147     if not cond:
00148         raise Exception(msg)
00149         exit(1)

Here is the caller graph for this function:

def optimizejars.deoptimize (   JAR_LOG_DIR,
  IN_JAR_DIR,
  OUT_JAR_DIR 
)

Definition at line 346 of file optimizejars.py.

00346 
00347 def deoptimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR):
00348     if not os.path.exists(JAR_LOG_DIR):
00349         os.makedirs(JAR_LOG_DIR)
00350 
00351     ls = os.listdir(IN_JAR_DIR)
00352     for jarfile in ls:
00353         if not re.search(jar_regex, jarfile):
00354             continue
00355         injarfile = os.path.join(IN_JAR_DIR, jarfile)
00356         outjarfile = os.path.join(OUT_JAR_DIR, jarfile) 
00357         logfile = os.path.join(JAR_LOG_DIR, jarfile + ".log")
00358         log = optimizejar(injarfile, outjarfile, None)
00359         open(logfile, "wb").write("\n".join(log))

Here is the call graph for this function:

Here is the caller graph for this function:

def optimizejars.format_struct (   format)

Definition at line 93 of file optimizejars.py.

00093 
00094 def format_struct (format):
00095     string_fields = {}
00096     fmt = "<"
00097     for (name,value) in iter(format):
00098         try:
00099             fmt += type_mapping[value][0]
00100         except KeyError:
00101             string_fields[name] = value
00102     return (fmt, string_fields)

Here is the caller graph for this function:

Definition at line 360 of file optimizejars.py.

00360 
00361 def main():        
00362     MODE = sys.argv[1]
00363     JAR_LOG_DIR = sys.argv[2]
00364     IN_JAR_DIR = sys.argv[3]
00365     OUT_JAR_DIR = sys.argv[4]
00366     if MODE == "--optimize":
00367         optimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR)
00368     elif MODE == "--deoptimize":
00369         deoptimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR)
00370     else:
00371         print("Unknown mode %s" % MODE)
00372         exit(1)

Here is the call graph for this function:

def optimizejars.optimize (   JAR_LOG_DIR,
  IN_JAR_DIR,
  OUT_JAR_DIR 
)

Definition at line 334 of file optimizejars.py.

00334 
00335 def optimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR):
00336     ls = os.listdir(IN_JAR_DIR)
00337     for jarfile in ls:
00338         if not re.search(jar_regex, jarfile):
00339             continue
00340         injarfile = os.path.join(IN_JAR_DIR, jarfile)
00341         outjarfile = os.path.join(OUT_JAR_DIR, jarfile) 
00342         logfile = os.path.join(JAR_LOG_DIR, jarfile + ".log")
00343         if not os.path.isfile(logfile):
00344             logfile = None
00345         optimizejar(injarfile, outjarfile, logfile)

Here is the call graph for this function:

Here is the caller graph for this function:

def optimizejars.optimizejar (   jar,
  outjar,
  inlog = None 
)

Definition at line 186 of file optimizejars.py.

00186 
00187 def optimizejar(jar, outjar, inlog = None):
00188     if inlog is not None:
00189         inlog = open(inlog).read().rstrip()
00190         # in the case of an empty log still move the index forward
00191         if len(inlog) == 0:
00192             inlog = []
00193         else:
00194             inlog = inlog.split("\n")
00195     outlog = []
00196     jarblob = BinaryBlob(jar)
00197     dirend = jarblob.read_struct(cdir_end, jarblob.length - size_of(cdir_end))
00198     assert_true(dirend.signature == ENDSIG, "no signature in the end");
00199     cdir_offset = dirend.cdir_offset
00200     readahead = 0
00201     if inlog is None and cdir_offset == 4:
00202         readahead = struct.unpack("<I", jarblob.readAt(0, 4))[0]
00203         print("%s: startup data ends at byte %d" % (outjar, readahead));
00204 
00205     total_stripped = 0;
00206     jarblob.offset = cdir_offset
00207     central_directory = []
00208     for i in range(0, dirend.cdir_entries):
00209         entry = jarblob.read_struct(cdir_entry)
00210         if entry.filename[-1:] == "/":
00211             total_stripped += len(entry.pack())
00212         else:
00213             total_stripped += entry.extrafield_size
00214         central_directory.append(entry)
00215         
00216     reordered_count = 0
00217     if inlog is not None:
00218         dup_guard = set()
00219         for ordered_name in inlog:
00220             if ordered_name in dup_guard:
00221                 continue
00222             else:
00223                 dup_guard.add(ordered_name)
00224             found = False
00225             for i in range(reordered_count, len(central_directory)):
00226                 if central_directory[i].filename == ordered_name:
00227                     # swap the cdir entries
00228                     tmp = central_directory[i]
00229                     central_directory[i] = central_directory[reordered_count]
00230                     central_directory[reordered_count] = tmp
00231                     reordered_count = reordered_count + 1
00232                     found = True
00233                     break
00234             if not found:
00235                 print( "Can't find '%s' in %s" % (ordered_name, jar))
00236 
00237     outfd = open(outjar, "wb")
00238     out_offset = 0
00239     if inlog is not None:
00240         # have to put central directory at offset 4 cos 0 confuses some tools.
00241         # This also lets us specify how many entries should be preread
00242         dirend.cdir_offset = 4
00243         # make room for central dir + end of dir + 4 extra bytes at front
00244         out_offset = dirend.cdir_offset + dirend.cdir_size + size_of(cdir_end) - total_stripped
00245         outfd.seek(out_offset)
00246 
00247     cdir_data = ""
00248     written_count = 0
00249     crc_mapping = {}
00250     dups_found = 0
00251     dupe_bytes = 0
00252     # store number of bytes suggested for readahead
00253     for entry in central_directory:
00254         # read in the header twice..first for comparison, second time for convenience when writing out
00255         jarfile = jarblob.read_struct(local_file_header, entry.offset)
00256         assert_true(jarfile.filename == entry.filename, "Directory/Localheader mismatch")
00257         # drop directory entries
00258         if entry.filename[-1:] == "/":
00259             total_stripped += len(jarfile.pack())
00260             dirend.cdir_entries -= 1
00261             continue
00262         # drop extra field data
00263         else:
00264             total_stripped += jarfile.extra_field_size;
00265         entry.extrafield = jarfile.extra_field = ""
00266         entry.extrafield_size = jarfile.extra_field_size = 0
00267         # January 1st, 2010
00268         entry.lastmod_date = jarfile.lastmod_date = ((2010 - 1980) << 9) | (1 << 5) | 1
00269         entry.lastmod_time = jarfile.lastmod_time = 0
00270         data = jarfile.pack()
00271         outfd.write(data)
00272         old_entry_offset = entry.offset
00273         entry.offset = out_offset
00274         out_offset = out_offset + len(data)
00275         entry_data = entry.pack()
00276         cdir_data += entry_data
00277         expected_len = entry.filename_size + entry.extrafield_size + entry.filecomment_size
00278         assert_true(len(entry_data) != expected_len,
00279                     "%s entry size - expected:%d got:%d" % (entry.filename, len(entry_data), expected_len))
00280         written_count += 1
00281 
00282         if entry.crc32 in crc_mapping:
00283             dups_found += 1
00284             dupe_bytes += entry.compressed_size + len(data) + len(entry_data)
00285             print("%s\n\tis a duplicate of\n%s\n---"%(entry.filename, crc_mapping[entry.crc32]))
00286         else:
00287             crc_mapping[entry.crc32] = entry.filename;
00288 
00289         if inlog is not None:
00290             if written_count == reordered_count:
00291                 readahead = out_offset
00292                 print("%s: startup data ends at byte %d"%( outjar, readahead));
00293             elif written_count < reordered_count:
00294                 pass
00295                 #print("%s @ %d" % (entry.filename, out_offset))
00296         elif readahead >= old_entry_offset + len(data):
00297             outlog.append(entry.filename)
00298             reordered_count += 1
00299 
00300     if inlog is None:
00301         dirend.cdir_offset = out_offset
00302 
00303     if dups_found > 0:
00304         print("WARNING: Found %d duplicate files taking %d bytes"%(dups_found, dupe_bytes))
00305 
00306     dirend.cdir_size = len(cdir_data)
00307     dirend.disk_entries = dirend.cdir_entries
00308     dirend_data = dirend.pack()
00309     assert_true(size_of(cdir_end) == len(dirend_data), "Failed to serialize directory end correctly. Serialized size;%d, expected:%d"%(len(dirend_data), size_of(cdir_end)));
00310 
00311     outfd.seek(dirend.cdir_offset)
00312     outfd.write(cdir_data)
00313     outfd.write(dirend_data)
00314 
00315     # for ordered jars the central directory is written in the begining of the file, so a second central-directory
00316     # entry has to be written in the end of the file
00317     if inlog is not None:
00318         outfd.seek(0)
00319         outfd.write(struct.pack("<I", readahead));
00320         outfd.seek(out_offset)
00321         outfd.write(dirend_data)
00322 
00323     print "Stripped %d bytes" % total_stripped
00324     print "%s %d/%d in %s" % (("Ordered" if inlog is not None else "Deoptimized"),
00325                               reordered_count, len(central_directory), outjar)
00326     outfd.close()
00327     return outlog
        

Here is the call graph for this function:

Here is the caller graph for this function:

def optimizejars.size_of (   format)

Definition at line 103 of file optimizejars.py.

00103 
00104 def size_of(format):
00105     return struct.calcsize(format_struct(format)[0])

Here is the call graph for this function:

Here is the caller graph for this function:


Variable Documentation

Initial value:
00001 [
00002     ("signature", "uint32"),
00003     ("disk_num", "uint16"),
00004     ("cdir_disk", "uint16"),
00005     ("disk_entries", "uint16"),
00006     ("cdir_entries", "uint16"),
00007     ("cdir_size", "uint32"),
00008     ("cdir_offset", "uint32"),
00009     ("comment_size", "uint16"),
00010 ]

Definition at line 80 of file optimizejars.py.

Initial value:
00001 [
00002     ("signature", "uint32"),
00003     ("creator_version", "uint16"),
00004     ("min_version", "uint16"),
00005     ("general_flag", "uint16"),
00006     ("compression", "uint16"),
00007     ("lastmod_time", "uint16"),
00008     ("lastmod_date", "uint16"),
00009     ("crc32", "uint32"),
00010     ("compressed_size", "uint32"),
00011     ("uncompressed_size", "uint32"),
00012     ("filename_size", "uint16"),
00013     ("extrafield_size", "uint16"),
00014     ("filecomment_size", "uint16"),
00015     ("disknum", "uint16"),
00016     ("internal_attr", "uint16"),
00017     ("external_attr", "uint32"),
00018     ("offset", "uint32"),
00019     ("filename", "filename_size"),
00020     ("extrafield", "extrafield_size"),
00021     ("filecomment", "filecomment_size"),
00022 ]

Definition at line 57 of file optimizejars.py.

int optimizejars.ENDSIG = 0x06054b50

Definition at line 143 of file optimizejars.py.

tuple optimizejars.jar_regex = re.compile("\\.jar?$")

Definition at line 332 of file optimizejars.py.

Initial value:
00001 [
00002     ("signature", "uint32"),
00003     ("min_version", "uint16"),
00004     ("general_flag", "uint16"),
00005     ("compression", "uint16"),
00006     ("lastmod_time", "uint16"),
00007     ("lastmod_date", "uint16"),
00008     ("crc32", "uint32"),
00009     ("compressed_size", "uint32"),
00010     ("uncompressed_size", "uint32"),
00011     ("filename_size", "uint16"),
00012     ("extra_field_size", "uint16"),
00013     ("filename", "filename_size"),
00014     ("extra_field", "extra_field_size"),
00015     ("data", "compressed_size")
00016 ]

Definition at line 40 of file optimizejars.py.

dictionary optimizejars.type_mapping = { "uint32":"I", "uint16":"H"}

Definition at line 91 of file optimizejars.py.