Back to index

python3.2  3.2.2
Functions | Variables
iobench Namespace Reference

Functions

def text_open
def get_file_sizes
def get_binary_files
def get_text_files
def with_open_mode
def with_sizes
def read_bytewise
def read_small_chunks
def read_big_chunks
def read_whole_file
def read_lines
def seek_forward_bytewise
def seek_forward_blockwise
def read_seek_bytewise
def read_seek_blockwise
def write_bytewise
def write_small_chunks
def write_medium_chunks
def write_large_chunks
def modify_bytewise
def modify_small_chunks
def modify_medium_chunks
def modify_seek_forward_bytewise
def modify_seek_forward_blockwise
def read_modify_bytewise
def read_modify_blockwise
def run_during
def warm_cache
def run_all_tests
def prepare_files
def main

Variables

 out = sys.stdout
string TEXT_ENCODING = 'utf8'
string NEWLINES = 'lf'
 xrange = range
list read_tests
list write_tests
list modify_tests

Function Documentation

Definition at line 36 of file iobench.py.

00036 
00037 def get_binary_files():
00038     return ((name + ".bin", size) for name, size in get_file_sizes())

Here is the call graph for this function:

Here is the caller graph for this function:

Definition at line 30 of file iobench.py.

00030 
00031 def get_file_sizes():
00032     for s in ['20 KB', '400 KB', '10 MB']:
00033         size, unit = s.split()
00034         size = int(size) * {'KB': 1024, 'MB': 1024 ** 2}[unit]
00035         yield s.replace(' ', ''), size

Here is the caller graph for this function:

Definition at line 39 of file iobench.py.

00039 
00040 def get_text_files():
00041     return (("%s-%s-%s.txt" % (name, TEXT_ENCODING, NEWLINES), size)
00042         for name, size in get_file_sizes())

Here is the call graph for this function:

Here is the caller graph for this function:

def iobench.main ( void  )

Definition at line 406 of file iobench.py.

00406 
00407 def main():
00408     global TEXT_ENCODING, NEWLINES
00409 
00410     usage = "usage: %prog [-h|--help] [options]"
00411     parser = OptionParser(usage=usage)
00412     parser.add_option("-b", "--binary",
00413                       action="store_true", dest="binary", default=False,
00414                       help="run binary I/O tests")
00415     parser.add_option("-t", "--text",
00416                       action="store_true", dest="text", default=False,
00417                       help="run text I/O tests")
00418     parser.add_option("-r", "--read",
00419                       action="store_true", dest="read", default=False,
00420                       help="run read tests")
00421     parser.add_option("-w", "--write",
00422                       action="store_true", dest="write", default=False,
00423                       help="run write & modify tests")
00424     parser.add_option("-E", "--encoding",
00425                       action="store", dest="encoding", default=None,
00426                       help="encoding for text tests (default: %s)" % TEXT_ENCODING)
00427     parser.add_option("-N", "--newlines",
00428                       action="store", dest="newlines", default='lf',
00429                       help="line endings for text tests "
00430                            "(one of: {lf (default), cr, crlf, all})")
00431     parser.add_option("-m", "--io-module",
00432                       action="store", dest="io_module", default=None,
00433                       help="io module to test (default: builtin open())")
00434     options, args = parser.parse_args()
00435     if args:
00436         parser.error("unexpected arguments")
00437     NEWLINES = options.newlines.lower()
00438     if NEWLINES not in ('lf', 'cr', 'crlf', 'all'):
00439         parser.error("invalid 'newlines' option: %r" % NEWLINES)
00440 
00441     test_options = ""
00442     if options.read:
00443         test_options += "r"
00444     if options.write:
00445         test_options += "w"
00446     elif not options.read:
00447         test_options += "rw"
00448     if options.text:
00449         test_options += "t"
00450     if options.binary:
00451         test_options += "b"
00452     elif not options.text:
00453         test_options += "tb"
00454 
00455     if options.encoding:
00456         TEXT_ENCODING = options.encoding
00457 
00458     if options.io_module:
00459         globals()['open'] = __import__(options.io_module, {}, {}, ['open']).open
00460 
00461     prepare_files()
00462     run_all_tests(test_options)

Here is the call graph for this function:

def iobench.modify_bytewise (   f,
  source 
)
modify one unit at a time 

Definition at line 166 of file iobench.py.

00166 
00167 def modify_bytewise(f, source):
00168     """ modify one unit at a time """
00169     f.seek(0)
00170     for i in xrange(0, len(source)):
00171         f.write(source[i:i+1])
00172 
00173 @with_open_mode("w+")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.modify_medium_chunks (   f,
  source 
)
modify 4096 units at a time 

Definition at line 182 of file iobench.py.

00182 
00183 def modify_medium_chunks(f, source):
00184     """ modify 4096 units at a time """
00185     f.seek(0)
00186     for i in xrange(0, len(source), 4096):
00187         f.write(source[i:i+4096])
00188 
00189 @with_open_mode("wb+")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.modify_seek_forward_blockwise (   f,
  source 
)
alternate write & seek 1000 units 

Definition at line 199 of file iobench.py.

00199 
00200 def modify_seek_forward_blockwise(f, source):
00201     """ alternate write & seek 1000 units """
00202     f.seek(0)
00203     for i in xrange(0, len(source), 2000):
00204         f.write(source[i:i+1000])
00205         f.seek(i+2000)
00206 
00207 # XXX the 2 following tests don't work with py3k's text IO
00208 @with_open_mode("wb+")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.modify_seek_forward_bytewise (   f,
  source 
)
alternate write & seek one unit 

Definition at line 190 of file iobench.py.

00190 
00191 def modify_seek_forward_bytewise(f, source):
00192     """ alternate write & seek one unit """
00193     f.seek(0)
00194     for i in xrange(0, len(source), 2):
00195         f.write(source[i:i+1])
00196         f.seek(i+2)
00197 
00198 @with_open_mode("wb+")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.modify_small_chunks (   f,
  source 
)
modify 20 units at a time 

Definition at line 174 of file iobench.py.

00174 
00175 def modify_small_chunks(f, source):
00176     """ modify 20 units at a time """
00177     f.seek(0)
00178     for i in xrange(0, len(source), 20):
00179         f.write(source[i:i+20])
00180 
00181 @with_open_mode("w+")
@with_sizes("medium")

Here is the call graph for this function:

Definition at line 364 of file iobench.py.

00364 
00365 def prepare_files():
00366     print("Preparing files...")
00367     # Binary files
00368     for name, size in get_binary_files():
00369         if os.path.isfile(name) and os.path.getsize(name) == size:
00370             continue
00371         with open(name, "wb") as f:
00372             f.write(os.urandom(size))
00373     # Text files
00374     chunk = []
00375     with text_open(__file__, "rU", encoding='utf8') as f:
00376         for line in f:
00377             if line.startswith("# <iobench text chunk marker>"):
00378                 break
00379         else:
00380             raise RuntimeError(
00381                 "Couldn't find chunk marker in %s !" % __file__)
00382         if NEWLINES == "all":
00383             it = itertools.cycle(["\n", "\r", "\r\n"])
00384         else:
00385             it = itertools.repeat(
00386                 {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES])
00387         chunk = "".join(line.replace("\n", next(it)) for line in f)
00388         if isinstance(chunk, bytes):
00389             chunk = chunk.decode('utf8')
00390         chunk = chunk.encode(TEXT_ENCODING)
00391     for name, size in get_text_files():
00392         if os.path.isfile(name) and os.path.getsize(name) == size:
00393             continue
00394         head = chunk * (size // len(chunk))
00395         tail = chunk[:size % len(chunk)]
00396         # Adjust tail to end on a character boundary
00397         while True:
00398             try:
00399                 tail.decode(TEXT_ENCODING)
00400                 break
00401             except UnicodeDecodeError:
00402                 tail = tail[:-1]
00403         with open(name, "wb") as f:
00404             f.write(head)
00405             f.write(tail)

Here is the call graph for this function:

Here is the caller graph for this function:

read 4096 units at a time 

Definition at line 76 of file iobench.py.

00076 
00077 def read_big_chunks(f):
00078     """ read 4096 units at a time """
00079     f.seek(0)
00080     while f.read(4096):
00081         pass
00082 
00083 @with_open_mode("r")
@with_sizes("small", "medium", "large")

Here is the call graph for this function:

def iobench.read_bytewise (   f)
read one unit at a time 

Definition at line 60 of file iobench.py.

00060 
00061 def read_bytewise(f):
00062     """ read one unit at a time """
00063     f.seek(0)
00064     while f.read(1):
00065         pass
00066 
00067 @with_open_mode("r")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.read_lines (   f)
read one line at a time 

Definition at line 92 of file iobench.py.

00092 
00093 def read_lines(f):
00094     """ read one line at a time """
00095     f.seek(0)
00096     for line in f:
00097         pass
00098 
00099 @with_open_mode("r")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.read_modify_blockwise (   f,
  source 
)
alternate read & write 1000 units 

Definition at line 218 of file iobench.py.

00218 
00219 def read_modify_blockwise(f, source):
00220     """ alternate read & write 1000 units """
00221     f.seek(0)
00222     for i in xrange(0, len(source), 2000):
00223         f.read(1000)
00224         f.write(source[i+1000:i+2000])
00225 

def iobench.read_modify_bytewise (   f,
  source 
)
alternate read & write one unit 

Definition at line 209 of file iobench.py.

00209 
00210 def read_modify_bytewise(f, source):
00211     """ alternate read & write one unit """
00212     f.seek(0)
00213     for i in xrange(0, len(source), 2):
00214         f.read(1)
00215         f.write(source[i+1:i+2])
00216 
00217 @with_open_mode("wb+")
@with_sizes("medium")

Here is the call graph for this function:

alternate read & seek 1000 units 

Definition at line 128 of file iobench.py.

00128 
00129 def read_seek_blockwise(f):
00130     """ alternate read & seek 1000 units """
00131     f.seek(0)
00132     while f.read(1000):
00133         f.seek(1000, 1)
00134 
00135 
00136 @with_open_mode("w")
@with_sizes("small")

Here is the call graph for this function:

alternate read & seek one unit 

Definition at line 120 of file iobench.py.

00120 
00121 def read_seek_bytewise(f):
00122     """ alternate read & seek one unit """
00123     f.seek(0)
00124     while f.read(1):
00125         f.seek(1, 1)
00126 
00127 @with_open_mode("rb")
@with_sizes("medium")

Here is the call graph for this function:

read 20 units at a time 

Definition at line 68 of file iobench.py.

00068 
00069 def read_small_chunks(f):
00070     """ read 20 units at a time """
00071     f.seek(0)
00072     while f.read(20):
00073         pass
00074 
00075 @with_open_mode("r")
@with_sizes("medium")

Here is the call graph for this function:

read whole contents at once 

Definition at line 84 of file iobench.py.

00084 
00085 def read_whole_file(f):
00086     """ read whole contents at once """
00087     f.seek(0)
00088     while f.read():
00089         pass
00090 
00091 @with_open_mode("rt")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.run_all_tests (   options)

Definition at line 264 of file iobench.py.

00264 
00265 def run_all_tests(options):
00266     def print_label(filename, func):
00267         name = re.split(r'[-.]', filename)[0]
00268         out.write(
00269             ("[%s] %s... "
00270                 % (name.center(7), func.__doc__.strip())
00271             ).ljust(52))
00272         out.flush()
00273 
00274     def print_results(size, n, real, cpu):
00275         bw = n * float(size) / 1024 ** 2 / real
00276         bw = ("%4d MB/s" if bw > 100 else "%.3g MB/s") % bw
00277         out.write(bw.rjust(12) + "\n")
00278         if cpu < 0.90 * real:
00279             out.write("   warning: test above used only %d%% CPU, "
00280                 "result may be flawed!\n" % (100.0 * cpu / real))
00281 
00282     def run_one_test(name, size, open_func, test_func, *args):
00283         mode = test_func.file_open_mode
00284         print_label(name, test_func)
00285         if "w" not in mode or "+" in mode:
00286             warm_cache(name)
00287         with open_func(name) as f:
00288             n, real, cpu = run_during(1.5, lambda: test_func(f, *args))
00289         print_results(size, n, real, cpu)
00290 
00291     def run_test_family(tests, mode_filter, files, open_func, *make_args):
00292         for test_func in tests:
00293             if test_func is None:
00294                 out.write("\n")
00295                 continue
00296             if mode_filter in test_func.file_open_mode:
00297                 continue
00298             for s in test_func.file_sizes:
00299                 name, size = files[size_names[s]]
00300                 #name += file_ext
00301                 args = tuple(f(name, size) for f in make_args)
00302                 run_one_test(name, size,
00303                     open_func, test_func, *args)
00304 
00305     size_names = {
00306         "small": 0,
00307         "medium": 1,
00308         "large": 2,
00309     }
00310 
00311     binary_files = list(get_binary_files())
00312     text_files = list(get_text_files())
00313     if "b" in options:
00314         print("Binary unit = one byte")
00315     if "t" in options:
00316         print("Text unit = one character (%s-decoded)" % TEXT_ENCODING)
00317 
00318     # Binary reads
00319     if "b" in options and "r" in options:
00320         print("\n** Binary input **\n")
00321         run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb"))
00322 
00323     # Text reads
00324     if "t" in options and "r" in options:
00325         print("\n** Text input **\n")
00326         run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r"))
00327 
00328     # Binary writes
00329     if "b" in options and "w" in options:
00330         print("\n** Binary append **\n")
00331         def make_test_source(name, size):
00332             with open(name, "rb") as f:
00333                 return f.read()
00334         run_test_family(write_tests, "t", binary_files,
00335             lambda fn: open(os.devnull, "wb"), make_test_source)
00336 
00337     # Text writes
00338     if "t" in options and "w" in options:
00339         print("\n** Text append **\n")
00340         def make_test_source(name, size):
00341             with text_open(name, "r") as f:
00342                 return f.read()
00343         run_test_family(write_tests, "b", text_files,
00344             lambda fn: text_open(os.devnull, "w"), make_test_source)
00345 
00346     # Binary overwrites
00347     if "b" in options and "w" in options:
00348         print("\n** Binary overwrite **\n")
00349         def make_test_source(name, size):
00350             with open(name, "rb") as f:
00351                 return f.read()
00352         run_test_family(modify_tests, "t", binary_files,
00353             lambda fn: open(fn, "r+b"), make_test_source)
00354 
00355     # Text overwrites
00356     if "t" in options and "w" in options:
00357         print("\n** Text overwrite **\n")
00358         def make_test_source(name, size):
00359             with text_open(name, "r") as f:
00360                 return f.read()
00361         run_test_family(modify_tests, "b", text_files,
00362             lambda fn: open(fn, "r+"), make_test_source)
00363 

Here is the call graph for this function:

Here is the caller graph for this function:

def iobench.run_during (   duration,
  func 
)

Definition at line 244 of file iobench.py.

00244 
00245 def run_during(duration, func):
00246     _t = time.time
00247     n = 0
00248     start = os.times()
00249     start_timestamp = _t()
00250     real_start = start[4] or start_timestamp
00251     while True:
00252         func()
00253         n += 1
00254         if _t() - start_timestamp > duration:
00255             break
00256     end = os.times()
00257     real = (end[4] if start[4] else time.time()) - real_start
00258     return n, real, sum(end[0:2]) - sum(start[0:2])

Here is the call graph for this function:

Here is the caller graph for this function:

seek forward 1000 units at a time 

Definition at line 110 of file iobench.py.

00110 
00111 def seek_forward_blockwise(f):
00112     """ seek forward 1000 units at a time """
00113     f.seek(0, 2)
00114     size = f.tell()
00115     f.seek(0, 0)
00116     for i in xrange(0, size - 1, 1000):
00117         f.seek(i, 0)
00118 
00119 @with_open_mode("rb")
@with_sizes("medium")

Here is the call graph for this function:

seek forward one unit at a time 

Definition at line 100 of file iobench.py.

00100 
00101 def seek_forward_bytewise(f):
00102     """ seek forward one unit at a time """
00103     f.seek(0, 2)
00104     size = f.tell()
00105     f.seek(0, 0)
00106     for i in xrange(0, size - 1):
00107         f.seek(i, 0)
00108 
00109 @with_open_mode("r")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.text_open (   fn,
  mode,
  encoding = None 
)

Definition at line 24 of file iobench.py.

00024 
00025 def text_open(fn, mode, encoding=None):
00026     try:
00027         return open(fn, mode, encoding=encoding or TEXT_ENCODING)
00028     except TypeError:
00029         return open(fn, mode)

Here is the caller graph for this function:

def iobench.warm_cache (   filename)

Definition at line 259 of file iobench.py.

00259 
00260 def warm_cache(filename):
00261     with open(filename, "rb") as f:
00262         f.read()
00263 

Here is the caller graph for this function:

def iobench.with_open_mode (   mode)

Definition at line 43 of file iobench.py.

00043 
00044 def with_open_mode(mode):
00045     def decorate(f):
00046         f.file_open_mode = mode
00047         return f
00048     return decorate

Here is the caller graph for this function:

def iobench.with_sizes (   sizes)

Definition at line 49 of file iobench.py.

00049 
00050 def with_sizes(*sizes):
00051     def decorate(f):
00052         f.file_sizes = sizes
00053         return f
00054     return decorate
00055 
00056 
00057 # Here begin the tests
00058 
00059 @with_open_mode("r")
@with_sizes("medium")

Here is the call graph for this function:

Here is the caller graph for this function:

def iobench.write_bytewise (   f,
  source 
)
write one unit at a time 

Definition at line 137 of file iobench.py.

00137 
00138 def write_bytewise(f, source):
00139     """ write one unit at a time """
00140     for i in xrange(0, len(source)):
00141         f.write(source[i:i+1])
00142 
00143 @with_open_mode("w")
@with_sizes("medium")

Here is the call graph for this function:

def iobench.write_large_chunks (   f,
  source 
)
write 1e6 units at a time 

Definition at line 158 of file iobench.py.

00158 
00159 def write_large_chunks(f, source):
00160     """ write 1e6 units at a time """
00161     for i in xrange(0, len(source), 1000000):
00162         f.write(source[i:i+1000000])
00163 
00164 
00165 @with_open_mode("w+")
@with_sizes("small")

Here is the call graph for this function:

def iobench.write_medium_chunks (   f,
  source 
)
write 4096 units at a time 

Definition at line 151 of file iobench.py.

00151 
00152 def write_medium_chunks(f, source):
00153     """ write 4096 units at a time """
00154     for i in xrange(0, len(source), 4096):
00155         f.write(source[i:i+4096])
00156 
00157 @with_open_mode("w")
@with_sizes("large")

Here is the call graph for this function:

def iobench.write_small_chunks (   f,
  source 
)
write 20 units at a time 

Definition at line 144 of file iobench.py.

00144 
00145 def write_small_chunks(f, source):
00146     """ write 20 units at a time """
00147     for i in xrange(0, len(source), 20):
00148         f.write(source[i:i+20])
00149 
00150 @with_open_mode("w")
@with_sizes("medium")

Here is the call graph for this function:


Variable Documentation

Initial value:
00001 [
00002     modify_bytewise, modify_small_chunks, modify_medium_chunks,
00003     None,
00004     modify_seek_forward_bytewise, modify_seek_forward_blockwise,
00005     read_modify_bytewise, read_modify_blockwise,
00006 ]

Definition at line 237 of file iobench.py.

Definition at line 16 of file iobench.py.

iobench.out = sys.stdout

Definition at line 13 of file iobench.py.

Initial value:
00001 [
00002     read_bytewise, read_small_chunks, read_lines, read_big_chunks,
00003     None, read_whole_file, None,
00004     seek_forward_bytewise, seek_forward_blockwise,
00005     read_seek_bytewise, read_seek_blockwise,
00006 ]

Definition at line 226 of file iobench.py.

Definition at line 15 of file iobench.py.

Initial value:
00001 [
00002     write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks,
00003 ]

Definition at line 233 of file iobench.py.

iobench.xrange = range

Definition at line 22 of file iobench.py.