Back to index

python-biopython  1.60
do2to3.py
Go to the documentation of this file.
00001 """Helper script for building and installing Biopython on Python 3.
00002 
00003 Note that we can't just use distutils.command.build_py function build_py_2to3
00004 in setup.py since (as far as I can see) that does not allow us to alter the
00005 2to3 options. In particular, we need to turn off the long fixer for some of
00006 our files.
00007 
00008 This code is intended to be called from setup.py automatically under Python 3,
00009 and is not intended for end users. The basic idea follows the approach taken
00010 by NumPy with their setup.py file calling tools/py3tool.py to do the 2to3
00011 conversion automatically.
00012 
00013 This calls the lib2to3 library functions to convert the Biopython source code
00014 from Python 2 to Python 3, tracking changes to files so that unchanged files
00015 need not be reconverted making development much easier (i.e. if you edit one
00016 source file, doing 'python setup.py install' will only reconvert the one file).
00017 This is done by the last modified date stamps (which will be updated by git if
00018 you switch branches).
00019 
00020 NOTE - This is intended to be run under Python 3 (not under Python 2), but
00021 care has been taken to make it run under Python 2 enough to give a clear error
00022 message. In particular, this meant avoiding with statements etc.
00023 """
00024 import sys
00025 if sys.version_info[0] < 3:
00026     sys.stderr.write("Please run this under Python 3\n")
00027     sys.exit(1)
00028 
00029 import shutil
00030 import os
00031 import lib2to3.main
00032 from io import StringIO
00033 
00034 
00035 def run2to3(filenames):
00036     stderr = sys.stderr
00037     handle = StringIO()
00038     try:
00039         #Want to capture stderr (otherwise too noisy)
00040         sys.stderr = handle
00041         while filenames:
00042             filename = filenames.pop(0)
00043             print("Converting %s" % filename)
00044             #TODO - Configurable options per file?
00045             args = ["--nofix=long", "--no-diffs", "-n", "-w"]
00046             e = lib2to3.main.main("lib2to3.fixes", args + [filename])
00047             if e != 0:
00048                 sys.stderr = stderr
00049                 sys.stderr.write(handle.getvalue())
00050                 os.remove(filename) #Don't want a half edited file!
00051                 raise RuntimeError("Error %i from 2to3 on %s" \
00052                                    % (e, filename))
00053             #And again for any doctests,
00054             e = lib2to3.main.main("lib2to3.fixes", args + ["-d", filename])
00055             if e != 0:
00056                 sys.stderr = stderr
00057                 sys.stderr.write(handle.getvalue())
00058                 os.remove(filename) #Don't want a half edited file!
00059                 raise RuntimeError("Error %i from 2to3 (doctests) on %s" \
00060                                    % (e, filename))
00061     except KeyboardInterrupt:
00062         sys.stderr = stderr
00063         sys.stderr.write("Interrupted during %s\n" % filename)
00064         os.remove(filename) #Don't want a half edited file!
00065         for filename in filenames:
00066             if os.path.isfile(filename):
00067                 #Don't want uncoverted files left behind:
00068                 os.remove(filename)
00069         sys.exit(1)
00070     finally:
00071         #Restore stderr
00072         sys.stderr = stderr
00073 
00074 
00075 def do_update(py2folder, py3folder, verbose=False):
00076     if not os.path.isdir(py2folder):
00077         raise ValueError("Python 2 folder %r does not exist" % py2folder)
00078     if not os.path.isdir(py3folder):
00079         os.mkdir(py3folder)
00080     #First remove any files from the 3to2 conversion which no
00081     #longer existing the Python 2 origin (only expected to happen
00082     #on a development machine).
00083     for dirpath, dirnames, filenames in os.walk(py3folder):
00084         relpath = os.path.relpath(dirpath, py3folder)
00085         for d in dirnames:
00086             new = os.path.join(py3folder, relpath, d)
00087             old = os.path.join(py2folder, relpath, d)
00088             if not os.path.isdir(old):
00089                 print("Removing %s" % new)
00090                 shutil.rmtree(new)
00091         for f in filenames:
00092             new = os.path.join(py3folder, relpath, f)
00093             old = os.path.join(py2folder, relpath, f)
00094             if not os.path.isfile(old):
00095                 print("Removing %s" % new)
00096                 os.remove(new)
00097     #Check all the Python 2 original files have been copied/converted
00098     #Note we need to do all the conversions *after* copying the files
00099     #so that 2to3 can detect local imports successfully.
00100     to_convert = []
00101     for dirpath, dirnames, filenames in os.walk(py2folder):
00102         if verbose: print("Processing %s" % dirpath)
00103         relpath = os.path.relpath(dirpath, py2folder)
00104         #This is just to give cleaner filenames
00105         if relpath[:2] == "/.":
00106             relpath = relpath[2:]
00107         elif relpath == ".":
00108             relpath = ""
00109         for d in dirnames:
00110             new = os.path.join(py3folder, relpath, d)
00111             if not os.path.isdir(new):
00112                 os.mkdir(new)
00113         for f in filenames:
00114             if f.startswith("."):
00115                 #Ignore hidden files
00116                 continue
00117             elif f.endswith("~") or f.endswith(".bak") \
00118             or f.endswith(".swp"):
00119                 #Ignore backup files
00120                 continue
00121             elif f.endswith(".pyc") or f.endswith("$py.class"):
00122                 #Ignore compiled python
00123                 continue
00124             old = os.path.join(py2folder, relpath, f)
00125             new = os.path.join(py3folder, relpath, f)
00126             #The filesystem can (in Linux) record nanoseconds, but
00127             #when copying only microsecond accuracy is used.
00128             #See http://bugs.python.org/issue10148
00129             #Compare modified times down to milliseconds only. In theory
00130             #might able to use times down to microseconds (10^-6), but
00131             #that doesn't work on this Windows machine I'm testing on.
00132             if os.path.isfile(new) \
00133             and round(os.stat(new).st_mtime*1000) >= \
00134                 round(os.stat(old).st_mtime*1000):
00135                 if verbose: print("Current: %s" % new)
00136                 continue
00137             #Python, C code, data files, etc - copy with date stamp etc
00138             shutil.copy2(old, new)
00139             assert abs(os.stat(old).st_mtime-os.stat(new).st_mtime)<0.0001, \
00140                    "Modified time not copied! %0.8f vs %0.8f, diff %f" \
00141                    % (os.stat(old).st_mtime, os.stat(new).st_mtime,
00142                       abs(os.stat(old).st_mtime-os.stat(new).st_mtime))
00143             if f.endswith(".py"):
00144                 #Also run 2to3 on it
00145                 to_convert.append(new)
00146                 if verbose: print("Will convert %s" % new)
00147             else:
00148                 if verbose: print("Updated %s" % new)
00149     if to_convert:
00150         print("Have %i python files to convert" % len(to_convert))
00151         run2to3(to_convert)
00152 
00153             
00154 def main(python2_source, python3_source,
00155          children=["Bio", "BioSQL", "Tests", "Scripts", "Doc"]):
00156     #Note want to use different folders for Python 3.1, 3.2, etc
00157     #since the 2to3 libraries have changed so the conversion
00158     #may differ slightly.
00159     print("The 2to3 library will be called automatically now,")
00160     print("and the converted files cached under %s" % python3_source)
00161     if not os.path.isdir(python3_source):
00162         os.mkdir(python3_source)
00163     for child in children:
00164         print("Processing %s" % child)
00165         do_update(os.path.join(python2_source, child),
00166                   os.path.join(python3_source, child))
00167     print("Python 2to3 processing done.")
00168               
00169 if __name__ == "__main__":
00170     python2_source = "."
00171     python3_source = "build/py%i.%i" % sys.version_info[:2]
00172     main(python2_source, python3_source)