Back to index

python-biopython  1.60
run_tests.py
Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 # This code is part of the Biopython distribution and governed by its
00003 # license.  Please see the LICENSE file that should have been included
00004 # as part of this package.
00005 """Run a set of PyUnit-based regression tests.
00006 
00007 This will find all modules whose name is "test_*.py" in the test
00008 directory, and run them.  Various command line options provide
00009 additional facilities.
00010 
00011 Command line options:
00012 
00013 --help        -- show usage info
00014 --offline     -- skip tests which require internet access
00015 -g;--generate -- write the output file for a test instead of comparing it.
00016                  The name of the test to write the output for must be
00017                  specified.
00018 -v;--verbose  -- run tests with higher verbosity (does not affect our
00019                  print-and-compare style unit tests).
00020 <test_name>   -- supply the name of one (or more) tests to be run.
00021                  The .py file extension is optional.
00022 doctest       -- run the docstring tests.
00023 By default, all tests are run.
00024 """
00025 
00026 # The default verbosity (not verbose)
00027 VERBOSITY = 0
00028 
00029 # standard modules
00030 import sys
00031 import cStringIO
00032 import os
00033 import re
00034 import getopt
00035 import time
00036 import traceback
00037 import unittest
00038 import doctest
00039 import distutils.util
00040 import gc
00041 
00042 def is_pypy():
00043     import platform
00044     try:
00045         if platform.python_implementation()=='PyPy':
00046             return True
00047     except AttributeError:
00048         #New in Python 2.6, not in Jython yet either
00049         pass
00050     return False
00051 
00052 def is_numpy():
00053     if is_pypy():
00054         return False
00055     try:
00056         import numpy
00057         del numpy
00058         return True
00059     except ImportError:
00060         return False
00061 
00062 # This is the list of modules containing docstring tests.
00063 # If you develop docstring tests for other modules, please add
00064 # those modules here. Please sort names alphabetically.
00065 DOCTEST_MODULES = [
00066                    "Bio.Align",
00067                    "Bio.Align.Generic",
00068                    "Bio.Align.Applications._Clustalw",
00069                    "Bio.Align.Applications._ClustalOmega",
00070                    "Bio.Align.Applications._Mafft",
00071                    "Bio.Align.Applications._Muscle",
00072                    "Bio.Align.Applications._Probcons",
00073                    "Bio.Align.Applications._Prank",
00074                    "Bio.Align.Applications._TCoffee",
00075                    "Bio.AlignIO",
00076                    "Bio.AlignIO.StockholmIO",
00077                    "Bio.Alphabet",
00078                    "Bio.Application",
00079                    "Bio.bgzf",
00080                    "Bio.Blast.Applications",
00081                    "Bio.Emboss.Applications",
00082                    "Bio.GenBank",
00083                    "Bio.KEGG.Compound",
00084                    "Bio.KEGG.Enzyme",
00085                    "Bio.Motif",
00086                    "Bio.pairwise2",
00087                    "Bio.Phylo.Applications._Raxml",
00088                    "Bio.Seq",
00089                    "Bio.SeqIO",
00090                    "Bio.SeqIO.AceIO",
00091                    "Bio.SeqIO.PhdIO",
00092                    "Bio.SeqIO.QualityIO",
00093                    "Bio.SeqIO.SffIO",
00094                    "Bio.SeqFeature",
00095                    "Bio.SeqRecord",
00096                    "Bio.SeqUtils",
00097                    "Bio.SeqUtils.MeltingTemp",
00098                    "Bio.Sequencing.Applications._Novoalign",
00099                    "Bio.Wise",
00100                    "Bio.Wise.psw",
00101                   ]
00102 #Silently ignore any doctests for modules requiring numpy!
00103 if is_numpy():
00104     DOCTEST_MODULES.extend(["Bio.Statistics.lowess",
00105                             "Bio.PDB.Polypeptide",
00106                             "Bio.PDB.Selection"
00107                             ])
00108 
00109 
00110 try:
00111     import sqlite3
00112     del sqlite3
00113 except ImportError:
00114     #Missing on Jython or Python 2.4
00115     DOCTEST_MODULES.remove("Bio.SeqIO")
00116 
00117 #Skip Bio.Seq doctest under Python 3, see http://bugs.python.org/issue7490
00118 if sys.version_info[0] == 3:
00119     DOCTEST_MODULES.remove("Bio.Seq")
00120 
00121 system_lang = os.environ.get('LANG', 'C') #Cache this
00122 
00123 def main(argv):
00124     """Run tests, return number of failures (integer)."""
00125     # insert our paths in sys.path:
00126     # ../build/lib.*
00127     # ..
00128     # Q. Why this order?
00129     # A. To find the C modules (which are in ../build/lib.*/Bio)
00130     # Q. Then, why ".."?
00131     # A. Because Martel may not be in ../build/lib.*
00132     test_path = sys.path[0] or "."
00133     source_path = os.path.abspath("%s/.." % test_path)
00134     sys.path.insert(1, source_path)
00135     build_path = os.path.abspath("%s/../build/lib.%s-%s" % (
00136         test_path, distutils.util.get_platform(), sys.version[:3]))
00137     if os.access(build_path, os.F_OK):
00138         sys.path.insert(1, build_path)
00139 
00140     # Using "export LANG=C" (which should work on Linux and similar) can
00141     # avoid problems detecting optional command line tools on
00142     # non-English OS (we may want 'command not found' in English).
00143     # HOWEVER, we do not want to change the default encoding which is
00144     # rather important on Python 3 with unicode.
00145     #lang = os.environ['LANG']
00146     
00147     # get the command line options
00148     try:
00149         opts, args = getopt.getopt(argv, 'gv', ["generate", "verbose",
00150             "doctest", "help", "offline"])
00151     except getopt.error, msg:
00152         print msg
00153         print __doc__
00154         return 2
00155 
00156     verbosity = VERBOSITY
00157 
00158     # deal with the options
00159     for o, a in opts:
00160         if o == "--help":
00161             print __doc__
00162             return 0
00163         if o == "--offline":
00164             print "Skipping any tests requiring internet access"
00165             #This is a bit of a hack...
00166             import requires_internet
00167             requires_internet.check.available = False
00168             #The check() function should now report internet not available
00169         if o == "-g" or o == "--generate":
00170             if len(args) > 1:
00171                 print "Only one argument (the test name) needed for generate"
00172                 print __doc__
00173                 return 2
00174             elif len(args) == 0:
00175                 print "No test name specified to generate output for."
00176                 print __doc__
00177                 return 2
00178             # strip off .py if it was included
00179             if args[0][-3:] == ".py":
00180                 args[0] = args[0][:-3]
00181 
00182             test = ComparisonTestCase(args[0])
00183             test.generate_output()
00184             return 0
00185 
00186         if o == "-v" or o == "--verbose":
00187             verbosity = 2
00188 
00189     # deal with the arguments, which should be names of tests to run
00190     for arg_num in range(len(args)):
00191         # strip off the .py if it was included
00192         if args[arg_num][-3:] == ".py":
00193             args[arg_num] = args[arg_num][:-3]
00194 
00195     print "Python version:", sys.version
00196     print "Operating system:", os.name, sys.platform
00197 
00198     # run the tests
00199     runner = TestRunner(args, verbosity)
00200     return runner.run()
00201 
00202 
00203 class ComparisonTestCase(unittest.TestCase):
00204     """Run a print-and-compare test and compare its output against expected output.
00205     """
00206 
00207     def __init__(self, name, output=None):
00208         """Initialize with the test to run.
00209 
00210         Arguments:
00211         o name - The name of the test. The expected output should be
00212           stored in the file output/name.
00213         o output - The output that was generated when this test was run.
00214         """
00215         unittest.TestCase.__init__(self)
00216         self.name = name
00217         self.output = output
00218 
00219     def shortDescription(self):
00220         return self.name
00221 
00222     def runTest(self):
00223         # check the expected output to be consistent with what
00224         # we generated
00225         outputdir = os.path.join(TestRunner.testdir, "output")
00226         outputfile = os.path.join(outputdir, self.name)
00227         try:
00228             if sys.version_info[0] >= 3:
00229                 #Python 3 problem: Can't use utf8 on output/test_geo
00230                 #due to micro (\xb5) and degrees (\xb0) symbols
00231                 expected = open(outputfile, encoding="latin")
00232             else:
00233                 expected = open(outputfile, 'rU')
00234         except IOError:
00235             self.fail("Warning: Can't open %s for test %s" % (outputfile, self.name))
00236 
00237         self.output.seek(0)
00238         # first check that we are dealing with the right output
00239         # the first line of the output file is the test name
00240         expected_test = expected.readline().strip()
00241 
00242         if expected_test != self.name:
00243             expected.close()
00244             raise ValueError("\nOutput:   %s\nExpected: %s" \
00245                   % (self.name, expected_test))
00246 
00247         # now loop through the output and compare it to the expected file
00248         while True:
00249             expected_line = expected.readline()
00250             output_line = self.output.readline()
00251 
00252             # stop looping if either of the info handles reach the end
00253             if not(expected_line) or not(output_line):
00254                 # make sure both have no information left
00255                 assert expected_line == '', "Unread: %s" % expected_line
00256                 assert output_line == '', "Extra output: %s" % output_line
00257                 break
00258 
00259             # normalize the newlines in the two lines
00260             expected_line = expected_line.strip("\r\n")
00261             output_line = output_line.strip("\r\n")
00262 
00263             # if the line is a doctest or PyUnit time output like:
00264             # Ran 2 tests in 0.285s
00265             # ignore it, so we don't have problems with different running times
00266             if re.compile("^Ran [0-9]+ tests? in ").match(expected_line):
00267                 pass
00268             # otherwise make sure the two lines are the same
00269             elif expected_line != output_line:
00270                 expected.close()
00271                 raise ValueError("\nOutput  : %s\nExpected: %s" \
00272                       % (repr(output_line), repr(expected_line)))
00273         expected.close()
00274 
00275     def generate_output(self):
00276         """Generate the golden output for the specified test.
00277         """
00278         outputdir = os.path.join(TestRunner.testdir, "output")
00279         outputfile = os.path.join(outputdir, self.name)
00280 
00281         output_handle = open(outputfile, 'w')
00282 
00283         # write the test name as the first line of the output
00284         output_handle.write(self.name + "\n")
00285 
00286         # remember standard out so we can reset it after we are done
00287         save_stdout = sys.stdout
00288         try:
00289             # write the output from the test into a string
00290             sys.stdout = output_handle
00291             __import__(self.name)
00292         finally:
00293             output_handle.close()
00294             # return standard out to its normal setting
00295             sys.stdout = save_stdout
00296 
00297 
00298 class TestRunner(unittest.TextTestRunner):
00299 
00300     if __name__ == '__main__':
00301         file = sys.argv[0]
00302     else:
00303         file = __file__
00304     testdir = os.path.dirname(file) or os.curdir
00305 
00306     def __init__(self, tests=[], verbosity=0):
00307         # if no tests were specified to run, we run them all
00308         # including the doctests
00309         self.tests = tests
00310         if not self.tests:
00311             # Make a list of all applicable test modules.
00312             names = os.listdir(TestRunner.testdir)
00313             for name in names:
00314                 if name[:5] == "test_" and name[-3:] == ".py":
00315                     self.tests.append(name[:-3])
00316             self.tests.sort()
00317             self.tests.append("doctest")
00318         if "doctest" in self.tests:
00319             self.tests.remove("doctest")
00320             self.tests.extend(DOCTEST_MODULES)
00321         stream = cStringIO.StringIO()
00322         unittest.TextTestRunner.__init__(self, stream,
00323                 verbosity=verbosity)
00324 
00325     def runTest(self, name):
00326         from Bio import MissingExternalDependencyError
00327         result = self._makeResult()
00328         output = cStringIO.StringIO()
00329         # Restore the language and thus default encoding (in case a prior
00330         # test changed this, e.g. to help with detecting command line tools)
00331         global system_lang
00332         os.environ['LANG']=system_lang
00333         # Note the current directory:
00334         cur_dir = os.path.abspath(".")
00335         try:
00336             stdout = sys.stdout
00337             sys.stdout = output
00338             if name.startswith("test_"):
00339                 sys.stderr.write("%s ... " % name)
00340                 #It's either a unittest or a print-and-compare test
00341                 suite = unittest.TestLoader().loadTestsFromName(name)
00342                 if suite.countTestCases()==0:
00343                     # This is a print-and-compare test instead of a
00344                     # unittest-type test.
00345                     test = ComparisonTestCase(name, output)
00346                     suite = unittest.TestSuite([test])
00347             else:
00348                 #It's a doc test
00349                 sys.stderr.write("%s docstring test ... " % name)
00350                 #Can't use fromlist=name.split(".") until python 2.5+
00351                 module = __import__(name, None, None, name.split("."))
00352                 suite = doctest.DocTestSuite(module)
00353                 del module
00354             suite.run(result)
00355             if cur_dir != os.path.abspath("."):
00356                 sys.stderr.write("FAIL\n")
00357                 result.stream.write(result.separator1+"\n")
00358                 result.stream.write("ERROR: %s\n" % name)
00359                 result.stream.write(result.separator2+"\n")
00360                 result.stream.write("Current directory changed\n")
00361                 result.stream.write("Was: %s\n" % cur_dir)
00362                 result.stream.write("Now: %s\n" % os.path.abspath("."))
00363                 os.chdir(cur_dir)
00364                 if not result.wasSuccessful():
00365                     result.printErrors()
00366                 return False
00367             elif result.wasSuccessful():
00368                 sys.stderr.write("ok\n")
00369                 return True
00370             else:
00371                 sys.stderr.write("FAIL\n")
00372                 result.printErrors()
00373             return False
00374         except MissingExternalDependencyError, msg:
00375             sys.stderr.write("skipping. %s\n" % msg)
00376             return True
00377         except Exception, msg:
00378             # This happened during the import
00379             sys.stderr.write("ERROR\n")
00380             result.stream.write(result.separator1+"\n")
00381             result.stream.write("ERROR: %s\n" % name)
00382             result.stream.write(result.separator2+"\n")
00383             result.stream.write(traceback.format_exc())
00384             return False
00385         except KeyboardInterrupt, err:
00386             # Want to allow this, and abort the test
00387             # (see below for special case)
00388             raise err
00389         except:
00390             # This happens in Jython with java.lang.ClassFormatError:
00391             # Invalid method Code length ...
00392             sys.stderr.write("ERROR\n")
00393             result.stream.write(result.separator1+"\n")
00394             result.stream.write("ERROR: %s\n" % name)
00395             result.stream.write(result.separator2+"\n")
00396             result.stream.write(traceback.format_exc())
00397             return False
00398         finally:
00399             sys.stdout = stdout
00400             #Running under PyPy we were leaking file handles...
00401             gc.collect()
00402 
00403     def run(self):
00404         """Run tests, return number of failures (integer)."""
00405         failures = 0
00406         startTime = time.time()
00407         for test in self.tests:
00408             ok = self.runTest(test)
00409             if not ok:
00410                 failures += 1
00411         total = len(self.tests)
00412         stopTime = time.time()
00413         timeTaken = stopTime - startTime
00414         sys.stderr.write(self.stream.getvalue())
00415         sys.stderr.write('-' * 70 + "\n")
00416         sys.stderr.write("Ran %d test%s in %.3f seconds\n" %
00417                             (total, total != 1 and "s" or "", timeTaken))
00418         sys.stderr.write("\n")
00419         if failures:
00420             sys.stderr.write("FAILED (failures = %d)\n" % failures)
00421         return failures
00422 
00423 
00424 if __name__ == "__main__":
00425     errors = main(sys.argv[1:])
00426     if errors:
00427         #Doing a sys.exit(...) isn't nice if run from IDLE...
00428         sys.exit(1)