Back to index

python3.2  3.2.2
Classes | Functions | Variables
ftpmirror Namespace Reference

Classes

class  LoggingFile

Functions

def usage
def main
def mirrorsubdir
def remove
def raw_input
def askabout
def makedir
def writedict

Variables

int verbose = 1
int interactive = 0
int mac = 0
int rmok = 0
int nologin = 0
list skippats = ['.', '..', '.mirrorinfo']

Function Documentation

def ftpmirror.askabout (   filetype,
  filename,
  pwd 
)

Definition at line 361 of file ftpmirror.py.

00361 
00362 def askabout(filetype, filename, pwd):
00363     prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
00364     while 1:
00365         reply = raw_input(prompt).strip().lower()
00366         if reply in ['y', 'ye', 'yes']:
00367             return 1
00368         if reply in ['', 'n', 'no', 'nop', 'nope']:
00369             return 0
00370         print('Please answer yes or no.')
00371 
00372 # Create a directory if it doesn't exist.  Recursively create the
# parent directory as well if needed.

Here is the call graph for this function:

Here is the caller graph for this function:

def ftpmirror.main ( void  )

Definition at line 44 of file ftpmirror.py.

00044 
00045 def main():
00046     global verbose, interactive, mac, rmok, nologin
00047     try:
00048         opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
00049     except getopt.error as msg:
00050         usage(msg)
00051     login = ''
00052     passwd = ''
00053     account = ''
00054     if not args: usage('hostname missing')
00055     host = args[0]
00056     port = 0
00057     if ':' in host:
00058         host, port = host.split(':', 1)
00059         port = int(port)
00060     try:
00061         auth = netrc.netrc().authenticators(host)
00062         if auth is not None:
00063             login, account, passwd = auth
00064     except (netrc.NetrcParseError, IOError):
00065         pass
00066     for o, a in opts:
00067         if o == '-l': login = a
00068         if o == '-p': passwd = a
00069         if o == '-a': account = a
00070         if o == '-v': verbose = verbose + 1
00071         if o == '-q': verbose = 0
00072         if o == '-i': interactive = 1
00073         if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
00074         if o == '-n': nologin = 1
00075         if o == '-r': rmok = 1
00076         if o == '-s': skippats.append(a)
00077     remotedir = ''
00078     localdir = ''
00079     if args[1:]:
00080         remotedir = args[1]
00081         if args[2:]:
00082             localdir = args[2]
00083             if args[3:]: usage('too many arguments')
00084     #
00085     f = ftplib.FTP()
00086     if verbose: print("Connecting to '%s%s'..." % (host,
00087                                                    (port and ":%d"%port or "")))
00088     f.connect(host,port)
00089     if not nologin:
00090         if verbose:
00091             print('Logging in as %r...' % (login or 'anonymous'))
00092         f.login(login, passwd, account)
00093     if verbose: print('OK.')
00094     pwd = f.pwd()
00095     if verbose > 1: print('PWD =', repr(pwd))
00096     if remotedir:
00097         if verbose > 1: print('cwd(%s)' % repr(remotedir))
00098         f.cwd(remotedir)
00099         if verbose > 1: print('OK.')
00100         pwd = f.pwd()
00101         if verbose > 1: print('PWD =', repr(pwd))
00102     #
00103     mirrorsubdir(f, localdir)
00104 
# Core logic: mirror one subdirectory (recursively)

Here is the call graph for this function:

Here is the caller graph for this function:

def ftpmirror.makedir (   pathname)

Definition at line 373 of file ftpmirror.py.

00373 
00374 def makedir(pathname):
00375     if os.path.isdir(pathname):
00376         return
00377     dirname = os.path.dirname(pathname)
00378     if dirname: makedir(dirname)
00379     os.mkdir(pathname, 0o777)
00380 
00381 # Write a dictionary to a file in a way that can be read back using
00382 # rval() but is still somewhat readable (i.e. not a single long line).
# Also creates a backup file.

Here is the caller graph for this function:

def ftpmirror.mirrorsubdir (   f,
  localdir 
)

Definition at line 105 of file ftpmirror.py.

00105 
00106 def mirrorsubdir(f, localdir):
00107     pwd = f.pwd()
00108     if localdir and not os.path.isdir(localdir):
00109         if verbose: print('Creating local directory', repr(localdir))
00110         try:
00111             makedir(localdir)
00112         except os.error as msg:
00113             print("Failed to establish local directory", repr(localdir))
00114             return
00115     infofilename = os.path.join(localdir, '.mirrorinfo')
00116     try:
00117         text = open(infofilename, 'r').read()
00118     except IOError as msg:
00119         text = '{}'
00120     try:
00121         info = eval(text)
00122     except (SyntaxError, NameError):
00123         print('Bad mirror info in', repr(infofilename))
00124         info = {}
00125     subdirs = []
00126     listing = []
00127     if verbose: print('Listing remote directory %r...' % (pwd,))
00128     f.retrlines('LIST', listing.append)
00129     filesfound = []
00130     for line in listing:
00131         if verbose > 1: print('-->', repr(line))
00132         if mac:
00133             # Mac listing has just filenames;
00134             # trailing / means subdirectory
00135             filename = line.strip()
00136             mode = '-'
00137             if filename[-1:] == '/':
00138                 filename = filename[:-1]
00139                 mode = 'd'
00140             infostuff = ''
00141         else:
00142             # Parse, assuming a UNIX listing
00143             words = line.split(None, 8)
00144             if len(words) < 6:
00145                 if verbose > 1: print('Skipping short line')
00146                 continue
00147             filename = words[-1].lstrip()
00148             i = filename.find(" -> ")
00149             if i >= 0:
00150                 # words[0] had better start with 'l'...
00151                 if verbose > 1:
00152                     print('Found symbolic link %r' % (filename,))
00153                 linkto = filename[i+4:]
00154                 filename = filename[:i]
00155             infostuff = words[-5:-1]
00156             mode = words[0]
00157         skip = 0
00158         for pat in skippats:
00159             if fnmatch(filename, pat):
00160                 if verbose > 1:
00161                     print('Skip pattern', repr(pat), end=' ')
00162                     print('matches', repr(filename))
00163                 skip = 1
00164                 break
00165         if skip:
00166             continue
00167         if mode[0] == 'd':
00168             if verbose > 1:
00169                 print('Remembering subdirectory', repr(filename))
00170             subdirs.append(filename)
00171             continue
00172         filesfound.append(filename)
00173         if filename in info and info[filename] == infostuff:
00174             if verbose > 1:
00175                 print('Already have this version of',repr(filename))
00176             continue
00177         fullname = os.path.join(localdir, filename)
00178         tempname = os.path.join(localdir, '@'+filename)
00179         if interactive:
00180             doit = askabout('file', filename, pwd)
00181             if not doit:
00182                 if filename not in info:
00183                     info[filename] = 'Not retrieved'
00184                 continue
00185         try:
00186             os.unlink(tempname)
00187         except os.error:
00188             pass
00189         if mode[0] == 'l':
00190             if verbose:
00191                 print("Creating symlink %r -> %r" % (filename, linkto))
00192             try:
00193                 os.symlink(linkto, tempname)
00194             except IOError as msg:
00195                 print("Can't create %r: %s" % (tempname, msg))
00196                 continue
00197         else:
00198             try:
00199                 fp = open(tempname, 'wb')
00200             except IOError as msg:
00201                 print("Can't create %r: %s" % (tempname, msg))
00202                 continue
00203             if verbose:
00204                 print('Retrieving %r from %r as %r...' % (filename, pwd, fullname))
00205             if verbose:
00206                 fp1 = LoggingFile(fp, 1024, sys.stdout)
00207             else:
00208                 fp1 = fp
00209             t0 = time.time()
00210             try:
00211                 f.retrbinary('RETR ' + filename,
00212                              fp1.write, 8*1024)
00213             except ftplib.error_perm as msg:
00214                 print(msg)
00215             t1 = time.time()
00216             bytes = fp.tell()
00217             fp.close()
00218             if fp1 != fp:
00219                 fp1.close()
00220         try:
00221             os.unlink(fullname)
00222         except os.error:
00223             pass            # Ignore the error
00224         try:
00225             os.rename(tempname, fullname)
00226         except os.error as msg:
00227             print("Can't rename %r to %r: %s" % (tempname, fullname, msg))
00228             continue
00229         info[filename] = infostuff
00230         writedict(info, infofilename)
00231         if verbose and mode[0] != 'l':
00232             dt = t1 - t0
00233             kbytes = bytes / 1024.0
00234             print(int(round(kbytes)), end=' ')
00235             print('Kbytes in', end=' ')
00236             print(int(round(dt)), end=' ')
00237             print('seconds', end=' ')
00238             if t1 > t0:
00239                 print('(~%d Kbytes/sec)' % \
00240                           int(round(kbytes/dt),))
00241             print()
00242     #
00243     # Remove files from info that are no longer remote
00244     deletions = 0
00245     for filename in list(info.keys()):
00246         if filename not in filesfound:
00247             if verbose:
00248                 print("Removing obsolete info entry for", end=' ')
00249                 print(repr(filename), "in", repr(localdir or "."))
00250             del info[filename]
00251             deletions = deletions + 1
00252     if deletions:
00253         writedict(info, infofilename)
00254     #
00255     # Remove local files that are no longer in the remote directory
00256     try:
00257         if not localdir: names = os.listdir(os.curdir)
00258         else: names = os.listdir(localdir)
00259     except os.error:
00260         names = []
00261     for name in names:
00262         if name[0] == '.' or name in info or name in subdirs:
00263             continue
00264         skip = 0
00265         for pat in skippats:
00266             if fnmatch(name, pat):
00267                 if verbose > 1:
00268                     print('Skip pattern', repr(pat), end=' ')
00269                     print('matches', repr(name))
00270                 skip = 1
00271                 break
00272         if skip:
00273             continue
00274         fullname = os.path.join(localdir, name)
00275         if not rmok:
00276             if verbose:
00277                 print('Local file', repr(fullname), end=' ')
00278                 print('is no longer pertinent')
00279             continue
00280         if verbose: print('Removing local file/dir', repr(fullname))
00281         remove(fullname)
00282     #
00283     # Recursively mirror subdirectories
00284     for subdir in subdirs:
00285         if interactive:
00286             doit = askabout('subdirectory', subdir, pwd)
00287             if not doit: continue
00288         if verbose: print('Processing subdirectory', repr(subdir))
00289         localsubdir = os.path.join(localdir, subdir)
00290         pwd = f.pwd()
00291         if verbose > 1:
00292             print('Remote directory now:', repr(pwd))
00293             print('Remote cwd', repr(subdir))
00294         try:
00295             f.cwd(subdir)
00296         except ftplib.error_perm as msg:
00297             print("Can't chdir to", repr(subdir), ":", repr(msg))
00298         else:
00299             if verbose: print('Mirroring as', repr(localsubdir))
00300             mirrorsubdir(f, localsubdir)
00301             if verbose > 1: print('Remote cwd ..')
00302             f.cwd('..')
00303         newpwd = f.pwd()
00304         if newpwd != pwd:
00305             print('Ended up in wrong directory after cd + cd ..')
00306             print('Giving up now.')
00307             break
00308         else:
00309             if verbose > 1: print('OK.')
00310 
# Helper to remove a file or directory tree

Here is the call graph for this function:

Here is the caller graph for this function:

def ftpmirror.raw_input (   prompt)

Definition at line 355 of file ftpmirror.py.

00355 
00356 def raw_input(prompt):
00357     sys.stdout.write(prompt)
00358     sys.stdout.flush()
00359     return sys.stdin.readline()
00360 
# Ask permission to download a file.

Here is the caller graph for this function:

def ftpmirror.remove (   fullname)

Definition at line 311 of file ftpmirror.py.

00311 
00312 def remove(fullname):
00313     if os.path.isdir(fullname) and not os.path.islink(fullname):
00314         try:
00315             names = os.listdir(fullname)
00316         except os.error:
00317             names = []
00318         ok = 1
00319         for name in names:
00320             if not remove(os.path.join(fullname, name)):
00321                 ok = 0
00322         if not ok:
00323             return 0
00324         try:
00325             os.rmdir(fullname)
00326         except os.error as msg:
00327             print("Can't remove local directory %r: %s" % (fullname, msg))
00328             return 0
00329     else:
00330         try:
00331             os.unlink(fullname)
00332         except os.error as msg:
00333             print("Can't remove local file %r: %s" % (fullname, msg))
00334             return 0
00335     return 1
00336 
# Wrapper around a file for writing to write a hash sign every block.

Here is the caller graph for this function:

def ftpmirror.usage (   args)

Definition at line 30 of file ftpmirror.py.

00030 
00031 def usage(*args):
00032     sys.stdout = sys.stderr
00033     for msg in args: print(msg)
00034     print(__doc__)
00035     sys.exit(2)

Here is the caller graph for this function:

def ftpmirror.writedict (   dict,
  filename 
)

Definition at line 383 of file ftpmirror.py.

00383 
00384 def writedict(dict, filename):
00385     dir, fname = os.path.split(filename)
00386     tempname = os.path.join(dir, '@' + fname)
00387     backup = os.path.join(dir, fname + '~')
00388     try:
00389         os.unlink(backup)
00390     except os.error:
00391         pass
00392     fp = open(tempname, 'w')
00393     fp.write('{\n')
00394     for key, value in dict.items():
00395         fp.write('%r: %r,\n' % (key, value))
00396     fp.write('}\n')
00397     fp.close()
00398     try:
00399         os.rename(filename, backup)
00400     except os.error:
00401         pass
00402     os.rename(tempname, filename)
00403 

Here is the call graph for this function:

Here is the caller graph for this function:


Variable Documentation

Definition at line 37 of file ftpmirror.py.

Definition at line 38 of file ftpmirror.py.

Definition at line 40 of file ftpmirror.py.

Definition at line 39 of file ftpmirror.py.

list ftpmirror.skippats = ['.', '..', '.mirrorinfo']

Definition at line 41 of file ftpmirror.py.

Definition at line 36 of file ftpmirror.py.