| #! /usr/bin/env python
|
|
|
| """Mirror a remote ftp subtree into a local directory tree.
|
|
|
| usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]
|
| [-l username [-p passwd [-a account]]]
|
| hostname[:port] [remotedir [localdir]]
|
| -v: verbose
|
| -q: quiet
|
| -i: interactive mode
|
| -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
|
| -n: don't log in
|
| -r: remove local files/directories no longer pertinent
|
| -l username [-p passwd [-a account]]: login info (default .netrc or anonymous)
|
| -s pat: skip files matching pattern
|
| hostname: remote host w/ optional port separated by ':'
|
| remotedir: remote directory (default initial)
|
| localdir: local directory (default current)
|
| """
|
|
|
| import os
|
| import sys
|
| import time
|
| import getopt
|
| import ftplib
|
| import netrc
|
| from fnmatch import fnmatch
|
|
|
| # Print usage message and exit
|
| def usage(*args):
|
| sys.stdout = sys.stderr
|
| for msg in args: print msg
|
| print __doc__
|
| sys.exit(2)
|
|
|
| verbose = 1 # 0 for -q, 2 for -v
|
| interactive = 0
|
| mac = 0
|
| rmok = 0
|
| nologin = 0
|
| skippats = ['.', '..', '.mirrorinfo']
|
|
|
| # Main program: parse command line and start processing
|
| def main():
|
| global verbose, interactive, mac, rmok, nologin
|
| try:
|
| opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
|
| except getopt.error, msg:
|
| usage(msg)
|
| login = ''
|
| passwd = ''
|
| account = ''
|
| if not args: usage('hostname missing')
|
| host = args[0]
|
| port = 0
|
| if ':' in host:
|
| host, port = host.split(':', 1)
|
| port = int(port)
|
| try:
|
| auth = netrc.netrc().authenticators(host)
|
| if auth is not None:
|
| login, account, passwd = auth
|
| except (netrc.NetrcParseError, IOError):
|
| pass
|
| for o, a in opts:
|
| if o == '-l': login = a
|
| if o == '-p': passwd = a
|
| if o == '-a': account = a
|
| if o == '-v': verbose = verbose + 1
|
| if o == '-q': verbose = 0
|
| if o == '-i': interactive = 1
|
| if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
|
| if o == '-n': nologin = 1
|
| if o == '-r': rmok = 1
|
| if o == '-s': skippats.append(a)
|
| remotedir = ''
|
| localdir = ''
|
| if args[1:]:
|
| remotedir = args[1]
|
| if args[2:]:
|
| localdir = args[2]
|
| if args[3:]: usage('too many arguments')
|
| #
|
| f = ftplib.FTP()
|
| if verbose: print "Connecting to '%s%s'..." % (host,
|
| (port and ":%d"%port or ""))
|
| f.connect(host,port)
|
| if not nologin:
|
| if verbose:
|
| print 'Logging in as %r...' % (login or 'anonymous')
|
| f.login(login, passwd, account)
|
| if verbose: print 'OK.'
|
| pwd = f.pwd()
|
| if verbose > 1: print 'PWD =', repr(pwd)
|
| if remotedir:
|
| if verbose > 1: print 'cwd(%s)' % repr(remotedir)
|
| f.cwd(remotedir)
|
| if verbose > 1: print 'OK.'
|
| pwd = f.pwd()
|
| if verbose > 1: print 'PWD =', repr(pwd)
|
| #
|
| mirrorsubdir(f, localdir)
|
|
|
| # Core logic: mirror one subdirectory (recursively)
|
| def mirrorsubdir(f, localdir):
|
| pwd = f.pwd()
|
| if localdir and not os.path.isdir(localdir):
|
| if verbose: print 'Creating local directory', repr(localdir)
|
| try:
|
| makedir(localdir)
|
| except os.error, msg:
|
| print "Failed to establish local directory", repr(localdir)
|
| return
|
| infofilename = os.path.join(localdir, '.mirrorinfo')
|
| try:
|
| text = open(infofilename, 'r').read()
|
| except IOError, msg:
|
| text = '{}'
|
| try:
|
| info = eval(text)
|
| except (SyntaxError, NameError):
|
| print 'Bad mirror info in', repr(infofilename)
|
| info = {}
|
| subdirs = []
|
| listing = []
|
| if verbose: print 'Listing remote directory %r...' % (pwd,)
|
| f.retrlines('LIST', listing.append)
|
| filesfound = []
|
| for line in listing:
|
| if verbose > 1: print '-->', repr(line)
|
| if mac:
|
| # Mac listing has just filenames;
|
| # trailing / means subdirectory
|
| filename = line.strip()
|
| mode = '-'
|
| if filename[-1:] == '/':
|
| filename = filename[:-1]
|
| mode = 'd'
|
| infostuff = ''
|
| else:
|
| # Parse, assuming a UNIX listing
|
| words = line.split(None, 8)
|
| if len(words) < 6:
|
| if verbose > 1: print 'Skipping short line'
|
| continue
|
| filename = words[-1].lstrip()
|
| i = filename.find(" -> ")
|
| if i >= 0:
|
| # words[0] had better start with 'l'...
|
| if verbose > 1:
|
| print 'Found symbolic link %r' % (filename,)
|
| linkto = filename[i+4:]
|
| filename = filename[:i]
|
| infostuff = words[-5:-1]
|
| mode = words[0]
|
| skip = 0
|
| for pat in skippats:
|
| if fnmatch(filename, pat):
|
| if verbose > 1:
|
| print 'Skip pattern', repr(pat),
|
| print 'matches', repr(filename)
|
| skip = 1
|
| break
|
| if skip:
|
| continue
|
| if mode[0] == 'd':
|
| if verbose > 1:
|
| print 'Remembering subdirectory', repr(filename)
|
| subdirs.append(filename)
|
| continue
|
| filesfound.append(filename)
|
| if info.has_key(filename) and info[filename] == infostuff:
|
| if verbose > 1:
|
| print 'Already have this version of',repr(filename)
|
| continue
|
| fullname = os.path.join(localdir, filename)
|
| tempname = os.path.join(localdir, '@'+filename)
|
| if interactive:
|
| doit = askabout('file', filename, pwd)
|
| if not doit:
|
| if not info.has_key(filename):
|
| info[filename] = 'Not retrieved'
|
| continue
|
| try:
|
| os.unlink(tempname)
|
| except os.error:
|
| pass
|
| if mode[0] == 'l':
|
| if verbose:
|
| print "Creating symlink %r -> %r" % (filename, linkto)
|
| try:
|
| os.symlink(linkto, tempname)
|
| except IOError, msg:
|
| print "Can't create %r: %s" % (tempname, msg)
|
| continue
|
| else:
|
| try:
|
| fp = open(tempname, 'wb')
|
| except IOError, msg:
|
| print "Can't create %r: %s" % (tempname, msg)
|
| continue
|
| if verbose:
|
| print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname)
|
| if verbose:
|
| fp1 = LoggingFile(fp, 1024, sys.stdout)
|
| else:
|
| fp1 = fp
|
| t0 = time.time()
|
| try:
|
| f.retrbinary('RETR ' + filename,
|
| fp1.write, 8*1024)
|
| except ftplib.error_perm, msg:
|
| print msg
|
| t1 = time.time()
|
| bytes = fp.tell()
|
| fp.close()
|
| if fp1 != fp:
|
| fp1.close()
|
| try:
|
| os.unlink(fullname)
|
| except os.error:
|
| pass # Ignore the error
|
| try:
|
| os.rename(tempname, fullname)
|
| except os.error, msg:
|
| print "Can't rename %r to %r: %s" % (tempname, fullname, msg)
|
| continue
|
| info[filename] = infostuff
|
| writedict(info, infofilename)
|
| if verbose and mode[0] != 'l':
|
| dt = t1 - t0
|
| kbytes = bytes / 1024.0
|
| print int(round(kbytes)),
|
| print 'Kbytes in',
|
| print int(round(dt)),
|
| print 'seconds',
|
| if t1 > t0:
|
| print '(~%d Kbytes/sec)' % \
|
| int(round(kbytes/dt),)
|
| print
|
| #
|
| # Remove files from info that are no longer remote
|
| deletions = 0
|
| for filename in info.keys():
|
| if filename not in filesfound:
|
| if verbose:
|
| print "Removing obsolete info entry for",
|
| print repr(filename), "in", repr(localdir or ".")
|
| del info[filename]
|
| deletions = deletions + 1
|
| if deletions:
|
| writedict(info, infofilename)
|
| #
|
| # Remove local files that are no longer in the remote directory
|
| try:
|
| if not localdir: names = os.listdir(os.curdir)
|
| else: names = os.listdir(localdir)
|
| except os.error:
|
| names = []
|
| for name in names:
|
| if name[0] == '.' or info.has_key(name) or name in subdirs:
|
| continue
|
| skip = 0
|
| for pat in skippats:
|
| if fnmatch(name, pat):
|
| if verbose > 1:
|
| print 'Skip pattern', repr(pat),
|
| print 'matches', repr(name)
|
| skip = 1
|
| break
|
| if skip:
|
| continue
|
| fullname = os.path.join(localdir, name)
|
| if not rmok:
|
| if verbose:
|
| print 'Local file', repr(fullname),
|
| print 'is no longer pertinent'
|
| continue
|
| if verbose: print 'Removing local file/dir', repr(fullname)
|
| remove(fullname)
|
| #
|
| # Recursively mirror subdirectories
|
| for subdir in subdirs:
|
| if interactive:
|
| doit = askabout('subdirectory', subdir, pwd)
|
| if not doit: continue
|
| if verbose: print 'Processing subdirectory', repr(subdir)
|
| localsubdir = os.path.join(localdir, subdir)
|
| pwd = f.pwd()
|
| if verbose > 1:
|
| print 'Remote directory now:', repr(pwd)
|
| print 'Remote cwd', repr(subdir)
|
| try:
|
| f.cwd(subdir)
|
| except ftplib.error_perm, msg:
|
| print "Can't chdir to", repr(subdir), ":", repr(msg)
|
| else:
|
| if verbose: print 'Mirroring as', repr(localsubdir)
|
| mirrorsubdir(f, localsubdir)
|
| if verbose > 1: print 'Remote cwd ..'
|
| f.cwd('..')
|
| newpwd = f.pwd()
|
| if newpwd != pwd:
|
| print 'Ended up in wrong directory after cd + cd ..'
|
| print 'Giving up now.'
|
| break
|
| else:
|
| if verbose > 1: print 'OK.'
|
|
|
| # Helper to remove a file or directory tree
|
| def remove(fullname):
|
| if os.path.isdir(fullname) and not os.path.islink(fullname):
|
| try:
|
| names = os.listdir(fullname)
|
| except os.error:
|
| names = []
|
| ok = 1
|
| for name in names:
|
| if not remove(os.path.join(fullname, name)):
|
| ok = 0
|
| if not ok:
|
| return 0
|
| try:
|
| os.rmdir(fullname)
|
| except os.error, msg:
|
| print "Can't remove local directory %r: %s" % (fullname, msg)
|
| return 0
|
| else:
|
| try:
|
| os.unlink(fullname)
|
| except os.error, msg:
|
| print "Can't remove local file %r: %s" % (fullname, msg)
|
| return 0
|
| return 1
|
|
|
| # Wrapper around a file for writing to write a hash sign every block.
|
| class LoggingFile:
|
| def __init__(self, fp, blocksize, outfp):
|
| self.fp = fp
|
| self.bytes = 0
|
| self.hashes = 0
|
| self.blocksize = blocksize
|
| self.outfp = outfp
|
| def write(self, data):
|
| self.bytes = self.bytes + len(data)
|
| hashes = int(self.bytes) / self.blocksize
|
| while hashes > self.hashes:
|
| self.outfp.write('#')
|
| self.outfp.flush()
|
| self.hashes = self.hashes + 1
|
| self.fp.write(data)
|
| def close(self):
|
| self.outfp.write('\n')
|
|
|
| # Ask permission to download a file.
|
| def askabout(filetype, filename, pwd):
|
| prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
|
| while 1:
|
| reply = raw_input(prompt).strip().lower()
|
| if reply in ['y', 'ye', 'yes']:
|
| return 1
|
| if reply in ['', 'n', 'no', 'nop', 'nope']:
|
| return 0
|
| print 'Please answer yes or no.'
|
|
|
| # Create a directory if it doesn't exist. Recursively create the
|
| # parent directory as well if needed.
|
| def makedir(pathname):
|
| if os.path.isdir(pathname):
|
| return
|
| dirname = os.path.dirname(pathname)
|
| if dirname: makedir(dirname)
|
| os.mkdir(pathname, 0777)
|
|
|
| # Write a dictionary to a file in a way that can be read back using
|
| # rval() but is still somewhat readable (i.e. not a single long line).
|
| # Also creates a backup file.
|
| def writedict(dict, filename):
|
| dir, fname = os.path.split(filename)
|
| tempname = os.path.join(dir, '@' + fname)
|
| backup = os.path.join(dir, fname + '~')
|
| try:
|
| os.unlink(backup)
|
| except os.error:
|
| pass
|
| fp = open(tempname, 'w')
|
| fp.write('{\n')
|
| for key, value in dict.items():
|
| fp.write('%r: %r,\n' % (key, value))
|
| fp.write('}\n')
|
| fp.close()
|
| try:
|
| os.rename(filename, backup)
|
| except os.error:
|
| pass
|
| os.rename(tempname, filename)
|
|
|
|
|
| if __name__ == '__main__':
|
| main()
|
