#! /usr/bin/python -- """transfer stuff to/from hosts described in ~/.transferrc TO-DO: dest-specific dir_basename config items defined exit codes exit with code 5 if .transferrc doesn't exist make directories on the target NOTE: Until :/.outgoing is supported, files copied from a gateway will appear to have originated at the gateway. (Therefore there is no need to create Transfer.incoming subdirectories whose files are received via a gateway.) Version: 1.2 """ import sys import os import os.path import re import platform from config_hierarchy import ConfigHierarchy # == definitions == self = os.path.basename(sys.argv[0]) config_file = os.path.expanduser("~/.transferrc") local = platform.node() dry_run = False incoming_basename = "Transfer.incoming" outgoing_basename = "Transfer.outgoing" # *** Classes *** class CommsWarning(UserWarning): pass class Rsync: # TO-DO: add option to use --links instead symlink_opts = ["--copy-links","--keep-dirlinks"] other_opts = ["-v",] standard_exclusions = (".svn", ".xvpics", "#*#", ".*.sw?", "*~", "*.bak") failure_errors = (12, # Error in rsync protocol data stream 255, # SSH failed (e.g. No route to host) ) incomplete_errors = (23, # Partial transfer due to error 24, # Partial transfer due to vanished source files 25, # The --max-delete limit stopped deletions ) def __init__(self, exclusions=()): self.exclusion_opts = ["--exclude=" + item for item in exclusions + self.standard_exclusions] def run_rsync(self, dest, *sources): args = ["-az",] + self.symlink_opts + self.other_opts + self.exclusion_opts + list(sources) + [dest,] if not dry_run: # run the program, passing its name as ARGV[0] exit_code = os.spawnvp(os.P_WAIT, "rsync", ["rsync",] + args) if exit_code in self.failure_errors: raise CommsWarning("failed") elif exit_code in self.incomplete_errors: raise CommsWarning("did not complete") elif exit_code != 0: raise RuntimeError("rsync barfed" ) else: os.spawnvp(os.P_WAIT, "echo", ["echo", "rsync"] + args) class GatewayWarning(UserWarning): pass # *** Functions *** def lookup_gateway(config, dest): """Returns the host for a given destination, or None if there isn't one defined. Raises GatewayWarning if there is a problem with the given gateway's definition.""" try: # if there's a gateway for the dest, use that gateway = config[("outgoing", "destinations", dest, "gateway")] try: gateway_host = config[("outgoing", "gateways", gateway, "host")] return gateway_host except KeyError: raise GatewayWarning("no details defined for gateway " + gateway) except KeyError: # couldn't look up a gateway for the dest, so assume it's direct return None def do_transfer(remote, pull): """Sends from local to remote if pull is False, otherwise receives from remote to local.""" try: # if there's a gateway for the dest, use that gateway_host = lookup_gateway(config, remote) if gateway_host is None: # couldn't look up a gateway for the remote, so assume it's direct if config.has_node(("hosts", remote)): try: remote_host = config[("hosts", remote, "addr")] # there's a host defined for this dest, so do a direct transfer if pull: # copy the files from the directory on the remote host local_rsyncpath = incoming_dir + "/" + remote remote_rsyncpath = remote_host + ":" + outgoing_basename + "/" + local rsync.run_rsync(local_rsyncpath, remote_rsyncpath + "/") else: # copy the files to the directory on the remote host local_rsyncpath = outgoing_dir + "/" + remote remote_rsyncpath = remote_host + ":" + incoming_basename + "/" + local rsync.run_rsync(remote_rsyncpath, local_rsyncpath + "/") except KeyError: print >>sys.stderr, "%s: no address/hostname defined for remote %s" % (self, remote) except CommsWarning, e: print >>sys.stderr, "%s: WARNING: communication with %s (%s) %s." % (self, remote, remote_host, e) except Exception, e: print >>sys.stderr, "fail (direct)", e sys.exit(3) else: print >>sys.stderr, "%s: no host or gateway defined for %s, skipping." % (self, remote) else: # use a gateway try: # -- rsync -- # (these files will be merged with any others in that directory) # TO-DO: add option to use :/.outgoing or # etc. if pull: # copy the files from the directory on the gateway host # whence they will be eventually copied (pull or push) to the remote host local_rsyncpath = incoming_dir + "/" + remote remote_rsyncpath = gateway_host + ":" + outgoing_basename + "/" + local rsync.run_rsync(local_rsyncpath, remote_rsyncpath + "/") else: # copy the files to the directory on the gateway host local_rsyncpath = outgoing_dir + "/" + remote remote_rsyncpath = gateway_host + ":" + outgoing_basename + "/" + remote rsync.run_rsync(remote_rsyncpath, local_rsyncpath + "/") except Exception, e: print >>sys.stderr, "fail (gateway = %s)" % gateway_host, e sys.exit(4) except GatewayWarning, e: print >>sys.stderr, self + ":", e # *** MAINLINE *** # == Parse config file == config = ConfigHierarchy() for line in file(config_file): if len(line) > 0 and not line.startswith("#"): m = re.match(r'\s*([^:\s]*)[:\s]*(.*)', line) leadin = m.group(1) levels = leadin.split(".") value = m.group(2) config[levels] = value # interpret paths in the rc file as relative to $HOME incoming_dir = os.path.expanduser(os.path.join("~", config.get(('incoming','dir_basename'), incoming_basename))) outgoing_dir = os.path.expanduser(os.path.join("~", config.get(('outgoing','dir_basename'), outgoing_basename))) rsync = Rsync(tuple(config.get(('global','excludes'), "").split())) # == Parse command-line options == # TO-DO: interpret paths supplied on the command-line as relative to $CWD # TO-DO: dry_run = True # TO-DO: add "-v" to other_opts (currently forced) ## print config.config_dict ## sys.exit(99) # == Perform outgoing transfers == for dest in os.listdir(outgoing_dir): print "\nTransferring to", dest + ":" do_transfer(dest, False) # == Perform incoming transfers == for source in os.listdir(incoming_dir): print "\nTransferring from", source + ":" do_transfer(source, True) # original plan was to use 'incoming.pull.rsync.sources'