#!/usr/bin/env python # -*- coding: UTF-8 -*- # # sfm.py - Simple FTP Mirror # # Copyright (c) 2009 joonis new media # Author: Thimo Kraemer # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. """ Usage: sfm [-v] [-q] [-u username [-p password [-a account]]] store|retrieve|remove|info hostname[:port] [remotedir [localdir]] -v: verbose (-vvv debug) -q: quiet -u username: ftp username (default anonymous) -p password: ftp password -a account: ftp account store: mirror the content of localdir to remotedir retrieve: mirror the content of remotedir to localdir remove: remove remotedir recursively info: prints some information about remote mirror hostname[:port]: remote host remotedir: remote directory (default initial) localdir: local directory (default current) Example: sfm -u myuser store ftp.mydomain.com /myfolder /my/folder Will mirror the content of local directory /my/folder to remote directory /myfolder """ import sys import os import getopt import getpass import ftplib import time import datetime import StringIO globals = { 'verbose': 1, 'status': { 'dirs_total': 0, 'dirs_created': 0, 'dirs_removed': 0, 'files_total': 0, 'files_created': 0, 'files_updated': 0, 'files_removed': 0, 'bytes_transfered': 0, 'bytes_total': 0, 'time_started': datetime.datetime.now(), 'time_finished': 0, }, } def log(msg, level=1, abort=False): if level <= globals['verbose'] or abort: if abort: sys.stdout = sys.stderr print print msg if abort: sys.exit(1) def strfbytes(value): units = ['Bytes', 'KB', 'MB', 'GB', 'TB'] value = float(value) for unit in units: if value < 1024: break value = value / 1024 if unit == units[0]: fmt = '%.0f %s' else: fmt = '%.2f %s' return fmt % (value, unit) class localHandler: """ Local file and directory functions""" def __init__(self, ftp, root): self.ftp = ftp self.root = root self.host = '' def storefile(self, src, dst): fh = open(dst, 'wb') self.ftp.retrbinary('RETR %s' % src, fh.write) fh.close() def storetext(self, text, dst): fh = open(dst, 'w') fh.write(text) fh.close() def readlines(self, path): fh = open(path, 'r') buffer = [line.strip() for line in fh.readlines()] fh.close() return buffer def list(self, dir, skip_mtime=False): dirs = [] files = {} for name in os.listdir(dir): path = os.path.join(dir, name) if os.path.isdir(path): dirs.append(name) else: if skip_mtime: mtime = 0 else: mtime = os.path.getmtime(path) files[name] = { 'size': os.path.getsize(path), 'mtime': mtime, } return (dirs, files) def makedir(self, path): log('--> Create directory %s' % path, 2) os.mkdir(path) globals['status']['dirs_created'] += 1 def removefile(self, path): log('--> Remove file %s' % path, 2) os.remove(path) globals['status']['files_removed'] += 1 def removedir(self, dir): for name in os.listdir(dir): path = os.path.join(dir, name) if os.path.isdir(path): self.removedir(path) else: self.removefile(path) log('--> Remove directory %s' % dir, 2) os.rmdir(dir) globals['status']['dirs_removed'] += 1 class remoteHandler: """Remote file and directory functions""" def __init__(self, ftp, root): self.ftp = ftp self.root = root self.host = ftp.host def storefile(self, src, dst): fh = open(src) self.ftp.storbinary('STOR %s' % dst, fh) fh.close() def storetext(self, text, dst): fh = StringIO.StringIO(text) self.ftp.storlines('STOR %s' % dst, fh) fh.close() def readlines(self, path): buffer = [] self.ftp.retrlines('RETR %s' % path, buffer.append) return buffer def list(self, dir, skip_mtime=False): month_to_int = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} try: buffer = [] self.ftp.dir('-a ', dir, buffer.append) except ftplib.error_temp: buffer = [] self.ftp.dir(dir, buffer.append) dirs = [] files = {} for line in buffer: cols = line.split(None, 8) name = os.path.split(cols[8])[1] if cols[0] == 'total' or name in ('.', '..'): continue if cols[0].startswith('d'): dirs.append(name) else: if skip_mtime: mtime = 0 else: month = month_to_int[cols[5]] day = int(cols[6]) if cols[7].find(':') == -1: year = int(cols[7]) hour = minute = 0 else: year = datetime.date.today().year hour, minute = [int(s) for s in cols[7].split(':')] mtime = datetime.datetime(year, month, day, hour, minute) mtime = int(time.mktime(mtime.timetuple())) size = int(cols[4]) files[name] = { 'size': size, 'mtime': mtime, } return (dirs, files) def makedir(self, path): log('--> Create directory %s' % path, 2) self.ftp.mkd(path) globals['status']['dirs_created'] += 1 def removefile(self, path): log('--> Remove file %s' % path, 2) self.ftp.delete(path) globals['status']['files_removed'] += 1 def removedir(self, path): dirs, files = self.list(path) for dir in dirs: self.removedir(os.path.join(path, dir)) for file in files: self.removefile(os.path.join(path, file)) if path == '/': return log('--> Remove directory %s' % path, 2) self.ftp.rmd(path) globals['status']['dirs_removed'] += 1 def mirror(src, dst, subdir=''): src_path = os.path.normpath('%s/%s' % (src.root, subdir)) dst_path = os.path.normpath('%s/%s' % (dst.root, subdir)) log('Working on %s%s' % (src.host, src_path)) src_dirs, src_files = src.list(src_path) if '.sfmstat' in src_files: del src_files['.sfmstat'] globals['status']['dirs_total'] += len(src_dirs) globals['status']['files_total'] += len(src_files) dst_dirs, dst_files = dst.list(dst_path, True) if '.sfmstat' in dst_files: sfmstat = dst.readlines(os.path.join(dst_path, '.sfmstat')) del dst_files['.sfmstat'] else: if dst_path == dst.root and (dst_dirs or dst_files): if globals['verbose']: abort = False else: abort = True log('New mirror, but target directory not empty!', abort=abort) result = raw_input('Do you really want to replace this directory? [y|n]: ') if result.lower() not in ('y', 'yes'): log('Aborted', abort=True) sfmstat = ['0 %s%s' % (src.host, src_path)] last_updated, mirror_path = sfmstat[0].split(None, 1) if mirror_path != (src.host + src_path): if globals['verbose']: abort = False else: abort = True error = 'Mirror mismatch!\n%s already contains another mirror of %s' % (dst_path, mirror_path) log(error, abort=abort) result = raw_input('Do you really want to replace this mirror? [y|n]: ') if result.lower() not in ('y', 'yes'): log('Aborted', abort=True) sfmstat = ['0 %s%s' % (src.host, src_path)] for line in sfmstat[1:]: mtime, file = line.split(None, 1) if file in dst_files: dst_files[file]['mtime'] = int(mtime) for dir in dst_dirs: if dir not in src_dirs: path = os.path.join(dst_path, dir) log('-> Remove directory %s' % path) dst.removedir(path) for file in dst_files: if file not in src_files: dst_file = os.path.join(dst_path, file) log('-> Remove file %s: %s' % (dst_file, strfbytes(dst_files[file]['size']))) dst.removefile(dst_file) newstat = ['%i %s%s' % (int(time.time()), src.host, src_path)] for file in src_files: if file not in dst_files or src_files[file]['mtime'] > dst_files[file]['mtime'] or src_files[file]['size'] != dst_files[file]['size']: src_file = os.path.join(src_path, file) dst_file = os.path.join(dst_path, file) if file in dst_files: log('-> Update file %s: %s' % (dst_file, strfbytes(src_files[file]['size']))) globals['status']['files_updated'] += 1 else: log('-> Create file %s: %s' % (dst_file, strfbytes(src_files[file]['size']))) globals['status']['files_created'] += 1 dst.storefile(src_file, dst_file) globals['status']['bytes_transfered'] += src_files[file]['size'] globals['status']['bytes_total'] += src_files[file]['size'] newstat.append('%i %s' % (src_files[file]['mtime'], file)) dst.storetext('\n'.join(newstat), os.path.join(dst_path, '.sfmstat')) for dir in src_dirs: if dir not in dst_dirs: dst_dir = os.path.join(dst_path, dir) log('-> Create directory %s' % dst_dir) dst.makedir(dst_dir) mirror(src, dst, os.path.join(subdir, dir)) def info(remote): try: sfmstat = remote.readlines(os.path.join(remote.root, '.sfmstat')) except ftplib.error_perm, err: if not err[0].startswith('550'): log(err, abort=True) sfmstat = None print if sfmstat: last_updated, mirror_path = sfmstat[0].split(None, 1) last_updated = datetime.datetime.fromtimestamp(float(last_updated)) print 'Mirror of', mirror_path print last_updated.strftime('Last updated on %A, %d. %B %Y at %H:%M:%S') else: print 'No mirror recognized' print print 'Content of %s%s:' % (remote.host, remote.root) remote.ftp.dir(remote.root) print def remove(remote): if globals['verbose']: info(remote) result = raw_input('Do you really want to remove this directory? [y|n]: ') if result.lower() not in ('y', 'yes'): log('Aborted', abort=True) remote.removedir(remote.root) def main(): username = '' password = '' account = '' try: opts, args = getopt.getopt(sys.argv[1:], 'vqu:p:a:') except getopt.GetoptError, msg: log('%s\n%s' % (msg, __doc__), abort=True) for opt, val in opts: if opt == '-v': globals['verbose'] += 1 if opt == '-q': globals['verbose'] = 0 if opt == '-u': username = val if opt == '-p': password = val if opt == '-a': account = val if not args: log('No action given\n' + __doc__, abort=True) action = args[0] if action not in ('store', 'retrieve', 'remove', 'info'): log('Unknown action: %s\n%s' % (action, __doc__), abort=True) if len(args) == 1: log('Missing hostname\n' + __doc__, abort=True) args[1] = args[1].split(':') host = args[1][0] if len(args[1]) == 1: port = 21 else: port = int(args[1][1]) remotedir = '/' if len(args) > 2: remotedir = os.path.normpath(args[2]) if not remotedir.startswith('/'): log('Invalid remotedir, must start with /', abort=True) localdir = os.getcwd() if len(args) > 3: if action not in ('store', 'retrieve'): log('Too many arguments\n%s' % __doc__, abort=True) localdir = os.path.abspath(args[3]) if not os.path.isdir(localdir): log('localdir does not exist: %s' % localdir, abort=True) if len(args) > 4: log('Too many arguments\n%s' % __doc__, abort=True) if not username: username = 'anonymous' elif not password and globals['verbose']: password = getpass.getpass('FTP Password: ') ftp = ftplib.FTP() if globals['verbose'] > 2: ftp.set_debuglevel(globals['verbose']-2) ftp.connect(host, port) ftp.login(username, password, account) try: ftp.cwd(remotedir) except ftplib.error_perm, err: if err[0].startswith('550'): log('remotedir does not exist: %s' % remotedir, abort=True) else: raise ftp.cwd('/') local = localHandler(ftp, localdir) remote = remoteHandler(ftp, remotedir) if action == 'store': mirror(local, remote) elif action == 'retrieve': mirror(remote, local) elif action == 'remove': remove(remote) elif action == 'info': info(remote) ftp.quit() return ftp.quit() log('Done') status = globals['status'] status['time_finished'] = datetime.datetime.now() print print '=' * 60 print 'Processing Summary' print '=' * 60 print '%-30s%30s' % ('Directories created', status['dirs_created']) print '%-30s%30s' % ('Directories removed', status['dirs_removed']) print '%-30s%30s' % ('Directories total', status['dirs_total']) print print '%-30s%30s' % ('Files created', status['files_created']) print '%-30s%30s' % ('Files updated', status['files_updated']) print '%-30s%30s' % ('Files removed', status['files_removed']) print '%-30s%30s' % ('Files total', status['files_total']) print print '%-30s%30s' % ('Bytes transfered', strfbytes(status['bytes_transfered'])) print '%-30s%30s' % ('Bytes total', strfbytes(status['bytes_total'])) print print '%-30s%30s' % ('Time started', status['time_started']) print '%-30s%30s' % ('Time finished', status['time_finished']) print '%-30s%30s' % ('Duration', status['time_finished']-status['time_started']) print '=' * 60 print if __name__ == '__main__': try: main() except KeyboardInterrupt: log('Aborted', abort=True) except SystemExit: raise except Exception, err: log(err, abort=True)