#! /usr/bin/env python
"""ftpmirror: mirror a remote directory tree from a local tree
Usage: ftpmirror.py dbname command [options]
dbname database name; dbname must be defined in an Add() call; uses xx.webmirror for the db file name
where command is one of:
-sync update db to the remote site and then send any differences
-update update db to remote site
-sendonly assume the db is up-to-date and send any differences
-diff display differences between db contents versus the local directory (no connection required)
-convert converts all line endings to unix style without sending files (no connection required)
where options is one or more of:
-vv very verbose output
-v verbose output
-q quiet (no output)
-unix converts all line endings to unix style for any sent files
-force send all files (consider all files as changed)
-h, -help display this message
defaults:
some output is printed
if the db file doesn't exist, the db file is updated from the remote site and
any differences are sent (this is the same as -sync)
if the db file exists, any differences between the db contents and the
local system are sent
"""
#TODO:
# - add '-remote' switch; this makes the remote site the master and reverses the mirror
# - need to put this as an internal option. Since the exclude/include parameters will be different for
# one direction vs another, the user cannot accidently put the -remote switch...
# - commandline: get password etc from commandline (from an @file??)
#ISSUES:
# - filenames can't have single quotes in them; check if ':' is ok
# - does not recover from FTP error, e.g. time out
# - does not handle very large .webmirror files very well
# - does not set permissions on remote files
# - for unix conversions, currently does an inplace conversion. Should it create a .bak file?
# - for unix conversions, executables and binaries are not automatically excluded?
# - if there are only files that match excluded patterns in a deleted directory,
# do we still delete the directory
# - if there is an exception thrown during an ftp operation, delete .webmirror
# and resync up. e.g. deleting a remote file will throw an exception if
# the remote file doesn't exist. The .webmirror is out of sync and needs to be
# refreshed...
# - if an exclude is given for the remote, then it applies for the local
# - if a remote file is changed, but the size remains the same, it doesn't detect a difference
# - need to compare remote datetime as well?
import sys
import os
import os.path
import string
import ftplib
from fnmatch import fnmatch
# ----------------------------------------------------------------
# Print usage message and exit
def usage(*args):
sys.stdout = sys.stderr
print "\n\n"
for msg in args: print msg
print __doc__
sys.exit(2)
# ----------------------------------------------------------------
#- holds all options necessary
#- this is a Singleton class. The single instance is held in gOptions
gOptions = None
class Options:
Password = ''
Userid = ''
Host = ''
DB = ''
InfoFilename = ''
SkipPatterns = []
Verbose = 1
Update = 0
Convert = 0
Force = 0
Unix = 0
Sync = 0
DiffOnly = 0
#---
def __init__(self):
self.DB = ''
self.InfoFilename = '.webmirror'
self.SkipPatterns = ['.', '..', '*.webmirror', self.InfoFilename]
self.Verbose = 1 # 0 for -q (quiet), 2 for -v (extra verbose)
self.Update = 0
self.Force = 0
self.Unix = 0
self.Convert = 0
self.Sync = 0
self.DiffOnly = 0
self.SendOnly = 0
#---
#- parse and set the command line
def Set(self, argv):
self.Verbose = 1
cmd = 0
db = 0
err = 0
for i in range(1, len(argv)):
arg = argv[i]
if cmp(arg, '-v') == 0: # very verbose
self.Verbose = 2
elif cmp(arg, '-vv') == 0: # very, very verbose
self.Verbose = 3
elif cmp(arg, '-q') == 0: # quiet
self.Verbose = 0
elif cmp(arg, '-update') == 0: # update local db file only
self.Update = 1
if (cmd) : err = 1; self.multiplecmderror()
cmd = 1
elif cmp(arg, '-sync') == 0: # update local db file and send any differences
self.Sync = 1
if (cmd) : err = 1; self.multiplecmderror()
cmd = 1
elif cmp(arg, '-diff') == 0: # diff between local and remote
self.DiffOnly = 1
if (cmd) : err = 1; self.multiplecmderror()
cmd = 1
elif cmp(arg, '-sendonly') == 0: # send without updating
self.SendOnly = 1
if (cmd) : err = 1; self.multiplecmderror()
cmd = 1
elif cmp(arg, '-convert') == 0: # convert to unix line endings w/o sending
self.Convert = 1
if (cmd) : err = 1; self.multiplecmderror()
cmd = 1
elif cmp(arg, '-unix') == 0: # convert to unix line endings
self.Unix = 1
elif cmp(arg, '-force') == 0: # send all files
self.Force = 1
elif cmp(arg, '-h') == 0 or cmp(arg, '-help') == 0: # display help
err = 1
elif (db == 0 and arg[0] != '-'):
self.DB = arg
self.InfoFilename = self.DB + '.webmirror'
db = 1
else:
err = 1
if (arg[0] == '-'):
print "ERROR: Unknown switch: '" + arg + "'"
elif (db == 1):
print "ERROR: Only one dbname allowed: '" + arg + "'"
else:
print "ERROR: Unknown argument: '" + arg + "'"
self.checkforerror(err)
if (self.Unix and not self.Sync):
print "ERROR: -unix can only be used with -sync"
err = 1
if (db == 0):
print "ERROR: You must provide a dbname."
err = 1
self.checkforerror(err)
self.dumpoptions()
#---
#- private: check for error and exit
def checkforerror(self, err):
if (err):
usage()
sys.exit(1)
#---
#- private: print warning
def multiplecmderror(self):
print "ERROR: Only one command allowed.\n\n"
#---
#- private: print all options
def dumpoptions(self):
if (self.Verbose <= 0): return
print 'Options: dbname ' + str(self.DB)
print 'Options: verbose ' + str(self.Verbose)
print 'Options: update ' + str(self.Update)
print 'Options: unix ' + str(self.Unix)
print 'Options: convert ' + str(self.Convert)
print 'Options: sync ' + str(self.Sync)
print 'Options: diff ' + str(self.DiffOnly)
print 'Options: sendonly ' + str(self.SendOnly)
print 'Options: force ' + str(self.Force)
print 'Options: userid ' + str(self.Userid)
print 'Options: host ' + str(self.Host)
print 'Options: file ' + str(self.InfoFilename)
print 'Options: skip ' + str(self.SkipPatterns)
# ----------------------------------------------------------------
def Trace(*args):
global gOptions
lvl = args[0]
if (gOptions is None or gOptions.Verbose < lvl): return
for i in range(1, len(args)): print args[i],
print
# ----------------------------------------------------------------
def ConvertFile(f):
"""convert line endings"""
if (fnmatch(f, "*.zip") or fnmatch(f, "*.jpg") or fnmatch(f, "*.ppt") or fnmatch(f, "*.doc")):
return;
try:
infh = open(f, 'rb')
inbuf = infh.read()
infh.close()
except IOError, msg:
return
os.rename(f, f + '~')
outfh = open(f, 'wb')
if inbuf.find('\x0D\x0A') != -1:
outfh.write(inbuf.replace('\x0D',''))
elif inbuf.find('\x0D') != -1:
outfh.write(inbuf.replace('\x0D','\x0A'))
else:
outfh.write(inbuf)
outfh.close()
os.remove(f + '~')
# ----------------------------------------------------------------
#- class to maintain file information
class FileInfo:
def __init__(self, **kwds):
self.__dict__.update(kwds)
#-- Non-file or unknown file
NullFileInfo = FileInfo(filesize=-1, localdate=-1)
DirectoryInfo = NullFileInfo
# ----------------------------------------------------------------
class ftpwrapper:
filesfound = []
subdirs = []
info = {}
#---
#- initialize everything
def init(self, host, login, passwd):
self.host = host
self.login = login
self.passwd = passwd
self.ftp = ftplib.FTP()
#---
#- open a connection to the host
def open(self):
Trace(1, 'Connecting to %s...' % `self.host`)
# find the url
try:
self.ftp.connect(self.host)
except ftplib.all_errors, resp:
Trace(0, 'ftp connect: ', resp) #will report bad url here
sys.exit(2)
# attempt the login
try:
account = ''
self.ftp.login(self.login, self.passwd, account)
except ftplib.error_perm, resp:
Trace(0, 'ftp login: ', resp) # will report bad userid/password
sys.exit(2)
#---
#- close connection to the host
def close(self):
try:
self.ftp.quit()
self.ftp.close()
except ftplib.all_errors, resp:
Trace(0, 'ftp quit/close: ', resp)
#keep going, even if there's an error
#---
#- gather all files in the given directory tree, save them in dict
def gatherfiles(self, root, dict, oldlist, matcher):
#if it's an excluded file/directory, go home
if (matcher.IsExclude(root)): return
#if we can't change directories into it, go home
if (self.cwd(root) == 0): return
dict[root] = DirectoryInfo
self.filelist(matcher)
sdirs = self.subdirs
if (len(self.info) > 0):
for key, value in self.info.items():
#if the key already exists, then reuse the localdate value
if oldlist.has_key(key):
dict[key] = FileInfo(filesize=value, localdate=oldlist[key].localdate)
else:
dict[key] = FileInfo(filesize=value, localdate=-1)
for dir in sdirs:
d = root + "/" + dir
self.gatherfiles(d, dict, oldlist, matcher)
#---
#- print working directory
def pwd(self):
try:
pwd = self.ftp.pwd()
return pwd
except ftplib.all_errors, resp:
Trace(0, 'ftp pwd: ', resp)
sys.exit(2) #if pwd fails, we're in rough shape
#---
#- change working directory to given dir
def cwd(self, remotedir):
Trace(2, 'cwd(%s)' % `remotedir`)
try:
self.ftp.cwd(remotedir)
return 1
except ftplib.error_perm, resp:
Trace(0, 'CWD resp: ', resp)
return 0 #can't change into given dir
#---
#- get list of files and print to stdout
def rawlist(self):
listing = []
try:
self.ftp.retrlines('LIST', listing.append)
except ftplib.all_errors, resp:
Trace(0, 'ftp rawlist LIST: ', resp)
sys.exit(2) #if retrlines fails, we're in rough shape
for line in listing:
print line
#---
#- convert the line endings in the current file to unix style
def convert(self, f):
ConvertFile(f);
#---
#- send a file from local 'path' to remote 'remotepath'
def sendfile(self, path, remotepath):
global gOptions
if (gOptions.Unix == 1):
self.convert(path)
try:
fp = open(path, 'rb')
except IOError, msg:
return
try:
resp = self.ftp.storbinary('STOR ' + remotepath, fp)
except ftplib.all_errors, resp:
Trace(0, 'STOR resp: ', resp)
sys.exit(2)
fp.close()
Trace(2, 'STOR resp: ', resp)
#---
#- delete a remote file 'remotepath'
def deletefile(self, remotepath):
try:
resp = self.ftp.sendcmd('DELE ' + remotepath)
Trace(2, 'DELE resp: ', resp)
except ftplib.error_perm, resp:
Trace(0, 'DELE resp: ', resp)
#---
#- create a remote directory 'remotedir'
def createdir(self, remotedir):
try:
resp = self.ftp.sendcmd('MKD ' + remotedir)
Trace(2, 'MKD resp: ', resp)
except ftplib.error_perm, resp:
Trace(0, 'MKD resp: ', resp)
#---
#- delete a remote directory 'remotedir'
def deletedir(self, remotedir):
try:
resp = self.ftp.sendcmd('RMD ' + remotedir)
Trace(2, 'RMD resp: ', resp)
except ftplib.error_perm, resp:
Trace(0, 'RMD resp: ', resp)
#---
#- get files from current directory
def filelist(self, matcher):
pwd = self.pwd()
self.subdirs = []
listing = []
self.info = {}
#Trace(1, 'Listing remote directory %s...' % `pwd`)
try:
self.ftp.retrlines('LIST -al', listing.append)
except ftplib.all_errors, resp:
Trace(0, 'ftp retrlines2 LIST: ', resp)
sys.exit(2)
self.filesfound = []
for line in listing:
Trace(3, 'line: ', `line`)
# Parse, assuming a UNIX listing
words = string.split(line, None, 8)
if len(words) < 6:
Trace(2, ' : Skipping short line')
continue
filename = string.lstrip(words[-1])
i = string.find(filename, " -> ")
if i >= 0:
# words[0] had better start with 'l'...
Trace(2, ' : Found symbolic link:', `filename`)
linkto = filename[i+4:]
filename = filename[:i]
continue
infostuff = words[-5:-1]
mode = words[0]
if (self.skipfile(filename)): continue
if (matcher.IsExclude(pwd + '/' + filename)): continue
if mode[0] == 'd':
Trace(2, ' : Found subdirectory: ', `filename`)
self.subdirs.append(filename)
continue
self.filesfound.append(pwd + '/' + filename)
self.info[pwd + '/' + filename] = infostuff[0]
#---
#- debug: print out all files and directories
def dump(self):
for f in self.filesfound:
print " file: " + f
for d in self.subdirs:
print " dir: " + d
for f in self.info.keys():
print " key: " + f + " : " + `self.info[f]`
#---
#- skip a file if it matches the skip patterns
def skipfile(self, filename):
global gOptions
skip = 0
for pat in gOptions.SkipPatterns:
if fnmatch(filename, pat):
skip = 1
if cmp(pat, '.') != 0 and cmp(pat, '..') != 0:
Trace(2, 'Skip pattern', `pat`, 'matches', `filename`)
break
return skip
# ----------------------------------------------------------------
class _FtpSingleton:
instance = None
mFtpWrapper = None
#---
def __init__(self):
global gOptions
self.mFtpWrapper = ftpwrapper()
self.mFtpWrapper.init(gOptions.Host, gOptions.Userid, gOptions.Password)
self.mFtpWrapper.open()
#---
def Get(self):
return self.mFtpWrapper
#---
def Close(self):
self.mFtpWrapper.close()
self.mFtpWrapper = None
# ----------------------------------------------------------------
#- create the ftp singleton.
def FtpSingleton():
if (_FtpSingleton.mFtpWrapper == None):
_FtpSingleton.instance = _FtpSingleton()
return _FtpSingleton.instance.Get()
#---
#- the destructor
def FtpSingletonCleanup():
if (_FtpSingleton.mFtpWrapper == None): return
_FtpSingleton.instance.Close()
_FtpSingleton.instance = None
# ----------------------------------------------------------------
#- Write a dictionary to a file in a way that can be read back using
#- rval() but is still somewhat readable (i.e. not a single long line).
#- Also creates a backup file.
def writedict(dict, filename):
dir, file = os.path.split(filename)
tempname = os.path.join(dir, '@' + file)
backup = os.path.join(dir, file + '~')
try:
os.unlink(backup)
except os.error:
pass
fp = open(tempname, 'w')
fp.write('{\n')
for key, value in dict.items():
if value == DirectoryInfo:
fp.write('\'%s\': DirectoryInfo,\n' % str(key))
else:
fp.write('\'%s\': FileInfo(filesize=%s, localdate=%s),\n' % (str(key), str(value.filesize), str(value.localdate)))
fp.write('}\n')
fp.close()
try:
os.rename(filename, backup)
except os.error:
pass
os.rename(tempname, filename)
# ----------------------------------------------------------------
#- read the file (written by writedict) into a dictionary
def readdict(filename):
global DirectoryInfo
dict = {}
text = ''
try:
text = open(filename, 'r').read()
except IOError, msg:
text = '{}'
try:
dict = eval(text)
except SyntaxError, ex:
print 'Bad mirror info in ', `filename`, ':', ex
dict = {}
return dict
# ----------------------------------------------------------------
#- compare two paths
def path_compare(path1, path2):
dir1, file1 = os.path.split(path1)
dir2, file2 = os.path.split(path2)
if (dir1 < dir2):
return -1
if (dir1 > dir2):
return 1
if (file1 < file2):
return -1
if (file1 > file2):
return 1
return 0
# ----------------------------------------------------------------
#- compare two paths. Opposite ordering from path_compare
def reverse_path_compare(path1, path2):
dir1, file1 = os.path.split(path1)
dir2, file2 = os.path.split(path2)
if (dir1 < dir2):
return 1
if (dir1 > dir2):
return -1
if (file1 < file2):
return 1
if (file1 > file2):
return -1
return 0
# ----------------------------------------------------------------
#- base class for getting information about entities on a file
#- system, e.g. local or remote via ftp
class FileSys:
mList = {}
mRoot = ''
mExclusions = []
#---
def Init(self, root, exclusions):
self.mList = {}
self.mRoot = root
self.mExclusions = exclusions
#---
#- get directory tree and file size information
def GetInformation(self, type, force):
Trace(1, "Getting ", type, " information...")
self.Gather(force)
self.Exclude()
self.Dump(type + " files")
#---
#- override in derived class for given file system
def Gather(self, force):
pass;
#---
#- exclude any files that match exclusion patterns
def Exclude(self):
for pat in self.mExclusions:
Trace(2, 'Exclude pattern:', `pat`)
for filename in self.mList.keys():
Trace(2, 'Exclude pattern:', `pat`, ' file:', `filename`)
if fnmatch(filename, pat):
del self.mList[filename]
Trace(2, 'Exclude pattern:', `pat`, ' matches ', `filename`)
#---
#- check if filename/directory name matches any excludes
def IsExclude(self, f):
for pat in self.mExclusions:
if fnmatch(f, pat):
Trace(2, 'Exclude pattern', `pat`, 'matches', `f`)
return 1
return 0
#---
#- print the file list to stdout
def Dump(self, title):
global gOptions
if (gOptions.Verbose < 3): return
print title, ":"
keylist = self.mList.keys()
keylist.sort(path_compare)
self.DumpList(keylist)
#- print the given file list to stdout
#---
def DumpFrom(self, keylist, title):
global gOptions
if (len(keylist) == 0):
if gOptions.DiffOnly == 1 or gOptions.Verbose > 0: print "No", title, "..."
return
if gOptions.DiffOnly != 1 and gOptions.Verbose < 2:
Trace(1, len(keylist), " ", title, "found")
return
print title, ":"
keylist.sort(path_compare)
self.DumpList(keylist)
#---
#- print the file list to stdout in reverse order
def DumpReverseFrom(self, keylist, title):
global gOptions
if (len(keylist) == 0):
if gOptions.DiffOnly == 1 or gOptions.Verbose > 0: print "No", title, "..."
return
if gOptions.DiffOnly != 1 and gOptions.Verbose < 2:
Trace(1, len(keylist), " ", title, "found")
return
print title, ":"
keylist.sort(reverse_path_compare)
self.DumpList(keylist)
#---
#- print the file list to stdout
def DumpList(self, keylist):
for key in keylist:
if self.mList[key] == DirectoryInfo :
print " dir : ", key
else:
print " file: ", key, " : ", self.mList[key].filesize, " ", self.mList[key].localdate
#---
#- find missing files
def FindFilesNotIn(self, other):
files = []
for item in self.mList.keys():
item2 = item.replace(self.mRoot, other.Root())
if item2 not in other.Keys() and self.mList[item] != DirectoryInfo:
files.append(item)
return files
#---
#- find missing directories
def FindDirsNotIn(self, other):
dirs = []
for item in self.mList.keys():
item2 = item.replace(self.mRoot, other.Root())
if item2 not in other.Keys() and self.mList[item] == DirectoryInfo:
dirs.append(item)
return dirs
#---
#- find common files & directories
def FindFilesIn(self, other):
files = []
for item in self.mList.keys():
item2 = item.replace(self.mRoot, other.Root())
if item2 in other.Keys():
files.append(item)
return files
#---
#-
def Keys(self):
return self.mList.keys()
#---
#-
def Root(self):
return self.mRoot
# ----------------------------------------------------------------
#- represents a unix file system on a remote server
class RemoteFileSys(FileSys):
mFtp = None
#---
#- get all file information using FTP
def Gather(self, force):
global gOptions
Trace(1, "Reading db...")
self.mList = readdict(gOptions.InfoFilename)
Trace(1, "Finished reading db.")
if (len(self.mList) == 0 or force == 1):
Trace(1, "Gathering Remote files...")
oldlist = self.mList
self.mList = {}
f = FtpSingleton()
f.gatherfiles(self.mRoot, self.mList, oldlist, self)
Trace(1, "Finished gathering Remote files.")
Trace(1, "Checkpointing DB...")
#checkpoint just in case...
writedict(self.mList, gOptions.InfoFilename)
Trace(1, "Finished checkpointing DB.")
# ----------------------------------------------------------------
#- represents a local windows file system
class LocalFileSys(FileSys):
#---
#- get all file information using a directory walk
def visit(self, flist, dirname, names):
d = dirname.replace('\\', '/')
if (self.IsExclude(d)): return
flist[d] = DirectoryInfo
for name in names:
p = os.path.join(dirname, name).replace('\\', '/')
if (self.IsExclude(p)): continue
flist[p] = FileInfo(filesize=os.path.getsize(p), localdate=os.path.getmtime(p))
#---
#- get all file information from local hard drive
def Gather(self, force):
os.path.walk(self.mRoot, self.visit, self.mList)
#---
#- convert all files in the tree
def ConvertOnly(self):
Trace(0, 'Converting to Unix line endings')
self.Gather(0)
for item in self.mList.keys():
if (self.mList[item] != DirectoryInfo) :
ConvertFile(item)
# ----------------------------------------------------------------
#- compares two directories and reports on new, changed,
#- or deleted files and directories
class Differences:
mDiffFound = 0
mRemote = None
mLocal = None
mNewFiles = []
mDeletedFiles = []
mChangedFiles = []
mNewDirectories = []
mDeletedDirectories = []
#---
def __init__(self, local, remote):
self.mDiffFound = 0
self.mLocal = local
self.mRemote = remote
self.mNewFiles = []
self.mDeletedFiles = []
self.mChangedFiles = []
self.mNewDirectories = []
self.mDeletedDirectories = []
#---
#- determine differences between two trees
def Determine(self):
Trace(1, "Determining differences...")
self.FindNewDirectories()
self.FindNewFiles()
self.FindDeletedFiles()
self.FindChangedFiles()
self.FindDeletedDirectories()
Trace(1, "Finished determining differences.")
#---
#- apply changes so that the remote tree looks like the local
def Apply(self):
global gOptions
if (self.mDiffFound == 0):
Trace(1, "No differences found, nothing to apply.")
return
Trace(1, "Applying differences...")
self.CreateNewDirectories()
self.SendNewFiles()
self.SendChangedFiles()
self.RemoveDeletedFiles()
self.RemoveDeletedDirectories()
Trace(1, "Finished applying differences.")
Trace(1, "Saving db...")
#save any changes against the Remote list
writedict(self.mRemote.mList, gOptions.InfoFilename)
Trace(1, "Finished saving db.")
#---
#- determine all new directories
def FindNewDirectories(self):
self.mNewDirectories = self.mLocal.FindDirsNotIn(self.mRemote)
self.mLocal.DumpFrom(self.mNewDirectories, "New Directories")
if (len(self.mNewDirectories) > 0): self.mDiffFound = 1
#---
#- determine all deleted directories
def FindDeletedDirectories(self):
self.mDeletedDirectories = self.mRemote.FindDirsNotIn(self.mLocal)
self.mRemote.DumpReverseFrom(self.mDeletedDirectories, "Deleted Directories")
if (len(self.mDeletedDirectories) > 0): self.mDiffFound = 1
#---
#- determine all new files
def FindNewFiles(self):
self.mNewFiles = self.mLocal.FindFilesNotIn(self.mRemote)
self.mLocal.DumpFrom(self.mNewFiles, "New Files")
if (len(self.mNewFiles) > 0): self.mDiffFound = 1
#---
#- determine all deleted files
def FindDeletedFiles(self):
self.mDeletedFiles = self.mRemote.FindFilesNotIn(self.mLocal)
self.mRemote.DumpFrom(self.mDeletedFiles, "Deleted Files")
if (len(self.mDeletedFiles) > 0): self.mDiffFound = 1
#---
#- determine all changed files
def FindChangedFiles(self):
global gOptions
if gOptions.Force == 1:
# it's a force, add them all.
for item in self.mLocal.Keys():
self.mChangedFiles.append(item)
else:
for item in self.mLocal.Keys():
localsize = self.mLocal.mList[item].filesize
localdate = self.mLocal.mList[item].localdate
item2 = item.replace(self.mLocal.Root(), self.mRemote.Root())
if item2 in self.mRemote.Keys():
remotesize = self.mRemote.mList[item2].filesize
remotedate = self.mRemote.mList[item2].localdate
if int(localsize) != int(remotesize):
Trace(2, str(item2) + ": size changed: " + str(localsize) + " remote=" + str(remotesize))
self.mChangedFiles.append(item)
elif localdate != remotedate: #check if file date changed
Trace(2, str(item2) + ": date changed: " + str(localdate) + " remote=" + str(remotedate))
self.mChangedFiles.append(item)
self.mLocal.DumpFrom(self.mChangedFiles, "Changed Files")
if (len(self.mChangedFiles) > 0): self.mDiffFound = 1
#---
#- create new directories on remote file system
def CreateNewDirectories(self):
if len(self.mNewDirectories) == 0:
Trace(1, "No new directories...")
return
Trace(1, "Creating new directories...")
ftp = FtpSingleton()
for d in self.mNewDirectories:
rdir = d.replace(self.mLocal.Root(), self.mRemote.Root())
currpath = ''
for currdir in rdir.split('/'):
if currdir == '': continue #skip empty item. split produces an empty string '' as the first item
currpath = currpath + '/' + currdir #rebuild the path one node at a time
if len(currpath) < len(self.mRemote.Root()): continue #no use looking at anything less than the root
if self.mRemote.mList.has_key(currpath): continue #no use creating the dir if we've already created it
Trace(2, "Creating remote dir: ", currpath)
ftp.createdir(currpath)
self.mRemote.mList[currpath] = DirectoryInfo;
#---
#- send new files to remote file system
def SendNewFiles(self):
if len(self.mNewFiles) == 0:
Trace(1, "No new files...")
return
Trace(1, "Sending new files...")
ftp = FtpSingleton()
for file in self.mNewFiles:
rfile = file.replace(self.mLocal.Root(), self.mRemote.Root())
Trace(2, "sending file: ", file, " to:", rfile)
ftp.sendfile(file, rfile)
self.mRemote.mList[rfile] = self.mLocal.mList[file];
#---
#- send changed files to remote file system
def SendChangedFiles(self):
if len(self.mChangedFiles) == 0:
Trace(1, "No changed files...")
return
Trace(1, "Sending changed files...")
ftp = FtpSingleton()
for file in self.mChangedFiles:
rfile = file.replace(self.mLocal.Root(), self.mRemote.Root())
Trace(2, "sending file: ", file, " to:", rfile)
ftp.sendfile(file, rfile)
self.mRemote.mList[rfile] = self.mLocal.mList[file];
#---
#- remove old files (i.e. deleted locally) on remote file system
def RemoveDeletedFiles(self):
if len(self.mDeletedFiles) == 0:
Trace(1, "No deleted files...")
return
Trace(1, "Removing deleted files...")
ftp = FtpSingleton()
for file in self.mDeletedFiles:
rfile = file.replace(self.mLocal.Root(), self.mRemote.Root())
Trace(2, "deleting file: ", rfile)
ftp.deletefile(rfile)
del self.mRemote.mList[rfile]
#---
#- remove old directories (i.e. deleted locally) on remote file system
def RemoveDeletedDirectories(self):
if len(self.mDeletedDirectories) == 0:
Trace(1, "No deleted directories...")
return
ftp = FtpSingleton()
Trace(1, "Removing deleted directories...")
#ensure we start at the leaves of the directory tree
self.mDeletedDirectories.sort(reverse_path_compare)
for dir in self.mDeletedDirectories:
rdir = dir.replace(self.mLocal.Root(), self.mRemote.Root())
Trace(2, "deleting directory: ", rdir)
ftp.deletedir(rdir)
del self.mRemote.mList[rdir]
# ----------------------------------------------------------------
#- the main driver class
class App:
mRemote = RemoteFileSys()
mLocal = LocalFileSys()
mDbList = {};
#---
#- save all database info in the list
def Add(self, name, remoteroot, remoteexclusions, localroot, localexclusions):
self.mDbList[name] = [remoteroot, remoteexclusions, localroot, localexclusions];
#---
#- private init
def Init(self):
global gOptions
if self.mDbList.has_key(gOptions.DB):
remoteroot, remoteexclusions, localroot, localexclusions = self.mDbList[gOptions.DB]
Trace(1, "Database options: ")
Trace(1, " Remote root : ", remoteroot)
Trace(1, " Remote exclusions: ", remoteexclusions)
Trace(1, " Local root : ", localroot)
Trace(1, " Local exclusions : ", localexclusions)
else:
print 'Database ' + str(gOptions.DB) + ' not found. These are available: '
for name in self.mDbList.keys():
print ' ' + name
sys.exit(2)
self.mRemote.Init(remoteroot, remoteexclusions)
self.mLocal.Init(localroot, localexclusions)
#---
#- do the actual processing
def Run(self, options):
global gOptions
gOptions = options
self.Init()
if (gOptions.Update):
self.mRemote.GetInformation("Remote", 1) #load from remote site, save in db
elif (gOptions.Convert):
self.mLocal.GetInformation("Local", 0) #load from local hard drive
self.mLocal.ConvertOnly();
else:
if (gOptions.Sync):
self.mRemote.GetInformation("Remote", 1) #load from remote site, save in db
else: # sendonly, diff
self.mRemote.GetInformation("Remote", 0) #load from db
self.mLocal.GetInformation("Local", 0) #load from local hard drive
diffs = Differences(self.mLocal, self.mRemote)
diffs.Determine()
if (gOptions.DiffOnly == 0):
diffs.Apply()
FtpSingletonCleanup()
#---
# Main program: parse command line and start processing
def main():
#you'll need these imports
#import jFtpMirror
#import sys
# options = jFtpMirror.Options()
options = Options()
options.Host = 'a.server.name' # example: www.bob.com
options.Userid = 'your_userid'
options.Password = 'your_password'
options.DB = 'your_dbname'
options.Set(sys.argv)
# app = jFtpMirror.App()
app = App()
#the db name. Named on the command line as -dbxx
#the remote root directory (relative to where your ftp logs into);
#the remote exclusion patterns, if any; see fnmatch for details
#the local root directory
#the local exclusion patterns, if any; see fnmatch for details
# DB remote excludes local excludes
app.Add('mytest', '/mytest', ['*/stats*', '*/err.log'], '/projects/web/mytest', ['*/.svn/*', '*/.svn', '*/err.log'])
app.Add('mysite', '/mysite', [], '/projects/web/mysite', ['*/.cvs/*'])
app.Add('myblah', '/myblah', ['err.log'], '/projects/web/myblah', ['err.log'])
app.Run(options)
if __name__ == '__main__':
main()
|