[Zope3-checkins] CVS: Zope3/src/zope/fssync - __init__.py:1.1 compare.py:1.1 fssync.py:1.1 main.py:1.1
Guido van Rossum
guido@python.org
Fri, 9 May 2003 16:54:15 -0400
Update of /cvs-repository/Zope3/src/zope/fssync
In directory cvs.zope.org:/tmp/cvs-serv15618
Added Files:
__init__.py compare.py fssync.py main.py
Log Message:
New fssync command line utility. Very fresh, but basics work.
=== Added File Zope3/src/zope/fssync/__init__.py ===
#
# This file is necessary to make this directory a package.
=== Added File Zope3/src/zope/fssync/compare.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Tools to compare parallel trees as written by toFS().
$Id: compare.py,v 1.1 2003/05/09 20:54:15 gvanrossum Exp $
"""
from __future__ import generators
import os
import filecmp
from os.path import exists, isfile, isdir, join, normcase
from zope.xmlpickle import loads
def checkUptodate(working, current):
"""Up-to-date check before committing changes.
Given a working tree containing the user's changes and Original
subtrees, and a current tree containing the current state of the
database (for the same object tree), decide whether all the
Original entries in the working tree match the entries in the
current tree. Return a list of error messages if something's
wrong, [] if everything is up-to-date.
"""
if not isdir(current):
return []
if not isdir(working):
return ["missing working directory %r" % working]
errors = []
for (left, right, common, lentries, rentries, ldirs, lnondirs,
rdirs, rnondirs) in treeComparisonWalker(working, current):
if rentries:
# Current has entries that working doesn't (the reverse
# means things added to working, which is fine)
for x in rentries:
errors.append("missing working entry for %r" % join(left, x))
for x in common:
nx = normcase(x)
if nx in rnondirs:
# Compare files (directories are compared by the walk)
lfile = join(left, "@@Zope", "Original", x)
rfile = join(right, x)
if not isfile(lfile):
errors.append("missing working original file %r" % lfile)
elif not filecmp.cmp(lfile, rfile, shallow=False):
errors.append("files %r and %r differ" % (lfile, rfile))
# Compare extra data (always)
lextra = join(left, "@@Zope", "Extra", x)
rextra = join(right, "@@Zope", "Extra", x)
errors.extend(checkUptodate(lextra, rextra))
# Compare annotations (always)
lann = join(left, "@@Zope", "Annotations", x)
rann = join(right, "@@Zope", "Annotations", x)
errors.extend(checkUptodate(lann, rann))
return errors
def treeComparisonWalker(left, right):
"""Generator that walks two parallel trees created by toFS().
Each item yielded is a tuple of 9 items:
left -- left directory path
right -- right directory path
common -- dict mapping common entry names to (left, right) entry dicts
lentries -- entry dicts unique to left
rentries -- entry dicts unique to right
ldirs -- names of subdirectories of left
lnondirs -- nondirectory names in left
rdirs -- names subdirectories of right
rnondirs -- nondirectory names in right
It's okay for the caller to modify the dicts to affect the rest of
the walk.
IOError exceptions may be raised.
"""
# XXX There may be problems on a case-insensitive filesystem when
# the Entries.xml file mentions two objects whose name only
# differs in case. Otherwise, case-insensitive filesystems are
# handled correctly.
queue = [(left, right)]
while queue:
left, right = queue.pop(0)
lentries = loadEntries(left)
rentries = loadEntries(right)
common = {}
for key in lentries.keys():
if key in rentries:
common[key] = lentries[key], rentries[key]
del lentries[key], rentries[key]
ldirs, lnondirs = classifyContents(left)
rdirs, rnondirs = classifyContents(right)
yield (left, right,
common, lentries, rentries,
ldirs, lnondirs, rdirs, rnondirs)
commonkeys = common.keys()
commonkeys.sort()
for x in commonkeys:
nx = normcase(x)
if nx in ldirs and nx in rdirs:
queue.append((ldirs[nx], rdirs[nx]))
# XXX Need to push @@Zope/Annotations/ and @@Zope/Extra/ as well.
nczope = normcase("@@Zope") # Constant used by classifyContents
def classifyContents(path):
"""Classify contents of a directory into directories and non-directories.
Return a pair of dicts, the first containing directory names, the
second containing names of non-directories. Each dict maps the
normcase'd version of the name to the path formed by concatenating
the path with the original name. '@@Zope' is excluded.
"""
dirs = {}
nondirs = {}
for name in os.listdir(path):
ncname = normcase(name)
if ncname == nczope:
continue
full = join(path, name)
if isdir(full):
dirs[ncname] = full
else:
nondirs[ncname] = full
return dirs, nondirs
def loadEntries(dir):
"""Return the Entries.xml file as a dict; default to {}."""
filename = join(dir, "@@Zope", "Entries.xml")
if exists(filename):
f = open(filename)
data = f.read()
f.close()
return loads(data)
else:
return {}
=== Added File Zope3/src/zope/fssync/fssync.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Support classes for fssync.
$Id: fssync.py,v 1.1 2003/05/09 20:54:15 gvanrossum Exp $
"""
import os
import base64
import shutil
import urllib
import filecmp
import htmllib
import httplib
import commands
import tempfile
import urlparse
import formatter
from StringIO import StringIO
from os.path import exists, isfile, isdir, islink
from os.path import dirname, basename, split, join
from os.path import realpath, normcase, normpath
from zope.xmlpickle import loads, dumps
from zope.fssync.compare import treeComparisonWalker
class Error(Exception):
"""User-level error, e.g. non-existent file.
This can be used in several ways:
1) raise Error("message")
2) raise Error("message %r %r" % (arg1, arg2))
3) raise Error("message %r %r", arg1, arg2)
4) raise Error("message", arg1, arg2)
- Forms 2-4 are equivalent.
- Form 4 assumes that "message" contains no % characters.
- When using forms 2 and 3, all % formats are supported.
- Form 2 has the disadvantage that when you specify a single
argument that happens to be a tuple, it may get misinterpreted.
- The message argument is required.
- Any number of arguments after that is allowed.
"""
def __init__(self, msg, *args):
self.msg = msg
self.args = args
def __str__(self):
msg, args = self.msg, self.args
if args:
if "%" in msg:
msg = msg % args
else:
msg += " ".join(map(repr, args))
return str(msg)
def __repr__(self):
return "%s%r" % (self.__class__.__name__, (self.msg,)+self.args)
class FSSync(object):
def __init__(self, topdir, verbose=False):
self.topdir = topdir
self.verbose = verbose
self.rooturl = self.findrooturl()
def setrooturl(self, rooturl):
self.rooturl = rooturl
def checkout(self):
fspath = self.topdir
if not self.rooturl:
raise Error("root url not found nor explicitly set")
if os.path.exists(fspath):
raise Error("can't checkout into existing directory", fspath)
url = self.rooturl
if not url.endswith("/"):
url += "/"
url += "@@toFS.zip?writeOriginals=True"
filename, headers = urllib.urlretrieve(url)
if headers["Content-Type"] != "application/zip":
raise Error("The request didn't return a zipfile; contents:\n%s",
self.slurptext(self.readfile(filename),
headers).strip())
try:
os.mkdir(fspath)
sts = os.system("cd %s; unzip -q %s" % (fspath, filename))
if sts:
raise Error("unzip command failed")
self.saverooturl()
finally:
os.unlink(filename)
def commit(self):
fspath = self.topdir
if not self.rooturl:
raise Error("root url not found")
(scheme, netloc, url, params,
query, fragment) = urlparse.urlparse(self.rooturl)
if scheme != "http":
raise Error("root url must start with http", rooturl)
user_passwd, host_port = urllib.splituser(netloc)
zipfile = tempfile.mktemp(".zip")
sts = os.system("cd %s; zip -q -r %s ." % (fspath, zipfile))
if sts:
raise Error("zip command failed")
zipdata = self.readfile(zipfile, "rb")
os.unlink(zipfile)
h = httplib.HTTP(host_port)
h.putrequest("POST", url + "/@@fromFS.zip")
h.putheader("Content-Type", "application/zip")
h.putheader("Content-Length", str(len(zipdata)))
if user_passwd:
auth = base64.encodestring(user_passwd).strip()
h.putheader('Authorization', 'Basic %s' % auth)
h.putheader("Host", host_port)
h.endheaders()
h.send(zipdata)
errcode, errmsg, headers = h.getreply()
if errcode != 200:
raise Error("HTTP error %s (%s); error document:\n%s",
errcode, errmsg,
self.slurptext(h.getfile().read(), headers))
if headers["Content-Type"] != "application/zip":
raise Error("The request didn't return a zipfile; contents:\n%s",
self.slurptext(h.getfile().read(), headers))
f = open(zipfile, "wb")
shutil.copyfileobj(h.getfile(), f)
f.close()
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
sts = os.system("cd %s; unzip -q %s" % (tmpdir, zipfile))
if sts:
raise Error("unzip command failed")
self.merge(self.topdir, tmpdir)
shutil.rmtree(tmpdir)
os.unlink(zipfile)
print "All done"
def update(self):
url = self.rooturl
if not url.endswith("/"):
url += "/"
url += "@@toFS.zip?writeOriginals=False"
filename, headers = urllib.urlretrieve(url)
try:
if headers["Content-Type"] != "application/zip":
raise Error("The request didn't return a zipfile; "
"contents:\n%s",
self.slurptext(self.readfile(filename),
headers).strip())
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
try:
sts = os.system("cd %s; unzip -q %s" % (tmpdir, filename))
if sts:
raise Error("unzip command failed")
self.merge(self.topdir, tmpdir)
print "All done"
finally:
shutil.rmtree(tmpdir)
finally:
os.unlink(filename)
def merge(self, ours, server):
# XXX This method is way too long, and still not complete :-(
for (left, right, common, lentries, rentries, ldirs, lnondirs,
rdirs, rnondirs) in treeComparisonWalker(ours, server):
origdir = join(left, "@@Zope", "Original")
lextradir = join(left, "@@Zope", "Extra")
rextradir = join(right, "@@Zope", "Extra")
lanndir = join(left, "@@Zope", "Annotations")
ranndir = join(right, "@@Zope", "Annotations")
weirdos = ldirs.copy() # This is for flagging "?" files
weirdos.update(lnondirs)
for x in common: # Compare matching stuff
nx = normpath(x)
if nx in weirdos:
del weirdos[nx]
if nx in rdirs:
if nx in lnondirs:
print "file '%s' is in the way of a directory"
elif nx not in ldirs:
print "restoring directory '%s'"
os.mkdir(join(left, x))
elif nx in rnondirs:
if nx in ldirs:
print "directory '%s' is in the way of a file"
else:
# Merge files
rx = rnondirs[nx]
origx = join(origdir, x)
if nx in lnondirs:
lx = lnondirs[nx]
else:
lx = join(left, x)
print "restoring lost file '%s'" % lx
self.copyfile(origx, lx)
if self.cmp(origx, rx):
# Unchanged on server
if self.cmp(lx, origx):
if self.verbose:
print "=", lx
else:
print "M", lx
elif self.cmp(lx, origx):
# Unchanged locally
self.copyfile(rx, lx)
self.copyfile(rx, origx)
print "U", lx
elif self.cmp(lx, rx):
# Only the original is out of date
print "file '%s' already contains changes" % lx
self.copyfile(rx, origx)
print "U", lx
else:
# Conflict! Must do a 3-way merge
print "merging changes into '%s'" % lx
self.copyfile(rx, origx)
sts = os.system("merge %s %s %s" %
(commands.mkarg(lx),
commands.mkarg(origx),
commands.mkarg(rx)))
if sts:
print "C", lx
else:
print "M", lx
# In all cases, merge Extra stuff if any
lx = join(lextradir, x)
rx = join(rextradir, x)
if isdir(rx):
self.ensuredir(lx)
self.merge(lx, rx)
# And merge Annotations if any
lx = join(lanndir, x)
rx = join(ranndir, x)
if isdir(rx):
self.ensuredir(lx)
self.merge(lx, rx)
entries = self.loadentries(left)
entries_changed = False
for x in rentries: # Copy new stuff from server
entries[x] = rentries[x]
entries_changed = True
nx = normpath(x)
if nx in rdirs:
del weirdos[nx]
# New directory; traverse into it
if nx in lnondirs:
print ("file '%s' is in the way of a new directory" %
lnondirs[nx])
else:
common[x] = ({}, rentries[x])
del rentries[x]
if nx not in ldirs:
lfull = join(left, x)
os.mkdir(lx)
ldirs[nx] = lx
elif nx in rnondirs:
if nx in ldirs:
print ("directory '%s' is in the way of a new file" %
ldirs[nx])
elif nx in lnondirs:
if self.cmp(rnondirs[nx], lnondirs[nx]):
print "U", lnondirs[nx]
del weirdos[nx]
else:
print ("file '%s' is in the way of a new file" %
lnondirs[nx])
else:
# New file; copy it
lx = join(left, x)
rx = join(right, x)
self.copyfile(rx, lx)
# And copy to Original
self.ensuredir(origdir)
self.copyfile(rx, join(origdir, x))
print "U", lx
# In all cases, copy Extra stuff if any
lx = join(lextradir, x)
rx = join(rextradir, x)
if isdir(rx):
self.ensuredir(lx)
self.merge(lx, rx)
# And copy Annotations if any
lx = join(lanndir, x)
rx = join(ranndir, x)
if isdir(rx):
self.ensuredir(lx)
self.merge(lx, rx)
if entries_changed:
self.dumpentries(entries, left)
for x in lentries: # Flag new stuff in the working directory
# XXX Could be deleted on server too!!!
nx = normpath(x)
if nx in weirdos:
print "A", weirdos[nx]
del weirdos[nx]
else:
lx = join(left, x)
print "newborn '%s' is missing" % lx
# XXX How about Annotations and Extra for these?
# Flag anything not yet noted
for nx in weirdos:
if not self.ignore(nx):
print "?", weirdos[nx]
def ignore(self, path):
return path.endswith("~")
def cmp(self, f1, f2):
try:
return filecmp.cmp(f1, f2, shallow=False)
except (os.error, IOError):
return False
def copyfile(self, src, dst):
shutil.copyfile(src, dst)
def ensuredir(self, dir):
if not isdir(dir):
os.makedirs(dir)
def slurptext(self, data, headers):
ctype = headers["content-type"]
if ctype == "text/html":
s = StringIO()
f = formatter.AbstractFormatter(formatter.DumbWriter(s))
p = htmllib.HTMLParser(f)
p.feed(data)
p.close()
return s.getvalue()
if ctype.startswith("text/"):
return data
return "Content-Type %r" % ctype
def findrooturl(self):
dir = self.topdir
while dir:
zopedir = join(dir, "@@Zope")
rootfile = join(zopedir, "Root")
try:
data = self.readfile(rootfile)
return data.strip()
except IOError:
pass
dir = self.parent(dir)
return None
def saverooturl(self):
if self.rooturl:
self.writefile(self.rooturl + "\n",
join(self.topdir, "@@Zope", "Root"))
def loadentries(self, dir):
file = join(dir, "@@Zope", "Entries.xml")
try:
return self.loadfile(file)
except IOError:
return {}
def dumpentries(self, entries, dir):
file = join(dir, "@@Zope", "Entries.xml")
self.dumpfile(entries, file)
def loadfile(self, file):
data = self.readfile(file)
return loads(data)
def dumpfile(self, obj, file):
data = dumps(obj)
self.writefile(data, file)
def readfile(self, file, mode="r"):
f = open(file, mode)
try:
return f.read()
finally:
f.close()
def writefile(self, data, file, mode="w"):
f = open(file, mode)
try:
f.write(data)
finally:
f.close()
def parent(self, path):
anomalies = ("", os.curdir, os.pardir)
head, tail = split(path)
if tail not in anomalies:
return head
head, tail = split(normpath(path))
if tail not in anomalies:
return head
head, tail = split(realpath(path))
if tail not in anomalies:
return head
return None
=== Added File Zope3/src/zope/fssync/main.py ===
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""New fssync utility.
Connects to the database using HTTP (using the toFS.zip view for
checkout and update and the fromFS.form view for commit).
An attempt is made to make the behavior similar to that of cvs.
Command line syntax summary:
fssync checkout URL TARGETDIR
fssync update [FILE_OR_DIR ...]
fssync status [FILE_OR_DIR ...]
fssync commit [FILE_OR_DIR ...]
fssync diff [FILE_OR_DIR ...]
$Id: main.py,v 1.1 2003/05/09 20:54:15 gvanrossum Exp $
"""
import os
import sys
import getopt
from os.path import dirname, join, realpath
# Find the zope root directory.
# XXX This assumes this script is <root>/src/zope/fssync/sync.py
scriptfile = sys.argv[0]
scriptdir = realpath(dirname(scriptfile))
rootdir = dirname(dirname(dirname(scriptdir)))
# Hack to fix the module search path
try:
import zope.xmlpickle
# All is well
except ImportError:
# Fix the path to include <root>/src
srcdir = join(rootdir, "src")
sys.path.append(srcdir)
from zope.xmlpickle import loads, dumps
from zope.fssync.fssync import Error, FSSync
class Usage(Error):
"""Subclass for usage error (command-line syntax).
This should set an exit status of 2 rather than 1.
"""
def main(argv=None):
try:
if argv is None:
argv = sys.argv
# XXX getopt
args = argv[1:]
command = args[0]
# XXX more getopt
args = args[1:]
if command in ("checkout", "co"):
url, fspath = args
checkout(url, fspath)
elif command in ("update", "up"):
args = args or [os.curdir]
for fspath in args:
print "update(%r)" % fspath
update(fspath)
elif command in ("commit", "com"):
args = args or [os.curdir]
[fspath] = args
commit(fspath)
else:
raise Usage("command %r not recognized" % command)
except Usage, msg:
print msg
print "for help use --help"
return 2
except Error, msg:
print msg
return 1
else:
return None
def checkout(url, fspath, writeOriginals=True):
fs = FSSync(fspath)
fs.setrooturl(url)
fs.checkout()
def commit(fspath):
fs = FSSync(fspath)
fs.commit()
def update(fspath):
fs = FSSync(fspath)
fs.update()
if __name__ == "__main__":
sys.exit(main(sys.argv))