[Zope-CVS] CVS: Packages/PartitionedFileStorage - PartitionedFileStorage_2_6.patch:1.1 PartitionedFileStorage_HEAD.patch:1.1 FileStorage_py_1_129.patch:NONE FileStorage_py_1_98_2_7.patch:NONE

Shane Hathaway shane@zope.com
Thu, 15 May 2003 15:45:05 -0400


Update of /cvs-repository/Packages/PartitionedFileStorage
In directory cvs.zope.org:/tmp/cvs-serv23920

Added Files:
	PartitionedFileStorage_2_6.patch 
	PartitionedFileStorage_HEAD.patch 
Removed Files:
	FileStorage_py_1_129.patch FileStorage_py_1_98_2_7.patch 
Log Message:
Updated to the latest HEAD and 2_6 branch.  PFS now works with the new pack
implementation.


=== Added File Packages/PartitionedFileStorage/PartitionedFileStorage_2_6.patch ===
Index: FileStorage.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/FileStorage.py,v
retrieving revision 1.98.2.9
diff -u -u -r1.98.2.9 FileStorage.py
--- FileStorage.py	13 May 2003 18:04:21 -0000	1.98.2.9
+++ FileStorage.py	15 May 2003 19:40:39 -0000
@@ -196,6 +196,43 @@
 
 packed_version='FS21'
 
+
+class FileOperations:
+    '''This class implements some basic operations on files.
+    PartitionedFileOperations provides an alternate implementation.
+    '''
+    def open(self, name, mode='r', bufsize=0):
+        global open
+        return open(name, mode, bufsize)
+
+    def exists(self, name):
+        return os.path.exists(name)
+
+    def rename(self, oldname, newname):
+        os.rename(oldname, newname)
+
+    def remove(self, name):
+        os.remove(name)
+
+    def fsync(self, file):
+        global fsync
+        if fsync is not None:
+            fsync(file.fileno())
+
+defaultFops = FileOperations()
+
+
+# Set the PARTITIONED_FILE_SIZE environment variable to use
+# PartitionedFile support.  Note: To convert a partitioned file back
+# to a normal file, just 'cat' (Unix) or 'copy' (DOS/Windows) the
+# partitions together in order.
+PARTITIONED_FILE_SIZE = long(os.environ.get('PARTITIONED_FILE_SIZE') or 0)
+if PARTITIONED_FILE_SIZE > 0:
+    from PartitionedFile import PartitionedFileOperations
+    defaultFops = PartitionedFileOperations(partlen=PARTITIONED_FILE_SIZE)
+
+
+
 class FileStorage(BaseStorage.BaseStorage,
                   ConflictResolution.ConflictResolvingStorage):
     # default pack time is 0
@@ -204,7 +241,10 @@
     _records_before_save = 10000
 
     def __init__(self, file_name, create=0, read_only=0, stop=None,
-                 quota=None):
+                 quota=None, fops=None):
+        if fops is None:
+            fops = defaultFops
+        self.fops = fops
 
         if read_only:
             self._is_read_only = 1
@@ -246,7 +286,8 @@
         self._file = None
         if not create:
             try:
-                self._file = open(file_name, read_only and 'rb' or 'r+b')
+                self._file = self.fops.open(
+                    file_name, read_only and 'rb' or 'r+b')
             except IOError, exc:
                 if exc.errno == errno.EFBIG:
                     # The file is too big to open.  Fail visibly.
@@ -257,15 +298,15 @@
                 # If something else went wrong, it's hard to guess
                 # what the problem was.  If the file does not exist,
                 # create it.  Otherwise, fail.
-                if os.path.exists(file_name):
+                if self.fops.exists(file_name):
                     raise
                 else:
                     create = 1
 
         if self._file is None and create:
-            if os.path.exists(file_name):
-                os.remove(file_name)
-            self._file = open(file_name, 'w+b')
+            if self.fops.exists(file_name):
+                self.fops.remove(file_name)
+            self._file = self.fops.open(file_name, 'w+b')
             self._file.write(packed_version)
 
         r = self._restore_index()
@@ -1003,7 +1044,7 @@
             file.write(self._tstatus)
             file.flush()
 
-            if fsync is not None: fsync(file.fileno())
+            self.fops.fsync(file)
 
             self._pos=nextpos
 
@@ -1498,7 +1539,8 @@
         p = FileStoragePacker(self._file_name, stop,
                               self._lock_acquire, self._lock_release,
                               self._commit_lock_acquire,
-                              self._commit_lock_release)
+                              self._commit_lock_release,
+                              self.fops)
         try:
             opos = p.pack()
             if opos is None:
@@ -1506,16 +1548,16 @@
             oldpath = self._file_name + ".old"
             self._file.close()
             try:
-                if os.path.exists(oldpath):
-                    os.remove(oldpath)
-                os.rename(self._file_name, oldpath)
+                if self.fops.exists(oldpath):
+                    self.fops.remove(oldpath)
+                self.fops.rename(self._file_name, oldpath)
             except Exception, msg:
-                self._file = open(self._file_name, 'r+b')
+                self._file = self.fops.open(self._file_name, 'r+b')
                 raise
 
             # OK, we're beyond the point of no return
-            os.rename(self._file_name + '.pack', self._file_name)
-            self._file = open(self._file_name, 'r+b')
+            self.fops.rename(self._file_name + '.pack', self._file_name)
+            self._file = self.fops.open(self._file_name, 'r+b')
             self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
             self._pos = opos
             self._save_index()
@@ -1688,8 +1730,10 @@
 
     return p, s
 
-def recover(file_name):
-    file=open(file_name, 'r+b')
+def recover(file_name, fops=None):
+    if fops is None:
+        fops = defaultFops
+    file=fops.open(file_name, 'r+b')
     index={}
     vindex={}
     tindex={}
@@ -1939,7 +1983,9 @@
     # seek to transaction header, where tid is first 8 bytes
     return file.read(8)
 
-def _truncate(file, name, pos):
+def _truncate(file, name, pos, fops=None):
+    if fops is None:
+        fops = defaultFops
     seek=file.seek
     seek(0,2)
     file_size=file.tell()
@@ -1947,11 +1993,11 @@
         i=0
         while 1:
             oname='%s.tr%s' % (name, i)
-            if os.path.exists(oname):
+            if fops.exists(oname):
                 i=i+1
             else:
                 warn("Writing truncated data from %s to %s", name, oname)
-                o=open(oname,'wb')
+                o = fops.open(oname, 'wb')
                 seek(pos)
                 cp(file, o, file_size-pos)
                 o.close()
@@ -1986,9 +2032,11 @@
     _ltid = z64
     _file = None
 
-    def __init__(self, file, start=None, stop=None):
+    def __init__(self, file, start=None, stop=None, fops=None):
+        if fops is None:
+            fops = defaultFops
         if isinstance(file, StringType):
-            file = open(file, 'rb')
+            file = fops.open(file, 'rb')
         self._file = file
         if file.read(4) != packed_version:
             raise FileStorageFormatError, file.name
Index: fspack.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/fspack.py,v
retrieving revision 1.5.4.1
diff -u -u -r1.5.4.1 fspack.py
--- fspack.py	13 May 2003 18:04:21 -0000	1.5.4.1
+++ fspack.py	15 May 2003 19:40:40 -0000
@@ -606,9 +606,10 @@
 
 class FileStoragePacker(FileStorageFormatter):
 
-    def __init__(self, path, stop, la, lr, cla, clr):
+    def __init__(self, path, stop, la, lr, cla, clr, fops):
+        self.fops = fops
         self._name = path
-        self._file = open(path, "rb")
+        self._file = fops.open(path, "rb")
         self._stop = stop
         self._packt = None
         self.locked = 0
@@ -659,7 +660,7 @@
 
         # Setup the destination file and copy the metadata.
         # XXX rename from _tfile to something clearer
-        self._tfile = open(self._name + ".pack", "w+b")
+        self._tfile = self.fops.open(self._name + ".pack", "w+b")
         self._file.seek(0)
         self._tfile.write(self._file.read(self._metadata_size))
 
@@ -671,7 +672,7 @@
         if ipos == opos:
             # pack didn't free any data.  there's no point in continuing.
             self._tfile.close()
-            os.remove(self._name + ".pack")
+            self.fops.remove(self._name + ".pack")
             return None
         if ipos < self.file_end:
             self.copyRest(ipos)


=== Added File Packages/PartitionedFileStorage/PartitionedFileStorage_HEAD.patch ===
Index: FileStorage.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/FileStorage.py,v
retrieving revision 1.131
diff -u -u -r1.131 FileStorage.py
--- FileStorage.py	1 May 2003 17:10:19 -0000	1.131
+++ FileStorage.py	15 May 2003 19:36:48 -0000
@@ -202,6 +202,43 @@
 
 packed_version='FS21'
 
+
+class FileOperations:
+    '''This class implements some basic operations on files.
+    PartitionedFileOperations provides an alternate implementation.
+    '''
+    def open(self, name, mode='r', bufsize=0):
+        global open
+        return open(name, mode, bufsize)
+
+    def exists(self, name):
+        return os.path.exists(name)
+
+    def rename(self, oldname, newname):
+        os.rename(oldname, newname)
+
+    def remove(self, name):
+        os.remove(name)
+
+    def fsync(self, file):
+        global fsync
+        if fsync is not None:
+            fsync(file.fileno())
+
+defaultFops = FileOperations()
+
+
+# Set the PARTITIONED_FILE_SIZE environment variable to use
+# PartitionedFile support.  Note: To convert a partitioned file back
+# to a normal file, just 'cat' (Unix) or 'copy' (DOS/Windows) the
+# partitions together in order.
+PARTITIONED_FILE_SIZE = long(os.environ.get('PARTITIONED_FILE_SIZE') or 0)
+if PARTITIONED_FILE_SIZE > 0:
+    from PartitionedFile import PartitionedFileOperations
+    defaultFops = PartitionedFileOperations(partlen=PARTITIONED_FILE_SIZE)
+
+
+
 class FileStorage(BaseStorage.BaseStorage,
                   ConflictResolution.ConflictResolvingStorage):
     # default pack time is 0
@@ -210,7 +247,10 @@
     _records_before_save = 10000
 
     def __init__(self, file_name, create=0, read_only=0, stop=None,
-                 quota=None):
+                 quota=None, fops=None):
+        if fops is None:
+            fops = defaultFops
+        self.fops = fops
 
         if read_only:
             self._is_read_only = 1
@@ -242,7 +282,8 @@
         self._file = None
         if not create:
             try:
-                self._file = open(file_name, read_only and 'rb' or 'r+b')
+                self._file = self.fops.open(
+                    file_name, read_only and 'rb' or 'r+b')
             except IOError, exc:
                 if exc.errno == errno.EFBIG:
                     # The file is too big to open.  Fail visibly.
@@ -253,15 +294,15 @@
                 # If something else went wrong, it's hard to guess
                 # what the problem was.  If the file does not exist,
                 # create it.  Otherwise, fail.
-                if os.path.exists(file_name):
+                if self.fops.exists(file_name):
                     raise
                 else:
                     create = 1
 
         if self._file is None and create:
-            if os.path.exists(file_name):
-                os.remove(file_name)
-            self._file = open(file_name, 'w+b')
+            if self.fops.exists(file_name):
+                self.fops.remove(file_name)
+            self._file = self.fops.open(file_name, 'w+b')
             self._file.write(packed_version)
 
         r = self._restore_index()
@@ -1009,7 +1050,7 @@
             file.write(self._tstatus)
             file.flush()
 
-            if fsync is not None: fsync(file.fileno())
+            self.fops.fsync(file)
 
             self._pos = nextpos
             
@@ -1454,7 +1495,8 @@
         p = FileStoragePacker(self._file_name, stop,
                               self._lock_acquire, self._lock_release,
                               self._commit_lock_acquire,
-                              self._commit_lock_release)
+                              self._commit_lock_release,
+                              self.fops)
         try:
             opos = p.pack()
             if opos is None:
@@ -1462,16 +1504,16 @@
             oldpath = self._file_name + ".old"
             self._file.close()
             try:
-                if os.path.exists(oldpath):
-                    os.remove(oldpath)
-                os.rename(self._file_name, oldpath)
+                if self.fops.exists(oldpath):
+                    self.fops.remove(oldpath)
+                self.fops.rename(self._file_name, oldpath)
             except Exception, msg:
-                self._file = open(self._file_name, 'r+b')
+                self._file = self.fops.open(self._file_name, 'r+b')
                 raise
 
             # OK, we're beyond the point of no return
-            os.rename(self._file_name + '.pack', self._file_name)
-            self._file = open(self._file_name, 'r+b')
+            self.fops.rename(self._file_name + '.pack', self._file_name)
+            self._file = self.fops.open(self._file_name, 'r+b')
             self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
             self._pos = opos
             self._save_index()
@@ -1649,8 +1691,10 @@
 
     return p, s
 
-def recover(file_name):
-    file=open(file_name, 'r+b')
+def recover(file_name, fops=None):
+    if fops is None:
+        fops = defaultFops
+    file=fops.open(file_name, 'r+b')
     index={}
     vindex={}
     tindex={}
@@ -1898,7 +1942,9 @@
     # seek to transaction header, where tid is first 8 bytes
     return file.read(8)
 
-def _truncate(file, name, pos):
+def _truncate(file, name, pos, fops=None):
+    if fops is None:
+        fops = defaultFops
     seek=file.seek
     seek(0,2)
     file_size=file.tell()
@@ -1906,11 +1952,11 @@
         i=0
         while 1:
             oname='%s.tr%s' % (name, i)
-            if os.path.exists(oname):
+            if fops.exists(oname):
                 i=i+1
             else:
                 warn("Writing truncated data from %s to %s", name, oname)
-                o=open(oname,'wb')
+                o = fops.open(oname, 'wb')
                 seek(pos)
                 cp(file, o, file_size-pos)
                 o.close()
@@ -1945,9 +1991,11 @@
     _ltid = z64
     _file = None
 
-    def __init__(self, file, start=None, stop=None):
+    def __init__(self, file, start=None, stop=None, fops=None):
+        if fops is None:
+            fops = defaultFops
         if isinstance(file, StringType):
-            file = open(file, 'rb')
+            file = fops.open(file, 'rb')
         self._file = file
         if file.read(4) != packed_version:
             raise FileStorageFormatError, file.name
Index: fspack.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/fspack.py,v
retrieving revision 1.5
diff -u -u -r1.5 fspack.py
--- fspack.py	13 May 2003 16:29:21 -0000	1.5
+++ fspack.py	15 May 2003 19:36:49 -0000
@@ -605,9 +605,10 @@
 
 class FileStoragePacker(FileStorageFormatter):
 
-    def __init__(self, path, stop, la, lr, cla, clr):
+    def __init__(self, path, stop, la, lr, cla, clr, fops):
+        self.fops = fops
         self._name = path
-        self._file = open(path, "rb")
+        self._file = fops.open(path, "rb")
         self._stop = stop
         self._packt = None
         self.locked = 0
@@ -658,7 +659,7 @@
 
         # Setup the destination file and copy the metadata.
         # XXX rename from _tfile to something clearer
-        self._tfile = open(self._name + ".pack", "w+b")
+        self._tfile = self.fops.open(self._name + ".pack", "w+b")
         self._file.seek(0)
         self._tfile.write(self._file.read(self._metadata_size))
 
@@ -670,7 +671,7 @@
         if ipos == opos:
             # pack didn't free any data.  there's no point in continuing.
             self._tfile.close()
-            os.remove(self._name + ".pack")
+            self.fops.remove(self._name + ".pack")
             return None
         if ipos < self.file_end:
             self.copyRest(ipos)

=== Removed File Packages/PartitionedFileStorage/FileStorage_py_1_129.patch ===

=== Removed File Packages/PartitionedFileStorage/FileStorage_py_1_98_2_7.patch ===