[Zope-CVS] CVS: Products/AdaptableStorage/gateway_fs - .cvsignore:1.1 FSAutoId.py:1.1 FSConnection.py:1.1 FSDirectoryItems.py:1.1 FSFileData.py:1.1 FSSectionData.py:1.1 TransactionalWrites.py:1.1 __init__.py:1.1 exceptions.py:1.1 public.py:1.1 serial_public.py:1.1
Shane Hathaway
shane@zope.com
Wed, 27 Nov 2002 13:37:07 -0500
Update of /cvs-repository/Products/AdaptableStorage/gateway_fs
In directory cvs.zope.org:/tmp/cvs-serv12157/gateway_fs
Added Files:
.cvsignore FSAutoId.py FSConnection.py FSDirectoryItems.py
FSFileData.py FSSectionData.py TransactionalWrites.py
__init__.py exceptions.py public.py serial_public.py
Log Message:
Moved the latest AdaptableStorage work out of the private repository.
It took a long time, but I moved it as soon as all the unit tests
passed and I felt that all the interface names and conventions were
good enough.
Documentation is still minimal, but now I think the system is finally
straight enough in my head to write down. :-) If you want a sneak
peek, the interfaces have some docstrings, if you're looking for a
"tree" view, while the OpenOffice diagram presents something of a
"forest" view.
Also note that I'm trying a new coding convention. The "public"
module in each package defines exactly which objects should be
exported from the package. This solves a few problems with imports
such as doubling of names and shadowing of modules. Overall, the
"public" module makes it easier to tell which classes are supposed to
be used by other packages, and makes it easier for other packages to
use the public classes. See what you think.
=== Added File Products/AdaptableStorage/gateway_fs/.cvsignore ===
*.pyc
=== Added File Products/AdaptableStorage/gateway_fs/FSAutoId.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Automatic ID gateway based on the key of the item.
$Id: FSAutoId.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
from serial_public import IGateway, RecordSchema
class FSAutoId:
__implements__ = IGateway
schema = RecordSchema()
schema.addColumn('id', 'string')
def getSchema(self):
return self.schema
def getIdFrom(self, key):
pos = key.rfind('/')
if pos >= 0:
return key[pos + 1:]
else:
return key
def load(self, object_mapper, key):
id = self.getIdFrom(key)
return ((id,),), id
def store(self, object_mapper, key, state):
id = self.getIdFrom(key)
assert state[0][0] == id, 'Mismatched file ID'
return id
=== Added File Products/AdaptableStorage/gateway_fs/FSConnection.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Filesystem persistence.
$Id: FSConnection.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
import os
import re
from shutil import rmtree
from types import StringType
from interfaces.public import IFSConnection
from exceptions import FSWriteError
from TransactionalWrites import getTransactionalWrites
# Try to decipher this one ;-)
# It basically matches "\n[sectionname]...\n", where len(sectionname) > 0.
section_re = re.compile(r'^\[([^\[\]\n]+)\][^\r\n]*(?:\r\n|\r|\n)',
re.MULTILINE)
NODE_TYPE_SECTION = '@node_type'
DATA_SECTION = '@data'
class FSConnection:
"""Reads / writes files with 'sections'.
The required 'type' section specifies whether the object is a file or
a directory. The optional 'data' section specifies either the main
file contents or the names of the files in the directory. All other
sections get stored in a '.properties' file. The properties file uses
square-bracket section headers and encodes sections by doubling
left-square brackets.
"""
__implements__ = IFSConnection
basepath = ''
def __init__(self, basepath):
self.basepath = basepath
def sortKey(self):
return self.basepath
def expandPath(self, subpath, check_exists=0):
if self.basepath:
while subpath.startswith('/') or subpath.startswith('\\'):
subpath = subpath[1:]
path = os.path.join(self.basepath, subpath)
else:
# unchanged.
path = subpath
if check_exists:
assert os.path.exists(path), path
return path
def checkSectionName(self, section_name):
assert isinstance(section_name, StringType)
assert '[' not in section_name
assert ']' not in section_name
assert '\n' not in section_name
assert section_name != NODE_TYPE_SECTION
assert section_name != DATA_SECTION
def _write(self, subpath, section_name, data):
# XXX We should be checking for '..'
path = self.expandPath(subpath)
# Do some early checking.
if os.path.exists(path):
v = os.access(path, os.W_OK)
if not v:
raise FSWriteError(
"Can't get write access to %s" % subpath)
getTransactionalWrites(self).record(subpath, section_name, data)
def writeSection(self, subpath, section_name, data):
self.checkSectionName(section_name)
self._write(subpath, section_name, data)
def writeNodeType(self, subpath, data):
path = self.expandPath(subpath)
# Do some early checking.
if os.path.exists(path):
want_dir = (data == 'd')
if (want_dir != (not not os.path.isdir(path))):
raise FSWriteError(
"Can't mix file and directory at %s" % subpath)
getTransactionalWrites(self).record(
subpath, NODE_TYPE_SECTION, data)
def writeData(self, subpath, data):
self._write(subpath, DATA_SECTION, data)
def readSection(self, subpath, section_name, default=None):
self.checkSectionName(section_name)
path = self.expandPath(subpath, 1)
sections = self.getPropertiesFromFile(path)
return sections.get(section_name, default)
def readNodeType(self, subpath):
path = self.expandPath(subpath, 1)
return os.path.isdir(path) and 'd' or 'f'
def readData(self, subpath):
path = self.expandPath(subpath, 1)
isdir = os.path.isdir(path)
# Read either the directory listing or the file contents.
if isdir:
names = []
for name in os.listdir(path):
if not name.startswith('.'):
names.append(name)
# Return a sequence instead of a string.
return names
else:
f = open(path, 'rb')
try:
return f.read()
finally:
f.close()
def getPropertiesPath(self, path):
if os.path.isdir(path):
props_fn = os.path.join(path, '.properties')
else:
dirname, filename = os.path.split(path)
props_fn = os.path.join(dirname, '.%s.properties' % filename)
return props_fn
def getPropertiesFromFile(self, path):
"""Read a properties file next to path."""
props_fn = self.getPropertiesPath(path)
if not os.path.exists(props_fn):
return {}
f = open(props_fn, 'rb')
try:
data = f.read()
finally:
f.close()
pos = 0
prev_section_name = None
res = {}
while 1:
match = section_re.search(data, pos)
if match is None:
endpos = len(data)
else:
endpos = match.start()
if prev_section_name is not None:
# get the data and decode.
section = data[pos:endpos].replace('[[', '[')
res[prev_section_name] = section
if match is None:
break
else:
prev_section_name = match.group(1)
pos = match.end()
return res
def writeFinal(self, subpath, sections):
# sections is a mapping.
path = self.expandPath(subpath)
t = sections[NODE_TYPE_SECTION]
if t == 'd' and not os.path.exists(path):
os.mkdir(path)
props_fn = self.getPropertiesPath(path)
items = sections.items()
items.sort()
props_f = open(props_fn, 'wb')
try:
for name, data in items:
if name == NODE_TYPE_SECTION:
continue
elif name == DATA_SECTION:
if t == 'd':
# Change the list of subobjects.
# Here we only have to delete.
# Subobjects will be created later.
# XXX we might check for dotted names here.
self.removeUnlinkedItems(path, data)
else:
# Change file contents.
f = open(path, 'wb')
try:
f.write(data)
finally:
f.close()
else:
if not data.endswith('\n'):
data = data + '\n'
props_f.write('\n[%s]\n' % name)
props_f.write(data.replace('[', '[['))
finally:
props_f.close()
def removeUnlinkedItems(self, path, items):
linked = {}
for name in items:
linked[name] = 1
existing = os.listdir(path)
for fn in existing:
if not fn.startswith('.') and not linked.get(fn):
item_fn = os.path.join(path, fn)
if os.path.isdir(item_fn):
rmtree(item_fn)
else:
os.remove(item_fn)
item_pfn = self.getPropertiesPath(item_fn)
if os.path.exists(item_pfn):
os.remove(item_pfn)
def beforeWrite(self, items):
non_containers = {}
for subpath, sections in items:
# type must be provided and must always be either 'd' or 'f'.
if not sections.has_key(NODE_TYPE_SECTION):
raise FSWriteError('node type not specified for %s' % subpath)
t = sections[NODE_TYPE_SECTION]
if t not in 'df':
raise FSWriteError(
'node type must be "d" or "f" at %s' % subpath)
dir = os.path.dirname(subpath)
if non_containers.get(dir):
raise FSWriteError(
"Not a directory: %s" % dir)
if t == 'f':
non_containers[subpath] = 1
else:
if isinstance(sections[DATA_SECTION], StringType):
raise FSWriteError(
'Data for a directory must be a list or tuple at %s'
% subpath)
=== Added File Products/AdaptableStorage/gateway_fs/FSDirectoryItems.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Read/write objects in a filesystem directory.
$Id: FSDirectoryItems.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
from serial_public import IGateway, RecordSchema
class FSDirectoryItems:
__implements__ = IGateway
schema = RecordSchema()
schema.addColumn('id', 'string', 1)
schema.addColumn('classification', 'classification')
def __init__(self, fs_conn):
self.fs_conn = fs_conn
def getSchema(self):
return self.schema
def load(self, object_mapper, key):
c = self.fs_conn
assert c.readNodeType(key) == 'd'
names = c.readData(key)
names.sort()
res = []
serial = []
for name in names:
subkey = '%s/%s' % (key, name)
ctext = c.readSection(subkey, 'classification', None)
classification = {}
if ctext:
lines = ctext.split('\n')
for line in lines:
if '=' in line:
k, v = line.split('=', 1)
classification[k.strip()] = v.strip()
if not classification:
classifier = object_mapper.getClassifier()
isdir = (c.readNodeType(subkey) == 'd')
classification, mapper_name = classifier.classifyFilename(
name, isdir)
res.append((name, classification))
items = classification.items()
items.sort()
serial.append((name, items))
res.sort()
serial.sort()
return tuple(res), serial
def store(self, object_mapper, key, state):
c = self.fs_conn
c.writeNodeType(key, 'd')
names = []
serial = []
for name, classification in state:
names.append(name)
subkey = '%s/%s' % (key, name)
items = classification.items()
items.sort()
serial.append((name, items))
text = []
for k, v in items:
text.append('%s=%s' % (k, v))
text = '\n'.join(text)
c.writeSection(subkey, 'classification', text)
names.sort()
serial.sort()
c.writeData(key, names)
return serial
=== Added File Products/AdaptableStorage/gateway_fs/FSFileData.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""File data gateway, where data is a string.
$Id: FSFileData.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
from serial_public import IGateway, RecordSchema
class FSFileData:
__implements__ = IGateway
schema = RecordSchema()
schema.addColumn('data', 'string')
def __init__(self, fs_conn):
self.fs_conn = fs_conn
def getSchema(self):
return self.schema
def load(self, object_mapper, key):
c = self.fs_conn
assert c.readNodeType(key) == 'f'
data = c.readData(key, '')
state = ((data,),)
return state, state
def store(self, object_mapper, key, state):
c = self.fs_conn
assert len(state) == 1
assert len(state[0]) == 1
c.writeNodeType(key, 'f')
c.writeData(key, state[0][0])
return state
=== Added File Products/AdaptableStorage/gateway_fs/FSSectionData.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Gateway of data in a filesystem properties section.
$Id: FSSectionData.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
from serial_public import IGateway, RecordSchema
class FSSectionData:
__implements__ = IGateway
schema = RecordSchema()
schema.addColumn('data', 'string')
def __init__(self, fs_conn, section):
self.fs_conn = fs_conn
self.section = section
def getSchema(self):
return self.schema
def load(self, object_mapper, key):
c = self.fs_conn
data = c.readSection(key, self.section, '')
state = ((data,),)
return state, state
def store(self, object_mapper, key, state):
c = self.fs_conn
assert len(state) == 1
assert len(state[0]) == 1
c.writeSection(key, self.section, state[0][0])
return state
=== Added File Products/AdaptableStorage/gateway_fs/TransactionalWrites.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Transactional writing for FSConnection.
It's not fully transactional, but close enough for now. :-)
We'd need help from the OS to achieve true transactions.
$Id: TransactionalWrites.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
import thread
import ZODB # For get_transaction()
from exceptions import FSWriteError
_pending = {} # { transaction -> TransactionalWrites }
class TransactionalWrites:
final = 0
def __init__(self, ident, fs_conn):
self.ident = ident
self.fs_conn = fs_conn
self.data = {} # { subpath string -> { section_name -> data } }
def record(self, subpath, section_name, data):
m = self.data
sections = m.get(subpath)
if sections is None:
sections = {}
m[subpath] = sections
if sections.has_key(section_name):
if sections[section_name] != data:
raise FSWriteError(
'Conflicting data storage at %s (%s)' %
(subpath, section_name))
else:
sections[section_name] = data
def removeSelf(self):
try:
del _pending[self.ident]
except KeyError:
pass
def tpc_begin(self, transaction):
pass
def tpc_vote(self, transaction):
"""Do some early verification
This is done while the transaction can still be vetoed safely.
"""
items = self.data.items()
items.sort() # Ensure that base directories come first.
self.fs_conn.beforeWrite(items)
self.final = 1
def tpc_abort(self, transaction):
self.removeSelf()
def tpc_finish(self, transaction):
if self.final:
try:
items = self.data.items()
items.sort() # Ensure that base directories come first.
for subpath, sections in items:
self.fs_conn.writeFinal(subpath, sections)
finally:
self.removeSelf()
def commit(self, self_again, transaction):
pass
def abort(self, self_again, transaction):
pass
def sortKey(self):
return self.fs_conn.sortKey()
def getTransactionalWrites(fs_conn):
ident = (id(fs_conn), thread.get_ident())
res = _pending.get(ident)
if res is None:
res = TransactionalWrites(ident, fs_conn)
_pending[ident] = res
get_transaction().register(res)
return res
=== Added File Products/AdaptableStorage/gateway_fs/__init__.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Filesystem gateway package.
$Id: __init__.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
=== Added File Products/AdaptableStorage/gateway_fs/exceptions.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""gateway_fs exception types
$Id: exceptions.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
class FSWriteError (Exception):
"""Unable to write data"""
=== Added File Products/AdaptableStorage/gateway_fs/public.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""gateway_fs public names
$Id: public.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
from interfaces.public import *
from exceptions import *
from FSAutoId import FSAutoId
from FSConnection import FSConnection
from FSDirectoryItems import FSDirectoryItems
from FSFileData import FSFileData
from FSSectionData import FSSectionData
=== Added File Products/AdaptableStorage/gateway_fs/serial_public.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Import of the public classes and interfaces from the serial package.
$Id: serial_public.py,v 1.1 2002/11/27 18:37:05 shane Exp $
"""
from Products.AdaptableStorage.serial.public import *