[Checkins] SVN: gocept.zeoraid/trunk/ - Added deployment documentation.

Christian Theune ct at gocept.com
Thu Mar 27 12:09:33 EDT 2008


Log message for revision 84968:
  - Added deployment documentation.
  - Added buildout recipe for installing ZEO servers with the ZEORAID management
    utility.
  

Changed:
  U   gocept.zeoraid/trunk/base.cfg
  U   gocept.zeoraid/trunk/setup.py
  U   gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/controller.py
  A   gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.py
  A   gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.txt
  A   gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/tests.py

-=-
Modified: gocept.zeoraid/trunk/base.cfg
===================================================================
--- gocept.zeoraid/trunk/base.cfg	2008-03-27 15:35:44 UTC (rev 84967)
+++ gocept.zeoraid/trunk/base.cfg	2008-03-27 16:09:32 UTC (rev 84968)
@@ -1,9 +1,8 @@
 [buildout]
 develop = . externals/ZODB
-parts = zodbscripts test client server1 server2 zeo
+parts = zodbscripts test server1 server2 zeo
 find-links = http://download.zope.org/distribution/
 
-
 [test]
 recipe = zc.recipe.testrunner
 eggs = gocept.zeoraid
@@ -36,12 +35,14 @@
     path ${storage2:path}
     </filestorage>
 
+[storage3]
+recipe = zc.recipe.filestorage
+
 [zeo]
-recipe = zc.zodbrecipes:server
-eggs = gocept.zeoraid
+recipe = gocept.zeoraid:server
 zeo.conf =
     <zeo>
-    address 8100
+    address 127.0.0.1:8100
     </zeo>
     %import gocept.zeoraid
     <raidstorage 1>
@@ -54,13 +55,12 @@
             storage 1
         </zeoclient>
     </raidstorage>
+    <raidstorage 2>
+        <filestorage>
+            path ${storage3:path}
+        </filestorage>
+    </raidstorage>
 
 [zodbscripts]
 recipe = zc.recipe.egg:scripts
 eggs = ZODB3
-       gocept.zeoraid
-
-[client]
-recipe = zc.recipe.egg
-eggs = gocept.zeoraid
-interpreter = client

Modified: gocept.zeoraid/trunk/setup.py
===================================================================
--- gocept.zeoraid/trunk/setup.py	2008-03-27 15:35:44 UTC (rev 84967)
+++ gocept.zeoraid/trunk/setup.py	2008-03-27 16:09:32 UTC (rev 84968)
@@ -34,16 +34,17 @@
     package_dir = {'':'src'},
     namespace_packages = ['gocept'],
     install_requires = ['setuptools',
-                        'ZODB3>=3.9dev',
-                        'zope.component'],
-    extras_require = {
-        'recipe': ['zc.buildout']
-    },
+                        'zc.zodbrecipes',
+                        'ZODB3>=3.9dev'],
+    #extras_require = {
+    #    'recipe': ['zc.buildout',
+    #               'zc.zodbrecipes']
+    #},
     entry_points = """
         [zc.buildout]
-        default = %s.recipe:Recipe [recipe]
+        server = gocept.zeoraid.scripts.recipe:ZEORAIDServer
 
         [console_scripts]
         zeoraid = gocept.zeoraid.scripts.controller:main
-        """ % name,
+        """,
     )

Modified: gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/controller.py
===================================================================
--- gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/controller.py	2008-03-27 15:35:44 UTC (rev 84967)
+++ gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/controller.py	2008-03-27 16:09:32 UTC (rev 84968)
@@ -72,18 +72,18 @@
         print self.raid.raid_disable(storage)
 
 
-def main():
+def main(host="127.0.0.1", port=8100, storage="1"):
     usage = "usage: %prog [options] command [command-options]"
     description = ("Connect to a RAIDStorage on a ZEO server and perform "
                    "maintenance tasks. Available commands: status, details, "
                    "recover <STORAGE>, disable <STORAGE>")
 
     parser = optparse.OptionParser(usage=usage, description=description)
-    parser.add_option("-S", "--storage", default="1",
+    parser.add_option("-S", "--storage", default=storage,
                       help="Use STORAGE on ZEO server. Default: %default")
-    parser.add_option("-H", "--host", default="127.0.0.1",
+    parser.add_option("-H", "--host", default=host,
                       help="Connect to HOST. Default: %default")
-    parser.add_option("-p", "--port", type="int", default=8100,
+    parser.add_option("-p", "--port", type="int", default=port,
                       help="Connect to PORT. Default: %default")
     options, args = parser.parse_args()
 

Added: gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.py
===================================================================
--- gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.py	                        (rev 0)
+++ gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.py	2008-03-27 16:09:32 UTC (rev 84968)
@@ -0,0 +1,95 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Recipe to create a ZEO server with additional RAID management scripts.
+
+The recipe is compatible to (and uses internally) zc.zodbrecipes:server.
+
+"""
+
+
+import ZConfig.schemaless
+import cStringIO
+import os.path
+import zc.zodbrecipes
+
+
+class ZEORAIDServer(object):
+
+    def __init__(self, buildout, name, options):
+        self.name = name
+        self.options = options
+        self.buildout = buildout
+
+        options['eggs'] = options.get('eggs', '')
+        options['eggs'] += '\ngocept.zeoraid'
+
+        # Other recipes we facilitate
+        self.egg = zc.recipe.egg.Egg(buildout, name, options)
+        self.zeo = zc.zodbrecipes.StorageServer(buildout, name, options)
+
+    def install(self):
+        paths = self.zeo.install()
+        paths.extend(self.egg.install())
+
+        if self.zeo.deployment:
+            # Mimick zc.zodbrecipes here
+            rc = self.options['deployment-name'] + '-' + self.name
+        else:
+            rc = self.name
+
+        # Analyse zeo.conf template for storages and the host/port
+        zeo_conf = self.options.get('zeo.conf', '')+'\n'
+        zeo_conf = ZConfig.schemaless.loadConfigFile(
+            cStringIO.StringIO(zeo_conf))
+
+        # Determine host/port
+        zeo_section = [s for s in zeo_conf.sections if s.type == 'zeo']
+        if not zeo_section:
+            raise zc.buildout.UserError('No zeo section was defined.')
+        if len(zeo_section) > 1:
+            raise zc.buildout.UserError('Too many zeo sections.')
+        zeo_section = zeo_section[0]
+        if not 'address' in zeo_section:
+            raise zc.buildout.UserError('No ZEO address was specified.')
+        address = zeo_section['address'][0].split(':')
+        if len(address) == 2:
+            host, port = address
+        elif len(address) == 1:
+            host, port = '127.0.0.1', 8100
+        else:
+            raise zc.buildout.UserError(
+                'Invalid ZEO address %r was specified.' %
+                zeo_section['address'][0])
+
+        # Determine the RAID storages
+        storages = [s.name for s in zeo_conf.sections
+                    if s.type == 'raidstorage'
+                    ]
+
+        # Create RAID control scripts for all RAID storages
+        requirements, working_set = self.egg.working_set()
+        for storage in storages:
+            script_name = rc + '-raid-' + storage
+            paths.append(os.path.join(self.options['rc-directory'],
+                                      script_name))
+            zc.buildout.easy_install.scripts(
+                requirements, working_set,
+                self.options['executable'],
+                self.options['rc-directory'],
+                scripts={'zeoraid': script_name},
+                arguments='port=%s, host="%s", storage="%s"' % (
+                    port, host, storage))
+        return paths
+
+    update = install


Property changes on: gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.py
___________________________________________________________________
Name: svn:eol-style
   + native

Added: gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.txt
===================================================================
--- gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.txt	                        (rev 0)
+++ gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.txt	2008-03-27 16:09:32 UTC (rev 84968)
@@ -0,0 +1,231 @@
+Defining ZEORAID storage servers
+================================
+
+The `gocept.zeoraid:server` recipe can be used to define ZEO storage servers
+with management utilities for ZEORAID.
+
+This recipe is internally built on top of `zc.zodbrecipes:server` and
+instruments it, adding the generation of management scripts.
+
+To define a storage server, you define a part for the server and specify
+configuration data.  
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = zodb server
+    ...
+    ... [zodb]
+    ... recipe = zc.recipe.egg:script
+    ... eggs = ZODB3
+    ... 
+    ... [server]
+    ... recipe = gocept.zeoraid:server
+    ... zeo.conf = 
+    ...    <zeo>
+    ...       address 8100
+    ...       monitor-address 8101
+    ...       transaction-timeout 300
+    ...    </zeo>
+    ...    %import gocept.zeoraid
+    ...    <raidstorage 1>
+    ...       <zeoclient 1>
+    ...           address 8101
+    ...       </zeoclient>
+    ...    </raidstorage>
+    ... ''')
+
+Here we specified a minimal ZEORAID configuration. When we run the buildout:
+
+    >>> print system(buildout),
+    Installing zodb.
+    Generated script '/sample-buildout/bin/mkzeoinst'.
+    Generated script '/sample-buildout/bin/fstail'.
+    Generated script '/sample-buildout/bin/zeopack'.
+    Generated script '/sample-buildout/bin/runzeo'.
+    Generated script '/sample-buildout/bin/zeopasswd'.
+    Generated script '/sample-buildout/bin/zeoctl'.
+    Generated script '/sample-buildout/bin/fsdump'.
+    Generated script '/sample-buildout/bin/fsrefs'.
+    Generated script '/sample-buildout/bin/repozo'.
+    Generated script '/sample-buildout/bin/fsoids'.
+    Installing server.
+    Generated script '/sample-buildout/bin/server'.
+    Generated script '/sample-buildout/bin/server-raid-1'.
+
+We get 2 things.  We get a directory in parts containing ZEO and
+zdaemon configuration files:
+
+    >>> ls('parts', 'server')
+    -  zdaemon.conf
+    -  zeo.conf
+
+Let's look at the configuration files:
+
+    >>> cat('parts', 'server', 'zeo.conf')
+    %import gocept.zeoraid
+    <BLANKLINE>
+    <zeo>
+      address 8100
+      monitor-address 8101
+      transaction-timeout 300
+    </zeo>
+    <BLANKLINE>
+    <raidstorage 1>
+      <zeoclient 1>
+        address 8101
+      </zeoclient>
+    </raidstorage>
+    <BLANKLINE>
+    <eventlog>
+      <logfile>
+        path STDOUT
+      </logfile>
+    </eventlog>
+
+We see the same data we input with the addition of an eventlog section that
+directs logging to standard out.
+
+Let's look at the scripts that were generated in our bin directory:
+
+    >>> ls('bin')
+    -  buildout
+    -  fsdump
+    -  fsoids
+    -  fsrefs
+    -  fstail
+    -  mkzeoinst
+    -  repozo
+    -  runzeo
+    -  server
+    -  server-raid-1
+    -  zeoctl
+    -  zeopack
+    -  zeopasswd
+
+
+We have an additional RAID control script:
+
+    >>> cat('bin', 'server-raid-1')
+    #!/.../python
+    <BLANKLINE>
+    import sys
+    sys.path[0:0] = [
+      '/home/ctheune/Development/gocept.zeoraid/src',
+      '/home/ctheune/Development/gocept.zeoraid/externals/ZODB/src',
+      '/sample-buildout/eggs/zc.zodbrecipes-0.4.0-py2.4.egg',
+      '/sample-buildout/eggs/setuptools-0.6c8-py2.4.egg',
+      '/sample-buildout/eggs/transaction-1.0a1-py2.4.egg',
+      '/sample-buildout/eggs/zdaemon-2.0.1-py2.4.egg',
+      '/sample-buildout/eggs/ZConfig-2.5.1-py2.4.egg',
+      '/sample-buildout/eggs/zope.testing-3.5.1-py2.4.egg',
+      '/sample-buildout/eggs/zope.proxy-3.4.0-py2.4-linux-i686.egg',
+      '/sample-buildout/eggs/zope.interface-3.4.1-py2.4-linux-i686.egg',
+      '/sample-buildout/eggs/zc.recipe.egg-1.1.0dev_r84019-py2.4.egg',
+      '/sample-buildout/eggs/zc.buildout-1.0.0-py2.4.egg',
+      ]
+    <BLANKLINE>
+    import gocept.zeoraid.scripts.controller
+    <BLANKLINE>
+    if __name__ == '__main__':
+        gocept.zeoraid.scripts.controller.main(port=8100, host="127.0.0.1", storage="1")
+
+
+This is a console script with customized defaults.  We can use this to control
+the RAID storage aspects of the ZEO server.
+
+Unix deployment support
+=======================
+
+The management script's name is compatible with `zc.recipe.deployment`.
+
+Let's create a deployment environmont and add a deployment section to our
+buildout:
+
+    >>> for d in 'cron', 'etc', 'log', 'rotate', 'rc', 'run':
+    ...     mkdir(d)
+    ...     globals()[d] = join(sample_buildout, d)
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = zodb server
+    ... 
+    ... [zodb]
+    ... recipe = zc.recipe.egg:script
+    ... eggs = ZODB3
+    ... 
+    ... [server]
+    ... recipe = gocept.zeoraid:server
+    ... zeo.conf = 
+    ...    <zeo>
+    ...       address 8100
+    ...       monitor-address 8101
+    ...       transaction-timeout 300
+    ...    </zeo>
+    ...    %%import gocept.zeoraid
+    ...    <raidstorage main>
+    ...       <zeoclient 1>
+    ...         address 8101
+    ...       </zeoclient>
+    ...    </raidstorage>
+    ... deployment = demo
+    ...
+    ... [demo]
+    ... crontab-directory = %(cron)s
+    ... etc-directory = %(etc)s
+    ... log-directory = %(log)s
+    ... logrotate-directory = %(rotate)s
+    ... rc-directory = %(rc)s
+    ... run-directory = %(run)s
+    ... user = bob
+    ... ''' % globals())
+
+    >>> print system(buildout),
+    Uninstalling server.
+    Updating zodb.
+    Installing server.
+    Generated script '/sample-buildout/rc/demo-server'.
+    Generated script '/sample-buildout/rc/demo-server-raid-main'.
+
+Now, the parts directory and the control script will be gone:
+
+    >>> import os
+    >>> os.path.exists(join('parts', 'server'))
+    False
+    >>> os.path.exists(join('bin', 'server'))
+    False
+
+Instead, the control script will be in the rc directory:
+
+    >>> ls('rc')
+    -  demo-server
+    -  demo-server-raid-main
+
+    >>> cat('rc', 'demo-server-raid-main')
+    #!/.../python
+    <BLANKLINE>
+    import sys
+    sys.path[0:0] = [
+      '/home/ctheune/Development/gocept.zeoraid/src',
+      '/home/ctheune/Development/gocept.zeoraid/externals/ZODB/src',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/zc.zodbrecipes-0.4.0-py2.4.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/setuptools-0.6c8-py2.4.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/transaction-1.0a1-py2.4.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/zdaemon-2.0.1-py2.4.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/ZConfig-2.5.1-py2.4.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/zope.testing-3.5.1-py2.4.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/zope.proxy-3.4.0-py2.4-linux-i686.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/zope.interface-3.4.1-py2.4-linux-i686.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/zc.recipe.egg-1.1.0dev_r84019-py2.4.egg',
+      '/tmp/tmpfeWuYRbuildoutSetUp/_TEST_/sample-buildout/eggs/zc.buildout-1.0.0-py2.4.egg',
+      ]
+    <BLANKLINE>
+    import gocept.zeoraid.scripts.controller
+    <BLANKLINE>
+    if __name__ == '__main__':
+        gocept.zeoraid.scripts.controller.main(port=8100, host="127.0.0.1", storage="main")
+
+
+The run-control script name now combines the deployment name and the
+script name.


Property changes on: gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/recipe.txt
___________________________________________________________________
Name: svn:eol-style
   + native

Added: gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/tests.py
===================================================================
--- gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/tests.py	                        (rev 0)
+++ gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/tests.py	2008-03-27 16:09:32 UTC (rev 84968)
@@ -0,0 +1,57 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import re
+import zc.buildout.testing
+
+import unittest
+from zope.testing import doctest, renormalizing
+
+
+def setUp(test):
+    zc.buildout.testing.buildoutSetUp(test)
+    zc.buildout.testing.install_develop('gocept.zeoraid', test)
+    zc.buildout.testing.install('zc.zodbrecipes', test)
+    zc.buildout.testing.install('zope.testing', test)
+    zc.buildout.testing.install('zc.recipe.egg', test)
+    zc.buildout.testing.install('zdaemon', test)
+    zc.buildout.testing.install('ZConfig', test)
+    zc.buildout.testing.install('ZODB3', test)
+    zc.buildout.testing.install('transaction', test)
+    zc.buildout.testing.install('zope.proxy', test)
+    zc.buildout.testing.install('zope.interface', test)
+
+
+checker = renormalizing.RENormalizing([
+    zc.buildout.testing.normalize_path,
+    (re.compile(
+    "Couldn't find index page for '[a-zA-Z0-9.]+' "
+    "\(maybe misspelled\?\)"
+    "\n"
+    ), ''),
+    (re.compile('#![^\n]+\n'), ''),                
+    (re.compile('-\S+-py\d[.]\d(-\S+)?.egg'),
+     '-pyN.N.egg',
+    ),
+    ])
+
+def test_suite():
+    return unittest.TestSuite((
+        doctest.DocFileSuite(
+            'recipe.txt',
+            setUp=setUp, tearDown=zc.buildout.testing.buildoutTearDown,
+            checker=checker,
+            optionflags=doctest.REPORT_NDIFF|doctest.ELLIPSIS
+            ),
+        ))


Property changes on: gocept.zeoraid/trunk/src/gocept/zeoraid/scripts/tests.py
___________________________________________________________________
Name: svn:eol-style
   + native



More information about the Checkins mailing list