[Zope-Checkins] CVS: Zope/utilities - check_catalog.py:1.1

[Andreas Jung]

Update of /cvs-repository/Zope/utilities
In directory cvs.zope.org:/tmp/cvs-serv5628/utilities

Added Files:
	check_catalog.py 
Log Message:
initial release of a utility to check the consistency of a ZCatalog


=== Added File Zope/utilities/check_catalog.py ===
#!/usr/bin/env python2.1

##############################################################################
# 
# Zope Public License (ZPL) Version 1.0
# -------------------------------------
# 
# Copyright (c) Digital Creations.  All rights reserved.
# 
# This license has been certified as Open Source(tm).
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 
# 1. Redistributions in source code must retain the above copyright
#    notice, this list of conditions, and the following disclaimer.
# 
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions, and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
# 
# 3. Digital Creations requests that attribution be given to Zope
#    in any manner possible. Zope includes a "Powered by Zope"
#    button that is installed by default. While it is not a license
#    violation to remove this button, it is requested that the
#    attribution remain. A significant investment has been put
#    into Zope, and this effort will continue if the Zope community
#    continues to grow. This is one way to assure that growth.
# 
# 4. All advertising materials and documentation mentioning
#    features derived from or use of this software must display
#    the following acknowledgement:
# 
#      "This product includes software developed by Digital Creations
#      for use in the Z Object Publishing Environment
#      (http://www.zope.org/)."
# 
#    In the event that the product being advertised includes an
#    intact Zope distribution (with copyright and license included)
#    then this clause is waived.
# 
# 5. Names associated with Zope or Digital Creations must not be used to
#    endorse or promote products derived from this software without
#    prior written permission from Digital Creations.
# 
# 6. Modified redistributions of any form whatsoever must retain
#    the following acknowledgment:
# 
#      "This product includes software developed by Digital Creations
#      for use in the Z Object Publishing Environment
#      (http://www.zope.org/)."
# 
#    Intact (re-)distributions of any official Zope release do not
#    require an external acknowledgement.
# 
# 7. Modifications are encouraged but must be packaged separately as
#    patches to official Zope releases.  Distributions that do not
#    clearly separate the patches from the original work must be clearly
#    labeled as unofficial distributions.  Modifications which do not
#    carry the name Zope may be packaged in any form, as long as they
#    conform to all of the clauses above.
# 
# 
# Disclaimer
# 
#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
#   SUCH DAMAGE.
# 
# 
# This software consists of contributions made by Digital Creations and
# many individuals on behalf of Digital Creations.  Specific
# attributions are listed in the accompanying credits file.
# 
##############################################################################

""" script to consistency of a ZCatalog """

__version__='$Revision: 1.1 $'[11:-2]


import Zope    
import os,sys,re,getopt
from types import IntType
from BTrees.IIBTree import IISet,difference,intersection

def checkCatalog(path,indexes):
    """ perform some consistency checks on a ZCatalog instance"""

    root = Zope.app()

    try:
        catalog = root.unrestrictedTraverse(path)
    except AttributeError:
        print 'Error: catalog object not found'
        sys.exit(1)

    # get Catalog instance
    _cat = catalog._catalog

    # check Catalog internal BTrees
    l_data  = list(_cat.data.keys())
    l_data.sort()
    l_uids  = list(_cat.uids.values())
    l_uids.sort()
    l_paths = list(_cat.data.keys())
    l_paths.sort()

    print "Checking catalog internal BTrees"
    print "\tINFO: Mapping data:  %d entries" % len(l_data)
    print "\tINFO: Mapping uids:  %d entries" % len(l_uids)
    print "\tINFO: Mapping paths: %d entries" % len(l_paths)

    if l_data == l_uids:
        print "\tOK:  self.data==self.uids"
    else:
        print "\tERR: self.data!=self.uids"

    if l_data == l_paths:
        print "\tOK:  self.data==self.paths"
    else:
        print "\tERR: self.data!=self.paths"
    

    # check BTrees of indexes

    for id,idx in _cat.indexes.items():

        if indexes and not idx.meta_type in indexes: continue

        print "Checking index '%s' (type: %s)" % (id, idx.meta_type)

        if idx.meta_type in ['FieldIndex','KeywordIndex']:

            # check forward entries
            RIDS = IISet() 
            for key, rids in idx._index.items():
                if isinstance(rids,IntType): 
                    RIDS.insert(  rids  )
                else:
                    map(RIDS.insert , rids.keys())

            diff = difference(RIDS, IISet(_cat.data.keys()))
            if len(diff)!=0:
                print '\tERR: Problem with forward entries' 
            else:    
                print '\tOK:  Forward entries' 


            # check backward entries
            RIDS = IISet(idx._unindex.keys())
            diff = difference(RIDS, IISet(_cat.data.keys()))
            if len(diff)!=0:
                print '\tERR: Problem with backward entries' 
            else:
                print '\tOK:  Backward entries' 

        else:
            print "\tWARN: no check implemented yet"
        



def usage():
    print "Usage: %s [--FieldIndex] /path/to/ZCatalog" % \
                os.path.basename(sys.argv[0])
    print 
    print "This scripts checks the consistency of the internal"
    print "BTrees of a ZCatalog and its indexes."
    sys.exit(1)
    

def main():

    opts,args = getopt.getopt(sys.argv[1:],'h',['help','FieldIndex'])

    indexes = []

    for o,v in opts:
        if o in ['-h','--help']: usage()
        if o=='--FieldIndex':   indexes.append(o[2:])

    checkCatalog(args,indexes)


if __name__=='__main__':
    main()