From sidnei at x3ng.com.br Sat Mar 15 13:51:23 2003 From: sidnei at x3ng.com.br (Sidnei da Silva) Date: Sun Aug 10 17:02:10 2008 Subject: [zopeorg-checkins] CVS: Products/ZopeOrg-NV/Extensions - change_modules.py:1.1 NZOMigrate.py:1.17 Message-ID: <200303151851.h2FIpNx29285@cvs.baymountain.com> Update of /cvs-zopeorg/Products/ZopeOrg-NV/Extensions In directory cvs.zope.org:/tmp/cvs-serv29269/Extensions Modified Files: NZOMigrate.py Added Files: change_modules.py Log Message: Adding Shane's change_modules ExternalMethod. Also refactored the migration to be more verbose about exceptions. === Added File Products/ZopeOrg-NV/Extensions/change_modules.py === _trans = { ('OFS.ObjectManager', 'Folder'): ('CMFCore.PortalFolder', 'PortalFolder'), } def changeClassModules(self, root_object=None): get_transaction().commit(1) # Make sure the current state is stored jar = self._p_jar if root_object is None: root_object = jar.root() root_oid = root_object._p_oid stats = [-1, -1] translation_func = lambda class_spec: _trans.get( class_spec, class_spec) jar.onCommitAction('_changeClassDuringCommit', translation_func, root_oid, stats) get_transaction().commit(1) return ('Done. Visited %d objects and changed %d. ' 'See the event log for more details.' % tuple(stats)) if 1: from cStringIO import StringIO from cPickle import Pickler, Unpickler from ZODB.ExportImport import Ghost, persistent_id from zLOG import LOG, INFO, WARNING def _changeClassDuringCommit(self, transaction, translation_func, root_oid, stats=None): '''Recursively changes the class of persistent objects. Invoked as a method of Connection by the transaction manager mid subtransaction commit. ''' to_visit = [root_oid] seen = {root_oid:1} storage = self._storage trigger = [0] change_count = 0 def persistent_load(oid, # Bound to the function: to_visit=to_visit, seen=seen, Ghost=Ghost, trigger=trigger, translation_func=translation_func, TupleType=type(())): "Adds to the list of oids to visit." if isinstance(oid, TupleType): simple_oid, class_spec = oid new_class_spec = translation_func(class_spec) if new_class_spec != class_spec: # Change this reference. trigger[0] = 1 oid = (simple_oid, new_class_spec) else: simple_oid = oid if not seen.has_key(simple_oid): seen[simple_oid] = 1 to_visit.append(simple_oid) g = Ghost() g.oid = oid return g def find_global(module, name, # Bound to the function: translation_func=translation_func, trigger=trigger, _silly=('__doc__',), _globals={}): class_spec = (module, name) new_class_spec = translation_func(class_spec) if new_class_spec != class_spec: # Change this reference. trigger[0] = 1 module, name = new_class_spec m = __import__(module, _globals, _globals, _silly) return getattr(m, name) version = self._version while to_visit: oid = to_visit.pop() p, serial = storage.load(oid, version) convert_ok = 1 trigger[0] = 0 pfile = StringIO(p) unpickler = Unpickler(pfile) unpickler.persistent_load = persistent_load unpickler.find_global = find_global class_spec, args = unpickler.load() try: # Get persistent_load() and find_global() called. state = unpickler.load() except (ImportError, AttributeError): # There are problems with this object. LOG('changeClasses', WARNING, 'Could not load state for OID ' '%s, class %s' % (repr(oid), repr(class_spec))) convert_ok = 0 if convert_ok: # Figure out the new class. new_class_spec = translation_func(class_spec) if new_class_spec != class_spec: # Change this reference. trigger[0] = 1 if trigger[0]: # Store the object with changes to classes. newp = StringIO() pickler = Pickler(newp, 1) pickler.persistent_id = persistent_id pickler.dump((new_class_spec, args)) pickler.dump(state) data = newp.getvalue() self.invalidate(oid) self._invalidating.append(oid) if self._cache.has_key(oid): # Ghostify. ob = self._cache[oid] if not ob._p_changed: ob._p_changed = None del self._cache[oid] storage.store(oid, serial, data, version, transaction) change_count = change_count + 1 if class_spec != new_class_spec: LOG('changeClass', INFO, 'Changed OID %s from class %s to class %s' % ( repr(oid), repr(class_spec), repr(new_class_spec))) else: LOG('changeClass', INFO, 'Rewrote OID %s with new references' % repr(oid)) if stats is not None: stats[:] = [len(seen), change_count] from ZODB.Connection import Connection Connection._changeClassDuringCommit = _changeClassDuringCommit === Products/ZopeOrg-NV/Extensions/NZOMigrate.py 1.16 => 1.17 === (820/920 lines abridged) +import sys from StringIO import StringIO from ZODB.POSException import POSKeyError from Acquisition import aq_base, aq_parent, aq_inner from Products.CMFCore.utils import getToolByName +from Products.CMFCollector.CollectorIssue import CollectorIssue from DateTime import DateTime +from DocumentTemplate.DT_HTML import HTML as DTML +from zExceptions.ExceptionFormatter import format_exception import zLOG -def migrate(source, dest, ignore_path=None, type_map=None): +def _send_update_notice(self, *args, **kw): + pass + +def migrate(source, dest, ignore_path=None, type_map=None, exceptions=None): dest_path = dest.getPhysicalPath() if ignore_path is None: ignore_path = [] if not dest_path in ignore_path: @@ -36,16 +43,140 @@ "BackTalk Book": "CMF BackTalk Book", "BackTalk Document": "CMF BackTalk Document" } - - tmut = Transmutator(source, dest, ignore_path, type_map) - tmut.log('Migration script started at %s\n.' % DateTime().ISO()) + + if exceptions is None: + exceptions = {} + exceptions['Resources/ZSP/zsp.xml'] = 'ZSP2ZopeOrgZSP' + exceptions['Resources/CaseStudies'] = 'CaseStudies2ZopeOrgCaseStudies' + + tmut = Transmutator(source, dest, ignore_path, type_map, exceptions) + tmut.log('Migration script started at %s.\n' % DateTime().ISO()) result = tmut.run() - tmut.log('Migration script finished at %s\n.' % DateTime().ISO()) + tmut.log('Migration script finished at %s.\n' % DateTime().ISO()) return result def normalizeMetaType(meta_type): return meta_type.replace(' ', '') +def dumpTrackerIssue(issue): + kw = {} + kw['id'] = issue.getId() + kw['resolution'] = issue.stage + traits = {} + for key, value in issue.traitVals: + traits[key] = value + kw['classification'] = traits.get('type', '') [-=- -=- -=- 820 lines omitted -=- -=- -=-] self.log('\n'.join(res) + '\n') - new_obj = self.fixModificationDate(obj, new_obj) + + + if hasattr(obj, 'objectIds') and new_obj is not None: + ignore = WikiMethodsCleanup(obj, obj_url, ignore) + ignore = DefaultDTMLCleanup(obj, obj_url, ignore) + self.log(Transmutator(obj, new_obj, \ + ignore, self._type_map, \ + self._exceptions, \ + self._count).run(), dup=0) + + if source_id == 'Members': + res = recursiveOwnerFix(new_obj, oid, 1) + setLocalRoles(new_obj, (oid, ), 'Owner', obj_url) + + massPublish(new_obj) + updateRoleMappings(new_obj) + + new_obj = fixModificationDate(obj, new_obj) ct = getToolByName(new_obj, 'portal_catalog', None) if ct is not None: try: ct.reindexObject(obj) except: pass - self.log('%s: Success.\n' % obj.absolute_url(relative=1)) - - if hasattr(obj, 'objectIds') and new_obj is not None: - ignore = self.WikiMethodsCleanup(obj, ignore) - ignore = self.DefaultDTMLCleanup(obj, ignore) - self.log(Transmutator(obj, new_obj, \ - ignore, self._type_map, \ - self._level + 1).run(), dup=0) - if self._level <= 2: - get_transaction().commit() - else: - get_transaction().commit(1) - return 'Ok.\n' + if self._count % 100: + get_transaction().commit() + elif self._count % 50: + get_transaction().commit(1) + + self.log('%s: Success.\n' % obj_url) + return 'Ok.\n' if __name__ == '__main__': print 'ok' From sidnei at x3ng.com.br Sun Mar 16 07:05:06 2003 From: sidnei at x3ng.com.br (Sidnei da Silva) Date: Sun Aug 10 17:02:10 2008 Subject: [zopeorg-checkins] CVS: Products/ZopeOrg-NV/Extensions - NZOMigrate.py:1.18 Message-ID: <200303161205.h2GC56w01978@cvs.baymountain.com> Update of /cvs-zopeorg/Products/ZopeOrg-NV/Extensions In directory cvs.zope.org:/tmp/cvs-serv1964 Modified Files: NZOMigrate.py Log Message: Some typoes. === Products/ZopeOrg-NV/Extensions/NZOMigrate.py 1.17 => 1.18 === setattr(new, str(value), []) else: try: - __traceback_info__ = ('copyProperties/getProperty(new)', prop) + __traceback_info__ = ('copyProperties/getProperty(new)', \ + prop) value = orig.getProperty(prop) except: log(pretty_exc(sys.exc_info())) value = '' - try: - __traceback_info__ = ('copyProperties/setProperty(new)', prop, value, type) - new._setProperty(prop, value, type) - except: - log(pretty_exc(sys.exc_info())) + try: + __traceback_info__ = ('copyProperties/setProperty(new)', \ + prop, value, type) + new._setProperty(prop, value, type) + except: + log(pretty_exc(sys.exc_info())) else: try: - __traceback_info__ = ('copyProperties/getProperty(existing)', prop) + __traceback_info__ = ('copyProperties/getProperty(existing)', + prop) value = orig.getProperty(prop) except: log(pretty_exc(sys.exc_info())) value = '' try: - __traceback_info__ = ('copyProperties/setProperty(existing)', prop, value) + __traceback_info__ = ('copyProperties/setProperty(existing)', \ + prop, value) new._setPropValue(prop, value) except: log(pretty_exc(sys.exc_info())) @@ -579,11 +583,11 @@ for id in ['case_facts', 'problem', 'solution', 'summary', \ 'external_url', 'title']: values[id] = '' - prop = getattr(src, id, None) + prop = getattr(source, id, None) if prop is not None: if callable(prop): try: - values[id] = prop(src) + values[id] = prop(source) except: try: values[id] = prop.raw() @@ -597,7 +601,7 @@ new_obj.setSolution('text/html', values['solution']) new_obj.setExternalURL(values['external_url']) new_obj.setTitle(values['title']) - if hasattr(aq_base(src), 'logo'): + if hasattr(aq_base(source), 'logo'): new_obj.setLogo('logo') return (new_obj, dest) @@ -1059,6 +1063,8 @@ if res != []: self.log('\n'.join(res) + '\n') + massPublish(new_obj) + updateRoleMappings(new_obj) if hasattr(obj, 'objectIds') and new_obj is not None: ignore = WikiMethodsCleanup(obj, obj_url, ignore) @@ -1071,9 +1077,6 @@ if source_id == 'Members': res = recursiveOwnerFix(new_obj, oid, 1) setLocalRoles(new_obj, (oid, ), 'Owner', obj_url) - - massPublish(new_obj) - updateRoleMappings(new_obj) new_obj = fixModificationDate(obj, new_obj) From sidnei at x3ng.com.br Sun Mar 16 08:20:37 2003 From: sidnei at x3ng.com.br (Sidnei da Silva) Date: Sun Aug 10 17:02:10 2008 Subject: [zopeorg-checkins] CVS: Products/ZopeOrg-NV/Extensions - NZOMigrate.py:1.19 change_modules.py:1.2 Message-ID: <200303161320.h2GDKbC12211@cvs.baymountain.com> Update of /cvs-zopeorg/Products/ZopeOrg-NV/Extensions In directory cvs.zope.org:/tmp/cvs-serv12194 Modified Files: NZOMigrate.py change_modules.py Log Message: Identation may kill you sometimes. Always use emacs ;) === Products/ZopeOrg-NV/Extensions/NZOMigrate.py 1.18 => 1.19 === if hasattr(obj, 'objectValues'): for subob in obj.objectValues(): massPublish(subob) - if hasattr(obj, 'keys'): + elif hasattr(obj, 'keys'): for id in obj.keys(): - massPublish(obj[id]) + try: + massPublish(obj[id]) + except (KeyError, AttributeError): + log(pretty_exc(sys.exc_info())) + pass + def filterList(alist): dict = {} @@ -426,12 +431,6 @@ 'custom_html_footer', 'local_exit'): if dm in obj_ids: ignore_path.append(obj_path + (dm,)) - # Exception: when migrating case studies we dont want to bring - # the dtml methods to the new site. Only images and files. - if obj_url.find('Resources/CaseStudies/') == 0: - for o in obj.objectValues(): - if o.meta_type not in ['Image', 'File']: - ignore_path.append(obj_path + (o.getId(),)) return ignore_path def fixOwnership(orig, new): @@ -454,14 +453,14 @@ def copyProperties(orig, new): if not hasattr(aq_base(orig), 'propertyIds') or \ - not hasattr(aq_base(new), 'propertiIds'): + not hasattr(aq_base(new), 'propertyIds'): return new for prop in orig.propertyIds(): if not new.hasProperty(prop) and not hasattr(aq_base(new), prop): type = orig.getPropertyType(prop) if type in ('selection', 'multiple selection'): for p in orig._properties: - if p['id'] == prop: + if p['id'] == str(prop): value = str(p['select_variable']) break if not hasattr(aq_base(new), value): @@ -474,12 +473,12 @@ except: log(pretty_exc(sys.exc_info())) value = '' - try: - __traceback_info__ = ('copyProperties/setProperty(new)', \ - prop, value, type) - new._setProperty(prop, value, type) - except: - log(pretty_exc(sys.exc_info())) + try: + __traceback_info__ = ('copyProperties/setProperty(new)', \ + prop, value, type) + new._setProperty(prop, value, type) + except: + log(pretty_exc(sys.exc_info())) else: try: __traceback_info__ = ('copyProperties/getProperty(existing)', @@ -583,11 +582,11 @@ for id in ['case_facts', 'problem', 'solution', 'summary', \ 'external_url', 'title']: values[id] = '' - prop = getattr(source, id, None) + prop = getattr(obj, id, None) if prop is not None: if callable(prop): try: - values[id] = prop(source) + values[id] = prop(obj) except: try: values[id] = prop.raw() @@ -601,7 +600,7 @@ new_obj.setSolution('text/html', values['solution']) new_obj.setExternalURL(values['external_url']) new_obj.setTitle(values['title']) - if hasattr(aq_base(source), 'logo'): + if hasattr(aq_base(obj), 'logo'): new_obj.setLogo('logo') return (new_obj, dest) @@ -713,11 +712,11 @@ r = getattr(f, obj.getId()) platform = obj.platform maturity = obj.status - info_url = obj.info_url - license_url = obj.license_url - changes_url = obj.changes_url - installation_url = obj.installation_url - license = obj.license + info_url = getattr(aq_base(obj), 'info_url', '') + license_url = getattr(aq_base(obj), 'license_url', '') + changes_url = getattr(aq_base(obj), 'changes_url', '') + installation_url = getattr(aq_base(obj), 'installation_url', '') + license = getattr(aq_base(obj), 'license', '') data = str(obj.data) size = len(data) content_type = obj._get_content_type(obj.data, data, obj.__name__, === Products/ZopeOrg-NV/Extensions/change_modules.py 1.1 => 1.2 === -_trans = { - ('OFS.ObjectManager', 'Folder'): - ('CMFCore.PortalFolder', 'PortalFolder'), - } +_trans = -def changeClassModules(self, root_object=None): +def changeClassModules(self, root_object=None, + _trans={('OFS.Folder', 'Folder'): + ('Products.CMFCore.PortalFolder', 'PortalFolder'), + }): get_transaction().commit(1) # Make sure the current state is stored jar = self._p_jar if root_object is None: @@ -97,7 +97,7 @@ try: # Get persistent_load() and find_global() called. state = unpickler.load() - except (ImportError, AttributeError): + except (ImportError, AttributeError, UnicodeError): # There are problems with this object. LOG('changeClasses', WARNING, 'Could not load state for OID ' '%s, class %s' % (repr(oid), repr(class_spec)))