[Checkins]
SVN: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/
Implemented resource manager capable of bundling resources
based on library intersection and resource content digests.
This is explained in detail in README.txt under the section
Resource bundles.
Malthe Borch
mborch at gmail.com
Tue Feb 5 14:37:28 EST 2008
Log message for revision 83553:
Implemented resource manager capable of bundling resources based on library intersection and resource content digests. This is explained in detail in README.txt under the section Resource bundles.
Changed:
U zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/README.txt
U zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/__init__.py
A zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/interfaces.py
U zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/publication.py
D zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resourcelibrary.py
A zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resources.py
U zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tal.py
U zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tests/test_unit.py
U zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/zcml.py
-=-
Modified: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/README.txt
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/README.txt 2008-02-05 19:35:49 UTC (rev 83552)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/README.txt 2008-02-05 19:37:27 UTC (rev 83553)
@@ -324,6 +324,52 @@
>>> browser.contents.count('src="http://localhost/@@/my-lib/included.js"')
1
+Resource bundles
+----------------
+
+The library is capable of bundling similar resources into
+bundles. They're made cache-friendly by giving them a filename that is
+a digest of the contents.
+
+Resources are concatenated only if they're of the same type and such
+that the intersection of any two bundles is empty. This is taken care
+of automatically by the library in the following way:
+
+ * If some resource 'x' is in both resource library 'A' and 'B', then
+ 'x' is split out as its own bundle, while the remaining resources
+ in each library are bundled together.
+
+ * If there are several shared resources, say, 'x' and 'y', they will
+ be bundled together if and only if any resources that requires
+ either, requires both.
+
+First we need to initialize the resource manager. In an application setup
+this happens automatically right after the application process is started.
+
+ >>> from zc.resourcelibrary import resources, interfaces
+ >>> resources.initializeResourceManager()
+
+It is then installed as a utility:
+
+ >>> from zope.component import getUtility
+ >>> manager = getUtility(interfaces.IResourceManager)
+
+We can list resource bundles by library (dependant):
+
+ >>> manager.by_dependant # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ {u'dependent':
+ [<zc.resourcelibrary.resources.ResourceBundle object at ...>],
+ u'dependency': [<zc.resourcelibrary.resources.ResourceBundle object at ...>],
+ u'my-lib': [<zc.resourcelibrary.resources.ResourceBundle object at ...>]}
+
+Or get the bundles themselves by bundle digest:
+
+ >>> manager.by_digest # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ {'9ea06e8b82c7c4f5cf1f3d367562eb19ec3cb403':
+ <zc.resourcelibrary.resources.ResourceBundle object at ...>,
+ 'be1bdec0aa74b4dcb079943e70528096cca985f8':
+ <zc.resourcelibrary.resources.ResourceBundle object at ...>}
+
Future Work
-----------
Modified: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/__init__.py
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/__init__.py 2008-02-05 19:35:49 UTC (rev 83552)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/__init__.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -1 +1,31 @@
-from resourcelibrary import getRequired, getIncluded, need
+import resources
+
+import zope.security.management
+import zope.security.interfaces
+
+from zope.publisher.interfaces import IRequest
+
+def getRequest():
+ try:
+ i = zope.security.management.getInteraction() # raises NoInteraction
+ except zope.security.interfaces.NoInteraction:
+ return
+
+ for p in i.participations:
+ if IRequest.providedBy(p):
+ return p
+
+def need(library_name):
+ request = getRequest()
+ # only take note of needed libraries if there is a request, and it is
+ # capable of handling resource librarys
+ if request and hasattr(request, 'resource_libraries'):
+ if not library_name in request.resource_libraries:
+ request.resource_libraries.append(library_name)
+
+def getRequired(name):
+ return resources.library_info[name].required
+
+def getIncluded(name):
+ return resources.library_info[name].included
+
Added: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/interfaces.py
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/interfaces.py (rev 0)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/interfaces.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -0,0 +1,19 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from zope.interface import Interface
+
+class IResourceManager(Interface):
+ def getBundlesForLibrary(library_name):
+ """Return the list of bundles that are required by the library."""
Modified: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/publication.py
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/publication.py 2008-02-05 19:35:49 UTC (rev 83552)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/publication.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -65,7 +65,7 @@
return response
def _implicitResult(self, body):
- #figure out the content type
+ # figure out the content type
content_type = self.getHeader('content-type')
if content_type is None:
if isHTML(body):
Deleted: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resourcelibrary.py
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resourcelibrary.py 2008-02-05 19:35:49 UTC (rev 83552)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resourcelibrary.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -1,48 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-from zope.publisher.interfaces import IRequest
-import zope.security.management
-import zope.security.interfaces
-
-library_info = {}
-
-class LibraryInfo(object):
- def __init__(self):
- self.included = []
- self.required = []
-
-
-def getRequest():
- try:
- i = zope.security.management.getInteraction() # raises NoInteraction
- except zope.security.interfaces.NoInteraction:
- return
-
- for p in i.participations:
- if IRequest.providedBy(p):
- return p
-
-def need(library_name):
- request = getRequest()
- # only take note of needed libraries if there is a request, and it is
- # capable of handling resource librarys
- if request and hasattr(request, 'resource_libraries'):
- if not library_name in request.resource_libraries:
- request.resource_libraries.append(library_name)
-
-def getRequired(name):
- return library_info[name].required
-
-def getIncluded(name):
- return library_info[name].included
Copied: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resources.py (from rev 83053, zc.resourcelibrary/trunk/src/zc/resourcelibrary/resourcelibrary.py)
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resources.py (rev 0)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/resources.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -0,0 +1,240 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from zope import component
+from zope import interface
+
+import zope.app.appsetup.interfaces
+
+import threading
+import os
+import sha
+
+from interfaces import IResourceManager
+
+library_info = {}
+
+class LibraryInfo(object):
+ def __init__(self):
+ self.path = u""
+ self.included = []
+ self.required = []
+
+class ResourceInfo(object):
+ def __init__(self, path, _type, dependants=None):
+ self.path = path
+ self.type = _type
+
+ if dependants is None:
+ self.dependants = []
+ else:
+ self.dependants = dependants
+
+ f = open(path, 'rb')
+
+ h = sha.sha()
+
+ for line in f:
+ h.update(line)
+
+ f.close()
+
+ self.digest = h.digest()
+
+ def __repr__(self):
+ return '<ResourceInfo path="%s" type="%s" dependants="%s">' % \
+ (self.path, self.type, self.dependants)
+
+ def addDependant(self, name):
+ self.dependants.append(name)
+
+class ResourceBundle(object):
+ def __init__(self, *resources):
+ self.paths = [resource.path for resource in resources]
+
+ assert len(resources) > 0
+
+ self.type = resources[0].type
+ self.dependants = resources[0].dependants
+
+ h = sha.sha()
+ map(h.update, (resource.digest for resource in resources))
+
+ self.digest = h.hexdigest()
+
+class ResourceManager(threading.local):
+ """The resource manager is a utility class to manage resource bundles."""
+
+ interface.implements(IResourceManager)
+
+ def __init__(self, libraries={}):
+ resources = self._unique_resources(libraries)
+ bundles = self._compute_bundles(resources)
+
+ self.by_dependant = self._by_dependant(resources, bundles)
+ self.by_digest = bundles
+
+ def getBundlesForLibrary(self, library_name):
+ return self.by_dependant[library_name]
+
+ def _unique_resources(self, libraries):
+ """Process resources for each library:
+
+ * Determine file type
+ * Extract SHA digest
+
+ Returned is a dict of ``ResourceInfo`` objects
+ indexed by digest.
+
+ Note: If two resources share digest (~ are equal), only one
+ will be used.
+
+ >>> linfo = LibraryInfo()
+ >>> import zc.resourcelibrary
+ >>> prefix = os.path.dirname(zc.resourcelibrary.__file__)
+ >>> path = "/tests/example/my-lib/".replace('/', os.path.sep)
+
+ >>> linfo.included = [prefix+path+"included.css",
+ ... prefix+path+"included.js"]
+
+ >>> manager = ResourceManager()
+ >>> resources = manager._unique_resources({'my-lib': linfo})
+ >>> len(resources)
+ 2
+ >>> rinfo = resources[0]
+ >>> rinfo.path # doctest: +ELLIPSIS
+ '.../example/my-lib/included.js'
+ >>> rinfo.dependants
+ ['my-lib']
+
+ """
+ resources = {}
+
+ for name, info in libraries.items():
+ included = info.included
+
+ for filename in included:
+ filetype = os.path.splitext(filename)[-1][1:]
+
+ if info.path:
+ filename = info.path + os.path.sep + filename
+
+ rinfo = ResourceInfo(filename, type)
+
+ digest = rinfo.digest
+
+ if digest not in resources:
+ resources[digest] = rinfo
+
+ resources[digest].addDependant(name)
+
+ return resources.values()
+
+ def _compute_bundles(self, resources):
+ groups = {}
+
+ # group resources based on type and dependants
+ for resource in resources:
+ digest = resource.digest
+
+ signature = tuple(sorted(resource.dependants)) + ('.%s' % resource.type,)
+
+ if signature not in groups:
+ groups[signature] = [resource]
+ else:
+ groups[signature].append(resource)
+
+ bundles = {}
+
+ for resources in groups.values():
+ bundle = ResourceBundle(*resources)
+ bundles[bundle.digest] = bundle
+
+ return bundles
+
+ def _list_dependants(self, resources):
+ dependants = set()
+
+ # maintain list of dependants
+ for resource in resources:
+ map(dependants.add, resource.dependants)
+
+ return dependants
+
+ def _by_dependant(self, resources, bundles):
+ """
+ We'll patch the built-in ``open`` method to define mock resources
+ that do not correspond to actual files.
+
+ >>> save_open = __builtins__['open']
+ >>> class MockFile(unicode):
+ ... def close(self): pass
+ >>> def mock_open(path, mode):
+ ... return MockFile(path)
+ >>> __builtins__['open'] = mock_open
+
+ Now we can define a few resources.
+
+ >>> resources = (ResourceInfo('/path/a', 1, ['a']),
+ ... ResourceInfo('/path/aa', 1, ['a']),
+ ... ResourceInfo('/path/b', 1, ['a', 'b']),
+ ... ResourceInfo('/path/bb', 1, ['a', 'b']),
+ ... ResourceInfo('/path/c', 1, ['a', 'b', 'c']))
+
+ Restore the ``open`` method.
+
+ >>> __builtins__['open'] = save_open
+
+ Now we're ready to compute the bundles per dependant:
+
+ >>> manager = ResourceManager({})
+ >>> bundles = manager._compute_bundles(resources)
+ >>> by_dependant = manager._by_dependant(resources, bundles)
+
+ Let's examine the bundles required for library 'a'. We expect
+ three bundles:
+
+ >>> bundles = by_dependant['a']
+ >>> len(bundles)
+ 3
+ >>> [bundle.paths for bundle in bundles]
+ [['/path/b', '/path/bb'], ['/path/c'], ['/path/a', '/path/aa']]
+ >>> [bundle.type for bundle in bundles]
+ [1, 1, 1]
+ >>> [bundle.digest for bundle in bundles] # doctest: +NORMALIZE_WHITESPACE
+ ['7309e08f51cab77e6855288001c30c2ab9e05400',
+ '2d5021510fc774edb7b698a9cd31a9059b51858e',
+ '11f36f3b56dd892d7af6757c925bcecf19549af1']
+ """
+
+ dependants = self._list_dependants(resources)
+
+ # compute bundles per library
+ by_dependant = {}
+
+ for name in dependants:
+ required_bundles = []
+
+ for digest, bundle in bundles.items():
+ if name in bundle.dependants:
+ required_bundles.append(bundle)
+
+ by_dependant[name] = required_bundles
+
+ return by_dependant
+
+ at component.adapter(zope.app.appsetup.interfaces.IProcessStartingEvent)
+def initializeResourceManager(*args):
+ manager = ResourceManager(library_info)
+ component.provideUtility(manager, IResourceManager)
Modified: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tal.py
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tal.py 2008-02-05 19:35:49 UTC (rev 83552)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tal.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -16,11 +16,12 @@
$Id: tal.py 3268 2005-08-22 23:31:27Z benji $
"""
from zope.tales.expressions import StringExpr
-from zc.resourcelibrary import resourcelibrary
+import zc.resourcelibrary
+
class ResourceLibraryExpression(StringExpr):
"""Resource library expression handler class"""
def __call__(self, econtext):
- resourcelibrary.need(self._expr)
+ zc.resourcelibrary.need(self._expr)
return ''
Modified: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tests/test_unit.py
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tests/test_unit.py 2008-02-05 19:35:49 UTC (rev 83552)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/tests/test_unit.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -2,13 +2,13 @@
import unittest
from zope.testing.doctestunit import DocTestSuite
-from zc.resourcelibrary import resourcelibrary
-from zc.resourcelibrary.resourcelibrary import LibraryInfo
+from zc.resourcelibrary import resources
+from zc.resourcelibrary.resources import LibraryInfo
+def setUp(test):
+ test.old_library_info = resources.library_info
+ resources.library_info = library_info = {}
-def setUp(test):
- test.old_library_info = resourcelibrary.library_info
- resourcelibrary.library_info = library_info = {}
# Dependencies:
#
# libA libD
@@ -27,9 +27,8 @@
def tearDown(test):
- resourcelibrary.library_info = test.old_library_info
+ resources.library_info = test.old_library_info
-
def doctest_dependency_resolution():
"""Test Response._addDependencies
@@ -66,6 +65,7 @@
return unittest.TestSuite(
(
DocTestSuite(setUp=setUp, tearDown=tearDown),
+ DocTestSuite('zc.resourcelibrary.resources'),
DocTestSuite('zc.resourcelibrary.publication',
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
Modified: zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/zcml.py
===================================================================
--- zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/zcml.py 2008-02-05 19:35:49 UTC (rev 83552)
+++ zc.resourcelibrary/branches/resources-bundles-branch/src/zc/resourcelibrary/zcml.py 2008-02-05 19:37:27 UTC (rev 83553)
@@ -11,7 +11,8 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-from zc.resourcelibrary.resourcelibrary import LibraryInfo, library_info
+
+from zc.resourcelibrary.resources import LibraryInfo, library_info
from zope.app import zapi
from zope.app.publisher.browser import directoryresource
from zope.app.publisher.browser.metadirectives import IBasicResourceInformation
@@ -104,10 +105,13 @@
'Resource library doesn\'t know how to include this '
'file: "%s".' % file_name)
+ library_info[self.name].path = _context.path(source)
+
# remember which files should be included in the HTML when this library
# is referenced
library_info[self.name].included.extend(include)
+ # register this directory as a browser resource
factory = directoryresource.DirectoryResourceFactory(
source, self.checker, self.name)
More information about the Checkins
mailing list