[Checkins] SVN: lovely.recipe/trunk/ recipe for importchecker

Jürgen Kartnaller juergen at kartnaller.at
Tue Jun 12 02:24:23 EDT 2007


Log message for revision 76632:
  recipe for importchecker
  

Changed:
  U   lovely.recipe/trunk/CHANGES.txt
  U   lovely.recipe/trunk/setup.py
  A   lovely.recipe/trunk/src/lovely/recipe/importchecker/
  A   lovely.recipe/trunk/src/lovely/recipe/importchecker/README.txt
  A   lovely.recipe/trunk/src/lovely/recipe/importchecker/__init__.py
  A   lovely.recipe/trunk/src/lovely/recipe/importchecker/app.py
  A   lovely.recipe/trunk/src/lovely/recipe/importchecker/importchecker.py
  A   lovely.recipe/trunk/src/lovely/recipe/importchecker/tests.py

-=-
Modified: lovely.recipe/trunk/CHANGES.txt
===================================================================
--- lovely.recipe/trunk/CHANGES.txt	2007-06-12 05:50:59 UTC (rev 76631)
+++ lovely.recipe/trunk/CHANGES.txt	2007-06-12 06:24:23 UTC (rev 76632)
@@ -3,6 +3,12 @@
 =========================
 
 
+2007/06/04 0.3.0a1:
+===================
+
+- recipe for importchecker
+
+
 2007/06/04 0.2.0a1:
 ===================
 

Modified: lovely.recipe/trunk/setup.py
===================================================================
--- lovely.recipe/trunk/setup.py	2007-06-12 05:50:59 UTC (rev 76631)
+++ lovely.recipe/trunk/setup.py	2007-06-12 06:24:23 UTC (rev 76632)
@@ -5,16 +5,17 @@
 mkdir = lovely.recipe.fs.mkdir:Mkdir
 mkfile = lovely.recipe.fs.mkfile:Mkfile
 i18n = lovely.recipe.i18n.i18n:I18n
+importchecker = lovely.recipe.importchecker.app:ImportChecker
 """
 
 setup (
     name='lovely.recipe',
-    version='0.2.0a1',
+    version='0.3.0a1',
     author = "Lovely Systems",
     author_email = "office at lovelysystems.com",
     license = "ZPL 2.1",
-    keywords = "buildout recipe filesystem",
-    url = 'svn://svn.zope.org/repos/main/lovely.recipe.fs',
+    keywords = "buildout recipe filesystem i18n importchecker",
+    url = 'svn://svn.zope.org/repos/main/lovely.recipe',
     packages = find_packages('src'),
     include_package_data = True,
     package_dir = {'':'src'},

Added: lovely.recipe/trunk/src/lovely/recipe/importchecker/README.txt
===================================================================
--- lovely.recipe/trunk/src/lovely/recipe/importchecker/README.txt	                        (rev 0)
+++ lovely.recipe/trunk/src/lovely/recipe/importchecker/README.txt	2007-06-12 06:24:23 UTC (rev 76632)
@@ -0,0 +1,44 @@
+=================
+i18n Tools Recipe
+=================
+
+This recipe creates an importchecker instance in the bin directory.
+
+
+Creating The Tools
+==================
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... parts = checker
+    ...
+    ... offline = true
+    ...
+    ... [checker]
+    ... recipe = lovely.recipe:importchecker
+    ... path = src/lovely
+    ... """)
+    >>> print system(buildout),
+    Installing checker.
+    checker: setting up importchecker
+    Generated script 'bin/importchecker'.
+
+    >>> import os
+    >>> ls(os.path.join(sample_buildout, 'bin'))
+    -  buildout
+    -  importchecker
+
+    >>> cat('bin', 'importchecker')
+    #!/opt/local/bin/python
+    <BLANKLINE>
+    import sys
+    sys.path[0:0] = [
+      '/Users/jukart/sandbox/lovely.recipe/src',
+      ]
+    <BLANKLINE>
+    import lovely.recipe.importchecker.importchecker
+    <BLANKLINE>
+    if __name__ == '__main__':
+        lovely.recipe.importchecker.importchecker.main(['importchecker', 'src/lovely'])
+


Property changes on: lovely.recipe/trunk/src/lovely/recipe/importchecker/README.txt
___________________________________________________________________
Name: svn:eol-style
   + native

Added: lovely.recipe/trunk/src/lovely/recipe/importchecker/__init__.py
===================================================================
--- lovely.recipe/trunk/src/lovely/recipe/importchecker/__init__.py	                        (rev 0)
+++ lovely.recipe/trunk/src/lovely/recipe/importchecker/__init__.py	2007-06-12 06:24:23 UTC (rev 76632)
@@ -0,0 +1 @@
+# Package


Property changes on: lovely.recipe/trunk/src/lovely/recipe/importchecker/__init__.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: lovely.recipe/trunk/src/lovely/recipe/importchecker/app.py
===================================================================
--- lovely.recipe/trunk/src/lovely/recipe/importchecker/app.py	                        (rev 0)
+++ lovely.recipe/trunk/src/lovely/recipe/importchecker/app.py	2007-06-12 06:24:23 UTC (rev 76632)
@@ -0,0 +1,57 @@
+##############################################################################
+#
+# Copyright (c) 2007 Lovely Systems and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id$
+"""
+__docformat__ = 'restructuredtext'
+
+import os
+import logging
+
+import zc.buildout
+import zc.recipe.egg
+
+import pkg_resources
+
+this_loc = pkg_resources.working_set.find(
+    pkg_resources.Requirement.parse('lovely.recipe')).location
+
+
+class ImportChecker(object):
+
+    def __init__(self, buildout, name, options):
+        self.buildout = buildout
+        self.name = name
+        self.options = options
+        if 'eggs' not in self.options:
+            self.options['eggs'] = ''
+        self.egg = zc.recipe.egg.Egg(buildout, name, options)
+
+    def install(self):
+        logging.getLogger(self.name).info('setting up importchecker')
+
+        requirements, ws = self.egg.working_set()
+
+        path = self.options.get('path', 'src')
+        arguments = ['importchecker', path]
+        zc.buildout.easy_install.scripts(
+            [('importchecker', 'lovely.recipe.importchecker.importchecker', 'main')],
+            ws, self.options['executable'], 'bin',
+            extra_paths = [this_loc],
+            arguments = arguments,
+            )
+
+        return ()
+
+


Property changes on: lovely.recipe/trunk/src/lovely/recipe/importchecker/app.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: lovely.recipe/trunk/src/lovely/recipe/importchecker/importchecker.py
===================================================================
--- lovely.recipe/trunk/src/lovely/recipe/importchecker/importchecker.py	                        (rev 0)
+++ lovely.recipe/trunk/src/lovely/recipe/importchecker/importchecker.py	2007-06-12 06:24:23 UTC (rev 76632)
@@ -0,0 +1,354 @@
+#!/usr/bin/env python2.4
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Import checker
+
+This utility finds is based on the zope importchecker script, prints
+out unused imports, imports that are only for tests packages and
+runtime imports.
+
+$Id$
+"""
+import compiler
+import os, os.path
+import sys
+
+def _findDottedNamesHelper(node, result):
+    more_node = node
+    name = node.__class__.__name__
+    if name == 'Getattr':
+        dotted = []
+        while name == 'Getattr':
+            dotted.append(node.attrname)
+            node = node.expr
+            name = node.__class__.__name__
+        if name == 'Name':
+            dotted.append(node.name)
+            dotted.reverse()
+            for i in range(1, len(dotted)):
+                result.append('.'.join(dotted[:i]))
+            result.append('.'.join(dotted))
+            return
+    elif name == 'Name':
+        result.append(node.name)
+        return
+    elif name == 'AssAttr':
+        return
+    for child in more_node.getChildNodes():
+        _findDottedNamesHelper(child, result)
+
+def findDottedNames(node):
+    """Find dotted names in an AST tree node
+    """
+    result = []
+    _findDottedNamesHelper(node, result)
+    return result
+
+class ImportFinder:
+    """An instance of this class will be used to walk over a compiler AST
+    tree (a module). During that operation, the appropriate methods of
+    this visitor will be called
+    """
+
+    def __init__(self):
+        self._map = {}
+
+    def visitFrom(self, stmt):
+        """Will be called for 'from foo import bar' statements
+        """
+        module_name, names = stmt.asList()
+        if module_name == '__future__':
+            # we don't care what's imported from the future
+            return
+        names_dict = {}
+        for orig_name, as_name in names:
+            # we don't care about from import *
+            if orig_name == '*':
+                continue
+            if as_name is None:
+                name = orig_name
+            else:
+                name = as_name
+            names_dict[name] = orig_name
+        self._map.setdefault(module_name, {'names': names_dict,
+                                           'lineno': stmt.lineno})
+
+    def visitImport(self, stmt):
+        """Will be called for 'import foo.bar' statements
+        """
+        for orig_name, as_name in stmt.names:
+            if as_name is None:
+                name = orig_name
+            else:
+                name = as_name
+            self._map.setdefault(orig_name, {'names': {name: orig_name},
+                                             'lineno': stmt.lineno})
+
+    def getMap(self):
+        return self._map
+
+def findImports(mod):
+    """Find import statements in module and put the result in a mapping.
+    """
+    visitor = ImportFinder()
+    compiler.walk(mod, visitor)
+    return visitor.getMap()
+
+class Module:
+    """This represents a python module.
+    """
+
+    def __init__(self, path):
+        mod = compiler.parseFile(path)
+        self._path = path
+        self._map = findImports(mod)
+        dottednames = {}
+        self._dottednames = findDottedNames(mod)
+
+    def getPath(self):
+        """Return the path to this module's file.
+        """
+        return self._path
+
+    def getImportedModuleNames(self):
+        """Return the names of imported modules.
+        """
+        return self._map.keys()
+
+    def getImportNames(self):
+        """Return the names of imports; add dottednames as well.
+        """
+        result = []
+        map = self._map
+        for module_name in map.keys():
+            for usedname, originalname in map[module_name]['names'].items():
+                result.append((originalname, module_name))
+                # add any other name that we could be using
+                for dottedname in self._dottednames:
+                    usednamedot = usedname + '.'
+                    if dottedname.startswith(usednamedot):
+                        attrname = dottedname[len(usednamedot):].split('.')[0]
+                        result.append((attrname, module_name))
+
+        return result
+
+    def getUnusedImports(self):
+        """Get unused imports of this module (the whole import info).
+        """
+        result = []
+        for value in self._map.values():
+            for usedname, originalname in value['names'].items():
+                if usedname not in self._dottednames:
+                    result.append((originalname, value['lineno']))
+        return result
+
+class ModuleFinder:
+
+    def __init__(self):
+        self._files = []
+
+    def visit(self, arg, dirname, names):
+        """This method will be called when we walk the filesystem
+        tree. It looks for python modules and stored their filenames.
+        """
+        for name in names:
+            # get all .py files that aren't weirdo emacs droppings
+            if name.endswith('.py') and not name.startswith('.#'):
+                self._files.append(os.path.join(dirname, name))
+
+    def getModuleFilenames(self):
+        return self._files
+
+def findModules(path):
+    """Find python modules in the given path and return their absolute
+    filenames in a sequence.
+    """
+    finder = ModuleFinder()
+    os.path.walk(path, finder.visit, ())
+    return finder.getModuleFilenames()
+
+class ImportDatabase:
+    """This database keeps tracks of imports.
+
+    It allows to NOT report cases where a module imports something
+    just so that another module can import it (import dependencies).
+    """
+
+    def __init__(self, root_path):
+        self._root_path = root_path
+        self._modules = {}
+        self._names = {}
+
+    def resolveDottedModuleName(self, dotted_name, module):
+        """Return path to file representing module, or None if no such
+        thing. Can do this relative from module.
+        """
+        dotted_path = dotted_name.replace('.', '/')
+        # try relative import first
+        path = os.path.join(os.path.dirname(module.getPath()), dotted_path)
+        path = self._resolveHelper(path)
+        if path is not None:
+            return path
+        # absolute import (assumed to be from this tree)
+        if os.path.isfile(os.path.join(self._root_path, '__init__.py')):
+            startpath, dummy = os.path.split(self._root_path)
+        else:
+            startpath = self._root_path
+        return self._resolveHelper(os.path.join(startpath, dotted_path))
+
+    def _resolveHelper(self, path):
+        if os.path.isfile(path + '.py'):
+            return path + '.py'
+        if os.path.isdir(path):
+            path = os.path.join(path, '__init__.py')
+            if os.path.isfile(path):
+                return path
+        return None
+
+    def findModules(self):
+        """Find modules in the given path.
+        """
+        for modulepath in findModules(self._root_path):
+            module = Module(modulepath)
+            self.addModule(module)
+
+    def addModule(self, module):
+        """Add information about a module to the database. A module in
+        this case is not a python module object, but an instance of
+        the above defined Module class.w
+        """
+        self_path = module.getPath()
+        # do nothing if we already know about it
+        if self._modules.has_key(self_path):
+            return
+
+        self._modules[self_path] = module
+
+        # add imported names to internal names mapping; this will
+        # allow us identify dependent imports later
+        names = self._names
+        for name, from_module_name in module.getImportNames():
+            path = self.resolveDottedModuleName(from_module_name, module)
+            t = (path, name)
+            modulepaths = names.get(t, {})
+            if not modulepaths.has_key(self_path):
+                modulepaths[self_path] = 1
+            names[t] = modulepaths
+
+    def getUnusedImports(self):
+        """Get unused imports of all known modules.
+        """
+        result = {}
+        for path, module in self._modules.items():
+            result[path] = self.getUnusedImportsInModule(module)
+        return result
+
+    def getImportedModuleNames(self, tests=False):
+        """returns all  names imported by modules"""
+        result = set()
+        import os
+        for path, module in self._modules.items():
+            # remove .py
+            parts = path[:-3].split(os.path.sep)
+            isTest = 'tests' in parts or 'testing' in parts \
+                     or 'ftests' in parts
+            if (tests and isTest) or (not tests and not isTest):
+                result.update(module.getImportedModuleNames())
+        return sorted(result)
+
+    def getUnusedImportsInModule(self, module):
+        """Get all unused imports in a module.
+        """
+        result = []
+        for name, lineno in module.getUnusedImports():
+            if not self.isNameImportedFrom(name, module):
+                result.append((name, lineno))
+        return result
+
+    def isNameImportedFrom(self, name, module):
+        """Return true if name is imported from module by another module.
+        """
+        return self._names.has_key((module.getPath(), name))
+
+    def getModulesImportingNameFrom(self, name, module):
+        """Return list of known modules that import name from module.
+        """
+        result = []
+        for path in self._names.get((module.getPath(), name), {}).keys():
+            result.append(self._modules[path])
+        return result
+
+def main(argv = sys.argv):
+    try:
+        path = os.path.abspath(argv[1])
+    except IndexError:
+        path = os.path.abspath(os.getcwd())
+        path = os.path.join(path, 'src')
+
+    if not os.path.exists(path):
+        print "Please provide a valid path %r" % path
+        sys.exit(1)
+    print "-"*79
+    print "Path: %r" % path
+    print "-"*79
+    print
+
+    path = os.path.abspath(path)
+    if not os.path.isdir(path):
+        print "Unknown path:", path
+        sys.exit(1)
+
+    db = ImportDatabase(path)
+    db.findModules()
+    unused_imports = db.getUnusedImports()
+    module_paths = unused_imports.keys()
+    module_paths.sort()
+    print "-"*79
+    print "Unused Imports:"
+    print "-"*79
+    for path in module_paths:
+        info = unused_imports[path]
+        if not info:
+            continue
+        line2names = {}
+        for name, line in info:
+            names = line2names.get(line, [])
+            names.append(name)
+            line2names[line] = names
+        lines = line2names.keys()
+        lines.sort()
+        for line in lines:
+            names = ', '.join(line2names[line])
+            print "%s:%s: %s" % (path, line, names)
+    testImports = db.getImportedModuleNames(tests=True)
+    installImports = db.getImportedModuleNames(tests=False)
+    print "-"*79
+    print
+    print "-"*79
+    print "Imports for 'tests' and 'testing' packages"
+    print "-"*79
+    for name in [name for name in testImports if name not in installImports]:
+        print name
+    print "-"*79
+    print
+    print "-"*79
+    print "Install imports"
+    print "-"*79
+    for name in installImports:
+        print name
+    print "-"*79
+
+if __name__ == '__main__':
+    main()
+


Property changes on: lovely.recipe/trunk/src/lovely/recipe/importchecker/importchecker.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: lovely.recipe/trunk/src/lovely/recipe/importchecker/tests.py
===================================================================
--- lovely.recipe/trunk/src/lovely/recipe/importchecker/tests.py	                        (rev 0)
+++ lovely.recipe/trunk/src/lovely/recipe/importchecker/tests.py	2007-06-12 06:24:23 UTC (rev 76632)
@@ -0,0 +1,37 @@
+##############################################################################
+#
+# Copyright (c) 2007 Lovely Systems and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id$
+"""
+__docformat__ = 'restructuredtext'
+
+from zc.buildout import testing
+import doctest, unittest
+from zope.testing import doctest, renormalizing
+
+def setUp(test):
+    testing.buildoutSetUp(test)
+    testing.install_develop('zc.recipe.egg', test)
+    testing.install_develop('lovely.recipe', test)
+
+def test_suite():
+    return unittest.TestSuite((
+        doctest.DocFileSuite('README.txt',
+                             setUp=setUp,
+                             tearDown=testing.buildoutTearDown,
+                             checker=renormalizing.RENormalizing([
+                                testing.normalize_path,
+                                testing.normalize_script,
+                                testing.normalize_egg_py])
+                             )))


Property changes on: lovely.recipe/trunk/src/lovely/recipe/importchecker/tests.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native



More information about the Checkins mailing list