[Zope-Checkins] CVS: Zope/lib/python/Products/ZCTextIndex - HTMLSplitter.py:1.10.42.1 Lexicon.py:1.17.42.1

Casey Duncan casey@zope.com
Mon, 2 Dec 2002 00:04:46 -0500


Update of /cvs-repository/Zope/lib/python/Products/ZCTextIndex
In directory cvs.zope.org:/tmp/cvs-serv5405

Modified Files:
      Tag: casey-zctextindex-locale-branch
	HTMLSplitter.py Lexicon.py 
Log Message:
Added locale support to splitters and a unit test as per collector #597


=== Zope/lib/python/Products/ZCTextIndex/HTMLSplitter.py 1.10 => 1.10.42.1 ===
--- Zope/lib/python/Products/ZCTextIndex/HTMLSplitter.py:1.10	Thu May 23 11:05:34 2002
+++ Zope/lib/python/Products/ZCTextIndex/HTMLSplitter.py	Mon Dec  2 00:04:45 2002
@@ -21,14 +21,15 @@
 
     __implements__ = ISplitter
 
-    def process(self, text, wordpat=r"\w+"):
+    def process(self, text, wordpat=r"(?L)\w+"):
         splat = []
         for t in text:
             splat += self._split(t, wordpat)
         return splat
 
     def processGlob(self, text):
-        return self.process(text, r"\w+[\w*?]*") # see Lexicon.globToWordIds()
+        # see Lexicon.globToWordIds()
+        return self.process(text, r"(?L)\w+[\w*?]*")
 
     def _split(self, text, wordpat):
         text = text.lower()


=== Zope/lib/python/Products/ZCTextIndex/Lexicon.py 1.17 => 1.17.42.1 ===
--- Zope/lib/python/Products/ZCTextIndex/Lexicon.py:1.17	Thu May 23 11:05:34 2002
+++ Zope/lib/python/Products/ZCTextIndex/Lexicon.py	Mon Dec  2 00:04:45 2002
@@ -156,8 +156,8 @@
 class Splitter:
 
     import re
-    rx = re.compile(r"\w+")
-    rxGlob = re.compile(r"\w+[\w*?]*") # See globToWordIds() above
+    rx = re.compile(r"(?L)\w+")
+    rxGlob = re.compile(r"(?L)\w+[\w*?]*") # See globToWordIds() above
 
     def process(self, lst):
         result = []