Converted 'catalog' module as package
Added 'index' module to handle zopyx.txng3 full-text index
Added zopyx.txng3.core and hurry.query packages dependencies
--- a/.pydevproject Tue Mar 16 01:29:17 2010 +0100
+++ b/.pydevproject Wed Mar 17 01:04:49 2010 +0100
@@ -18,6 +18,15 @@
<path>/var/local/eggs/zope.publisher-3.4.6-py2.5.egg</path>
<path>/var/local/eggs/pytz-2007k-py2.5.egg</path>
<path>/var/local/eggs/zope.schema-3.4.0-py2.5.egg</path>
+<path>/var/local/eggs/zope.annotation-3.4.1-py2.5.egg</path>
+<path>/var/local/eggs/zope.app.catalog-3.5.1-py2.5.egg</path>
+<path>/var/local/eggs/zope.app.container-3.5.6-py2.5-linux-i686.egg</path>
+<path>/var/local/eggs/zope.app.file-3.4.4-py2.5.egg</path>
+<path>/var/local/eggs/zope.app.intid-3.4.1-py2.5.egg</path>
+<path>/var/local/eggs/zope.index-3.4.1-py2.5.egg</path>
+<path>/var/local/eggs/zopyx.txng3.core-3.3.4-py2.5.egg</path>
+<path>/var/local/eggs/hurry.query-0.9.2-py2.5.egg</path>
+<path>/var/local/eggs/ZODB3-3.8.5-py2.5-linux-i686.egg</path>
</pydev_pathproperty>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Python 2.5</pydev_property>
</pydev_project>
--- a/.settings/org.eclipse.core.resources.prefs Tue Mar 16 01:29:17 2010 +0100
+++ b/.settings/org.eclipse.core.resources.prefs Wed Mar 17 01:04:49 2010 +0100
@@ -1,7 +1,8 @@
-#Thu Mar 04 12:01:13 CET 2010
+#Wed Mar 17 00:02:25 CET 2010
eclipse.preferences.version=1
encoding//ztfy/utils/__init__.py=utf-8
-encoding//ztfy/utils/catalog.py=utf-8
+encoding//ztfy/utils/catalog/__init__.py=utf-8
+encoding//ztfy/utils/catalog/index.py=utf-8
encoding//ztfy/utils/file.py=utf-8
encoding//ztfy/utils/html.py=utf-8
encoding//ztfy/utils/protocol/xmlrpc.py=utf-8
--- a/buildout.cfg Tue Mar 16 01:29:17 2010 +0100
+++ b/buildout.cfg Wed Mar 17 01:04:49 2010 +0100
@@ -7,6 +7,8 @@
eggs = ztfy.utils
zc.set
zope.app.zapi
+ zopyx.txng3.core
+ hurry.query
[i18n]
recipe = z3c.recipe.i18n:i18n
--- a/ztfy.utils.egg-info/SOURCES.txt Tue Mar 16 01:29:17 2010 +0100
+++ b/ztfy.utils.egg-info/SOURCES.txt Wed Mar 17 01:04:49 2010 +0100
@@ -34,7 +34,6 @@
ztfy/utils/request.py
ztfy/utils/security.py
ztfy/utils/text.py
-ztfy/utils/timezone.py
ztfy/utils/traversing.py
ztfy/utils/unicode.py
ztfy/utils/docs/HISTORY.txt
@@ -51,4 +50,8 @@
ztfy/utils/tal/text.py
ztfy/utils/tests/__init__.py
ztfy/utils/tests/test_utilsdocs.py
-ztfy/utils/tests/test_utilsdocstrings.py
\ No newline at end of file
+ztfy/utils/tests/test_utilsdocstrings.py
+ztfy/utils/timezone/__init__.py
+ztfy/utils/timezone/interfaces.py
+ztfy/utils/timezone/schema.py
+ztfy/utils/timezone/utility.py
\ No newline at end of file
--- a/ztfy/utils/catalog.py Tue Mar 16 01:29:17 2010 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,160 +0,0 @@
-### -*- coding: utf-8 -*- ####################################################
-##############################################################################
-#
-# Copyright (c) 2008-2009 Thierry Florac <tflorac AT ulthar.net>
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-__docformat__ = "restructuredtext"
-
-# import standard packages
-
-# import Zope3 interfaces
-from zope.annotation.interfaces import IAnnotations
-from zope.app.catalog.interfaces import ICatalog
-from zope.app.container.interfaces import IContainer
-from zope.app.file.interfaces import IFile
-from zope.app.intid.interfaces import IIntIds
-
-# import local interfaces
-
-# import Zope3 packages
-from zope.app import zapi
-
-# import local packages
-import request as request_utils
-
-from ztfy.utils import _
-
-
-#
-# IntIds utility functions
-#
-
-def getIntIdUtility(name='', request=None, context=None):
- """Look for a named IIntIds utility"""
- if request is None:
- request = request_utils.getRequest()
- intids = request_utils.getRequestData('IntIdsUtility::' + name, request)
- if intids is None:
- intids = zapi.queryUtility(IIntIds, name, context=context)
- if intids is not None:
- request_utils.setRequestData('IntIdsUtility::' + name, intids, request)
- return intids
-
-
-def getObjectId(object, intids_name='', request=None, context=None):
- """Look for an object Id as recorded by given IIntIds utility"""
- if request is None:
- request = request_utils.getRequest()
- intids = getIntIdUtility(intids_name, request, context)
- if intids is not None:
- return intids.queryId(object)
- return None
-
-
-def getObject(id, intids_name='', request=None, context=None):
- """Look for an object recorded by given IIntIds utility and id"""
- if request is None:
- request = request_utils.getRequest()
- intids = getIntIdUtility(intids_name, request, context)
- if intids is not None:
- return intids.queryObject(id)
- return None
-
-
-#
-# Catalog utility functions
-#
-
-def queryCatalog(name='', context=None):
- """Look for a registered catalog"""
- return zapi.queryUtility(ICatalog, name, context=context)
-
-
-def indexObject(object, catalog_name='', index_name='', request=None, context=None):
- """Index object into a registered catalog"""
- if request is None:
- request = request_utils.getRequest()
- intids = getIntIdUtility('', request, context)
- if intids is not None:
- if ICatalog.providedBy(catalog_name):
- catalog = catalog_name
- else:
- catalog = queryCatalog(catalog_name, context)
- if catalog is not None:
- id = intids.register(object)
- if index_name:
- catalog[index_name].index_doc(id, object)
- else:
- catalog.index_doc(id, object)
- return True
- return False
-
-
-def unindexObject(object, catalog_name='', index_name='', request=None, context=None):
- """Remove object from a registered catalog"""
- if request is None:
- request = request_utils.getRequest()
- id = getObjectId(object, '', request, context)
- if id is not None:
- if ICatalog.providedBy(catalog_name):
- catalog = catalog_name
- else:
- catalog = queryCatalog(catalog_name, context)
- if catalog is not None:
- if index_name:
- catalog[index_name].unindex_doc(id)
- else:
- catalog.unindex_doc(id)
- return True
- return False
-
-
-def _indexObject(object, intids, catalogs):
- """Index object data into given set of catalogs"""
- id = intids.register(object)
- for catalog in catalogs:
- catalog.index_doc(id, object)
-
-def _indexObjectValues(object, intids, catalogs):
- """Index object values into given set of catalogs"""
- container = IContainer(object, None)
- if container is not None:
- for subobject in container.values():
- _indexAllObject(subobject, intids, catalogs)
-
-def _indexObjectAnnotations(object, intids, catalogs):
- """Index object annotations into given set of catalogs"""
- annotations = IAnnotations(object, None)
- if annotations is not None:
- keys = annotations.keys()
- for key in keys:
- _indexAllObject(annotations[key], intids, catalogs)
- file = IFile(annotations[key], None)
- if file is not None:
- _indexObject(file, intids, catalogs)
-
-def _indexAllObject(object, intids, catalogs):
- """Index object, object values and annotations into given set of catalogs"""
- _indexObject(object, intids, catalogs)
- _indexObjectValues(object, intids, catalogs)
- _indexObjectAnnotations(object, intids, catalogs)
-
-def indexAllObjectValues(object, context=None):
- """Reindex a whole container properties and contents (including annotations) into site's catalogs"""
- if context is None:
- context = object
- intids = zapi.queryUtility(IIntIds, context=context)
- if intids is not None:
- catalogs = zapi.getAllUtilitiesRegisteredFor(ICatalog, context)
- if catalogs:
- _indexAllObject(object, intids, catalogs)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ztfy/utils/catalog/__init__.py Wed Mar 17 01:04:49 2010 +0100
@@ -0,0 +1,160 @@
+### -*- coding: utf-8 -*- ####################################################
+##############################################################################
+#
+# Copyright (c) 2008-2009 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+__docformat__ = "restructuredtext"
+
+# import standard packages
+
+# import Zope3 interfaces
+from zope.annotation.interfaces import IAnnotations
+from zope.app.catalog.interfaces import ICatalog
+from zope.app.container.interfaces import IContainer
+from zope.app.file.interfaces import IFile
+from zope.app.intid.interfaces import IIntIds
+
+# import local interfaces
+
+# import Zope3 packages
+from zope.app import zapi
+
+# import local packages
+from ztfy.utils import request as request_utils
+
+from ztfy.utils import _
+
+
+#
+# IntIds utility functions
+#
+
+def getIntIdUtility(name='', request=None, context=None):
+ """Look for a named IIntIds utility"""
+ if request is None:
+ request = request_utils.getRequest()
+ intids = request_utils.getRequestData('IntIdsUtility::' + name, request)
+ if intids is None:
+ intids = zapi.queryUtility(IIntIds, name, context=context)
+ if intids is not None:
+ request_utils.setRequestData('IntIdsUtility::' + name, intids, request)
+ return intids
+
+
+def getObjectId(object, intids_name='', request=None, context=None):
+ """Look for an object Id as recorded by given IIntIds utility"""
+ if request is None:
+ request = request_utils.getRequest()
+ intids = getIntIdUtility(intids_name, request, context)
+ if intids is not None:
+ return intids.queryId(object)
+ return None
+
+
+def getObject(id, intids_name='', request=None, context=None):
+ """Look for an object recorded by given IIntIds utility and id"""
+ if request is None:
+ request = request_utils.getRequest()
+ intids = getIntIdUtility(intids_name, request, context)
+ if intids is not None:
+ return intids.queryObject(id)
+ return None
+
+
+#
+# Catalog utility functions
+#
+
+def queryCatalog(name='', context=None):
+ """Look for a registered catalog"""
+ return zapi.queryUtility(ICatalog, name, context=context)
+
+
+def indexObject(object, catalog_name='', index_name='', request=None, context=None):
+ """Index object into a registered catalog"""
+ if request is None:
+ request = request_utils.getRequest()
+ intids = getIntIdUtility('', request, context)
+ if intids is not None:
+ if ICatalog.providedBy(catalog_name):
+ catalog = catalog_name
+ else:
+ catalog = queryCatalog(catalog_name, context)
+ if catalog is not None:
+ id = intids.register(object)
+ if index_name:
+ catalog[index_name].index_doc(id, object)
+ else:
+ catalog.index_doc(id, object)
+ return True
+ return False
+
+
+def unindexObject(object, catalog_name='', index_name='', request=None, context=None):
+ """Remove object from a registered catalog"""
+ if request is None:
+ request = request_utils.getRequest()
+ id = getObjectId(object, '', request, context)
+ if id is not None:
+ if ICatalog.providedBy(catalog_name):
+ catalog = catalog_name
+ else:
+ catalog = queryCatalog(catalog_name, context)
+ if catalog is not None:
+ if index_name:
+ catalog[index_name].unindex_doc(id)
+ else:
+ catalog.unindex_doc(id)
+ return True
+ return False
+
+
+def _indexObject(object, intids, catalogs):
+ """Index object data into given set of catalogs"""
+ id = intids.register(object)
+ for catalog in catalogs:
+ catalog.index_doc(id, object)
+
+def _indexObjectValues(object, intids, catalogs):
+ """Index object values into given set of catalogs"""
+ container = IContainer(object, None)
+ if container is not None:
+ for subobject in container.values():
+ _indexAllObject(subobject, intids, catalogs)
+
+def _indexObjectAnnotations(object, intids, catalogs):
+ """Index object annotations into given set of catalogs"""
+ annotations = IAnnotations(object, None)
+ if annotations is not None:
+ keys = annotations.keys()
+ for key in keys:
+ _indexAllObject(annotations[key], intids, catalogs)
+ file = IFile(annotations[key], None)
+ if file is not None:
+ _indexObject(file, intids, catalogs)
+
+def _indexAllObject(object, intids, catalogs):
+ """Index object, object values and annotations into given set of catalogs"""
+ _indexObject(object, intids, catalogs)
+ _indexObjectValues(object, intids, catalogs)
+ _indexObjectAnnotations(object, intids, catalogs)
+
+def indexAllObjectValues(object, context=None):
+ """Reindex a whole container properties and contents (including annotations) into site's catalogs"""
+ if context is None:
+ context = object
+ intids = zapi.queryUtility(IIntIds, context=context)
+ if intids is not None:
+ catalogs = zapi.getAllUtilitiesRegisteredFor(ICatalog, context)
+ if catalogs:
+ _indexAllObject(object, intids, catalogs)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ztfy/utils/catalog/configure.zcml Wed Mar 17 01:04:49 2010 +0100
@@ -0,0 +1,13 @@
+<configure
+ xmlns="http://namespaces.zope.org/zope"
+ i18n_domain="ztfy.utils">
+
+ <class class=".index.TextIndexNG">
+ <require
+ interface="zope.app.catalog.interfaces.IAttributeIndex
+ zope.index.interfaces.IStatistics"
+ set_schema="zope.app.catalog.interfaces.IAttributeIndex"
+ permission="zope.ManageServices" />
+ </class>
+
+</configure>
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/ztfy/utils/catalog/index.py Wed Mar 17 01:04:49 2010 +0100
@@ -0,0 +1,152 @@
+### -*- coding: utf-8 -*- ####################################################
+##############################################################################
+#
+# Copyright (c) 2008-2010 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+__docformat__ = "restructuredtext"
+
+# import standard packages
+import re
+from persistent import Persistent
+from BTrees import IFBTree
+
+# import Zope3 interfaces
+from zope.index.interfaces import IInjection, IStatistics, IIndexSearch
+from zopyx.txng3.core.interfaces import IStorageWithTermFrequency
+from zopyx.txng3.core.interfaces.ting import ITingIndex
+
+# import local interfaces
+
+# import Zope3 packages
+from zope.app import zapi
+from zope.app.catalog.attribute import AttributeIndex
+from zope.app.container.contained import Contained
+from zope.interface import implements
+from zopyx.txng3.core import config
+from zopyx.txng3.core.index import Index
+
+# import local packages
+from hurry.query.query import IndexTerm
+
+
+class TextIndexNG(AttributeIndex, Persistent, Contained):
+ """Adaptation of zopyx.txng3.core for use zope.app.catalog index"""
+
+ implements(IInjection, IStatistics, IIndexSearch, ITingIndex)
+
+ def __init__(self,
+ field_name=None,
+ interface=None,
+ field_callable=False,
+ use_stemmer=config.defaults['use_stemmer'],
+ dedicated_storage=config.defaults['dedicated_storage'],
+ ranking=config.defaults['ranking'],
+ use_normalizer=config.defaults['use_normalizer'],
+ languages=config.DEFAULT_LANGUAGE,
+ use_stopwords=config.defaults['use_stopwords'],
+ autoexpand_limit=config.defaults['autoexpand_limit'],
+ splitter=config.DEFAULT_SPLITTER,
+ index_unknown_languages=config.defaults['index_unknown_languages'],
+ query_parser=config.DEFAULT_PARSER,
+ lexicon=config.DEFAULT_LEXICON,
+ splitter_additional_chars=config.defaults['splitter_additional_chars'],
+ storage=config.DEFAULT_STORAGE,
+ splitter_casefolding=config.defaults['splitter_casefolding']):
+ spaces = re.compile(r'\s+')
+ if ranking:
+ util = zapi.createObject(storage)
+ if not IStorageWithTermFrequency.providedBy(util):
+ raise ValueError("This storage cannot be used for ranking")
+ _fields = spaces.split(field_name)
+ AttributeIndex.__init__(self, _fields[0], interface, field_callable)
+ if len(_fields) < 2:
+ dedicated_storage = False
+ self._index = Index(fields=_fields,
+ languages=spaces.split(languages),
+ use_stemmer=use_stemmer,
+ dedicated_storage=dedicated_storage,
+ ranking=ranking,
+ use_normalizer=use_normalizer,
+ use_stopwords=use_stopwords,
+ storage=storage,
+ autoexpand_limit=autoexpand_limit,
+ splitter=splitter,
+ lexicon=lexicon,
+ index_unknown_languages=index_unknown_languages,
+ query_parser=query_parser,
+ splitter_additional_chars=splitter_additional_chars,
+ splitter_casefolding=splitter_casefolding)
+ self.languages = languages
+ self.use_stemmer = use_stemmer
+ self.dedicated_storage = dedicated_storage
+ self.ranking = ranking
+ self.use_normalizer = use_normalizer
+ self.use_stopwords = use_stopwords
+ self.interface = interface
+ self.storage = storage
+ self.autoexpand_limit = autoexpand_limit
+ self.default_field = _fields[0]
+ self._fields = _fields
+ self.splitter = splitter
+ self.lexicon = lexicon
+ self.index_unknown_languages = index_unknown_languages
+ self.query_parser = query_parser
+ self.splitter_additional_chars = splitter_additional_chars
+ self.splitter_casefolding = splitter_casefolding
+
+ def clear(self):
+ self._index.clear()
+
+ def documentCount(self):
+ """See interface IStatistics"""
+ return len(self._index.getStorage(self.default_field))
+
+ def wordCount(self):
+ """See interface IStatistics"""
+ return len(self._index.getLexicon())
+
+ def index_doc(self, docid, value):
+ """See interface IInjection"""
+ v = self.interface(value, None)
+ if v is not None:
+ self.unindex_doc(docid)
+ self._index.index_object(v, docid)
+
+ def unindex_doc(self, docid):
+ """See interface IInjection"""
+ self._index.unindex_object(docid)
+
+ def apply(self, query):
+ if isinstance(query, dict):
+ kw = query
+ query = kw['query']
+ del kw['query']
+ ting_rr = self._index.search(query, **kw)
+ return ting_rr.getDocids().keys()
+
+
+class Text(IndexTerm):
+ """hurry.query search term"""
+
+ def __init__(self, index_id, text):
+ super(Text, self).__init__(index_id)
+ self.text = text
+
+ def getIndex(self):
+ index = super(Text, self).getIndex()
+ assert ITingIndex.providedBy(index)
+ return index
+
+ def apply(self):
+ index = self.getIndex()
+ return IFBTree.IFSet(index.apply(self.text))
--- a/ztfy/utils/configure.zcml Tue Mar 16 01:29:17 2010 +0100
+++ b/ztfy/utils/configure.zcml Wed Mar 17 01:04:49 2010 +0100
@@ -8,4 +8,6 @@
<include package=".tal" />
<include package=".timezone" />
+ <include package=".catalog" />
+
</configure>