--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/.hgignore Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,19 @@
+
+syntax: regexp
+^develop-eggs$
+syntax: regexp
+^parts$
+syntax: regexp
+^bin$
+syntax: regexp
+^\.installed\.cfg$
+syntax: regexp
+^\.settings$
+syntax: regexp
+^build$
+syntax: regexp
+^dist$
+syntax: regexp
+^\.idea$
+syntax: regexp
+.*\.pyc$
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/LICENSE Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,42 @@
+Zope Public License (ZPL) Version 2.1
+=====================================
+
+A copyright notice accompanies this license document that identifies
+the copyright holders.
+
+This license has been certified as open source. It has also been designated
+as GPL compatible by the Free Software Foundation (FSF).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions in source code must retain the accompanying copyright
+ notice, this list of conditions, and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the accompanying copyright
+ notice, this list of conditions, and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Names of the copyright holders must not be used to endorse or promote
+ products derived from this software without prior written permission
+ from the copyright holders.
+ 4. The right to distribute this software or to use it for any purpose does
+ not give you the right to use Servicemarks (sm) or Trademarks (tm) of the
+ copyright holders. Use of them is covered by separate agreement with the
+ copyright holders.
+ 5. If any files are modified, you must cause the modified files to carry
+ prominent notices stating that you changed the files and the date of any
+ change.
+
+
+Disclaimer
+==========
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/MANIFEST.in Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,5 @@
+include *.txt
+recursive-include docs *
+recursive-include src *
+global-exclude *.pyc
+global-exclude *.*~
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/bootstrap.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,178 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+"""
+
+import os
+import shutil
+import sys
+import tempfile
+
+from optparse import OptionParser
+
+tmpeggs = tempfile.mkdtemp()
+
+usage = '''\
+[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
+
+Bootstraps a buildout-based project.
+
+Simply run this script in a directory containing a buildout.cfg, using the
+Python that you want bin/buildout to use.
+
+Note that by using --find-links to point to local resources, you can keep
+this script from going over the network.
+'''
+
+parser = OptionParser(usage=usage)
+parser.add_option("-v", "--version", help="use a specific zc.buildout version")
+
+parser.add_option("-t", "--accept-buildout-test-releases",
+ dest='accept_buildout_test_releases',
+ action="store_true", default=False,
+ help=("Normally, if you do not specify a --version, the "
+ "bootstrap script and buildout gets the newest "
+ "*final* versions of zc.buildout and its recipes and "
+ "extensions for you. If you use this flag, "
+ "bootstrap and buildout will get the newest releases "
+ "even if they are alphas or betas."))
+parser.add_option("-c", "--config-file",
+ help=("Specify the path to the buildout configuration "
+ "file to be used."))
+parser.add_option("-f", "--find-links",
+ help=("Specify a URL to search for buildout releases"))
+parser.add_option("--allow-site-packages",
+ action="store_true", default=False,
+ help=("Let bootstrap.py use existing site packages"))
+
+
+options, args = parser.parse_args()
+
+######################################################################
+# load/install setuptools
+
+try:
+ if options.allow_site_packages:
+ import setuptools
+ import pkg_resources
+ from urllib.request import urlopen
+except ImportError:
+ from urllib2 import urlopen
+
+ez = {}
+exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
+
+if not options.allow_site_packages:
+ # ez_setup imports site, which adds site packages
+ # this will remove them from the path to ensure that incompatible versions
+ # of setuptools are not in the path
+ import site
+ # inside a virtualenv, there is no 'getsitepackages'.
+ # We can't remove these reliably
+ if hasattr(site, 'getsitepackages'):
+ for sitepackage_path in site.getsitepackages():
+ sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
+
+setup_args = dict(to_dir=tmpeggs, download_delay=0)
+ez['use_setuptools'](**setup_args)
+import setuptools
+import pkg_resources
+
+# This does not (always?) update the default working set. We will
+# do it.
+for path in sys.path:
+ if path not in pkg_resources.working_set.entries:
+ pkg_resources.working_set.add_entry(path)
+
+######################################################################
+# Install buildout
+
+ws = pkg_resources.working_set
+
+cmd = [sys.executable, '-c',
+ 'from setuptools.command.easy_install import main; main()',
+ '-mZqNxd', tmpeggs]
+
+find_links = os.environ.get(
+ 'bootstrap-testing-find-links',
+ options.find_links or
+ ('http://downloads.buildout.org/'
+ if options.accept_buildout_test_releases else None)
+ )
+if find_links:
+ cmd.extend(['-f', find_links])
+
+setuptools_path = ws.find(
+ pkg_resources.Requirement.parse('setuptools')).location
+
+requirement = 'zc.buildout'
+version = options.version
+if version is None and not options.accept_buildout_test_releases:
+ # Figure out the most recent final version of zc.buildout.
+ import setuptools.package_index
+ _final_parts = '*final-', '*final'
+
+ def _final_version(parsed_version):
+ for part in parsed_version:
+ if (part[:1] == '*') and (part not in _final_parts):
+ return False
+ return True
+ index = setuptools.package_index.PackageIndex(
+ search_path=[setuptools_path])
+ if find_links:
+ index.add_find_links((find_links,))
+ req = pkg_resources.Requirement.parse(requirement)
+ if index.obtain(req) is not None:
+ best = []
+ bestv = None
+ for dist in index[req.project_name]:
+ distv = dist.parsed_version
+ if _final_version(distv):
+ if bestv is None or distv > bestv:
+ best = [dist]
+ bestv = distv
+ elif distv == bestv:
+ best.append(dist)
+ if best:
+ best.sort()
+ version = best[-1].version
+if version:
+ requirement = '=='.join((requirement, version))
+cmd.append(requirement)
+
+import subprocess
+if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
+ raise Exception(
+ "Failed to execute command:\n%s" % repr(cmd)[1:-1])
+
+######################################################################
+# Import and run buildout
+
+ws.add_entry(tmpeggs)
+ws.require(requirement)
+import zc.buildout.buildout
+
+if not [a for a in args if '=' not in a]:
+ args.append('bootstrap')
+
+# if -c was provided, we push it back into args for buildout' main function
+if options.config_file is not None:
+ args[0:0] = ['-c', options.config_file]
+
+zc.buildout.buildout.main(args)
+shutil.rmtree(tmpeggs)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/buildout.cfg Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,63 @@
+[buildout]
+eggs-directory = /var/local/env/pyams/eggs
+
+socket-timeout = 3
+show-picked-versions = true
+newest = false
+
+allow-hosts =
+ bitbucket.org
+ *.python.org
+ *.sourceforge.net
+ github.com
+
+#extends = http://download.ztfy.org/webapp/ztfy.webapp.dev.cfg
+versions = versions
+newest = false
+#allow-picked-versions = false
+
+src = src
+develop = .
+ ../pyams_utils
+
+parts =
+ package
+ i18n
+ pyflakes
+ test
+
+[package]
+recipe = zc.recipe.egg
+eggs =
+ hypatia
+ nltk
+ pyams_catalog
+ pyams_utils
+ pyramid
+ zope.component
+ zope.interface
+
+[i18n]
+recipe = zc.recipe.egg
+eggs =
+ babel
+ lingua
+
+[pyflakes]
+recipe = zc.recipe.egg
+eggs = pyflakes
+scripts = pyflakes
+entry-points = pyflakes=pyflakes.scripts.pyflakes:main
+initialization = if not sys.argv[1:]: sys.argv[1:] = ["${buildout:src}"]
+
+[pyflakesrun]
+recipe = collective.recipe.cmd
+on_install = true
+cmds = ${buildout:develop}/bin/${pyflakes:scripts}
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = pyams_catalog [test]
+
+[versions]
+pyams_catalog = 0.1.0
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/setup.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,69 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+"""
+This module contains pyams_catalog package
+"""
+import os
+from setuptools import setup, find_packages
+
+DOCS = os.path.join(os.path.dirname(__file__),
+ 'docs')
+
+README = os.path.join(DOCS, 'README.txt')
+HISTORY = os.path.join(DOCS, 'HISTORY.txt')
+
+version = '0.1.0'
+long_description = open(README).read() + '\n\n' + open(HISTORY).read()
+
+tests_require = []
+
+setup(name='pyams_catalog',
+ version=version,
+ description="PyAMS catalog tools",
+ long_description=long_description,
+ classifiers=[
+ "License :: OSI Approved :: Zope Public License",
+ "Development Status :: 4 - Beta",
+ "Programming Language :: Python",
+ "Framework :: Pyramid",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+ ],
+ keywords='Pyramid PyAMS catalog',
+ author='Thierry Florac',
+ author_email='tflorac@ulthar.net',
+ url='http://hg.ztfy.org/pyams/pyams_catalog',
+ license='ZPL',
+ packages=find_packages('src'),
+ package_dir={'': 'src'},
+ namespace_packages=[],
+ include_package_data=True,
+ package_data={'': ['*.zcml', '*.txt', '*.pt', '*.pot', '*.po', '*.mo', '*.png', '*.gif', '*.jpeg', '*.jpg', '*.css', '*.js']},
+ zip_safe=False,
+ # uncomment this to be able to run tests with setup.py
+ test_suite="pyams_catalog.tests.test_utilsdocs.test_suite",
+ tests_require=tests_require,
+ extras_require=dict(test=tests_require),
+ install_requires=[
+ 'setuptools',
+ # -*- Extra requirements: -*-
+ 'hypatia',
+ 'nltk',
+ 'pyams_utils',
+ 'pyramid',
+ 'zope.component',
+ 'zope.interface',
+ ],
+ entry_points="""
+ # -*- Entry points: -*-
+ """,
+ )
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/PKG-INFO Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,18 @@
+Metadata-Version: 1.1
+Name: pyams-catalog
+Version: 0.1.0
+Summary: PyAMS catalog tools
+Home-page: http://hg.ztfy.org/pyams/pyams_catalog
+Author: Thierry Florac
+Author-email: tflorac@ulthar.net
+License: ZPL
+Description:
+
+
+Keywords: Pyramid PyAMS catalog
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: Zope Public License
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Framework :: Pyramid
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/SOURCES.txt Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,23 @@
+MANIFEST.in
+setup.py
+docs/HISTORY.txt
+docs/README.txt
+src/pyams_catalog/__init__.py
+src/pyams_catalog/configure.zcml
+src/pyams_catalog/include.py
+src/pyams_catalog/index.py
+src/pyams_catalog/nltk.py
+src/pyams_catalog/site.py
+src/pyams_catalog.egg-info/PKG-INFO
+src/pyams_catalog.egg-info/SOURCES.txt
+src/pyams_catalog.egg-info/dependency_links.txt
+src/pyams_catalog.egg-info/entry_points.txt
+src/pyams_catalog.egg-info/namespace_packages.txt
+src/pyams_catalog.egg-info/not-zip-safe
+src/pyams_catalog.egg-info/requires.txt
+src/pyams_catalog.egg-info/top_level.txt
+src/pyams_catalog/doctests/README.txt
+src/pyams_catalog/interfaces/__init__.py
+src/pyams_catalog/tests/__init__.py
+src/pyams_catalog/tests/test_utilsdocs.py
+src/pyams_catalog/tests/test_utilsdocstrings.py
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/dependency_links.txt Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,1 @@
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/entry_points.txt Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,3 @@
+
+ # -*- Entry points: -*-
+
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/namespace_packages.txt Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,1 @@
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/not-zip-safe Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,1 @@
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/requires.txt Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,9 @@
+setuptools
+hypatia
+nltk
+pyams_utils
+pyramid
+zope.component
+zope.interface
+
+[test]
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog.egg-info/top_level.txt Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,1 @@
+pyams_catalog
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/__init__.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+__docformat__ = 'restructuredtext'
+
+
+from pyramid.i18n import TranslationStringFactory
+_ = TranslationStringFactory('pyams_catalog')
+
+
+def includeme(config):
+ """Pyramid include"""
+
+ from .include import include_package
+ include_package(config)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/configure.zcml Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,6 @@
+<configure
+ xmlns="http://pylonshq.com/pyramid">
+
+ <include package="pyramid_zcml" />
+
+</configure>
\ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/doctests/README.txt Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,3 @@
+=====================
+pyams_catalog package
+=====================
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/include.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+__docformat__ = 'restructuredtext'
+
+
+# import standard library
+
+# import interfaces
+
+# import packages
+
+
+def include_package(config):
+ """Pyramid include"""
+
+ # add translations
+ config.add_translation_dirs('pyams_catalog:locales')
+
+ # load registry components
+ config.scan()
+
+ if hasattr(config, 'load_zcml'):
+ config.load_zcml('configure.zcml')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/index.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,98 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+__docformat__ = 'restructuredtext'
+
+
+# import standard library
+
+# import interfaces
+
+# import packages
+from hypatia.facet import FacetIndex
+from hypatia.field import FieldIndex
+from hypatia.keyword import KeywordIndex
+from hypatia.text import TextIndex
+from hypatia.text.lexicon import Lexicon
+from hypatia.util import BaseIndexMixin
+from persistent import Persistent
+from pyams_catalog.nltk import NltkFullTextProcessor
+from ZODB.broken import Broken
+
+
+_marker = object()
+
+
+class InterfaceSupportIndexMixin(BaseIndexMixin):
+ """Custom index mixin handling objects interfaces"""
+
+ def __init__(self, interface):
+ self.interface = interface
+
+ def discriminate(self, obj, default):
+ """See interface IIndexInjection"""
+ if self.interface is not None:
+ obj = self.interface(obj, None)
+ if obj is None:
+ return default
+
+ if callable(self.discriminator):
+ value = self.discriminator(obj, _marker)
+ else:
+ value = getattr(obj, self.discriminator, _marker)
+ if callable(value):
+ value = value(obj)
+
+ if value is _marker:
+ return default
+
+ if isinstance(value, Persistent):
+ raise ValueError('Catalog cannot index persistent object {0!r}'.format(value))
+
+ if isinstance(value, Broken):
+ raise ValueError('Catalog cannot index broken object {0!r}'.format(value))
+
+ return value
+
+
+class FieldIndexWithInterface(InterfaceSupportIndexMixin, FieldIndex):
+ """Field index with interface support"""
+
+ def __init__(self, interface, discriminator, family=None):
+ InterfaceSupportIndexMixin.__init__(self, interface)
+ FieldIndex.__init__(self, discriminator, family)
+
+
+class KeywordIndexWithInterface(InterfaceSupportIndexMixin, KeywordIndex):
+ """Keyword index with interface support"""
+
+ def __init__(self, interface, discriminator, family=None):
+ InterfaceSupportIndexMixin.__init__(self, interface)
+ KeywordIndex.__init__(self, discriminator, family)
+
+
+class FacetIndexWithInterface(InterfaceSupportIndexMixin, FacetIndex):
+ """Facet index with interface support"""
+
+ def __init__(self, interface, discriminator, facets, family=None):
+ InterfaceSupportIndexMixin.__init__(self, interface)
+ FacetIndex.__init__(self, discriminator, facets, family)
+
+
+class TextIndexWithInterface(InterfaceSupportIndexMixin, TextIndex):
+ """Text index with interface support"""
+
+ def __init__(self, interface, discriminator, lexicon=None, language='english', index=None, family=None):
+ InterfaceSupportIndexMixin.__init__(self, interface)
+ if lexicon is None:
+ lexicon = Lexicon(NltkFullTextProcessor(language))
+ TextIndex.__init__(self, discriminator, lexicon, index, family)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/interfaces/__init__.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+__docformat__ = 'restructuredtext'
+
+
+# import standard library
+
+# import interfaces
+
+# import packages
+
+
+NLTK_LANGUAGES = {'da': 'danish',
+ 'nl': 'dutch',
+ 'en': 'english',
+ 'fi': 'finnish',
+ 'fr': 'french',
+ 'de': 'german',
+ 'hu': 'hungarian',
+ 'it': 'italian',
+ 'no': 'norwegian',
+ 'po': 'porter',
+ 'pt': 'portuguese',
+ 'ro': 'romanian',
+ 'ru': 'russian',
+ 'es': 'spanish',
+ 'sv': 'swedish'}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/nltk.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+__docformat__ = 'restructuredtext'
+
+
+# import standard library
+
+# import interfaces
+from hypatia.text.interfaces import IPipelineElement
+
+# import packages
+import nltk
+from pyams_utils.unicode import translate_string
+from zope.interface import implementer
+
+
+@implementer(IPipelineElement)
+class NltkStemmedTextProcessor(object):
+ """NLTK based text processor using stemmer"""
+
+ def __init__(self, language='english'):
+ self.language = language
+ self.stemmer = nltk.stem.SnowballStemmer(language, ignore_stopwords=True)
+
+ def process(self, lst):
+ result = []
+ for s in lst:
+ translated = translate_string(s, keep_chars="'-").replace("'", ' ')
+ tokens = nltk.word_tokenize(translated, self.language)
+ result += [stem for stem in [self.stemmer.stem(token) for token in tokens
+ if token not in self.stemmer.stopwords]
+ if stem and (len(stem) > 1) and (stem not in self.stemmer.stopwords)]
+ return result
+
+ def processGlob(self, lst):
+ result = []
+ for s in lst:
+ translated = translate_string(s, keep_chars="'-*?").replace("'", ' ')
+ tokens = nltk.word_tokenize(translated, self.language)
+ result += [stem for stem in [self.stemmer.stem(token) for token in tokens
+ if token not in self.stemmer.stopwords]
+ if stem and (len(stem) > 1) and (stem not in self.stemmer.stopwords)]
+ return result
+
+
+@implementer(IPipelineElement)
+class NltkFullTextProcessor(object):
+ """NLTK based full text processor"""
+
+ def __init__(self, language='english'):
+ self.language = language
+
+ def process(self, lst):
+ result = []
+ for s in lst:
+ translated = translate_string(s, keep_chars="'-").replace("'", ' ')
+ result += [token for token in nltk.word_tokenize(translated, self.language)
+ if token and len(token) > 1]
+ return result
+
+ def processGlob(self, lst):
+ result = []
+ for s in lst:
+ translated = translate_string(s, keep_chars="'-*?").replace("'", ' ')
+ result += [token for token in nltk.word_tokenize(translated, self.language)
+ if token and len(token) > 1]
+ return result
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/site.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,109 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+__docformat__ = 'restructuredtext'
+
+
+# import standard library
+import logging
+logger = logging.getLogger('PyAMS (utils)')
+
+# import interfaces
+from hypatia.interfaces import ICatalog
+from persistent.interfaces import IPersistent
+from pyams_utils.interfaces.site import ISiteGenerations
+from zope.intid.interfaces import IIntIds
+from zope.lifecycleevent.interfaces import IObjectAddedEvent, IObjectRemovedEvent, IObjectModifiedEvent
+from zope.site.interfaces import INewLocalSite
+
+# import packages
+from hypatia.catalog import Catalog
+from pyams_utils.registry import utility_config, get_utilities_for, query_utility
+from pyams_utils.site import check_required_utilities
+from pyramid.events import subscriber
+from zope.interface.interface import InterfaceClass
+
+
+REQUIRED_UTILITIES = ((ICatalog, '', Catalog, 'Catalog'),)
+
+
+@subscriber(INewLocalSite)
+def handle_new_local_site(event):
+ """Create a new catalog when a site is created"""
+ site = event.manager.__parent__
+ check_required_utilities(site, REQUIRED_UTILITIES)
+
+
+@utility_config(name='PyAMS catalog', provides=ISiteGenerations)
+class CatalogGenerationsChecker(object):
+ """Catalog generations checker"""
+
+ generation = 1
+
+ def evolve(self, site, current=None):
+ """Check for required utilities"""
+ check_required_utilities(site, REQUIRED_UTILITIES)
+
+
+def check_required_indexes(site, indexes, catalog_name=''):
+ """Utility function to check for required catalog indexes
+
+ utilities argument is a tuple made of:
+ - the index name
+ - the index class
+ - index factory arguments
+ """
+ sm = site.getSiteManager()
+ catalog = sm.queryUtility(ICatalog, name=catalog_name)
+ if catalog is None:
+ logger.warning("No catalog found! Index check ignored...")
+ return
+ for name, klass, args in indexes:
+ index = catalog.get(name)
+ if index is None:
+ for key, value in args.copy().items():
+ if callable(value) and not isinstance(value, InterfaceClass):
+ args[key] = value()
+ index = klass(**args)
+ catalog[name] = index
+
+
+@subscriber(IObjectAddedEvent, context_selector=IPersistent)
+def handle_new_object(event):
+ """Index new persistent object"""
+ intids = query_utility(IIntIds)
+ if intids is not None:
+ object_id = intids.register(event.object)
+ for name, catalog in get_utilities_for(ICatalog):
+ catalog.index_doc(object_id, event.object)
+
+
+@subscriber(IObjectModifiedEvent, context_selector=IPersistent)
+def handle_modified_object(event):
+ """Update catalog for modified object"""
+ intids = query_utility(IIntIds)
+ if intids is not None:
+ object_id = intids.queryId(event.object)
+ if object_id is not None:
+ for name, catalog in get_utilities_for(ICatalog):
+ catalog.reindex_doc(object_id, event.object)
+
+
+@subscriber(IObjectRemovedEvent, context_selector=IPersistent)
+def handle_removed_object(event):
+ """Un-index removed object"""
+ intids = query_utility(IIntIds)
+ if intids is not None:
+ object_id = intids.queryId(event.object)
+ if object_id is not None:
+ for name, catalog in get_utilities_for(ICatalog):
+ catalog.unindex_doc(object_id)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/tests/__init__.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,1 @@
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/tests/test_utilsdocs.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+"""
+Generic Test case for pyams_catalog doctest
+"""
+__docformat__ = 'restructuredtext'
+
+import unittest
+import doctest
+import sys
+import os
+
+
+current_dir = os.path.dirname(__file__)
+
+def doc_suite(test_dir, setUp=None, tearDown=None, globs=None):
+ """Returns a test suite, based on doctests found in /doctest."""
+ suite = []
+ if globs is None:
+ globs = globals()
+
+ flags = (doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE |
+ doctest.REPORT_ONLY_FIRST_FAILURE)
+
+ package_dir = os.path.split(test_dir)[0]
+ if package_dir not in sys.path:
+ sys.path.append(package_dir)
+
+ doctest_dir = os.path.join(package_dir, 'doctests')
+
+ # filtering files on extension
+ docs = [os.path.join(doctest_dir, doc) for doc in
+ os.listdir(doctest_dir) if doc.endswith('.txt')]
+
+ for test in docs:
+ suite.append(doctest.DocFileSuite(test, optionflags=flags,
+ globs=globs, setUp=setUp,
+ tearDown=tearDown,
+ module_relative=False))
+
+ return unittest.TestSuite(suite)
+
+def test_suite():
+ """returns the test suite"""
+ return doc_suite(current_dir)
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='test_suite')
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_catalog/tests/test_utilsdocstrings.py Thu Mar 19 15:16:09 2015 +0100
@@ -0,0 +1,62 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+"""
+Generic Test case for pyams_catalog doc strings
+"""
+__docformat__ = 'restructuredtext'
+
+import unittest
+import doctest
+import sys
+import os
+
+
+current_dir = os.path.abspath(os.path.dirname(__file__))
+
+def doc_suite(test_dir, globs=None):
+ """Returns a test suite, based on doc tests strings found in /*.py"""
+ suite = []
+ if globs is None:
+ globs = globals()
+
+ flags = (doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE |
+ doctest.REPORT_ONLY_FIRST_FAILURE)
+
+ package_dir = os.path.split(test_dir)[0]
+ if package_dir not in sys.path:
+ sys.path.append(package_dir)
+
+ # filtering files on extension
+ docs = [doc for doc in
+ os.listdir(package_dir) if doc.endswith('.py')]
+ docs = [doc for doc in docs if not doc.startswith('__')]
+
+ for test in docs:
+ fd = open(os.path.join(package_dir, test))
+ content = fd.read()
+ fd.close()
+ if '>>> ' not in content:
+ continue
+ test = test.replace('.py', '')
+ location = 'pyams_catalog.%s' % test
+ suite.append(doctest.DocTestSuite(location, optionflags=flags,
+ globs=globs))
+
+ return unittest.TestSuite(suite)
+
+def test_suite():
+ """returns the test suite"""
+ return doc_suite(current_dir)
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='test_suite')