# HG changeset patch # User Thierry Florac # Date 1461255892 -7200 # Node ID 5af41c7a366f20e0d121576a6847d9c9458e14f9 First release diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/__init__.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,24 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +from pyramid.i18n import TranslationStringFactory +_ = TranslationStringFactory('pyams_content_es') + + +def includeme(config): + """Pyramid include""" + + from .include import include_package + include_package(config) diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/component/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/component/__init__.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,20 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces + +# import packages diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/component/extfile.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/component/extfile.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,47 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library +import base64 + +# import interfaces +from pyams_content.component.extfile.interfaces import IExtFileContainerTarget, IExtFileContainer +from pyams_content_es.interfaces import IDocumentIndexInfo + +# import packages +from pyams_utils.adapter import adapter_config + + +@adapter_config(name='extfile', context=IExtFileContainerTarget, provides=IDocumentIndexInfo) +def ExtFileContainerTargetIndexInfo(content): + """External files index info""" + result = [] + for extfile in IExtFileContainer(content).values(): + extfile_index = {'title': extfile.title, + 'description': extfile.description, + 'data': {}} + for lang, data in extfile.data.items(): + if data.content_type.startswith(b'image/') or \ + data.content_type.startswith(b'audio/') or \ + data.content_type.startswith(b'video/'): + continue + extfile_index['data'][lang] = { + '_content_type': data.content_type.decode(), + '_name': data.filename, + '_language': lang, + '_content': base64.encodebytes(data.data).decode() + } + result.append(extfile_index) + return {'extfile': result} diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/component/gallery.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/component/gallery.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,70 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content.component.gallery.interfaces import IGalleryContainerTarget, IGalleryContainer, IGallery, \ + IGalleryFileInfo +from pyams_content_es.interfaces import IDocumentIndexInfo + +# import packages +from pyams_utils.adapter import adapter_config + + +@adapter_config(context=IGallery, provides=IDocumentIndexInfo) +def GalleryIndexInfo(gallery): + """Gallery index info""" + info = {} + for lang, title in gallery.title.items(): + if title: + info.setdefault(lang, title) + for lang, description in gallery.description.items(): + if description: + new_info = '{old}\n{info}'.format(old=info.get(lang, ''), + info=description) + info[lang] = new_info + for image in gallery.values(): + image_info = IGalleryFileInfo(image, None) + if image_info is not None: + for lang, title in (image_info.title or {}).items(): + if title: + new_info = '{old}\n{info}'.format(old=info.get(lang, ''), + info=title) + info[lang] = new_info + for lang, description in (image_info.description or {}).items(): + if description: + new_info = '{old}\n{info}'.format(old=info.get(lang, ''), + info=description) + info[lang] = new_info + for lang, comments in (image_info.author_comments or {}).items(): + if comments: + new_info = '{old}\n{info}'.format(old=info.get(lang, ''), + info=comments) + info[lang] = new_info + return info + + +@adapter_config(name='gallery', context=IGalleryContainerTarget, provides=IDocumentIndexInfo) +def GalleryContainerTargetIndexInfo(content): + """Gallery container index info""" + body = {} + for gallery in IGalleryContainer(content).values(): + info = IDocumentIndexInfo(gallery, None) + if info is not None: + for lang, info_body in info.items(): + body[lang] = '{old}\n{body}'.format(old=body.get(lang, ''), + body=info_body) + return {'gallery': body} diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/component/paragraph.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/component/paragraph.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,68 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content.component.paragraph.interfaces import IParagraphContainer, IParagraphContainerTarget, IHTMLParagraph, \ + IIllustrationParagraph +from pyams_content_es.interfaces import IDocumentIndexInfo + +# import packages +from pyams_utils.adapter import adapter_config +from pyams_utils.html import html_to_text + + +@adapter_config(context=IHTMLParagraph, provides=IDocumentIndexInfo) +def HTMLParagraphIndexInfo(paragraph): + """HTML paragraph index info""" + info = {} + for lang, title in paragraph.title.items(): + if title: + info.setdefault(lang, title) + for lang, body in paragraph.body.items(): + if body: + new_body = '{old}\n{body}'.format(old=info.get(lang, ''), + body=html_to_text(body).replace('\r', '')) + info[lang] = new_body + return info + + +@adapter_config(context=IIllustrationParagraph, provides=IDocumentIndexInfo) +def IllustrationIndexInfo(paragraph): + """Illustration index info""" + info = {} + for lang, title in paragraph.title.items(): + if title: + info.setdefault(lang, title) + for lang, legend in paragraph.legend.items(): + if legend: + new_legend = '{old}\n{legend}'.format(old=info.get(lang, ''), + legend=legend) + info[lang] = new_legend + return info + + +@adapter_config(name='body', context=IParagraphContainerTarget, provides=IDocumentIndexInfo) +def ParagraphContainerTargetIndexInfo(content): + """Paragraph container index info""" + body = {} + for paragraph in IParagraphContainer(content).values(): + info = IDocumentIndexInfo(paragraph, None) + if info is not None: + for lang, info_body in info.items(): + body[lang] = '{old}\n{body}'.format(old=body.get(lang, ''), + body=info_body) + return {'body': body} diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/component/theme.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/component/theme.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,44 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +from pyams_utils.list import unique + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content.component.theme.interfaces import IThemesTarget, IThemesInfo +from pyams_content_es.interfaces import IDocumentIndexInfo + +# import packages +from pyams_utils.adapter import adapter_config + + +@adapter_config(name='themes', context=IThemesTarget, provides=IDocumentIndexInfo) +def ThemesTargetIndexInfo(content): + """Themes target index info""" + terms = [] + parents = [] + synonyms = [] + associations = [] + for term in IThemesInfo(content).themes or (): + terms.append(term.label) + if term.usage is not None: + terms.append(term.usage.label) + parents.extend([parent.label for parent in term.get_parents()]) + synonyms.extend([synonym.label for synonym in term.used_for]) + associations.extend([association.label for association in term.associations]) + return {'themes': {'terms': unique(terms), + 'parents': unique(parents), + 'synonyms': unique(synonyms), + 'associations': unique(associations)}} diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/component/workflow.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/component/workflow.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,32 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content_es.interfaces import IDocumentIndexInfo +from pyams_workflow.interfaces import IWorkflowState, IWorkflowPublicationSupport, IWorkflowInfo + +# import packages +from pyams_utils.adapter import adapter_config + + +@adapter_config(name='workflow', context=IWorkflowPublicationSupport, provides=IDocumentIndexInfo) +def WorkflowManagedContentIndexInfo(content): + """Workflow managed content index info""" + workflow_state = IWorkflowState(content) + return {'workflow': {'name': IWorkflowInfo(content).name, + 'status': workflow_state.state, + 'date': workflow_state.state_date}} diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/doctests/README.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/doctests/README.txt Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,3 @@ +======================== +pyams_content_es package +======================== diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/document.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/document.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,71 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content.shared.common.interfaces import IWfSharedContent +from pyams_content_es.interfaces import IDocumentIndexInfo, IDocumentIndexTarget +from pyams_sequence.interfaces import ISequentialIdInfo +from pyams_workflow.interfaces import IWorkflowState +from zope.intid.interfaces import IIntIds + +# import packages +from pyams_content.shared.common import WfSharedContent +from pyams_utils.adapter import adapter_config +from pyams_utils.registry import get_utility +from pyramid.threadlocal import get_current_registry +from pyramid_es.mixin import ElasticMixin as ElasticMixinBase, ESMapping, ESField +from zope.interface import classImplements + + +class ElasticMixin(ElasticMixinBase): + """ElasticSearch base mixin class""" + + @property + def id(self): + return '{oid}.{version}'.format(oid=ISequentialIdInfo(self).hex_oid, + version=IWorkflowState(self).version_id) + + @property + def internal_id(self): + intids = get_utility(IIntIds) + return intids.register(self) + + def elastic_mapping(self): + return IDocumentIndexInfo(self) + + def elastic_document(self): + document_info = super(ElasticMixin, self).elastic_document() + registry = get_current_registry() + for name, adapted in registry.getAdapters((self, ), IDocumentIndexInfo): + if not name: + continue + document_info.update(adapted) + return document_info + + +WfSharedContent.__bases__ += (ElasticMixin, ) +classImplements(WfSharedContent, IDocumentIndexTarget) + + +@adapter_config(context=IWfSharedContent, provides=IDocumentIndexInfo) +def WfSharedContentIndexInfo(content): + return ESMapping(analyzer='content', + properties=ESMapping(ESField('internal_id'), + ESField('title', boost=3.0), + ESField('short_name'), + ESField('description'), + ESField('keywords', boost=2.0))) diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/include.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/include.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,72 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library +import atexit +import logging +logger = logging.getLogger('PyAMS (content.es)') + +import sys + +# import interfaces +from pyams_content_es.interfaces import INDEXER_HANDLER_KEY +from pyramid.interfaces import IApplicationCreated + +# import packages +from pyams_content_es.process import ContentIndexerProcess, ContentIndexerMessageHandler +from pyams_zmq.process import process_exit_func +from pyramid.events import subscriber +from zope.component.globalregistry import getGlobalSiteManager + + +def include_package(config): + """Pyramid include""" + + # add translations + config.add_translation_dirs('pyams_content_es:locales') + + # load registry components + try: + import pyams_zmi + except ImportError: + config.scan(ignore='pyams_content_es.zmi') + else: + config.scan() + + +@subscriber(IApplicationCreated) +def handle_new_application(event): + """Start indexer process when application created""" + + # check for upgrade mode + if sys.argv[0].endswith('pyams_upgrade'): + return + + registry = getGlobalSiteManager() + settings = registry.settings + start_handler = settings.get(INDEXER_HANDLER_KEY, False) + if start_handler: + process = None + # create content indexer process + try: + process = ContentIndexerProcess(start_handler, ContentIndexerMessageHandler, registry) + logger.debug('Starting content indexer process {0!r}...'.format(process)) + process.start() + if process.is_alive(): + atexit.register(process_exit_func, process=process) + finally: + if process and not process.is_alive(): + process.terminate() + process.join() diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/index.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/index.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,66 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content_es.interfaces import IContentIndexerUtility, IDocumentIndexTarget +from transaction.interfaces import ITransactionManager +from zope.lifecycleevent.interfaces import IObjectAddedEvent, IObjectModifiedEvent, IObjectRemovedEvent + +# import packages +from pyams_utils.registry import query_utility +from pyramid.events import subscriber + + +# +# Documents events +# + +def index_document(status, document): + if not status: # aborted transaction + return + indexer = query_utility(IContentIndexerUtility) + if indexer is not None: + indexer.index_document(document) + + +def unindex_document(status, document): + if not status: # aborted transaction + return + indexer = query_utility(IContentIndexerUtility) + if indexer is not None: + indexer.unindex_document(document) + + +@subscriber(IObjectAddedEvent, context_selector=IDocumentIndexTarget) +def handle_added_document(event): + """Handle added document""" + document = event.object + ITransactionManager(document).get().addAfterCommitHook(index_document, kws={'document': document}) + + +@subscriber(IObjectModifiedEvent, context_selector=IDocumentIndexTarget) +def handle_modified_document(event): + """Handle modified document""" + document = event.object + ITransactionManager(document).get().addAfterCommitHook(index_document, kws={'document': document}) + + +@subscriber(IObjectRemovedEvent, context_selector=IDocumentIndexTarget) +def handle_removed_document(event): + """Handle removed document""" + document = event.object + ITransactionManager(document).get().addAfterCommitHook(unindex_document, kws={'document': document}) diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/interfaces/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/interfaces/__init__.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,66 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces + +# import packages +from zope.interface import Interface +from zope.schema import Choice + +from pyams_content_es import _ + + +# +# Indexer interfaces +# + +INDEXER_NAME = 'ElasticSearch content indexer' +INDEXER_HANDLER_KEY = 'pyams_content.es.tcp_handler' + + +# +# Utility interfaces +# + +class IContentIndexerUtility(Interface): + """Content indexer utility interface""" + + zeo_connection = Choice(title=_("ZEO connection name"), + description=_("Name of ZEO connection utility defining indexer connection"), + required=False, + vocabulary="PyAMS ZEO connections") + + def index_document(self, document): + """Index given document""" + + def unindex_document(self, document): + """Un-index given document""" + + def test_process(self): + """Send test request to indexer process""" + + +# +# Contents interfaces +# + +class IDocumentIndexInfo(Interface): + """Document index info""" + + +class IDocumentIndexTarget(Interface): + """Document index target marker interface""" diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/process.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/process.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,176 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + + +__docformat__ = 'restructuredtext' + + +# import standard library +import logging +logger = logging.getLogger('PyAMS (content.es)') + +from multiprocessing import Process +from pprint import pformat +from threading import Thread + +# import interfaces +from pyams_utils.interfaces import PYAMS_APPLICATION_SETTINGS_KEY, PYAMS_APPLICATION_DEFAULT_NAME +from transaction.interfaces import ITransactionManager +from zope.intid.interfaces import IIntIds + +# import packages +from pyams_utils.registry import set_local_registry, get_utility +from pyams_utils.zodb import ZEOConnection +from pyams_zmq.handler import ZMQMessageHandler +from pyams_zmq.process import ZMQProcess +from pyramid.threadlocal import manager as threadlocal_manager +from zope.component.globalregistry import getGlobalSiteManager + + +class BaseIndexerProcess(Process): + """Base indexer process""" + + def __init__(self, settings, group=None, target=None, name=None, *args, **kwargs): + Process.__init__(self, group, target, name, *args, **kwargs) + self.settings = settings + + def run(self): + logger.debug("Starting indexer thread...") + # Loading components registry + registry = getGlobalSiteManager() + threadlocal_manager.set({'request': None, 'registry': registry}) + logger.debug("Getting global registry: {0!r}".format(registry)) + # Get ES client + es_client = getattr(registry, 'pyramid_es_client', None) + if es_client is None: + logger.debug("Missing ElasticSearch client in registry!") + return + # Check settings + settings = self.settings + logger.debug("Checking index parameters: {0}".format(str(settings))) + zeo_settings = settings.get('zeo') + document_id = settings.get('document') + if not (zeo_settings and document_id): + logger.warning('Bad indexer request: {0}'.format(str(settings))) + return + # Open ZEO connection + manager = None + connection_info = ZEOConnection() + connection_info.update(zeo_settings) + logger.debug("Opening ZEO connection...") + storage, db = connection_info.get_connection(get_storage=True) + try: + connection = db.open() + root = connection.root() + logger.debug("Getting connection root {0!r}".format(root)) + application_name = registry.settings.get(PYAMS_APPLICATION_SETTINGS_KEY, PYAMS_APPLICATION_DEFAULT_NAME) + application = root.get(application_name) + logger.debug("Loading application {0!r} named {1}".format(application, application_name)) + if application is not None: + # set local registry + sm = application.getSiteManager() + set_local_registry(sm) + logger.debug("Setting local registry {0!r}".format(sm)) + # find document + intids = get_utility(IIntIds) + document = intids.queryObject(document_id) + if document is None: + logger.warning("Can't find requested document {0}!".format(document_id)) + return + # index document + logger.debug("Starting indexing for {0!r}".format(document)) + manager = ITransactionManager(document) + for attempt in manager.attempts(): + with attempt as t: + self.update_index(es_client, document) + if t.status == 'Committed': + break + finally: + if manager is not None: + manager.abort() + connection.close() + storage.close() + threadlocal_manager.pop() + + def update_index(self, client, document): + """Update index""" + raise NotImplementedError("Index threads must implement update_index method") + + +class IndexerProcess(BaseIndexerProcess): + """Content indexer process""" + + def update_index(self, client, document): + client.index_object(document) + + +class UnindexerProcess(BaseIndexerProcess): + """Content un-indexer process""" + + def update_index(self, client, document): + client.delete_object(document) + + +class IndexerThread(Thread): + """Content indexer thread""" + + def __init__(self, process): + Thread.__init__(self) + self.process = process + + def run(self): + self.process.start() + self.process.join() + + +class ContentIndexerHandler(object): + """Content indexer handler""" + + def index(self, settings): + IndexerThread(IndexerProcess(settings)).start() + return [200, 'Indexer process started'] + + def unindex(self, settings): + IndexerThread(UnindexerProcess(settings)).start() + return [200, 'Un-indexer process started'] + + def test(self, settings): + messages = ['Content indexer process ready to handle requests.', ''] + registry = getGlobalSiteManager() + es_client = getattr(registry, 'pyramid_es_client', None) + if es_client is None: + messages.append('WARNING: no ElasticSearch client defined!') + else: + messages.extend(['ElasticSearch client properties:', + '- servers: {0}'.format(es_client.es.transport.hosts), + '- index: {0}'.format(es_client.index), + '']) + ping = es_client.es.ping() + messages.append('Server ping: {0}'.format('OK' if ping else 'KO')) + if ping: + messages.extend(['', 'Server info:']) + messages.extend(pformat(es_client.es.info()).split('\n')) + return [200, '\n'.join(messages)] + + +class ContentIndexerMessageHandler(ZMQMessageHandler): + """Content indexer message handler""" + + handler = ContentIndexerHandler + + +class ContentIndexerProcess(ZMQProcess): + """Content indexer ZMQ process""" + + def __init__(self, zmq_address, handler, registry): + ZMQProcess.__init__(self, zmq_address, handler) + self.registry = registry diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/scripts/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/scripts/__init__.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,20 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces + +# import packages diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/scripts/index.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/scripts/index.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,44 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library +import argparse +import sys +import textwrap + +# import interfaces + +# import packages +from pyams_content_es.site import site_index +from pyramid.paster import bootstrap + + +def index_site(): + """Update all ElasticSearch indexes""" + usage = "usage: {0} config_uri".format(sys.argv[0]) + description = """Update all ElasticSearch indexes with all database contents.""" + + parser = argparse.ArgumentParser(usage=usage, + description=textwrap.dedent(description)) + parser.add_argument('config_uri', help='Name of configuration file') + args = parser.parse_args() + + config_uri = args.config_uri + env = bootstrap(config_uri) + settings, closer = env['registry'].settings, env['closer'] + try: + site_index(env['request']) + finally: + closer() diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/site.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/site.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,67 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library +import transaction + +# import interfaces +from pyams_content_es.interfaces import IContentIndexerUtility, INDEXER_NAME, IDocumentIndexTarget +from pyams_utils.interfaces.site import ISiteGenerations +from zope.site.interfaces import INewLocalSite + +# import packages +from pyams_content_es.utility import ContentIndexerUtility +from pyams_utils.container import find_objects_providing +from pyams_utils.registry import utility_config, set_local_registry, query_utility +from pyams_utils.site import check_required_utilities, site_factory +from pyramid.events import subscriber + + +REQUIRED_UTILITIES = ((IContentIndexerUtility, '', ContentIndexerUtility, INDEXER_NAME), ) + + +@subscriber(INewLocalSite) +def handle_new_local_site(event): + """Create a new indexer utility when a site is created""" + site = event.manager.__parent__ + check_required_utilities(site, REQUIRED_UTILITIES) + + +@utility_config(name='PyAMS content indexer', provides=ISiteGenerations) +class ContentIndexerGenerationsChecker(object): + """Content indexer utility generations checker""" + + generation = 1 + + def evolve(self, site, current=None): + """Check for required utilities""" + check_required_utilities(site, REQUIRED_UTILITIES) + + +def site_index(request): + """Index all site contents in ElasticSearch""" + application = site_factory(request) + if application is not None: + try: + set_local_registry(application.getSiteManager()) + indexer = query_utility(IContentIndexerUtility) + if indexer is not None: + for document in find_objects_providing(application, IDocumentIndexTarget): + print("Indexing: {0!r}".format(document)) + indexer.index_document(document) + finally: + set_local_registry(None) + transaction.commit() + return application diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/tests/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/tests/__init__.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,1 @@ + diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/tests/test_utilsdocs.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/tests/test_utilsdocs.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,59 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +""" +Generic Test case for pyams_content_es doctest +""" +__docformat__ = 'restructuredtext' + +import unittest +import doctest +import sys +import os + + +current_dir = os.path.dirname(__file__) + +def doc_suite(test_dir, setUp=None, tearDown=None, globs=None): + """Returns a test suite, based on doctests found in /doctest.""" + suite = [] + if globs is None: + globs = globals() + + flags = (doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | + doctest.REPORT_ONLY_FIRST_FAILURE) + + package_dir = os.path.split(test_dir)[0] + if package_dir not in sys.path: + sys.path.append(package_dir) + + doctest_dir = os.path.join(package_dir, 'doctests') + + # filtering files on extension + docs = [os.path.join(doctest_dir, doc) for doc in + os.listdir(doctest_dir) if doc.endswith('.txt')] + + for test in docs: + suite.append(doctest.DocFileSuite(test, optionflags=flags, + globs=globs, setUp=setUp, + tearDown=tearDown, + module_relative=False)) + + return unittest.TestSuite(suite) + +def test_suite(): + """returns the test suite""" + return doc_suite(current_dir) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') + diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/tests/test_utilsdocstrings.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/tests/test_utilsdocstrings.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,62 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +""" +Generic Test case for pyams_content_es doc strings +""" +__docformat__ = 'restructuredtext' + +import unittest +import doctest +import sys +import os + + +current_dir = os.path.abspath(os.path.dirname(__file__)) + +def doc_suite(test_dir, globs=None): + """Returns a test suite, based on doc tests strings found in /*.py""" + suite = [] + if globs is None: + globs = globals() + + flags = (doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | + doctest.REPORT_ONLY_FIRST_FAILURE) + + package_dir = os.path.split(test_dir)[0] + if package_dir not in sys.path: + sys.path.append(package_dir) + + # filtering files on extension + docs = [doc for doc in + os.listdir(package_dir) if doc.endswith('.py')] + docs = [doc for doc in docs if not doc.startswith('__')] + + for test in docs: + fd = open(os.path.join(package_dir, test)) + content = fd.read() + fd.close() + if '>>> ' not in content: + continue + test = test.replace('.py', '') + location = 'pyams_content_es.%s' % test + suite.append(doctest.DocTestSuite(location, optionflags=flags, + globs=globs)) + + return unittest.TestSuite(suite) + +def test_suite(): + """returns the test suite""" + return doc_suite(current_dir) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/utility.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/utility.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,81 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content_es.interfaces import IContentIndexerUtility, INDEXER_HANDLER_KEY +from pyams_utils.interfaces.zeo import IZEOConnection +from zope.intid.interfaces import IIntIds + +# import packages +from persistent import Persistent +from pyams_utils.registry import get_utility +from pyams_zmq.socket import zmq_socket, zmq_response +from pyramid.threadlocal import get_current_registry +from zope.container.contained import Contained +from zope.interface import implementer +from zope.schema.fieldproperty import FieldProperty + + +@implementer(IContentIndexerUtility) +class ContentIndexerUtility(Persistent, Contained): + """Content indexer utility""" + + zeo_connection = FieldProperty(IContentIndexerUtility['zeo_connection']) + + def _get_socket(self): + registry = get_current_registry() + handler = registry.settings.get(INDEXER_HANDLER_KEY, False) + if handler: + return zmq_socket(handler) + + def index_document(self, document): + """Send index request for given document""" + socket = self._get_socket() + if socket is None: + return [501, "No socket handler defined in configuration file"] + if not self.zeo_connection: + return [502, "Missing ZEO connection"] + zeo = get_utility(IZEOConnection, self.zeo_connection) + intids = get_utility(IIntIds) + settings = {'zeo': zeo.get_settings(), + 'document': intids.register(document)} + socket.send_json(['index', settings]) + return zmq_response(socket) + + def unindex_document(self, document): + """Send unindex request for given document""" + socket = self._get_socket() + if socket is None: + return [501, "No socket handler defined in configuration file"] + if not self.zeo_connection: + return [502, "Missing ZEO connection"] + zeo = get_utility(IZEOConnection, self.zeo_connection) + intids = get_utility(IIntIds) + settings = {'zeo': zeo.get_settings(), + 'document': intids.register(document)} + socket.send_json(['unindex', settings]) + return zmq_response(socket) + + def test_process(self): + """Send test request to indexer process""" + socket = self._get_socket() + if socket is None: + return [501, "No socket handler defined in configuration file"] + if not self.zeo_connection: + return [502, "Missing ZEO connection"] + socket.send_json(['test', {}]) + return zmq_response(socket) diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/zmi/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/zmi/__init__.py Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,136 @@ +# +# Copyright (c) 2008-2015 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + + +# import standard library + +# import interfaces +from pyams_content_es.interfaces import IContentIndexerUtility +from pyams_form.interfaces.form import IWidgetsSuffixViewletsManager +from pyams_skin.interfaces.viewlet import ITableItemColumnActionsMenu +from pyams_skin.layer import IPyAMSLayer +from pyams_utils.interfaces import VIEW_SYSTEM_PERMISSION, MANAGE_SYSTEM_PERMISSION + +# import packages +from pyams_form.form import AJAXEditForm, AJAXAddForm +from pyams_form.schema import CloseButton +from pyams_pagelet.pagelet import pagelet_config +from pyams_skin.viewlet.toolbar import ToolbarMenuItem +from pyams_template.template import template_config +from pyams_viewlet.viewlet import viewlet_config, Viewlet +from pyams_zmi.control_panel import UtilitiesTable +from pyams_zmi.form import AdminDialogEditForm, AdminDialogAddForm +from pyams_zmi.layer import IAdminLayer +from pyramid.view import view_config +from z3c.form import field, button +from zope.interface import Interface + +from pyams_content_es import _ + + +@pagelet_config(name='properties.html', context=IContentIndexerUtility, layer=IPyAMSLayer, + permission=VIEW_SYSTEM_PERMISSION) +class ContentIndexerUtilityPropertiesEditForm(AdminDialogEditForm): + """Content indexer utility properties edit form""" + + @property + def title(self): + return self.context.__name__ + + legend = _("Update content indexer properties") + + fields = field.Fields(IContentIndexerUtility).select('zeo_connection') + + ajax_handler = 'properties.json' + edit_permission = MANAGE_SYSTEM_PERMISSION + + +@view_config(name='properties.json', context=IContentIndexerUtility, request_type=IPyAMSLayer, + permission=MANAGE_SYSTEM_PERMISSION, renderer='json', xhr=True) +class ContentIndexerUtilityPropertiesAJAXEditForm(AJAXEditForm, ContentIndexerUtilityPropertiesEditForm): + """Content index utility properties edit form, JSON renderer""" + + +@viewlet_config(name='test-conversion-process.menu', context=IContentIndexerUtility, layer=IAdminLayer, + view=UtilitiesTable, manager=ITableItemColumnActionsMenu, permission=MANAGE_SYSTEM_PERMISSION) +class ContentIndexerProcessTestMenu(ToolbarMenuItem): + """Content indexer process test menu""" + + label = _("Test process connection...") + label_css_class = 'fa fa-fw fa-server' + url = 'test-indexer-process.html' + modal_target = True + stop_propagation = True + + +class IContentIndexerProcessTestButtons(Interface): + """Content indexer process test buttons""" + + close = CloseButton(name='close', title=_("Close")) + test = button.Button(name='test', title=_("Test connection")) + + +@pagelet_config(name='test-indexer-process.html', context=IContentIndexerUtility, layer=IPyAMSLayer, + permission=MANAGE_SYSTEM_PERMISSION) +class ContentIndexerProcessTestForm(AdminDialogAddForm): + """Content indexer process test form""" + + @property + def title(self): + return self.context.__name__ + + legend = _("Test content indexer process connection") + icon_css_class = 'fa fa-fw fa-server' + + prefix = 'test_form.' + fields = field.Fields(Interface) + buttons = button.Buttons(IContentIndexerProcessTestButtons) + ajax_handler = 'test-indexer-process.json' + edit_permission = MANAGE_SYSTEM_PERMISSION + + @property + def form_target(self): + return '#{0}_test_result'.format(self.id) + + def updateActions(self): + super(ContentIndexerProcessTestForm, self).updateActions() + if 'test' in self.actions: + self.actions['test'].addClass('btn-primary') + + def createAndAdd(self, data): + return self.context.test_process() + + +@viewlet_config(name='test-indexer-process.suffix', layer=IAdminLayer, manager=IWidgetsSuffixViewletsManager, + view=ContentIndexerProcessTestForm, weight=50) +@template_config(template='templates/process-test.pt') +class ContentIndexerProcessTestSuffix(Viewlet): + """Content indexer process test form suffix""" + + +@view_config(name='test-indexer-process.json', context=IContentIndexerUtility, request_type=IPyAMSLayer, + permission=MANAGE_SYSTEM_PERMISSION, renderer='json', xhr=True) +class ContentIndexerProcessAJAXTestForm(AJAXAddForm, ContentIndexerProcessTestForm): + """Content indexer process test form, JSON renderer""" + + def get_ajax_output(self, changes): + status, message = changes + if status == 200: + return {'status': 'success', + 'content': {'html': message}, + 'close_form': False} + else: + return {'status': 'info', + 'content': {'html': message}, + 'close_form': False} diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/zmi/templates/process-test.pt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/zmi/templates/process-test.pt Thu Apr 21 18:24:52 2016 +0200 @@ -0,0 +1,3 @@ +
+ 

+

diff -r 000000000000 -r 5af41c7a366f src/pyams_content_es/zmi/test.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_content_es/zmi/test.py	Thu Apr 21 18:24:52 2016 +0200
@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2008-2015 Thierry Florac 
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+
+__docformat__ = 'restructuredtext'
+
+
+# import standard library
+
+# import interfaces
+from pyams_i18n.interfaces import II18n
+from zope.intid.interfaces import IIntIds
+
+# import packages
+from pyams_utils.registry import get_utility
+from pyramid.response import Response
+from pyramid.view import view_config
+from pyramid_es import get_client
+
+
+@view_config(name='test-es.json', renderer='string')
+def es_test_view(context, request):
+
+    def get_response():
+        client = get_client(request)
+        # query = client.query() \
+        #               .filter_term('title.fr', 'breve') \
+        #               .add_term_aggregate('status', 'workflow.status') \
+        #               .size(100)
+        # return query.execute(fields=['title.fr']).raw
+
+        query = client.query() \
+                      .filter_term('_type', 'WfNewsEvent') \
+                      .filter_terms('workflow.status', ['published', 'retiring']) \
+                      .add_term_aggregate('status', 'workflow.status') \
+                      .add_date_aggregate('wf_date', 'workflow.date') \
+                      .size(100)
+        intids = get_utility(IIntIds)
+        for result in query.execute(fields=['internal_id']):
+            if not result.internal_id:
+                continue
+            target = intids.queryObject(result.internal_id[0])
+            yield II18n(target).query_attribute('title').encode() + b'\n'
+        # return pformat(query.execute(fields=['title.fr', 'workflow.status', 'internal_id']).raw)
+
+    return Response(app_iter=get_response())