# HG changeset patch # User Thierry Florac # Date 1551446913 -3600 # Node ID 16da0050b814ee5255e61c37b2f712e227b4f7e2 # Parent 81086f654208ca2e1855dd67690f86dacfde400e Added views to compare index with database contents diff -r 81086f654208 -r 16da0050b814 src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.mo Binary file src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.mo has changed diff -r 81086f654208 -r 16da0050b814 src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.po --- a/src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.po Fri Mar 01 14:27:06 2019 +0100 +++ b/src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.po Fri Mar 01 14:28:33 2019 +0100 @@ -5,7 +5,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE 1.0\n" -"POT-Creation-Date: 2018-01-11 17:19+0100\n" +"POT-Creation-Date: 2019-03-01 13:28+0100\n" "PO-Revision-Date: 2016-04-21 18:26+0200\n" "Last-Translator: Thierry Florac \n" "Language-Team: French\n" @@ -16,30 +16,72 @@ "Generated-By: Lingua 3.10.dev0\n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" -#: src/pyams_content_es/zmi/__init__.py:51 +#: src/pyams_content_es/interfaces/__init__.py:45 +msgid "ZODB connection name" +msgstr "Connexion ZODB" + +#: src/pyams_content_es/interfaces/__init__.py:46 +msgid "Name of ZODB connection defining indexer connection" +msgstr "Nom de la ZODB définissant les paramètres de connexion" + +#: src/pyams_content_es/zmi/db.py:53 +msgid "Check index contents..." +msgstr "Vérifier le contenu de l'index" + +#: src/pyams_content_es/zmi/db.py:79 +#: src/pyams_content_es/zmi/templates/index-ok.pt:9 +msgid "Check index contents" +msgstr "Vérifier le contenu de l'index" + +#: src/pyams_content_es/zmi/db.py:63 src/pyams_content_es/zmi/__init__.py:82 +#: src/pyams_content_es/zmi/templates/index-ok.pt:19 +#: src/pyams_content_es/zmi/templates/index-updater.pt:31 +msgid "Close" +msgstr "Fermer" + +#: src/pyams_content_es/zmi/db.py:64 +#: src/pyams_content_es/zmi/templates/index-ok.pt:24 +msgid "Check index" +msgstr "Vérifier l'index" + +#: src/pyams_content_es/zmi/db.py:185 +msgid "Requested contents have been re-indexed!" +msgstr "Les contenus indiqués ont été ré-indexés !" + +#: src/pyams_content_es/zmi/db.py:130 +#, python-format +msgid "Loading index data ({})..." +msgstr "Chargement de l'index ({})..." + +#: src/pyams_content_es/zmi/db.py:139 +#, python-format +msgid "Loading database contents ({})..." +msgstr "Chargement des contenus ({})..." + +#: src/pyams_content_es/zmi/__init__.py:53 msgid "Update content indexer properties" msgstr "Propriétés de l'indexeur de contenus" -#: src/pyams_content_es/zmi/__init__.py:70 +#: src/pyams_content_es/zmi/__init__.py:72 msgid "Test process connection..." msgstr "Tester la connexion" -#: src/pyams_content_es/zmi/__init__.py:93 +#: src/pyams_content_es/zmi/__init__.py:97 msgid "Test content indexer process connection" msgstr "Test de la connexion au processus d'indexation" -#: src/pyams_content_es/zmi/__init__.py:80 -msgid "Close" -msgstr "Fermer" - -#: src/pyams_content_es/zmi/__init__.py:81 +#: src/pyams_content_es/zmi/__init__.py:83 msgid "Test connection" msgstr "Tester la connexion" -#: src/pyams_content_es/interfaces/__init__.py:42 -msgid "ZODB connection name" -msgstr "Connexion ZODB" +#: src/pyams_content_es/zmi/templates/index-ok.pt:11 +msgid "Index is up-to-date compared to database contents." +msgstr "L'index est à jour au regard du contenu de la base de données." -#: src/pyams_content_es/interfaces/__init__.py:43 -msgid "Name of ZODB connection defining indexer connection" -msgstr "Nom de la ZODB définissant les paramètres de connexion" +#: src/pyams_content_es/zmi/templates/index-updater.pt:9 +msgid "Update index contents" +msgstr "Mise à jour de l'index" + +#: src/pyams_content_es/zmi/templates/index-updater.pt:35 +msgid "Update index" +msgstr "Mettre l'index à jour" diff -r 81086f654208 -r 16da0050b814 src/pyams_content_es/locales/pyams_content_es.pot --- a/src/pyams_content_es/locales/pyams_content_es.pot Fri Mar 01 14:27:06 2019 +0100 +++ b/src/pyams_content_es/locales/pyams_content_es.pot Fri Mar 01 14:28:33 2019 +0100 @@ -1,12 +1,12 @@ # # SOME DESCRIPTIVE TITLE # This file is distributed under the same license as the PACKAGE package. -# FIRST AUTHOR , 2018. +# FIRST AUTHOR , 2019. #, fuzzy msgid "" msgstr "" "Project-Id-Version: PACKAGE 1.0\n" -"POT-Creation-Date: 2018-01-11 17:19+0100\n" +"POT-Creation-Date: 2019-03-01 13:28+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" @@ -16,30 +16,73 @@ "Content-Transfer-Encoding: 8bit\n" "Generated-By: Lingua 3.10.dev0\n" -#: ./src/pyams_content_es/zmi/__init__.py:51 +#: ./src/pyams_content_es/interfaces/__init__.py:45 +msgid "ZODB connection name" +msgstr "" + +#: ./src/pyams_content_es/interfaces/__init__.py:46 +msgid "Name of ZODB connection defining indexer connection" +msgstr "" + +#: ./src/pyams_content_es/zmi/db.py:53 +msgid "Check index contents..." +msgstr "" + +#: ./src/pyams_content_es/zmi/db.py:79 +#: ./src/pyams_content_es/zmi/templates/index-ok.pt:9 +msgid "Check index contents" +msgstr "" + +#: ./src/pyams_content_es/zmi/db.py:63 +#: ./src/pyams_content_es/zmi/__init__.py:82 +#: ./src/pyams_content_es/zmi/templates/index-ok.pt:19 +#: ./src/pyams_content_es/zmi/templates/index-updater.pt:31 +msgid "Close" +msgstr "" + +#: ./src/pyams_content_es/zmi/db.py:64 +#: ./src/pyams_content_es/zmi/templates/index-ok.pt:24 +msgid "Check index" +msgstr "" + +#: ./src/pyams_content_es/zmi/db.py:185 +msgid "Requested contents have been re-indexed!" +msgstr "" + +#: ./src/pyams_content_es/zmi/db.py:130 +#, python-format +msgid "Loading index data ({})..." +msgstr "" + +#: ./src/pyams_content_es/zmi/db.py:139 +#, python-format +msgid "Loading database contents ({})..." +msgstr "" + +#: ./src/pyams_content_es/zmi/__init__.py:53 msgid "Update content indexer properties" msgstr "" -#: ./src/pyams_content_es/zmi/__init__.py:70 +#: ./src/pyams_content_es/zmi/__init__.py:72 msgid "Test process connection..." msgstr "" -#: ./src/pyams_content_es/zmi/__init__.py:93 +#: ./src/pyams_content_es/zmi/__init__.py:97 msgid "Test content indexer process connection" msgstr "" -#: ./src/pyams_content_es/zmi/__init__.py:80 -msgid "Close" -msgstr "" - -#: ./src/pyams_content_es/zmi/__init__.py:81 +#: ./src/pyams_content_es/zmi/__init__.py:83 msgid "Test connection" msgstr "" -#: ./src/pyams_content_es/interfaces/__init__.py:42 -msgid "ZODB connection name" +#: ./src/pyams_content_es/zmi/templates/index-ok.pt:11 +msgid "Index is up-to-date compared to database contents." msgstr "" -#: ./src/pyams_content_es/interfaces/__init__.py:43 -msgid "Name of ZODB connection defining indexer connection" +#: ./src/pyams_content_es/zmi/templates/index-updater.pt:9 +msgid "Update index contents" msgstr "" + +#: ./src/pyams_content_es/zmi/templates/index-updater.pt:35 +msgid "Update index" +msgstr "" diff -r 81086f654208 -r 16da0050b814 src/pyams_content_es/zmi/__init__.py --- a/src/pyams_content_es/zmi/__init__.py Fri Mar 01 14:27:06 2019 +0100 +++ b/src/pyams_content_es/zmi/__init__.py Fri Mar 01 14:28:33 2019 +0100 @@ -15,28 +15,27 @@ # import standard library -# import interfaces -from pyams_content_es.interfaces import IContentIndexerUtility -from pyams_form.interfaces.form import IWidgetsSuffixViewletsManager -from pyams_skin.interfaces.viewlet import ITableItemColumnActionsMenu -from pyams_skin.layer import IPyAMSLayer -from pyams_utils.interfaces import VIEW_SYSTEM_PERMISSION, MANAGE_SYSTEM_PERMISSION - -# import packages -from pyams_form.form import AJAXEditForm, AJAXAddForm -from pyams_form.schema import CloseButton -from pyams_pagelet.pagelet import pagelet_config -from pyams_skin.viewlet.toolbar import ToolbarMenuItem -from pyams_template.template import template_config -from pyams_viewlet.viewlet import viewlet_config, Viewlet -from pyams_zmi.form import AdminDialogEditForm, AdminDialogAddForm -from pyams_zmi.layer import IAdminLayer -from pyams_zmi.zmi.control_panel import UtilitiesTable from pyramid.view import view_config -from z3c.form import field, button +from z3c.form import button, field from zope.interface import Interface from pyams_content_es import _ +# import interfaces +from pyams_content_es.interfaces import IContentIndexerUtility +# import packages +from pyams_form.form import AJAXAddForm, AJAXEditForm, ajax_config +from pyams_form.interfaces.form import IWidgetsSuffixViewletsManager +from pyams_form.schema import CloseButton +from pyams_pagelet.pagelet import pagelet_config +from pyams_skin.interfaces.viewlet import ITableItemColumnActionsMenu +from pyams_skin.layer import IPyAMSLayer +from pyams_skin.viewlet.toolbar import ToolbarMenuItem +from pyams_template.template import template_config +from pyams_utils.interfaces import MANAGE_SYSTEM_PERMISSION, VIEW_SYSTEM_PERMISSION +from pyams_viewlet.viewlet import Viewlet, viewlet_config +from pyams_zmi.form import AdminDialogAddForm, AdminDialogEditForm +from pyams_zmi.layer import IAdminLayer +from pyams_zmi.zmi.control_panel import UtilitiesTable @pagelet_config(name='properties.html', context=IContentIndexerUtility, layer=IPyAMSLayer, @@ -85,6 +84,8 @@ @pagelet_config(name='test-indexer-process.html', context=IContentIndexerUtility, layer=IPyAMSLayer, permission=MANAGE_SYSTEM_PERMISSION) +@ajax_config(name='text-indexer-process.json', context=IContentIndexerUtility, layer=IPyAMSLayer, + base=AJAXAddForm) class ContentIndexerProcessTestForm(AdminDialogAddForm): """Content indexer process test form""" @@ -98,7 +99,7 @@ prefix = 'test_form.' fields = field.Fields(Interface) buttons = button.Buttons(IContentIndexerProcessTestButtons) - ajax_handler = 'test-indexer-process.json' + edit_permission = MANAGE_SYSTEM_PERMISSION @property @@ -113,19 +114,6 @@ def createAndAdd(self, data): return self.context.test_process() - -@viewlet_config(name='test-indexer-process.suffix', layer=IAdminLayer, manager=IWidgetsSuffixViewletsManager, - view=ContentIndexerProcessTestForm, weight=50) -@template_config(template='templates/process-test.pt') -class ContentIndexerProcessTestSuffix(Viewlet): - """Content indexer process test form suffix""" - - -@view_config(name='test-indexer-process.json', context=IContentIndexerUtility, request_type=IPyAMSLayer, - permission=MANAGE_SYSTEM_PERMISSION, renderer='json', xhr=True) -class ContentIndexerProcessAJAXTestForm(AJAXAddForm, ContentIndexerProcessTestForm): - """Content indexer process test form, JSON renderer""" - def get_ajax_output(self, changes): status, message = changes if status == 200: @@ -140,3 +128,10 @@ 'content': {'html': message}, 'close_form': False } + + +@viewlet_config(name='test-indexer-process.suffix', layer=IAdminLayer, manager=IWidgetsSuffixViewletsManager, + view=ContentIndexerProcessTestForm, weight=50) +@template_config(template='templates/process-test.pt') +class ContentIndexerProcessTestSuffix(Viewlet): + """Content indexer process test form suffix""" diff -r 81086f654208 -r 16da0050b814 src/pyams_content_es/zmi/db.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/zmi/db.py Fri Mar 01 14:28:33 2019 +0100 @@ -0,0 +1,185 @@ +# +# Copyright (c) 2008-2019 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + +from itertools import tee + +from elasticsearch_dsl import Q, Search +from pyramid.renderers import render +from pyramid.view import view_config +from pyramid_es import get_client +from z3c.form import button, field +from zope.interface import Interface, alsoProvides, implementer +from zope.intid import IIntIds + +from pyams_content.shared.common import IWfSharedContent +from pyams_content_es import _ +from pyams_content_es.interfaces import IContentIndexerUtility +from pyams_form.form import AJAXAddForm, ajax_config +from pyams_form.schema import CloseButton +from pyams_pagelet.pagelet import pagelet_config +from pyams_skin.interfaces.viewlet import ITableItemColumnActionsMenu +from pyams_skin.layer import IPyAMSLayer +from pyams_skin.viewlet.toolbar import ToolbarMenuItem +from pyams_utils.container import find_objects_providing +from pyams_utils.interfaces import MANAGE_SYSTEM_PERMISSION +from pyams_utils.interfaces.data import IObjectData +from pyams_utils.progress import init_progress_status, set_progress_status +from pyams_utils.registry import get_utility, query_utility +from pyams_viewlet.viewlet import viewlet_config +from pyams_workflow.interfaces import IWorkflowState +from pyams_zmi.form import AdminDialogAddForm +from pyams_zmi.layer import IAdminLayer +from pyams_zmi.zmi.control_panel import UtilitiesTable + + +@viewlet_config(name='check-index.menu', context=IContentIndexerUtility, layer=IAdminLayer, + view=UtilitiesTable, manager=ITableItemColumnActionsMenu, permission=MANAGE_SYSTEM_PERMISSION, + weight=10) +class ContentIndexerDatabaseCheckerMenu(ToolbarMenuItem): + """Content indexer database checker menu""" + + label = _("Check index contents...") + label_css_class = 'fa fa-fw fa-check' + url = 'check-index.html' + modal_target = True + stop_propagation = True + + +class IContentIndexerDatabaseCheckerButtons(Interface): + """Content indexer database checker buttons""" + + close = CloseButton(name='close', title=_("Close")) + check = button.Button(name='check', title=_("Check index")) + + +@pagelet_config(name='check-index.html', context=IContentIndexerUtility, layer=IPyAMSLayer, + permission=MANAGE_SYSTEM_PERMISSION) +@ajax_config(name='check-index.json', context=IContentIndexerUtility, layer=IPyAMSLayer, + base=AJAXAddForm) +@implementer(IObjectData) +class ContentIndexerDatabaseCheckerForm(AdminDialogAddForm): + """Content indexer database checker form""" + + @property + def title(self): + return self.context.__name__ + + legend = _("Check index contents") + icon_css_class = 'fa fa-fw fa-check' + + prefix = 'checker_form.' + fields = field.Fields(Interface) + buttons = button.Buttons(IContentIndexerDatabaseCheckerButtons) + + edit_permission = MANAGE_SYSTEM_PERMISSION + + object_data = { + 'ams-form-hide-submit-footer': True + } + + @property + def form_target(self): + return '#{}'.format(self.id) + + def updateActions(self): + super(ContentIndexerDatabaseCheckerForm, self).updateActions() + if 'check' in self.actions: + action = self.actions['check'] + action.addClass('btn-primary') + action.object_data = { + 'ams-progress-handler': 'get-progress-status.json' + } + alsoProvides(action, IObjectData) + + def createAndAdd(self, data): + request = self.request + translate = request.localizer.translate + client = get_client(request) + progress_id = request.params.get('progress_id') + if progress_id: + init_progress_status(progress_id, request.principal.id, "Elasticsearch index check") + results = {} + search = Search(using=client.es, index=client.index) \ + .query(Q('exists', **{'field': 'content_type'})) \ + .source(['internal_id', 'workflow.status']) + count, first, page_size = -1, 0, 100 + while count != 0: + count = 0 + items = search[first:first+page_size] + for index_item in items: + try: + results[index_item.internal_id] = index_item.workflow.status + except AttributeError: + pass + count += 1 + first += count + if progress_id: + set_progress_status(progress_id, + message=translate(_("Loading index data ({})...")).format(first)) + intids = get_utility(IIntIds) + for index, content in enumerate(find_objects_providing(request.root, IWfSharedContent)): + internal_id = intids.register(content) + state = IWorkflowState(content, None) + if (state is not None) and (results.get(internal_id) != state.state): + yield internal_id, content + if progress_id and not (index % 100): + set_progress_status(progress_id, + message=translate(_("Loading database contents ({})...")).format(index)) + if progress_id: + set_progress_status(progress_id, 'finished') + + def get_ajax_output(self, changes): + check, items = tee(changes) + try: + next(check) + except StopIteration: + return { + 'status': 'success', + 'content': { + 'html': render('templates/index-ok.pt', {'view': self}, + request=self.request) + }, + 'close_form': False + } + else: + return { + 'status': 'info', + 'content': { + 'html': render('templates/index-updater.pt', {'view': self, 'items': items}, + request=self.request) + }, + 'close_form': False + } + + @staticmethod + def get_version(content): + return IWorkflowState(content).version_id + + +@view_config(name='update-index.json', context=IContentIndexerUtility, request_type=IPyAMSLayer, + permission=MANAGE_SYSTEM_PERMISSION, renderer='json', xhr=True) +def update_index(request): + """Update given missing documents into index""" + indexer = request.context + intids = query_utility(IIntIds) + for name, value in request.params.items(): + if name != 'internal_id': + continue + content = intids.queryObject(int(value)) + if content is not None: + indexer.index_document(content) + return { + 'status': 'success', + 'message': request.localizer.translate(_("Requested contents have been re-indexed!")) + } diff -r 81086f654208 -r 16da0050b814 src/pyams_content_es/zmi/templates/index-ok.pt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/zmi/templates/index-ok.pt Fri Mar 01 14:28:33 2019 +0100 @@ -0,0 +1,26 @@ +
+
+
+ + + Check index contents + +
+ Index is up-to-date compared to database contents. +
+
+
+
+
+ + +
+
diff -r 81086f654208 -r 16da0050b814 src/pyams_content_es/zmi/templates/index-updater.pt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content_es/zmi/templates/index-updater.pt Fri Mar 01 14:28:33 2019 +0100 @@ -0,0 +1,37 @@ +
+
+
+ + + Update index contents + +
+ + +
+
+
+
+ +