Added views to compare index with database contents
authorThierry Florac <tflorac@ulthar.net>
Fri, 01 Mar 2019 14:28:33 +0100
changeset 150 16da0050b814
parent 149 81086f654208
child 151 9863acd4f318
Added views to compare index with database contents
src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.mo
src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.po
src/pyams_content_es/locales/pyams_content_es.pot
src/pyams_content_es/zmi/__init__.py
src/pyams_content_es/zmi/db.py
src/pyams_content_es/zmi/templates/index-ok.pt
src/pyams_content_es/zmi/templates/index-updater.pt
Binary file src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.mo has changed
--- a/src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.po	Fri Mar 01 14:27:06 2019 +0100
+++ b/src/pyams_content_es/locales/fr/LC_MESSAGES/pyams_content_es.po	Fri Mar 01 14:28:33 2019 +0100
@@ -5,7 +5,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE 1.0\n"
-"POT-Creation-Date: 2018-01-11 17:19+0100\n"
+"POT-Creation-Date: 2019-03-01 13:28+0100\n"
 "PO-Revision-Date: 2016-04-21 18:26+0200\n"
 "Last-Translator: Thierry Florac <tflorac@ulthar.net>\n"
 "Language-Team: French\n"
@@ -16,30 +16,72 @@
 "Generated-By: Lingua 3.10.dev0\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"
 
-#: src/pyams_content_es/zmi/__init__.py:51
+#: src/pyams_content_es/interfaces/__init__.py:45
+msgid "ZODB connection name"
+msgstr "Connexion ZODB"
+
+#: src/pyams_content_es/interfaces/__init__.py:46
+msgid "Name of ZODB connection defining indexer connection"
+msgstr "Nom de la ZODB définissant les paramètres de connexion"
+
+#: src/pyams_content_es/zmi/db.py:53
+msgid "Check index contents..."
+msgstr "Vérifier le contenu de l'index"
+
+#: src/pyams_content_es/zmi/db.py:79
+#: src/pyams_content_es/zmi/templates/index-ok.pt:9
+msgid "Check index contents"
+msgstr "Vérifier le contenu de l'index"
+
+#: src/pyams_content_es/zmi/db.py:63 src/pyams_content_es/zmi/__init__.py:82
+#: src/pyams_content_es/zmi/templates/index-ok.pt:19
+#: src/pyams_content_es/zmi/templates/index-updater.pt:31
+msgid "Close"
+msgstr "Fermer"
+
+#: src/pyams_content_es/zmi/db.py:64
+#: src/pyams_content_es/zmi/templates/index-ok.pt:24
+msgid "Check index"
+msgstr "Vérifier l'index"
+
+#: src/pyams_content_es/zmi/db.py:185
+msgid "Requested contents have been re-indexed!"
+msgstr "Les contenus indiqués ont été ré-indexés !"
+
+#: src/pyams_content_es/zmi/db.py:130
+#, python-format
+msgid "Loading index data ({})..."
+msgstr "Chargement de l'index ({})..."
+
+#: src/pyams_content_es/zmi/db.py:139
+#, python-format
+msgid "Loading database contents ({})..."
+msgstr "Chargement des contenus ({})..."
+
+#: src/pyams_content_es/zmi/__init__.py:53
 msgid "Update content indexer properties"
 msgstr "Propriétés de l'indexeur de contenus"
 
-#: src/pyams_content_es/zmi/__init__.py:70
+#: src/pyams_content_es/zmi/__init__.py:72
 msgid "Test process connection..."
 msgstr "Tester la connexion"
 
-#: src/pyams_content_es/zmi/__init__.py:93
+#: src/pyams_content_es/zmi/__init__.py:97
 msgid "Test content indexer process connection"
 msgstr "Test de la connexion au processus d'indexation"
 
-#: src/pyams_content_es/zmi/__init__.py:80
-msgid "Close"
-msgstr "Fermer"
-
-#: src/pyams_content_es/zmi/__init__.py:81
+#: src/pyams_content_es/zmi/__init__.py:83
 msgid "Test connection"
 msgstr "Tester la connexion"
 
-#: src/pyams_content_es/interfaces/__init__.py:42
-msgid "ZODB connection name"
-msgstr "Connexion ZODB"
+#: src/pyams_content_es/zmi/templates/index-ok.pt:11
+msgid "Index is up-to-date compared to database contents."
+msgstr "L'index est à jour au regard du contenu de la base de données."
 
-#: src/pyams_content_es/interfaces/__init__.py:43
-msgid "Name of ZODB connection defining indexer connection"
-msgstr "Nom de la ZODB définissant les paramètres de connexion"
+#: src/pyams_content_es/zmi/templates/index-updater.pt:9
+msgid "Update index contents"
+msgstr "Mise à jour de l'index"
+
+#: src/pyams_content_es/zmi/templates/index-updater.pt:35
+msgid "Update index"
+msgstr "Mettre l'index à jour"
--- a/src/pyams_content_es/locales/pyams_content_es.pot	Fri Mar 01 14:27:06 2019 +0100
+++ b/src/pyams_content_es/locales/pyams_content_es.pot	Fri Mar 01 14:28:33 2019 +0100
@@ -1,12 +1,12 @@
 #
 # SOME DESCRIPTIVE TITLE
 # This file is distributed under the same license as the PACKAGE package.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2018.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
 #, fuzzy
 msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE 1.0\n"
-"POT-Creation-Date: 2018-01-11 17:19+0100\n"
+"POT-Creation-Date: 2019-03-01 13:28+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -16,30 +16,73 @@
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Lingua 3.10.dev0\n"
 
-#: ./src/pyams_content_es/zmi/__init__.py:51
+#: ./src/pyams_content_es/interfaces/__init__.py:45
+msgid "ZODB connection name"
+msgstr ""
+
+#: ./src/pyams_content_es/interfaces/__init__.py:46
+msgid "Name of ZODB connection defining indexer connection"
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/db.py:53
+msgid "Check index contents..."
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/db.py:79
+#: ./src/pyams_content_es/zmi/templates/index-ok.pt:9
+msgid "Check index contents"
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/db.py:63
+#: ./src/pyams_content_es/zmi/__init__.py:82
+#: ./src/pyams_content_es/zmi/templates/index-ok.pt:19
+#: ./src/pyams_content_es/zmi/templates/index-updater.pt:31
+msgid "Close"
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/db.py:64
+#: ./src/pyams_content_es/zmi/templates/index-ok.pt:24
+msgid "Check index"
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/db.py:185
+msgid "Requested contents have been re-indexed!"
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/db.py:130
+#, python-format
+msgid "Loading index data ({})..."
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/db.py:139
+#, python-format
+msgid "Loading database contents ({})..."
+msgstr ""
+
+#: ./src/pyams_content_es/zmi/__init__.py:53
 msgid "Update content indexer properties"
 msgstr ""
 
-#: ./src/pyams_content_es/zmi/__init__.py:70
+#: ./src/pyams_content_es/zmi/__init__.py:72
 msgid "Test process connection..."
 msgstr ""
 
-#: ./src/pyams_content_es/zmi/__init__.py:93
+#: ./src/pyams_content_es/zmi/__init__.py:97
 msgid "Test content indexer process connection"
 msgstr ""
 
-#: ./src/pyams_content_es/zmi/__init__.py:80
-msgid "Close"
-msgstr ""
-
-#: ./src/pyams_content_es/zmi/__init__.py:81
+#: ./src/pyams_content_es/zmi/__init__.py:83
 msgid "Test connection"
 msgstr ""
 
-#: ./src/pyams_content_es/interfaces/__init__.py:42
-msgid "ZODB connection name"
+#: ./src/pyams_content_es/zmi/templates/index-ok.pt:11
+msgid "Index is up-to-date compared to database contents."
 msgstr ""
 
-#: ./src/pyams_content_es/interfaces/__init__.py:43
-msgid "Name of ZODB connection defining indexer connection"
+#: ./src/pyams_content_es/zmi/templates/index-updater.pt:9
+msgid "Update index contents"
 msgstr ""
+
+#: ./src/pyams_content_es/zmi/templates/index-updater.pt:35
+msgid "Update index"
+msgstr ""
--- a/src/pyams_content_es/zmi/__init__.py	Fri Mar 01 14:27:06 2019 +0100
+++ b/src/pyams_content_es/zmi/__init__.py	Fri Mar 01 14:28:33 2019 +0100
@@ -15,28 +15,27 @@
 
 # import standard library
 
-# import interfaces
-from pyams_content_es.interfaces import IContentIndexerUtility
-from pyams_form.interfaces.form import IWidgetsSuffixViewletsManager
-from pyams_skin.interfaces.viewlet import ITableItemColumnActionsMenu
-from pyams_skin.layer import IPyAMSLayer
-from pyams_utils.interfaces import VIEW_SYSTEM_PERMISSION, MANAGE_SYSTEM_PERMISSION
-
-# import packages
-from pyams_form.form import AJAXEditForm, AJAXAddForm
-from pyams_form.schema import CloseButton
-from pyams_pagelet.pagelet import pagelet_config
-from pyams_skin.viewlet.toolbar import ToolbarMenuItem
-from pyams_template.template import template_config
-from pyams_viewlet.viewlet import viewlet_config, Viewlet
-from pyams_zmi.form import AdminDialogEditForm, AdminDialogAddForm
-from pyams_zmi.layer import IAdminLayer
-from pyams_zmi.zmi.control_panel import UtilitiesTable
 from pyramid.view import view_config
-from z3c.form import field, button
+from z3c.form import button, field
 from zope.interface import Interface
 
 from pyams_content_es import _
+# import interfaces
+from pyams_content_es.interfaces import IContentIndexerUtility
+# import packages
+from pyams_form.form import AJAXAddForm, AJAXEditForm, ajax_config
+from pyams_form.interfaces.form import IWidgetsSuffixViewletsManager
+from pyams_form.schema import CloseButton
+from pyams_pagelet.pagelet import pagelet_config
+from pyams_skin.interfaces.viewlet import ITableItemColumnActionsMenu
+from pyams_skin.layer import IPyAMSLayer
+from pyams_skin.viewlet.toolbar import ToolbarMenuItem
+from pyams_template.template import template_config
+from pyams_utils.interfaces import MANAGE_SYSTEM_PERMISSION, VIEW_SYSTEM_PERMISSION
+from pyams_viewlet.viewlet import Viewlet, viewlet_config
+from pyams_zmi.form import AdminDialogAddForm, AdminDialogEditForm
+from pyams_zmi.layer import IAdminLayer
+from pyams_zmi.zmi.control_panel import UtilitiesTable
 
 
 @pagelet_config(name='properties.html', context=IContentIndexerUtility, layer=IPyAMSLayer,
@@ -85,6 +84,8 @@
 
 @pagelet_config(name='test-indexer-process.html', context=IContentIndexerUtility, layer=IPyAMSLayer,
                 permission=MANAGE_SYSTEM_PERMISSION)
+@ajax_config(name='text-indexer-process.json', context=IContentIndexerUtility, layer=IPyAMSLayer,
+             base=AJAXAddForm)
 class ContentIndexerProcessTestForm(AdminDialogAddForm):
     """Content indexer process test form"""
 
@@ -98,7 +99,7 @@
     prefix = 'test_form.'
     fields = field.Fields(Interface)
     buttons = button.Buttons(IContentIndexerProcessTestButtons)
-    ajax_handler = 'test-indexer-process.json'
+
     edit_permission = MANAGE_SYSTEM_PERMISSION
 
     @property
@@ -113,19 +114,6 @@
     def createAndAdd(self, data):
         return self.context.test_process()
 
-
-@viewlet_config(name='test-indexer-process.suffix', layer=IAdminLayer, manager=IWidgetsSuffixViewletsManager,
-                view=ContentIndexerProcessTestForm, weight=50)
-@template_config(template='templates/process-test.pt')
-class ContentIndexerProcessTestSuffix(Viewlet):
-    """Content indexer process test form suffix"""
-
-
-@view_config(name='test-indexer-process.json', context=IContentIndexerUtility, request_type=IPyAMSLayer,
-             permission=MANAGE_SYSTEM_PERMISSION, renderer='json', xhr=True)
-class ContentIndexerProcessAJAXTestForm(AJAXAddForm, ContentIndexerProcessTestForm):
-    """Content indexer process test form, JSON renderer"""
-
     def get_ajax_output(self, changes):
         status, message = changes
         if status == 200:
@@ -140,3 +128,10 @@
                 'content': {'html': message},
                 'close_form': False
             }
+
+
+@viewlet_config(name='test-indexer-process.suffix', layer=IAdminLayer, manager=IWidgetsSuffixViewletsManager,
+                view=ContentIndexerProcessTestForm, weight=50)
+@template_config(template='templates/process-test.pt')
+class ContentIndexerProcessTestSuffix(Viewlet):
+    """Content indexer process test form suffix"""
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_content_es/zmi/db.py	Fri Mar 01 14:28:33 2019 +0100
@@ -0,0 +1,185 @@
+#
+# Copyright (c) 2008-2019 Thierry Florac <tflorac AT ulthar.net>
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+
+__docformat__ = 'restructuredtext'
+
+from itertools import tee
+
+from elasticsearch_dsl import Q, Search
+from pyramid.renderers import render
+from pyramid.view import view_config
+from pyramid_es import get_client
+from z3c.form import button, field
+from zope.interface import Interface, alsoProvides, implementer
+from zope.intid import IIntIds
+
+from pyams_content.shared.common import IWfSharedContent
+from pyams_content_es import _
+from pyams_content_es.interfaces import IContentIndexerUtility
+from pyams_form.form import AJAXAddForm, ajax_config
+from pyams_form.schema import CloseButton
+from pyams_pagelet.pagelet import pagelet_config
+from pyams_skin.interfaces.viewlet import ITableItemColumnActionsMenu
+from pyams_skin.layer import IPyAMSLayer
+from pyams_skin.viewlet.toolbar import ToolbarMenuItem
+from pyams_utils.container import find_objects_providing
+from pyams_utils.interfaces import MANAGE_SYSTEM_PERMISSION
+from pyams_utils.interfaces.data import IObjectData
+from pyams_utils.progress import init_progress_status, set_progress_status
+from pyams_utils.registry import get_utility, query_utility
+from pyams_viewlet.viewlet import viewlet_config
+from pyams_workflow.interfaces import IWorkflowState
+from pyams_zmi.form import AdminDialogAddForm
+from pyams_zmi.layer import IAdminLayer
+from pyams_zmi.zmi.control_panel import UtilitiesTable
+
+
+@viewlet_config(name='check-index.menu', context=IContentIndexerUtility, layer=IAdminLayer,
+                view=UtilitiesTable, manager=ITableItemColumnActionsMenu, permission=MANAGE_SYSTEM_PERMISSION,
+                weight=10)
+class ContentIndexerDatabaseCheckerMenu(ToolbarMenuItem):
+    """Content indexer database checker menu"""
+
+    label = _("Check index contents...")
+    label_css_class = 'fa fa-fw fa-check'
+    url = 'check-index.html'
+    modal_target = True
+    stop_propagation = True
+
+
+class IContentIndexerDatabaseCheckerButtons(Interface):
+    """Content indexer database checker buttons"""
+
+    close = CloseButton(name='close', title=_("Close"))
+    check = button.Button(name='check', title=_("Check index"))
+
+
+@pagelet_config(name='check-index.html', context=IContentIndexerUtility, layer=IPyAMSLayer,
+                permission=MANAGE_SYSTEM_PERMISSION)
+@ajax_config(name='check-index.json', context=IContentIndexerUtility, layer=IPyAMSLayer,
+             base=AJAXAddForm)
+@implementer(IObjectData)
+class ContentIndexerDatabaseCheckerForm(AdminDialogAddForm):
+    """Content indexer database checker form"""
+
+    @property
+    def title(self):
+        return self.context.__name__
+
+    legend = _("Check index contents")
+    icon_css_class = 'fa fa-fw fa-check'
+
+    prefix = 'checker_form.'
+    fields = field.Fields(Interface)
+    buttons = button.Buttons(IContentIndexerDatabaseCheckerButtons)
+
+    edit_permission = MANAGE_SYSTEM_PERMISSION
+
+    object_data = {
+        'ams-form-hide-submit-footer': True
+    }
+
+    @property
+    def form_target(self):
+        return '#{}'.format(self.id)
+
+    def updateActions(self):
+        super(ContentIndexerDatabaseCheckerForm, self).updateActions()
+        if 'check' in self.actions:
+            action = self.actions['check']
+            action.addClass('btn-primary')
+            action.object_data = {
+                'ams-progress-handler': 'get-progress-status.json'
+            }
+            alsoProvides(action, IObjectData)
+
+    def createAndAdd(self, data):
+        request = self.request
+        translate = request.localizer.translate
+        client = get_client(request)
+        progress_id = request.params.get('progress_id')
+        if progress_id:
+            init_progress_status(progress_id, request.principal.id, "Elasticsearch index check")
+        results = {}
+        search = Search(using=client.es, index=client.index) \
+            .query(Q('exists', **{'field': 'content_type'})) \
+            .source(['internal_id', 'workflow.status'])
+        count, first, page_size = -1, 0, 100
+        while count != 0:
+            count = 0
+            items = search[first:first+page_size]
+            for index_item in items:
+                try:
+                    results[index_item.internal_id] = index_item.workflow.status
+                except AttributeError:
+                    pass
+                count += 1
+            first += count
+            if progress_id:
+                set_progress_status(progress_id,
+                                    message=translate(_("Loading index data ({})...")).format(first))
+        intids = get_utility(IIntIds)
+        for index, content in enumerate(find_objects_providing(request.root, IWfSharedContent)):
+            internal_id = intids.register(content)
+            state = IWorkflowState(content, None)
+            if (state is not None) and (results.get(internal_id) != state.state):
+                yield internal_id, content
+            if progress_id and not (index % 100):
+                set_progress_status(progress_id,
+                                    message=translate(_("Loading database contents ({})...")).format(index))
+        if progress_id:
+            set_progress_status(progress_id, 'finished')
+
+    def get_ajax_output(self, changes):
+        check, items = tee(changes)
+        try:
+            next(check)
+        except StopIteration:
+            return {
+                'status': 'success',
+                'content': {
+                    'html': render('templates/index-ok.pt', {'view': self},
+                                   request=self.request)
+                },
+                'close_form': False
+            }
+        else:
+            return {
+                'status': 'info',
+                'content': {
+                    'html': render('templates/index-updater.pt', {'view': self, 'items': items},
+                                   request=self.request)
+                },
+                'close_form': False
+            }
+
+    @staticmethod
+    def get_version(content):
+        return IWorkflowState(content).version_id
+
+
+@view_config(name='update-index.json', context=IContentIndexerUtility, request_type=IPyAMSLayer,
+             permission=MANAGE_SYSTEM_PERMISSION, renderer='json', xhr=True)
+def update_index(request):
+    """Update given missing documents into index"""
+    indexer = request.context
+    intids = query_utility(IIntIds)
+    for name, value in request.params.items():
+        if name != 'internal_id':
+            continue
+        content = intids.queryObject(int(value))
+        if content is not None:
+            indexer.index_document(content)
+    return {
+        'status': 'success',
+        'message': request.localizer.translate(_("Requested contents have been re-indexed!"))
+    }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_content_es/zmi/templates/index-ok.pt	Fri Mar 01 14:28:33 2019 +0100
@@ -0,0 +1,26 @@
+<form method="post" data-async
+	  class="ams-form form-horizontal"
+	  data-ams-form-hide-submit-footer="true"
+	  i18n:domain="pyams_content_es">
+	<div class="modal-viewport">
+		<fieldset>
+			<legend>
+				<i class="fa fa-fw fa-check"></i>
+				<i18n:var translate="">Check index contents</i18n:var>
+			</legend>
+			<div class="alert alert-success padding-10" i18n:translate="">
+				Index is up-to-date compared to database contents.
+			</div>
+			<div class="clearfix"></div>
+		</fieldset>
+	</div>
+	<footer>
+		<button class="btn" data-dismiss="modal"
+				i18n:translate="">Close</button>
+		<button class="submit btn btn-primary"
+				data-ams-form-action="${tales:absolute_url(view.context)}/"
+				data-ams-form-handler="check-index.json"
+				data-ams-progress-handler="get-progress-status.json"
+				i18n:translate="">Check index</button>
+	</footer>
+</form>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/pyams_content_es/zmi/templates/index-updater.pt	Fri Mar 01 14:28:33 2019 +0100
@@ -0,0 +1,37 @@
+<form method="post" data-async
+	  class="ams-form form-horizontal"
+	  tal:define="global nb_items 0"
+	  i18n:domain="pyams_content_es">
+	<div class="modal-viewport">
+		<fieldset>
+			<legend>
+				<i class="fa fa-fw fa-check"></i>
+				<i18n:var translate="">Update index contents</i18n:var>
+			</legend>
+			<div tal:repeat="(internal_id, item) items">
+				<label class="checkbox">
+					<input type="checkbox"
+						   checked="checked"
+						   name="internal_id"
+						   value="${internal_id}" />
+					<i></i>
+					<span tal:define="title i18n:item.title;
+									  oid tales:oid(item);
+									  version view.get_version(item);">
+						${title} (${oid}, v${version})
+					</span>
+				</label>
+				<tal:var define="global nb_items nb_items+1" />
+			</div>
+			<div class="clearfix"></div>
+		</fieldset>
+	</div>
+	<footer tal:condition="nb_items">
+		<button class="btn" data-dismiss="modal"
+				i18n:translate="">Close</button>
+		<button class="submit btn btn-primary"
+				data-ams-form-action="${tales:absolute_url(view.context)}/"
+				data-ams-form-handler="update-index.json"
+				i18n:translate="">Update index</button>
+	</footer>
+</form>