# HG changeset patch # User Thierry Florac # Date 1543305002 -3600 # Node ID 852aa448da04ad838eb2438d4549870a9939dfc3 # Parent 52ed8cc1c9478ad58aa0f706dbbb62c0923d7a42 Updated views and search folders engine diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/features/search/__init__.py --- a/src/pyams_content/features/search/__init__.py Mon Nov 26 17:06:03 2018 +0100 +++ b/src/pyams_content/features/search/__init__.py Tue Nov 27 08:50:02 2018 +0100 @@ -12,17 +12,23 @@ __docformat__ = 'restructuredtext' +from hypatia.interfaces import ICatalog +from hypatia.query import Contains, Or from zope.interface import implementer from zope.schema.fieldproperty import FieldProperty from pyams_content.component.illustration import IIllustrationTarget, ILinkIllustrationTarget from pyams_content.features.preview.interfaces import IPreviewTarget from pyams_content.features.search.interfaces import ISearchFolder, ISearchFolderRoles -from pyams_content.interfaces import MANAGE_SITE_PERMISSION, MANAGER_ROLE, GUEST_ROLE -from pyams_content.shared.view import WfView +from pyams_content.interfaces import GUEST_ROLE, MANAGER_ROLE, MANAGE_SITE_PERMISSION +from pyams_content.shared.view import IViewQuery, ViewQuery, WfView +from pyams_content.shared.view.interfaces import IViewUserQuery from pyams_form.interfaces.form import IFormContextPermissionChecker -from pyams_portal.interfaces import IPortalContext, DESIGNER_ROLE +from pyams_i18n.interfaces import INegotiator +from pyams_portal.interfaces import DESIGNER_ROLE, IPortalContext from pyams_utils.adapter import ContextAdapter, adapter_config +from pyams_utils.registry import get_utility +from pyams_utils.request import check_request from pyams_content import _ @@ -37,7 +43,6 @@ content_name = _("Search folder") - handle_content_url = True handle_header = True handle_description = True @@ -47,6 +52,7 @@ selected_content_types = FieldProperty(ISearchFolder['selected_content_types']) selected_datatypes = FieldProperty(ISearchFolder['selected_datatypes']) + order_by = FieldProperty(ISearchFolder['order_by']) visible_in_list = FieldProperty(ISearchFolder['visible_in_list']) navigation_title = FieldProperty(ISearchFolder['navigation_title']) @@ -54,9 +60,54 @@ def is_deletable(): return True + def get_results(self, context, sort_index=None, reverse=None, limit=None, + start=0, length=None, ignore_cache=False, get_count=False): + if not ignore_cache: + request = check_request() + ignore_cache = bool(request.params) + return super(SearchFolder, self).get_results(context, sort_index, reverse, limit, + start, length, ignore_cache, get_count) + @adapter_config(context=ISearchFolder, provides=IFormContextPermissionChecker) class SearchFolderPermissionChecker(ContextAdapter): """Search folder edit permission checker""" edit_permission = MANAGE_SITE_PERMISSION + + +@adapter_config(context=ISearchFolder, provides=IViewQuery) +class SearchFolderQuery(ViewQuery): + """Search folder query adapter""" + + def get_params(self, context): + params = super(SearchFolderQuery, self).get_params(context) + request = check_request() + registry = request.registry + for name, adapter in registry.getAdapters((self,), IViewUserQuery): + for user_param in adapter.get_user_params(request): + params &= user_param + return params + + +@adapter_config(name='user_search', context=SearchFolderQuery, provides=IViewUserQuery) +class SearchFolderUserQuery(ContextAdapter): + """Search folder user query""" + + @staticmethod + def get_user_params(request): + params = request.params + fulltext = params.get('user_search') + if fulltext: + catalog = get_utility(ICatalog) + negotiator = get_utility(INegotiator) + query_params = [] + for lang in {request.registry.settings.get('pyramid.default_locale_name', 'en'), + request.locale_name, + negotiator.server_language} | negotiator.offered_languages: + index_name = 'title:{0}'.format(lang) + if index_name in catalog: + index = catalog[index_name] + if index.check_query(fulltext): + query_params.append(Contains(index, ' and '.join((w + '*' for w in fulltext.split())))) + yield Or(*query_params) diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/features/search/interfaces.py --- a/src/pyams_content/features/search/interfaces.py Mon Nov 26 17:06:03 2018 +0100 +++ b/src/pyams_content/features/search/interfaces.py Tue Nov 27 08:50:02 2018 +0100 @@ -9,22 +9,21 @@ # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE. # + +__docformat__ = 'restructuredtext' + from zope.interface import Interface +from zope.schema import Bool, Choice, Set -from pyams_content.interfaces import MANAGER_ROLE, GUEST_ROLE +from pyams_content.interfaces import GUEST_ROLE, MANAGER_ROLE +from pyams_content.shared.common.interfaces import CONTENT_TYPES_VOCABULARY +from pyams_content.shared.common.interfaces.types import ALL_DATA_TYPES_VOCABULARY +from pyams_content.shared.site.interfaces import IBaseSiteItem, ISiteElement +from pyams_content.shared.view import IWfView +from pyams_content.shared.view.interfaces import RELEVANCE_ORDER, USER_VIEW_ORDER_VOCABULARY from pyams_i18n.schema import I18nTextLineField from pyams_portal.interfaces import DESIGNER_ROLE from pyams_security.schema import PrincipalsSet - - -__docformat__ = 'restructuredtext' - -from zope.schema import Choice, Set, Bool - -from pyams_content.shared.common.interfaces import CONTENT_TYPES_VOCABULARY -from pyams_content.shared.common.interfaces.types import ALL_DATA_TYPES_VOCABULARY -from pyams_content.shared.site.interfaces import ISiteElement, IBaseSiteItem -from pyams_content.shared.view import IWfView from pyams_sequence.interfaces import ISequentialIdTarget from pyams_content import _ @@ -53,6 +52,12 @@ class ISearchFolder(IBaseSiteItem, ISiteElement, IWfView, ISequentialIdTarget): """Search folder interface""" + order_by = Choice(title=_("Order by"), + description=_("Property to use to sort results"), + vocabulary=USER_VIEW_ORDER_VOCABULARY, + required=False, + default=RELEVANCE_ORDER) + visible_in_list = Bool(title=_("Visible in folders list"), description=_("If 'no', folder will not be displayed into folders list"), required=True, diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/features/search/portlet/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content/features/search/portlet/__init__.py Tue Nov 27 08:50:02 2018 +0100 @@ -0,0 +1,78 @@ +# +# Copyright (c) 2008-2018 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + +import math + +from zope.interface import implementer +from zope.schema.fieldproperty import FieldProperty + +from pyams_content.features.search import ISearchFolder +from pyams_content.features.search.portlet.interfaces import ISearchResultsPortletSettings +from pyams_content.shared.view.interfaces import RELEVANCE_ORDER +from pyams_portal.portlet import Portlet, PortletSettings, portlet_config +from pyams_utils.factory import factory_config +from pyams_utils.interfaces import VIEW_PERMISSION +from pyams_utils.request import check_request +from pyams_utils.traversing import get_parent + +from pyams_content import _ + + +SEARCH_RESULTS_PORTLET_NAME = 'pyams_content.portlet.search.results' + + +@implementer(ISearchResultsPortletSettings) +@factory_config(provided=ISearchResultsPortletSettings) +class SearchResultsPortletSettings(PortletSettings): + """Search results portlet settings""" + + title = FieldProperty(ISearchResultsPortletSettings['title']) + + @staticmethod + def get_items(request=None, limit=None, ignore_cache=False): + context = get_parent(request.context, ISearchFolder) + if context is None: + raise StopIteration + if request is None: + request = check_request() + params = request.params + sort_index = params.get('order_by', RELEVANCE_ORDER) + yield from context.get_results(context, sort_index, + reverse=sort_index != RELEVANCE_ORDER, + limit=limit, + start=int(params.get('start', 0)), + length=int(params.get('length', 10)), + ignore_cache=ignore_cache, + get_count=True) + + @staticmethod + def get_pages(request, count): + params = request.params + start = int(params.get('start', 0)) + 1 + length = int(params.get('length', 10)) + current = math.ceil(start / length) + nb_pages = math.ceil(count / length) + return current, nb_pages + + +@portlet_config(permission=VIEW_PERMISSION) +class SearchResultsPortlet(Portlet): + """Search results portlet""" + + name = SEARCH_RESULTS_PORTLET_NAME + label = _("Search results") + + toolbar_css_class = 'fa fa-fw fa-2x fa-search-plus' + + settings_factory = ISearchResultsPortletSettings diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/features/search/portlet/interfaces.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content/features/search/portlet/interfaces.py Tue Nov 27 08:50:02 2018 +0100 @@ -0,0 +1,26 @@ +# +# Copyright (c) 2008-2018 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + +from pyams_i18n.schema import I18nTextLineField +from pyams_portal.interfaces import IPortletSettings + +from pyams_content import _ + + +class ISearchResultsPortletSettings(IPortletSettings): + """Search results portlet settings""" + + title = I18nTextLineField(title=_("Title"), + description=_("Portlet main title"), + required=False) diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/features/search/portlet/zmi/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content/features/search/portlet/zmi/__init__.py Tue Nov 27 08:50:02 2018 +0100 @@ -0,0 +1,47 @@ +# +# Copyright (c) 2008-2018 Thierry Florac +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# + +__docformat__ = 'restructuredtext' + +from zope.interface import Interface + +from pyams_content.features.search.portlet import ISearchResultsPortletSettings +from pyams_form.form import AJAXEditForm +from pyams_pagelet.interfaces import IPagelet +from pyams_pagelet.pagelet import pagelet_config +from pyams_portal.interfaces import IPortletPreviewer +from pyams_portal.portlet import PortletPreviewer +from pyams_portal.zmi.portlet import PortletSettingsEditor +from pyams_skin.layer import IPyAMSLayer +from pyams_template.template import template_config +from pyams_utils.adapter import adapter_config +from pyams_utils.interfaces import VIEW_SYSTEM_PERMISSION + + +@pagelet_config(name='properties.html', context=ISearchResultsPortletSettings, layer=IPyAMSLayer, + permission=VIEW_SYSTEM_PERMISSION) +class SearchResultsPortletSettingsEditor(PortletSettingsEditor): + """Search results portlet settings editor""" + + settings = ISearchResultsPortletSettings + + +@adapter_config(name='properties.json', context=(ISearchResultsPortletSettings, IPyAMSLayer), provides=IPagelet) +class SearchresultsPortletSettingsAJAXEditor(AJAXEditForm, SearchResultsPortletSettingsEditor): + """Search results portlet settings editor, JSON renderer""" + + +@adapter_config(context=(Interface, IPyAMSLayer, Interface, ISearchResultsPortletSettings), + provides=IPortletPreviewer) +@template_config(template='templates/search-preview.pt', layer=IPyAMSLayer) +class SearchResultsPortletPreviewer(PortletPreviewer): + """Search results portlet previewer""" diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/features/search/portlet/zmi/templates/search-preview.pt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/pyams_content/features/search/portlet/zmi/templates/search-preview.pt Tue Nov 27 08:50:02 2018 +0100 @@ -0,0 +1,16 @@ +
+ ${i18n:settings.title} +
+ Extracted contents (limited to 10): +
    +
  • + ${i18n:item.title} + (${tales:oid(item)}) + +
    • + No result found +
+
+
diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/shared/view/__init__.py --- a/src/pyams_content/shared/view/__init__.py Mon Nov 26 17:06:03 2018 +0100 +++ b/src/pyams_content/shared/view/__init__.py Tue Nov 27 08:50:02 2018 +0100 @@ -13,10 +13,8 @@ __docformat__ = 'restructuredtext' import logging -logger = logging.getLogger("PyAMS (content)") - from datetime import datetime -from itertools import tee +from itertools import islice, tee from hypatia.catalog import CatalogQuery from hypatia.interfaces import ICatalog @@ -36,7 +34,8 @@ IWfSharedContent from pyams_content.shared.common.interfaces.types import IWfTypedSharedContent from pyams_content.shared.view.interfaces import IView, IWfView, IWfViewFactory, IViewQuery, \ - IViewQueryParamsExtension, IViewQueryFilterExtension, VIEW_CONTENT_TYPE, VIEW_CONTENT_NAME, IViewSettings + IViewQueryParamsExtension, IViewQueryFilterExtension, VIEW_CONTENT_TYPE, VIEW_CONTENT_NAME, IViewSettings, \ + RELEVANCE_ORDER from pyams_utils.adapter import adapter_config, ContextAdapter from pyams_utils.interfaces import ICacheKeyValue from pyams_utils.list import unique_iter @@ -45,6 +44,8 @@ from pyams_workflow.interfaces import IWorkflow +logger = logging.getLogger("PyAMS (content)") + VIEWS_CACHE_REGION = 'views' VIEWS_CACHE_NAME = 'PyAMS::view' @@ -103,7 +104,8 @@ data_types |= set(self.selected_datatypes) return list(data_types) - def get_results(self, context, sort_index=None, reverse=None, limit=None, ignore_cache=False): + def get_results(self, context, sort_index=None, reverse=None, + limit=None, start=0, length=999, ignore_cache=False, get_count=False): results = _MARKER if not ignore_cache: # check for cache @@ -115,6 +117,7 @@ cache_key = VIEW_CACHE_KEY.format(view=ICacheKeyValue(self)) try: results = views_cache.get_value(cache_key) + count = views_cache.get_value(cache_key + '::count') except KeyError: pass # Execute query @@ -123,19 +126,25 @@ adapter = registry.queryAdapter(self, IViewQuery, name='es') if adapter is None: adapter = registry.getAdapter(self, IViewQuery) - results = adapter.get_results(context, - sort_index or self.order_by, - reverse if reverse is not None else self.reversed_order, - limit or self.limit) - intids = get_utility(IIntIds) - cache, results = tee(results) + if sort_index == RELEVANCE_ORDER: + sort_index = None # keep natural order + else: + sort_index = self.order_by + results, count = adapter.get_results(context, + sort_index, + reverse if reverse is not None else self.reversed_order, + limit or self.limit) + count = min(count, limit or self.limit or 999) + cache, results = tee(islice(results, start, start + length)) if not ignore_cache: + intids = get_utility(IIntIds) views_cache.set_value(cache_key, [intids.queryId(item) for item in cache]) + views_cache.set_value(cache_key + '::count', count) logger.debug("Storing view items to cache key {0}".format(cache_key)) else: results = CatalogResultSet(results) logger.debug("Retrieving view items from cache key {0}".format(cache_key)) - return results + return (results, count) if get_count else results register_content_type(WfView) @@ -172,11 +181,11 @@ do_search = True for name, adapter in sorted(registry.getAdapters((view,), IViewQueryParamsExtension), key=lambda x: x[1].weight): - new_params = adapter.get_params(context) - if isinstance(new_params, tuple): - new_params, do_search = new_params - if new_params: - params &= new_params + for new_params in adapter.get_params(context): + if isinstance(new_params, tuple): + new_params, do_search = new_params + if new_params: + params &= new_params if not do_search: break # activate search @@ -197,14 +206,18 @@ catalog = get_utility(ICatalog) registry = get_current_registry() params = self.get_params(context) - items = CatalogResultSet(CatalogQuery(catalog).query(params, - sort_index=sort_index, - reverse=reverse, - limit=limit)) + if sort_index == RELEVANCE_ORDER: + sort_index = None + query = CatalogQuery(catalog).query(params, + sort_index=sort_index, + reverse=reverse, + limit=limit) + total_count = query[0] + items = CatalogResultSet(query) for name, adapter in sorted(registry.getAdapters((view,), IViewQueryFilterExtension), key=lambda x: x[1].weight): items = adapter.filter(context, items) - return unique_iter(items) + return unique_iter(items), total_count @subscriber(IObjectModifiedEvent, context_selector=IWfView) diff -r 52ed8cc1c947 -r 852aa448da04 src/pyams_content/shared/view/interfaces.py --- a/src/pyams_content/shared/view/interfaces.py Mon Nov 26 17:06:03 2018 +0100 +++ b/src/pyams_content/shared/view/interfaces.py Tue Nov 27 08:50:02 2018 +0100 @@ -30,6 +30,7 @@ VIEW_CONTENT_TYPE = 'view' VIEW_CONTENT_NAME = _('View') +RELEVANCE_ORDER = 'relevance' CREATION_DATE_ORDER = 'created_date' UPDATE_DATE_ORDER = 'modified_date' PUBLICATION_DATE_ORDER = 'publication_date' @@ -46,6 +47,18 @@ for item in VIEW_ORDERS]) +USER_VIEW_ORDERS = ( + {'id': RELEVANCE_ORDER, 'title': _("Relevance (on user search)")}, + {'id': CREATION_DATE_ORDER, 'title': _("Creation date")}, + {'id': UPDATE_DATE_ORDER, 'title': _("Last update date")}, + {'id': PUBLICATION_DATE_ORDER, 'title': _("Current publication date")}, + {'id': FIRSTPUBLICATION_DATE_ORDER, 'title': _("First publication date")} +) + +USER_VIEW_ORDER_VOCABULARY = SimpleVocabulary([SimpleTerm(item['id'], title=item['title']) + for item in USER_VIEW_ORDERS]) + + class IViewsManager(ISharedTool): """Views manager interface""" @@ -101,7 +114,8 @@ is_using_context = Attribute("Check if view is using context settings") - def get_results(self, context, sort_index=None, reverse=True, limit=None, ignore_cache=False): + def get_results(self, context, sort_index=None, reverse=True, limit=None, + start=0, length=999, ignore_cache=False, get_count=False): """Get results of catalog query""" @@ -122,8 +136,18 @@ class IViewQuery(Interface): """View query interface""" + def get_params(self, context): + """Get static view query params""" + def get_results(self, context, sort_index, reverse, limit): - """Get results of catalog query""" + """Get tuple of limited results and total results count""" + + +class IViewUserQuery(Interface): + """View user search query interface""" + + def get_user_params(self, request): + """Get dynamic user query params""" class IViewQueryExtension(Interface):