nucliadb 6.4.2.post4403__py3-none-any.whl → 6.5.0.post4408__py3-none-any.whl

migrations/pg/0006_catalog_title_indexes.py

@@ -0,0 +1,39 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from nucliadb.common.maindb.pg import PGTransaction
+
+
+async def migrate(txn: PGTransaction) -> None:
+    # Concurrent index must be created outside of a transaction but psycopg automatically
+    # creates transactions. We temporarily disable this for building indexes.
+    await txn.connection.commit()
+    try:
+        await txn.connection.set_autocommit(True)
+        # btree index on title for sorting
+        await txn.connection.execute("CREATE INDEX CONCURRENTLY ON catalog (title);")
+        # trigram index on title for fuzzy search
+        await txn.connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
+        await txn.connection.execute(
+            "CREATE INDEX CONCURRENTLY ON catalog USING gist (title gist_trgm_ops);"
+        )
+        await txn.connection.execute("ANALYZE catalog")
+    finally:
+        await txn.connection.set_autocommit(False)

nucliadb/search/api/v1/catalog.py

@@ -99,7 +99,7 @@ async def catalog_get(
     show: list[ResourceProperties] = fastapi_query(
         SearchParamDefaults.show, default=[ResourceProperties.BASIC, ResourceProperties.ERRORS]
     ),
-) -> Union[KnowledgeboxSearchResults, HTTPClientError]:
+) -> Union[CatalogResponse, HTTPClientError]:
     try:
         expr = (
             CatalogFilterExpression.model_validate_json(filter_expression) if filter_expression else None
@@ -151,7 +151,7 @@ async def catalog_post(
 async def catalog(
     kbid: str,
     item: CatalogRequest,
-):
+) -> Union[HTTPClientError, CatalogResponse]:
     """
     Catalog endpoint is a simplified version of the search endpoint, it only
     returns bm25 results on titles and it does not support vector search.

nucliadb/search/search/pgcatalog.py

@@ -27,6 +27,7 @@ from psycopg.rows import dict_row
 from nucliadb.common.maindb.pg import PGDriver
 from nucliadb.common.maindb.utils import get_driver
 from nucliadb.search.search.query_parser.models import CatalogExpression, CatalogQuery
+from nucliadb_models import search as search_models
 from nucliadb_models.labels import translate_system_to_alias_label
 from nucliadb_models.search import (
     ResourceResult,
@@ -115,14 +116,8 @@ def _prepare_query_filters(catalog_query: CatalogQuery) -> tuple[str, dict[str,
     filter_sql = ["kbid = %(kbid)s"]
     filter_params: dict[str, Any] = {"kbid": catalog_query.kbid}
 
-    if catalog_query.query:
-        # This is doing tokenization inside the SQL server (to keep the index updated). We could move it to
-        # the python code at update/query time if it ever becomes a problem but for now, a single regex
-        # executed per query is not a problem.
-        filter_sql.append(
-            "regexp_split_to_array(lower(title), '\\W') @> regexp_split_to_array(lower(%(query)s), '\\W')"
-        )
-        filter_params["query"] = catalog_query.query
+    if catalog_query.query and catalog_query.query.query:
+        filter_sql.append(_prepare_query_search(catalog_query.query, filter_params))
 
     if catalog_query.filters:
         filter_sql.append(_convert_filter(catalog_query.filters, filter_params))
@@ -133,6 +128,41 @@ def _prepare_query_filters(catalog_query: CatalogQuery) -> tuple[str, dict[str,
     )
 
 
+def _prepare_query_search(query: search_models.CatalogQuery, params: dict[str, Any]) -> str:
+    if query.match == search_models.CatalogQueryMatch.Exact:
+        params["query"] = query.query
+        return f"{query.field.value} = %(query)s"
+    elif query.match == search_models.CatalogQueryMatch.StartsWith:
+        params["query"] = query.query + "%"
+        if query.field == search_models.CatalogQueryField.Title:
+            # Insensitive search supported by pg_trgm for title
+            return f"{query.field.value} ILIKE %(query)s"
+        else:
+            # Sensitive search for slug (btree does not support ILIKE and slugs are all lowercase anyway)
+            return f"{query.field.value} LIKE %(query)s"
+    # The rest of operators only supported by title
+    elif query.match == search_models.CatalogQueryMatch.Words:
+        # This is doing tokenization inside the SQL server (to keep the index updated). We could move it to
+        # the python code at update/query time if it ever becomes a problem but for now, a single regex
+        # executed per query is not a problem.
+        params["query"] = query.query
+        return "regexp_split_to_array(lower(title), '\\W') @> regexp_split_to_array(lower(%(query)s), '\\W')"
+    elif query.match == search_models.CatalogQueryMatch.Fuzzy:
+        params["query"] = query.query
+        # Note: the operator is %>, We use %%> for psycopg escaping
+        return "title %%> %(query)s"
+    elif query.match == search_models.CatalogQueryMatch.EndsWith:
+        params["query"] = "%" + query.query
+        return "title ILIKE %(query)s"
+    elif query.match == search_models.CatalogQueryMatch.Contains:
+        params["query"] = "%" + query.query + "%"
+        return "title ILIKE %(query)s"
+    else:  # pragma: nocover
+        # This is a trick so mypy generates an error if this branch can be reached,
+        # that is, if we are missing some ifs
+        _a: int = "a"
+
+
 def _prepare_query(catalog_query: CatalogQuery) -> tuple[str, dict[str, Any]]:
     # Base query with all the filters
     query, filter_params = _prepare_query_filters(catalog_query)
@@ -188,7 +218,7 @@ async def pgcatalog_search(catalog_query: CatalogQuery) -> Resources:
                     if not (
                         facet.startswith("/n/s") or facet.startswith("/n/i") or facet.startswith("/l")
                     ):
-                        logger.warn(
+                        logger.warning(
                             f"Unexpected facet used at catalog: {facet}, kbid={catalog_query.kbid}"
                         )
 
@@ -239,7 +269,7 @@ async def pgcatalog_search(catalog_query: CatalogQuery) -> Resources:
             )
             for r in data
         ],
-        query=catalog_query.query,
+        query=catalog_query.query.query if catalog_query.query else "",
         total=total,
         page_number=catalog_query.page_number,
         page_size=catalog_query.page_size,

nucliadb/search/search/query_parser/models.py

@@ -172,7 +172,7 @@ class CatalogExpression:
 
 class CatalogQuery(BaseModel):
     kbid: str
-    query: str
+    query: Optional[search_models.CatalogQuery]
     filters: Optional[CatalogExpression]
     sort: search_models.SortOptions
     faceted: list[str]

nucliadb/search/search/query_parser/parsers/catalog.py

@@ -84,9 +84,20 @@ async def parse_catalog(kbid: str, item: search_models.CatalogRequest) -> Catalo
             limit=None,
         )
 
+    if isinstance(item.query, search_models.CatalogQuery):
+        query = item.query
+    elif isinstance(item.query, str) and len(item.query) > 0:
+        query = search_models.CatalogQuery(
+            field=search_models.CatalogQueryField.Title,
+            match=search_models.CatalogQueryMatch.Words,
+            query=item.query,
+        )
+    else:
+        query = None
+
     return CatalogQuery(
         kbid=kbid,
-        query=item.query,
+        query=query,
         filters=catalog_expr,
         sort=sort,
         faceted=item.faceted,

{nucliadb-6.4.2.post4403.dist-info → nucliadb-6.5.0.post4408.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.4.2.post4403
+Version: 6.5.0.post4408
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.4.2.post4403
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.2.post4403
-Requires-Dist: nucliadb-protos>=6.4.2.post4403
-Requires-Dist: nucliadb-models>=6.4.2.post4403
-Requires-Dist: nidx-protos>=6.4.2.post4403
+Requires-Dist: nucliadb-telemetry[all]>=6.5.0.post4408
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.5.0.post4408
+Requires-Dist: nucliadb-protos>=6.5.0.post4408
+Requires-Dist: nucliadb-models>=6.5.0.post4408
+Requires-Dist: nidx-protos>=6.5.0.post4408
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.4.2.post4403.dist-info → nucliadb-6.5.0.post4408.dist-info}/RECORD

@@ -37,6 +37,7 @@ migrations/pg/0002_catalog.py,sha256=Rsleecu351Ty19kYZgOpqX5G3MEAY8nMxCJrAeuS2Mw
 migrations/pg/0003_catalog_kbid_index.py,sha256=uKq_vtnuf73GVf0mtl2rhzdk_czAoEU1UdiVKVZpA0M,1044
 migrations/pg/0004_catalog_facets.py,sha256=FJFASHjfEHG3sNve9BP2HnnLO4xr7dnR6Qpctnmt4LE,2180
 migrations/pg/0005_purge_tasks_index.py,sha256=3mtyFgpcK0QQ_NONYay7V9xICijCLNkyTPuoc0PBjRg,1139
+migrations/pg/0006_catalog_title_indexes.py,sha256=n2OGxwE4oeCwHAYaxBkja4t10BmwTjZ2IoCyOdjEBSc,1710
 migrations/pg/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/__init__.py,sha256=_abCmDJ_0ku483Os4UAjPX7Nywm39cQgAV_DiyjsKeQ,891
 nucliadb/health.py,sha256=UIxxA4oms4HIsCRZM_SZsdkIZIlgzmOxw-qSHLlWuak,3465
@@ -215,7 +216,7 @@ nucliadb/search/utilities.py,sha256=9SsRDw0rJVXVoLBfF7rBb6q080h-thZc7u8uRcTiBeY,
 nucliadb/search/api/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/api/v1/__init__.py,sha256=DH16OYnw9jQ38OpKlmdXeoq2j40ZPXZRtGvClKOkMhw,1239
 nucliadb/search/api/v1/ask.py,sha256=b4tz33HNsfT5DXv_2DMc_jirnFsHuobreWkbAKkzj5o,5337
-nucliadb/search/api/v1/catalog.py,sha256=mVAPPf6CXimVOsBpbhPo63KXf8eXps--cifZOEQAIyk,7714
+nucliadb/search/api/v1/catalog.py,sha256=3SqLgwFkFFY8x-xBruHQaZ0EGpf7oKbSj-_PnobV68E,7747
 nucliadb/search/api/v1/feedback.py,sha256=kNLc4dHz2SXHzV0PwC1WiRAwY88fDptPcP-kO0q-FrQ,2620
 nucliadb/search/api/v1/find.py,sha256=iMjyq4y0JOMC_x1B8kUfVdkCoc9G9Ark58kPLLY4HDw,10824
 nucliadb/search/api/v1/graph.py,sha256=gthqxCOn9biE6D6s93jRGLglk0ono8U7OyS390kWiI8,4178
@@ -248,7 +249,7 @@ nucliadb/search/search/ingestion_agents.py,sha256=IK6yOPEF9rST_uoqspdVdPk0pldjDh
 nucliadb/search/search/merge.py,sha256=XiRBsxhYPshPV7lZXD-9E259KZOPIf4I2tKosY0lPo4,22470
 nucliadb/search/search/metrics.py,sha256=3I6IN0qDSmqIvUaWJmT3rt-Jyjs6LcvnKI8ZqCiuJPY,3501
 nucliadb/search/search/paragraphs.py,sha256=pNAEiYqJGGUVcEf7xf-PFMVqz0PX4Qb-WNG-_zPGN2o,7799
-nucliadb/search/search/pgcatalog.py,sha256=s_J98fsX_RuFXwpejpkGqG-tD9ELuzz4YQ6U3ew5h2g,9313
+nucliadb/search/search/pgcatalog.py,sha256=QtgArjoM-dW_B1oO0aXqp5au7GlLG8jAct9jevUHatw,10997
 nucliadb/search/search/predict_proxy.py,sha256=JwgBeEg1j4LnCjPCvTUrnmOd9LceJAt3iAu4m9cmJBo,3390
 nucliadb/search/search/query.py,sha256=0qIQdt548L3jtKOyKo06aGJ73SLBxAW3N38_Hc1M3Uw,11528
 nucliadb/search/search/rank_fusion.py,sha256=xZtXhbmKb_56gs73u6KkFm2efvTATOSMmpOV2wrAIqE,9613
@@ -265,11 +266,11 @@ nucliadb/search/search/chat/query.py,sha256=3jMPNbiFEOoS0ydMOPYkSx1qVlvAv51npzad
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=sVl9gRNzhE-s480LBBVkiXzNRbKhYRQN5F3it5tNNp8,939
 nucliadb/search/search/query_parser/fetcher.py,sha256=nP4EySj2BvH10QgCvgzvp13Nf22wwfHsdLbDoPlH2cQ,16831
-nucliadb/search/search/query_parser/models.py,sha256=k9cCjTpndP9ynr8A9J8MBmDYmjLBKL1UM4L0GXVuJw0,5031
+nucliadb/search/search/query_parser/models.py,sha256=kAslqX_-zaIdUpcpdNU2a5uQPQh7LC605qWLZ4aZ5T4,5064
 nucliadb/search/search/query_parser/old_filters.py,sha256=HircRqYEac_90bNCtFIJZ2RKA90kjbpNOQcp_ArBqR0,9083
 nucliadb/search/search/query_parser/parsers/__init__.py,sha256=ySCNSdbesLXGZyR88919njulA6UE10_3PhqMG_Yj1o4,1034
 nucliadb/search/search/query_parser/parsers/ask.py,sha256=eTz8wS-EJHuAagR384h6TT64itymFZRpfZJGX8r6aZM,2771
-nucliadb/search/search/query_parser/parsers/catalog.py,sha256=e89kh3nGV9JT9wjdWB8JbC2HPydn0rVk7WsKBo6q3gw,7122
+nucliadb/search/search/query_parser/parsers/catalog.py,sha256=JuDiBL2wdjAuEFEPo0e2nQ4VqWjF3FXakT0ziZk3Oes,7495
 nucliadb/search/search/query_parser/parsers/common.py,sha256=mJMPOKurBK7-A7s3oNlPLxHP_yIn4j5Uw8rh_OQtzS4,6339
 nucliadb/search/search/query_parser/parsers/find.py,sha256=lHVspg-i_eWXvu7BT9WfuFVGVKYhr380y4tDX5yfTD4,12735
 nucliadb/search/search/query_parser/parsers/graph.py,sha256=zyqdUg5Afmhb2_-hvj9FUCaoLh026MUP1fgY2j-lD7c,9385
@@ -369,8 +370,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.4.2.post4403.dist-info/METADATA,sha256=t1OV8EADzoCOzE8RN9vyuLl6APN2AwAXYANfpl6dVsg,4152
-nucliadb-6.4.2.post4403.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.4.2.post4403.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.4.2.post4403.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.4.2.post4403.dist-info/RECORD,,
+nucliadb-6.5.0.post4408.dist-info/METADATA,sha256=5PRi3_KM32DTF2_BSWmfndESWte8q1aH9fPh0SYJ6aE,4152
+nucliadb-6.5.0.post4408.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.5.0.post4408.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.5.0.post4408.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.5.0.post4408.dist-info/RECORD,,