nucliadb 6.7.2.post4889__py3-none-any.whl → 6.7.2.post4908__py3-none-any.whl

nucliadb/ingest/orm/processor/processor.py

@@ -28,6 +28,7 @@ from nidx_protos import noderesources_pb2, nodewriter_pb2
 from nidx_protos.noderesources_pb2 import Resource as PBBrainResource
 
 from nucliadb.common import datamanagers, locking
+from nucliadb.common.catalog import catalog_delete, catalog_update
 from nucliadb.common.cluster.settings import settings as cluster_settings
 from nucliadb.common.cluster.utils import get_shard_manager
 from nucliadb.common.external_index_providers.base import ExternalIndexManager
@@ -61,8 +62,6 @@ from nucliadb_utils.cache.pubsub import PubSubDriver
 from nucliadb_utils.storages.storage import Storage
 from nucliadb_utils.utilities import get_storage, has_feature
 
-from .pgcatalog import pgcatalog_delete, pgcatalog_update
-
 logger = logging.getLogger("ingest-processor")
 
 MESSAGE_TO_NOTIFICATION_SOURCE = {
@@ -227,7 +226,8 @@ class Processor:
                     shard = await kb.get_resource_shard(shard_id)
                     if shard is None:
                         raise AttributeError("Shard not available")
-                    await pgcatalog_delete(txn, message.kbid, uuid)
+
+                    await catalog_delete(txn, message.kbid, uuid)
                     external_index_manager = await get_external_index_manager(kbid=message.kbid)
                     if external_index_manager is not None:
                         await self.external_index_delete_resource(external_index_manager, uuid)
@@ -374,8 +374,7 @@ class Processor:
                             index_message.labels.remove(current_status[0])
                             index_message.labels.append("/n/s/ERROR")
 
-                    await pgcatalog_update(txn, kbid, resource, index_message)
-
+                    await catalog_update(txn, kbid, resource, index_message)
                     if transaction_check:
                         await sequence_manager.set_last_seqid(txn, partition, seqid)
                     await txn.commit()

nucliadb/ingest/settings.py

@@ -67,6 +67,11 @@ class DriverSettings(BaseSettings):
     )
 
 
+class CatalogConfig(Enum):
+    UNSET = "unset"
+    PG = "pg"
+
+
 # For use during migration from pull v1 to pull v2
 class ProcessingPullMode(Enum):
     OFF = "off"
@@ -75,6 +80,9 @@ class ProcessingPullMode(Enum):
 
 
 class Settings(DriverSettings):
+    # Catalog settings
+    catalog: CatalogConfig = Field(default=CatalogConfig.PG, description="Catalog backend")
+
     # Pull worker settings
     pull_time_error_backoff: int = 30
     pull_api_timeout: int = 60

nucliadb/search/api/v1/catalog.py

@@ -25,6 +25,7 @@ from fastapi import Request, Response
 from fastapi_versioning import version
 from pydantic import ValidationError
 
+from nucliadb.common.catalog import catalog_facets, catalog_search
 from nucliadb.common.datamanagers.exceptions import KnowledgeBoxNotFound
 from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.models.responses import HTTPClientError
@@ -33,7 +34,6 @@ from nucliadb.search.api.v1.router import KB_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
 from nucliadb.search.search import cache
 from nucliadb.search.search.merge import fetch_resources
-from nucliadb.search.search.pgcatalog import pgcatalog_facets, pgcatalog_search
 from nucliadb.search.search.query_parser.parsers import parse_catalog
 from nucliadb.search.search.utils import (
     maybe_log_request_payload,
@@ -164,7 +164,7 @@ async def catalog(
             query_parser = await parse_catalog(kbid, item)
 
             catalog_results = CatalogResponse()
-            catalog_results.fulltext = await pgcatalog_search(query_parser)
+            catalog_results.fulltext = await catalog_search(query_parser)
             catalog_results.resources = await fetch_resources(
                 resources=[r.rid for r in catalog_results.fulltext.results],
                 kbid=kbid,
@@ -205,7 +205,7 @@ async def catalog(
 )
 @requires(NucliaDBRoles.READER)
 @version(1)
-async def catalog_facets(
+async def catalog_facets_endpoint(
     request: Request, kbid: str, item: CatalogFacetsRequest
 ) -> CatalogFacetsResponse:
-    return CatalogFacetsResponse(facets=await pgcatalog_facets(kbid, item))
+    return CatalogFacetsResponse(facets=await catalog_facets(kbid, item))

nucliadb/search/search/query_parser/models.py

@@ -17,9 +17,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from dataclasses import dataclass
 from datetime import datetime
-from typing import Literal, Optional, Union
+from typing import Optional, Union
 
 from nidx_protos import nodereader_pb2
 from pydantic import BaseModel, ConfigDict, Field
@@ -153,33 +152,6 @@ class ParsedQuery(BaseModel):
     generation: Optional[Generation] = None
 
 
-### Catalog
-@dataclass
-class CatalogExpression:
-    @dataclass
-    class Date:
-        field: Union[Literal["created_at"], Literal["modified_at"]]
-        since: Optional[datetime]
-        until: Optional[datetime]
-
-    bool_and: Optional[list["CatalogExpression"]] = None
-    bool_or: Optional[list["CatalogExpression"]] = None
-    bool_not: Optional["CatalogExpression"] = None
-    date: Optional[Date] = None
-    facet: Optional[str] = None
-    resource_id: Optional[str] = None
-
-
-class CatalogQuery(BaseModel):
-    kbid: str
-    query: Optional[search_models.CatalogQuery]
-    filters: Optional[CatalogExpression]
-    sort: search_models.SortOptions
-    faceted: list[str]
-    page_size: int
-    page_number: int
-
-
 ### Graph
 
 

nucliadb/search/search/query_parser/parsers/ask.py

@@ -63,7 +63,7 @@ class _AskParser:
             )
         elif isinstance(self.item.max_tokens, MaxTokens):
             max_tokens = self.item.max_tokens
-        else:  # pragma: nocover
+        else:  # pragma: no cover
             # This is a trick so mypy generates an error if this branch can be reached,
             # that is, if we are missing some ifs
             _a: int = "a"

nucliadb/search/search/query_parser/parsers/catalog.py

@@ -19,13 +19,10 @@
 #
 
 from nucliadb.common import datamanagers
+from nucliadb.common.catalog.interface import CatalogExpression, CatalogQuery
 from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.common.filter_expression import FacetFilterTypes, facet_from_filter
 from nucliadb.search.search.filters import translate_label
-from nucliadb.search.search.query_parser.models import (
-    CatalogExpression,
-    CatalogQuery,
-)
 from nucliadb_models import search as search_models
 from nucliadb_models.filters import (
     And,
@@ -185,7 +182,7 @@ async def parse_filter_expression(expr: ResourceFilterExpression, kbid: str) ->
             if rid is None:
                 raise InvalidQueryError("slug", f"Cannot find slug {expr.slug}")
             cat.resource_id = rid
-        else:  # pragma: nocover
+        else:  # pragma: no cover
             # Cannot happen due to model validation
             raise ValueError("Resource needs id or slug")
     elif isinstance(expr, DateCreated):

nucliadb/search/search/query_parser/parsers/graph.py

@@ -153,7 +153,7 @@ def parse_path_query(expr: graph_requests.GraphPathQuery) -> nodereader_pb2.Grap
     elif isinstance(expr, graph_requests.Generated):
         _set_generated_to_pb(expr, pb)
 
-    else:  # pragma: nocover
+    else:  # pragma: no cover
         # This is a trick so mypy generates an error if this branch can be reached,
         # that is, if we are missing some ifs
         _a: int = "a"
@@ -182,7 +182,7 @@ def _parse_node_query(expr: graph_requests.GraphNodesQuery) -> nodereader_pb2.Gr
     elif isinstance(expr, graph_requests.Generated):
         _set_generated_to_pb(expr, pb)
 
-    else:  # pragma: nocover
+    else:  # pragma: no cover
         # This is a trick so mypy generates an error if this branch can be reached,
         # that is, if we are missing some ifs
         _a: int = "a"
@@ -212,7 +212,7 @@ def _parse_relation_query(
     elif isinstance(expr, graph_requests.Generated):
         _set_generated_to_pb(expr, pb)
 
-    else:  # pragma: nocover
+    else:  # pragma: no cover
         # This is a trick so mypy generates an error if this branch can be reached,
         # that is, if we are missing some ifs
         _a: int = "a"
@@ -230,7 +230,7 @@ def _set_node_to_pb(node: graph_requests.GraphNode, pb: nodereader_pb2.GraphQuer
             pb.fuzzy.kind = nodereader_pb2.GraphQuery.Node.MatchLocation.PREFIX
             pb.fuzzy.distance = 1
 
-        else:  # pragma: nocover
+        else:  # pragma: no cover
             # This is a trick so mypy generates an error if this branch can be reached,
             # that is, if we are missing some ifs
             _a: int = "a"
@@ -263,7 +263,7 @@ def _set_generated_to_pb(generated: graph_requests.Generated, pb: nodereader_pb2
 
         pb.facet.facet = facet
 
-    else:  # pragma: nocover
+    else:  # pragma: no cover
         # This is a trick so mypy generates an error if this branch can be reached,
         # that is, if we are missing some ifs
         _a: int = "a"

nucliadb/search/search/rerankers.py

@@ -181,7 +181,7 @@ def get_reranker(reranker: parser_models.Reranker) -> Reranker:
     elif isinstance(reranker, parser_models.PredictReranker):
         algorithm = PredictReranker(reranker.window)
 
-    else:  # pragma: nocover
+    else:  # pragma: no cover
         # This is a trick so mypy generates an error if this branch can be reached,
         # that is, if we are missing some ifs
         _a: int = "a"

{nucliadb-6.7.2.post4889.dist-info → nucliadb-6.7.2.post4908.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.7.2.post4889
+Version: 6.7.2.post4908
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4889
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4889
-Requires-Dist: nucliadb-protos>=6.7.2.post4889
-Requires-Dist: nucliadb-models>=6.7.2.post4889
-Requires-Dist: nidx-protos>=6.7.2.post4889
+Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4908
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4908
+Requires-Dist: nucliadb-protos>=6.7.2.post4908
+Requires-Dist: nucliadb-models>=6.7.2.post4908
+Requires-Dist: nidx-protos>=6.7.2.post4908
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.46.0
 Requires-Dist: uvicorn[standard]

{nucliadb-6.7.2.post4889.dist-info → nucliadb-6.7.2.post4908.dist-info}/RECORD

@@ -19,11 +19,11 @@ migrations/0019_upgrade_to_paragraphs_v3.py,sha256=XVuku1QLvbzTi9Akzh3a72qohXilh
 migrations/0020_drain_nodes_from_cluster.py,sha256=BeECAI0T8u14M2U5USl1fFNcsfmdMerNhisolYQN_eA,1411
 migrations/0021_overwrite_vectorsets_key.py,sha256=lXNkMsa_DWEceci29c0tDO4Y4WsJQZS6p26Sh-Ya9Mg,1586
 migrations/0022_fix_paragraph_deletion_bug.py,sha256=-tH342VXF-8xwc_h3P1cYaUtTT1wHSGf7ZoeVEpnaYs,1422
-migrations/0023_backfill_pg_catalog.py,sha256=bBSDpU55trAthgtvVrpRheZAlcnGbVb468AsNkfJ6hk,3105
+migrations/0023_backfill_pg_catalog.py,sha256=cg9pCVwOMIadpC2p2D0iX7ciuEhFETM7ex5KoT8X0sM,3212
 migrations/0025_assign_models_to_kbs_v2.py,sha256=YqzXvsru08gBsbRjuhgc3ttoEANwxaWkYeFw62J7nn0,4634
 migrations/0026_fix_high_cardinality_content_types.py,sha256=BsbBkvZDzjRHQfoouZNNtHA1xMxTKm8wOVnp_WAS9j4,2322
 migrations/0027_rollover_texts3.py,sha256=EWoUloOwv4alEqr1-DUA3tb5KLC5Vza-bBQDfZqy5jI,2818
-migrations/0028_extracted_vectors_reference.py,sha256=49DHCIlBpjofU8cYVHTdWv0EBIlnPTWV2WCezf0rJUo,2392
+migrations/0028_extracted_vectors_reference.py,sha256=HIykGAWMt5FfIM6QyNznLNsQ--3LrzXdM5oiXQg2q0c,2393
 migrations/0029_backfill_field_status.py,sha256=W3BWa3KsstDt2xundn0ED6FoG9sqRh7Y9IDjP1yTrLw,5651
 migrations/0030_label_deduplication.py,sha256=y14TxtCMi3-TBMz_eZoyyPDHNlZb29taJujlDuHumsA,2008
 migrations/0031_languages_deduplication.py,sha256=o6va6lP3oTRT1uSzp5MIhHHBFbhCxSZ-oNlXXpiAdUo,2340
@@ -33,7 +33,7 @@ migrations/0034_rollover_nidx_texts_3.py,sha256=t19QtWUgHxmTaBPoR1DooAby2IYmkLTQ
 migrations/0035_rollover_nidx_texts_4.py,sha256=W0_AUd01pjMpYMDC3yqF6HzDLgcnnPprL80kfyb1WZI,1187
 migrations/0036_backfill_catalog_slug.py,sha256=toYqxH_EfUFqoVn_cOdR5Fg8bWZU5BoFMfPBSf74LKU,2957
 migrations/0037_backfill_catalog_facets.py,sha256=IH7H4OZ4tzws6xEh7Qro0bPDHDYOoVViEUj-JwPPe1U,2791
-migrations/0038_backfill_catalog_field_labels.py,sha256=PUkgghFiaCSuGYOiOZqp7p352-C4mi7DNMAZ-B-S0SE,3385
+migrations/0038_backfill_catalog_field_labels.py,sha256=F519nYngJDb1Mtwf-OQpweDPWKPxAlqdxy5E-DyQrhA,3492
 migrations/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 migrations/pg/0001_bootstrap.py,sha256=3O_P17l0d0h48nebN6VQLXzM_B7S7zvDpaLR0koVgWE,1274
 migrations/pg/0002_catalog.py,sha256=Rsleecu351Ty19kYZgOpqX5G3MEAY8nMxCJrAeuS2Mw,1690
@@ -65,16 +65,21 @@ nucliadb/common/cache.py,sha256=FmMVPoxmKGODvOCyO_pxEHjAV3sYeefW9Jeh1cVLTU0,6538
 nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,869
 nucliadb/common/counters.py,sha256=8lOi3A2HeLDDlcNaS2QT1SfD3350VPBjiY3FkmHH1V8,977
 nucliadb/common/exceptions.py,sha256=_PJk_NfAhZBFBvmgAfvsJKZ9KuRt5Y1cNsH3-cXE07w,1120
-nucliadb/common/filter_expression.py,sha256=aRbGfg2pci7sUo1KgSqcm72Eu-3Ordy8knnW6DYId5o,6568
+nucliadb/common/filter_expression.py,sha256=-6buKY1SCVYpkrG_60Ui3ebSDWnXeF_xVmBCrNipoII,6569
 nucliadb/common/ids.py,sha256=4QjoIofes_vtKj2HsFWZf8VVIVWXxdkYtLpx1n618Us,8239
 nucliadb/common/locking.py,sha256=eZG47mI1OPnKbxSd95qa6jDXBhUoxVBIuSjxoEuBRWE,5894
 nucliadb/common/nidx.py,sha256=4WHcEkLdH54S2C0FcLwHvcQXMiRj2lZ7zqOEbD8eEM8,9189
-nucliadb/common/vector_index_config.py,sha256=LqGwhrDCp1q1vBow3scd1Chhr4GLYjYnGL72FKvOYYc,1552
+nucliadb/common/vector_index_config.py,sha256=DQrlraTWE5uUn68l9s10d3wobNeVtbP-ANEQmUfSWyo,1553
 nucliadb/common/back_pressure/__init__.py,sha256=paAcAZcfGRTyURF9lnn3vX0vcwakTEVswG_xcdGBH-U,928
 nucliadb/common/back_pressure/cache.py,sha256=ANvXglWzI5naAD6N4E_fNi17qS6KNyAhjLeh6WlZZ84,2931
 nucliadb/common/back_pressure/materializer.py,sha256=bXUalaaTMdrltm23ezkoymcRPJl7Ha8RVTj7xdVfHgQ,11468
 nucliadb/common/back_pressure/settings.py,sha256=3qNOzbI0KC6LMy-wMilXRSBfZu6CCpGHod26MTgAZ2o,3082
 nucliadb/common/back_pressure/utils.py,sha256=aZeP1XSkdgaRgZC76yR9Kje3511ZUCp7KB-XzcvhMYY,2018
+nucliadb/common/catalog/__init__.py,sha256=6ZTYyXuUcvUH4Z97CT-lBvy8OCKnA5Hkq7yHPp4lJOw,3250
+nucliadb/common/catalog/dummy.py,sha256=-LA29hSHm2XPTXjOTV2g5w4yNZm9XvGzIyO2he7rJDw,1490
+nucliadb/common/catalog/interface.py,sha256=IBgvlOEvZ46szcAc6FpIb3Fr0pelGVvjOzRDkA1tC10,3281
+nucliadb/common/catalog/pg.py,sha256=hRdiR3RDGyZnjAB0dWu4PZBgAfiZLjVIh08nCK34Nl8,20193
+nucliadb/common/catalog/utils.py,sha256=lQLTe3rN_ra0CiNjTQIiIaU4lhx5beRkx1Va10ZBk_Y,2222
 nucliadb/common/cluster/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/cluster/exceptions.py,sha256=t7v_l93t44l2tQpdQXgO_w-c4YZRcaayOz1A2i0w4RQ,1258
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=JkufazWzMA4KFEU8EBkMbiiDW4C8lLcRhiiCxP7aCQY,2949
@@ -136,7 +141,7 @@ nucliadb/ingest/partitions.py,sha256=c1OWrFWgadNtvghY3Fl-xlurdyV5hZpVJPEoRAsBt1k
 nucliadb/ingest/processing.py,sha256=IKXMZXIPuuojKQiXR2T5-5NwMvmUnIQIhBXUGgzyFFo,21551
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/ingest/serialize.py,sha256=hiddxbV5gxVk8uY8-Q1AEq2DhJx5fOBP34zq5ONGgcs,16240
-nucliadb/ingest/settings.py,sha256=8OJMjVVbI3OWIbZLrXBqpB79zHbbLkCSb9VJA0IzRss,4269
+nucliadb/ingest/settings.py,sha256=DouvoeZxu4uLgEIDLCLKNrzu8SLZpgQxTB--Ly6pRJI,4448
 nucliadb/ingest/utils.py,sha256=l1myURu3r8oA11dx3GpHw-gNTUc1AFX8xdPm9Lgl2rA,2275
 nucliadb/ingest/consumer/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/consumer/auditing.py,sha256=xK21DIa_ZAiOJVVbnkmT4jgCRGshNGyPyxsqhE6kROE,7204
@@ -156,7 +161,7 @@ nucliadb/ingest/fields/generic.py,sha256=elgtqv15aJUq3zY7X_g0bli_2BpcwPArVvzhe54
 nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJs,4172
 nucliadb/ingest/fields/text.py,sha256=2grxo8twWbpXEd_iwUMBw9q0dWorVmlPONmY5d1ThwQ,1684
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/orm/brain_v2.py,sha256=AhDjTmHAYPJe1xbhQGMgoqqCPMMaeEHKZcQPzB0HKrE,33646
+nucliadb/ingest/orm/brain_v2.py,sha256=8MAo1N_nhoGy73TvKBuaw-NaMxIanRPCNttw6dFY4jk,33647
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=kXyeF6XOpFKhEsGLcY-GLIk21Exp0cJst4XQQ9jJoug,14791
 nucliadb/ingest/orm/exceptions.py,sha256=gsp7TtVNQPiIEh-zf_UEJClwuFU0iu-5vzj0OrKMScg,1550
@@ -168,8 +173,7 @@ nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,
 nucliadb/ingest/orm/processor/__init__.py,sha256=xhDNKCxY0XNOlIVKEtM8QT75vDUkJIt7K-_VgGbbOQU,904
 nucliadb/ingest/orm/processor/auditing.py,sha256=gxn5v30KVaH0TnIjo715mWjzKGJ-DMviElEXJG9BNN4,4612
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
-nucliadb/ingest/orm/processor/pgcatalog.py,sha256=VPQ_Evme7xmmGoQ45zt0Am0yPkaD4hxN1r5rEaVt6s8,4633
-nucliadb/ingest/orm/processor/processor.py,sha256=vCLJ3e9ByW80aPFsA1lAFSjyDihULPVId1OQGyC7huw,33841
+nucliadb/ingest/orm/processor/processor.py,sha256=CPOf5KZgTcKLO6LxkFia2LueskcKBlv5bTifnTOZ1KE,33845
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=kUH0bCuM6NqpA0xSwfyb9igig3Btu57pc8VYnKggqx4,1693
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -222,7 +226,7 @@ nucliadb/search/utilities.py,sha256=9SsRDw0rJVXVoLBfF7rBb6q080h-thZc7u8uRcTiBeY,
 nucliadb/search/api/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/api/v1/__init__.py,sha256=DH16OYnw9jQ38OpKlmdXeoq2j40ZPXZRtGvClKOkMhw,1239
 nucliadb/search/api/v1/ask.py,sha256=hZUnk1opZuXp1IwTiingSatlUefg2CZ9r_Z9sUwZMaU,5698
-nucliadb/search/api/v1/catalog.py,sha256=5ZY3d8sVia1traUxVS0Q4aQJmgcOuXzbxis_uY4ulE4,8077
+nucliadb/search/api/v1/catalog.py,sha256=zMflTu9UKfvuEO_u4Et33Q2kSni0TIk2E5t-_Ad5BXM,8069
 nucliadb/search/api/v1/feedback.py,sha256=kNLc4dHz2SXHzV0PwC1WiRAwY88fDptPcP-kO0q-FrQ,2620
 nucliadb/search/api/v1/find.py,sha256=j6mxEyxjlLnZSqCT_N2LmOJlytsm1vkY4KFFmJRrtP8,10904
 nucliadb/search/api/v1/graph.py,sha256=gthqxCOn9biE6D6s93jRGLglk0ono8U7OyS390kWiI8,4178
@@ -255,11 +259,10 @@ nucliadb/search/search/ingestion_agents.py,sha256=IK6yOPEF9rST_uoqspdVdPk0pldjDh
 nucliadb/search/search/merge.py,sha256=XiRBsxhYPshPV7lZXD-9E259KZOPIf4I2tKosY0lPo4,22470
 nucliadb/search/search/metrics.py,sha256=3I6IN0qDSmqIvUaWJmT3rt-Jyjs6LcvnKI8ZqCiuJPY,3501
 nucliadb/search/search/paragraphs.py,sha256=pNAEiYqJGGUVcEf7xf-PFMVqz0PX4Qb-WNG-_zPGN2o,7799
-nucliadb/search/search/pgcatalog.py,sha256=0n_gDihZZhqrDLRHvHzS3IESvMRTcU6YShqizQMyE_Y,16807
 nucliadb/search/search/predict_proxy.py,sha256=Df8F5K-oS4TIXJc_y8UDViJTo7st5L0kMgxYPFZ39Vk,8806
 nucliadb/search/search/query.py,sha256=lYCesbUv-B7IyVFQoCCurcxl_Azc5nq3jtVQJ9tk1Ao,11552
 nucliadb/search/search/rank_fusion.py,sha256=xZtXhbmKb_56gs73u6KkFm2efvTATOSMmpOV2wrAIqE,9613
-nucliadb/search/search/rerankers.py,sha256=E2J1QdKAojqbhHM3KAyaOXKf6tJyETUxKs4tf_BEyqk,7472
+nucliadb/search/search/rerankers.py,sha256=2LNC0I28EXriffMuBlOYzjQq0vCTjpCxaK29f852n3s,7473
 nucliadb/search/search/shards.py,sha256=mc5DK-MoCv9AFhlXlOFHbPvetcyNDzTFOJ5rimK8PC8,2636
 nucliadb/search/search/summarize.py,sha256=3lLdwsM28W505bKvmK7JLXmz7kcjd8Hp70LQs391ofY,5087
 nucliadb/search/search/utils.py,sha256=ajRIXfdTF67dBVahQCXW-rSv6gJpUMPt3QhJrWqArTQ,2175
@@ -272,14 +275,14 @@ nucliadb/search/search/chat/query.py,sha256=AhOPMf68p2BRjKz7CdkcUIDMANtxr00oGt42
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=sVl9gRNzhE-s480LBBVkiXzNRbKhYRQN5F3it5tNNp8,939
 nucliadb/search/search/query_parser/fetcher.py,sha256=0Eg_7x9BaAQ1AuTK6NXQMUoGFAXIZiMRurR32tydeNM,17198
-nucliadb/search/search/query_parser/models.py,sha256=kAslqX_-zaIdUpcpdNU2a5uQPQh7LC605qWLZ4aZ5T4,5064
+nucliadb/search/search/query_parser/models.py,sha256=ARJYv88bj-u1G4-JxLKNbaz8gwC5sIyeZx_9Aai3KKI,4302
 nucliadb/search/search/query_parser/old_filters.py,sha256=GsU3T3-WiSPvjucP7evHkshzAWZOli8qsuXChvWRCY0,9092
 nucliadb/search/search/query_parser/parsers/__init__.py,sha256=ySCNSdbesLXGZyR88919njulA6UE10_3PhqMG_Yj1o4,1034
-nucliadb/search/search/query_parser/parsers/ask.py,sha256=mc4Rx7jLM101tRZYOxEQC-txO5C-fDJid_oMAtZTRug,2809
-nucliadb/search/search/query_parser/parsers/catalog.py,sha256=JuDiBL2wdjAuEFEPo0e2nQ4VqWjF3FXakT0ziZk3Oes,7495
+nucliadb/search/search/query_parser/parsers/ask.py,sha256=ySa3lBhUuammIchJvj7xodeGIYGkR0uyLnHuOtLfWE8,2810
+nucliadb/search/search/query_parser/parsers/catalog.py,sha256=DErVfWJ9a_F5a8Qbht1WY4Lm8r3ui9Y3C7oJMOOZOng,7474
 nucliadb/search/search/query_parser/parsers/common.py,sha256=jbQweWVufngbobr99qpHh1iiaGICOC6-e9AV33x0-Gk,6594
 nucliadb/search/search/query_parser/parsers/find.py,sha256=4xQwa0BxNucenUrW_iZ2jCGd15Dm4AKS_B91BE8sDi4,12773
-nucliadb/search/search/query_parser/parsers/graph.py,sha256=zyqdUg5Afmhb2_-hvj9FUCaoLh026MUP1fgY2j-lD7c,9385
+nucliadb/search/search/query_parser/parsers/graph.py,sha256=s7nCB7ly_4BZWds-8zce1R-r2fHSiEhAK8P-eL14wTk,9390
 nucliadb/search/search/query_parser/parsers/search.py,sha256=78KSJ9t3I7nFVY2Qk2fMw2P1RHUdGRsWzBf59FdAeTA,10503
 nucliadb/search/search/query_parser/parsers/unit_retrieval.py,sha256=xiOQ7_X6MkcZs3W_0DjdVfyk-G1AY6RBx3oG5hsq7ig,11455
 nucliadb/standalone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -376,8 +379,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vu1BGg4VqJ_x2P1u2BxqPKlSfw5orT_a3R-Ln5oPUpU,8483
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.7.2.post4889.dist-info/METADATA,sha256=n5UTXqF3fg0n3X1orrwnWa9yaPykhsM2ntFhyJm-Z8E,4158
-nucliadb-6.7.2.post4889.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.7.2.post4889.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.7.2.post4889.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.7.2.post4889.dist-info/RECORD,,
+nucliadb-6.7.2.post4908.dist-info/METADATA,sha256=-hdK9V9pdxcHO7mkQqqnfX3XYOR7RqrzKlgdVL4Tj7k,4158
+nucliadb-6.7.2.post4908.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.7.2.post4908.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.7.2.post4908.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.7.2.post4908.dist-info/RECORD,,

nucliadb/ingest/orm/processor/pgcatalog.py

@@ -1,129 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-from typing import cast
-
-from nidx_protos.noderesources_pb2 import Resource as IndexMessage
-
-from nucliadb.common.maindb.driver import Transaction
-from nucliadb.common.maindb.pg import PGDriver, PGTransaction
-from nucliadb.common.maindb.utils import get_driver
-from nucliadb_telemetry import metrics
-
-from ..resource import Resource
-
-observer = metrics.Observer("pg_catalog_write", labels={"type": ""})
-
-
-def _pg_transaction(txn: Transaction) -> PGTransaction:
-    return cast(PGTransaction, txn)
-
-
-def pgcatalog_enabled(kbid):
-    return isinstance(get_driver(), PGDriver)
-
-
-def extract_facets(labels):
-    facets = set()
-    for label in labels:
-        parts = label.split("/")
-        facet = ""
-        for part in parts[1:]:
-            facet += f"/{part}"
-            facets.add(facet)
-    return facets
-
-
-@observer.wrap({"type": "update"})
-async def pgcatalog_update(txn: Transaction, kbid: str, resource: Resource, index_message: IndexMessage):
-    if not pgcatalog_enabled(kbid):
-        return
-
-    if resource.basic is None:
-        raise ValueError("Cannot index into the catalog a resource without basic metadata ")
-
-    created_at = resource.basic.created.ToDatetime()
-    modified_at = resource.basic.modified.ToDatetime()
-    if modified_at < created_at:
-        modified_at = created_at
-
-    async with _pg_transaction(txn).connection.cursor() as cur:
-        # Do not index canceled labels
-        cancelled_labels = {
-            f"/l/{clf.labelset}/{clf.label}"
-            for clf in resource.basic.usermetadata.classifications
-            if clf.cancelled_by_user
-        }
-
-        # Labels from the resource and classification labels from each field
-        labels = [label for label in index_message.labels]
-        for classification in resource.basic.computedmetadata.field_classifications:
-            for clf in classification.classifications:
-                label = f"/l/{clf.labelset}/{clf.label}"
-                if label not in cancelled_labels:
-                    labels.append(label)
-
-        await cur.execute(
-            """
-            INSERT INTO catalog
-            (kbid, rid, title, created_at, modified_at, labels, slug)
-            VALUES
-            (%(kbid)s, %(rid)s, %(title)s, %(created_at)s, %(modified_at)s, %(labels)s, %(slug)s)
-            ON CONFLICT (kbid, rid) DO UPDATE SET
-            title = excluded.title,
-            created_at = excluded.created_at,
-            modified_at = excluded.modified_at,
-            labels = excluded.labels,
-            slug = excluded.slug""",
-            {
-                "kbid": resource.kb.kbid,
-                "rid": resource.uuid,
-                "title": resource.basic.title,
-                "created_at": created_at,
-                "modified_at": modified_at,
-                "labels": labels,
-                "slug": resource.basic.slug,
-            },
-        )
-        await cur.execute(
-            "DELETE FROM catalog_facets WHERE kbid = %(kbid)s AND rid = %(rid)s",
-            {
-                "kbid": resource.kb.kbid,
-                "rid": resource.uuid,
-            },
-        )
-        await cur.execute(
-            "INSERT INTO catalog_facets (kbid, rid, facet) SELECT %(kbid)s AS kbid, %(rid)s AS rid, unnest(%(facets)s::text[]) AS facet",
-            {
-                "kbid": resource.kb.kbid,
-                "rid": resource.uuid,
-                "facets": list(extract_facets(labels)),
-            },
-        )
-
-
-@observer.wrap({"type": "delete"})
-async def pgcatalog_delete(txn: Transaction, kbid: str, rid: str):
-    if not pgcatalog_enabled(kbid):
-        return
-    async with _pg_transaction(txn).connection.cursor() as cur:
-        await cur.execute(
-            "DELETE FROM catalog where kbid = %(kbid)s AND rid = %(rid)s", {"kbid": kbid, "rid": rid}
-        )