nucliadb 6.7.2.post4886__py3-none-any.whl → 6.7.2.post4908__py3-none-any.whl

migrations/0023_backfill_pg_catalog.py

@@ -28,9 +28,10 @@ import logging
 from typing import cast
 
 from nucliadb.common import datamanagers
+from nucliadb.common.catalog import catalog_update, get_catalog
+from nucliadb.common.catalog.pg import PGCatalog
 from nucliadb.common.maindb.pg import PGDriver, PGTransaction
 from nucliadb.ingest.orm.index_message import get_resource_index_message
-from nucliadb.ingest.orm.processor.pgcatalog import pgcatalog_update
 from nucliadb.migrator.context import ExecutionContext
 
 logger = logging.getLogger(__name__)
@@ -43,6 +44,9 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
     if not isinstance(context.kv_driver, PGDriver):
         return
 
+    if not isinstance(get_catalog(), PGCatalog):
+        return
+
     BATCH_SIZE = 100
     async with context.kv_driver.rw_transaction() as txn:
         txn = cast(PGTransaction, txn)
@@ -75,7 +79,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
                         continue
 
                     index_message = await get_resource_index_message(resource, reindex=False)
-                    await pgcatalog_update(txn, kbid, resource, index_message)
+                    await catalog_update(txn, kbid, resource, index_message)
 
                 await txn.commit()
                 continue_sql = f"AND key > '/kbs/{kbid}/r/{rid}'"

migrations/0028_extracted_vectors_reference.py

@@ -39,7 +39,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
     async with datamanagers.with_rw_transaction() as txn:
         vectorsets = [vs async for (_vid, vs) in datamanagers.vectorsets.iter(txn, kbid=kbid)]
 
-        if len(vectorsets) == 0:  # pragma: nocover
+        if len(vectorsets) == 0:  # pragma: no cover
             # should never happen, everyone should have at least one
             logger.warning(f"KB has no vectorsets!", extra={"kbid": kbid})
             return

migrations/0038_backfill_catalog_field_labels.py

@@ -28,9 +28,10 @@ import logging
 from typing import cast
 
 from nucliadb.common import datamanagers
+from nucliadb.common.catalog import catalog_update, get_catalog
+from nucliadb.common.catalog.pg import PGCatalog
 from nucliadb.common.maindb.pg import PGDriver, PGTransaction
 from nucliadb.ingest.orm.index_message import get_resource_index_message
-from nucliadb.ingest.orm.processor.pgcatalog import pgcatalog_update
 from nucliadb.migrator.context import ExecutionContext
 from nucliadb_protos import resources_pb2
 
@@ -44,6 +45,9 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
     if not isinstance(context.kv_driver, PGDriver):
         return
 
+    if not isinstance(get_catalog(), PGCatalog):
+        return
+
     BATCH_SIZE = 100
     async with context.kv_driver.rw_transaction() as txn:
         txn = cast(PGTransaction, txn)
@@ -84,7 +88,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
                         continue
 
                     index_message = await get_resource_index_message(resource, reindex=False)
-                    await pgcatalog_update(txn, kbid, resource, index_message)
+                    await catalog_update(txn, kbid, resource, index_message)
 
                 if to_index:
                     await txn.commit()

nucliadb/common/catalog/__init__.py

@@ -0,0 +1,79 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from nidx_protos.noderesources_pb2 import Resource as IndexMessage
+
+from nucliadb.common.catalog.dummy import DummyCatalog
+from nucliadb.common.catalog.interface import Catalog, CatalogQuery
+from nucliadb.common.catalog.pg import PGCatalog
+from nucliadb.common.catalog.utils import build_catalog_resource_data
+from nucliadb.common.maindb.driver import Transaction
+from nucliadb.ingest.orm.resource import Resource
+from nucliadb.ingest.settings import CatalogConfig, settings
+from nucliadb_models.search import CatalogFacetsRequest, Resources
+from nucliadb_utils.exceptions import ConfigurationError
+
+
+def get_catalog() -> Catalog:
+    if settings.catalog == CatalogConfig.UNSET:
+        return DummyCatalog()
+    elif settings.catalog == CatalogConfig.PG:
+        return PGCatalog()
+    else:
+        raise ConfigurationError(f"Unknown catalog configuration: {settings.catalog}")
+
+
+async def catalog_update(txn: Transaction, kbid: str, resource: Resource, index_message: IndexMessage):
+    catalog = get_catalog()
+    resource_data = build_catalog_resource_data(resource, index_message)
+    await catalog.update(txn, kbid, resource.uuid, resource_data)
+
+
+async def catalog_delete(txn: Transaction, kbid: str, rid: str):
+    catalog = get_catalog()
+    await catalog.delete(txn, kbid, rid)
+
+
+async def catalog_search(query: CatalogQuery) -> Resources:
+    catalog = get_catalog()
+    return await catalog.search(query)
+
+
+async def catalog_facets(kbid: str, request: CatalogFacetsRequest) -> dict[str, int]:
+    catalog = get_catalog()
+    return await catalog.facets(kbid, request)

nucliadb/common/catalog/dummy.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from nucliadb.common.catalog.interface import Catalog, CatalogQuery, CatalogResourceData
+from nucliadb.common.maindb.driver import Transaction
+from nucliadb_models.search import CatalogFacetsRequest, Resources
+
+
+class DummyCatalog(Catalog):
+    async def update(self, txn: Transaction, kbid: str, rid: str, data: CatalogResourceData):
+        return
+
+    async def delete(self, txn: Transaction, kbid: str, rid: str):
+        return
+
+    async def search(self, query: CatalogQuery) -> Resources:
+        return Resources(results=[], min_score=0.0)
+
+    async def facets(self, kbid: str, request: CatalogFacetsRequest) -> dict[str, int]:
+        return {}

nucliadb/common/catalog/interface.py

@@ -0,0 +1,85 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+import abc
+import datetime
+from dataclasses import dataclass
+from typing import Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+
+from nucliadb.common.maindb.driver import Transaction
+from nucliadb_models import search as search_models
+from nucliadb_models.search import CatalogFacetsRequest, Resources
+
+
+class CatalogResourceData(BaseModel):
+    """
+    Data extracted from a resource to be indexed in the catalog
+    """
+
+    title: str = Field(description="Resource title")
+    created_at: datetime.datetime = Field(description="Resource creation date")
+    modified_at: datetime.datetime = Field(description="Resource last modification date")
+    labels: list[str] = Field(
+        description="Resource labels. This includes labels at the resource level and all classification labels of its fields"
+    )
+    slug: str = Field(description="Resource slug")
+
+
+@dataclass
+class CatalogExpression:
+    @dataclass
+    class Date:
+        field: Union[Literal["created_at"], Literal["modified_at"]]
+        since: Optional[datetime.datetime]
+        until: Optional[datetime.datetime]
+
+    bool_and: Optional[list["CatalogExpression"]] = None
+    bool_or: Optional[list["CatalogExpression"]] = None
+    bool_not: Optional["CatalogExpression"] = None
+    date: Optional[Date] = None
+    facet: Optional[str] = None
+    resource_id: Optional[str] = None
+
+
+class CatalogQuery(BaseModel):
+    kbid: str
+    query: Optional[search_models.CatalogQuery] = Field(description="Full-text search query")
+    filters: Optional[CatalogExpression] = Field(description="Filters to apply to the search")
+    sort: search_models.SortOptions = Field(description="Sorting option")
+    faceted: list[str] = Field(description="List of facets to compute during the search")
+    page_size: int = Field(description="Used for pagination. Maximum page size is 100")
+    page_number: int = Field(description="Used for pagination. First page is 0")
+
+
+class Catalog(abc.ABC, metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    async def update(self, txn: Transaction, kbid: str, rid: str, data: CatalogResourceData): ...
+
+    @abc.abstractmethod
+    async def delete(self, txn: Transaction, kbid: str, rid: str): ...
+
+    @abc.abstractmethod
+    async def search(self, query: CatalogQuery) -> Resources: ...
+
+    @abc.abstractmethod
+    async def facets(self, kbid: str, request: CatalogFacetsRequest) -> dict[str, int]: ...