nucliadb 6.7.2.post4874__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

nucliadb/ingest/orm/processor/pgcatalog.py

@@ -1,129 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-from typing import cast
-
-from nidx_protos.noderesources_pb2 import Resource as IndexMessage
-
-from nucliadb.common.maindb.driver import Transaction
-from nucliadb.common.maindb.pg import PGDriver, PGTransaction
-from nucliadb.common.maindb.utils import get_driver
-from nucliadb_telemetry import metrics
-
-from ..resource import Resource
-
-observer = metrics.Observer("pg_catalog_write", labels={"type": ""})
-
-
-def _pg_transaction(txn: Transaction) -> PGTransaction:
-    return cast(PGTransaction, txn)
-
-
-def pgcatalog_enabled(kbid):
-    return isinstance(get_driver(), PGDriver)
-
-
-def extract_facets(labels):
-    facets = set()
-    for label in labels:
-        parts = label.split("/")
-        facet = ""
-        for part in parts[1:]:
-            facet += f"/{part}"
-            facets.add(facet)
-    return facets
-
-
-@observer.wrap({"type": "update"})
-async def pgcatalog_update(txn: Transaction, kbid: str, resource: Resource, index_message: IndexMessage):
-    if not pgcatalog_enabled(kbid):
-        return
-
-    if resource.basic is None:
-        raise ValueError("Cannot index into the catalog a resource without basic metadata ")
-
-    created_at = resource.basic.created.ToDatetime()
-    modified_at = resource.basic.modified.ToDatetime()
-    if modified_at < created_at:
-        modified_at = created_at
-
-    async with _pg_transaction(txn).connection.cursor() as cur:
-        # Do not index canceled labels
-        cancelled_labels = {
-            f"/l/{clf.labelset}/{clf.label}"
-            for clf in resource.basic.usermetadata.classifications
-            if clf.cancelled_by_user
-        }
-
-        # Labels from the resource and classification labels from each field
-        labels = [label for label in index_message.labels]
-        for classification in resource.basic.computedmetadata.field_classifications:
-            for clf in classification.classifications:
-                label = f"/l/{clf.labelset}/{clf.label}"
-                if label not in cancelled_labels:
-                    labels.append(label)
-
-        await cur.execute(
-            """
-            INSERT INTO catalog
-            (kbid, rid, title, created_at, modified_at, labels, slug)
-            VALUES
-            (%(kbid)s, %(rid)s, %(title)s, %(created_at)s, %(modified_at)s, %(labels)s, %(slug)s)
-            ON CONFLICT (kbid, rid) DO UPDATE SET
-            title = excluded.title,
-            created_at = excluded.created_at,
-            modified_at = excluded.modified_at,
-            labels = excluded.labels,
-            slug = excluded.slug""",
-            {
-                "kbid": resource.kb.kbid,
-                "rid": resource.uuid,
-                "title": resource.basic.title,
-                "created_at": created_at,
-                "modified_at": modified_at,
-                "labels": labels,
-                "slug": resource.basic.slug,
-            },
-        )
-        await cur.execute(
-            "DELETE FROM catalog_facets WHERE kbid = %(kbid)s AND rid = %(rid)s",
-            {
-                "kbid": resource.kb.kbid,
-                "rid": resource.uuid,
-            },
-        )
-        await cur.execute(
-            "INSERT INTO catalog_facets (kbid, rid, facet) SELECT %(kbid)s AS kbid, %(rid)s AS rid, unnest(%(facets)s::text[]) AS facet",
-            {
-                "kbid": resource.kb.kbid,
-                "rid": resource.uuid,
-                "facets": list(extract_facets(labels)),
-            },
-        )
-
-
-@observer.wrap({"type": "delete"})
-async def pgcatalog_delete(txn: Transaction, kbid: str, rid: str):
-    if not pgcatalog_enabled(kbid):
-        return
-    async with _pg_transaction(txn).connection.cursor() as cur:
-        await cur.execute(
-            "DELETE FROM catalog where kbid = %(kbid)s AND rid = %(rid)s", {"kbid": kbid, "rid": rid}
-        )

nucliadb/search/search/hydrator.py

@@ -1,197 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import asyncio
-import logging
-from contextlib import AsyncExitStack
-from typing import Optional
-
-from pydantic import BaseModel
-
-from nucliadb.common.external_index_providers.base import TextBlockMatch
-from nucliadb.common.ids import FieldId
-from nucliadb.common.maindb.utils import get_driver
-from nucliadb.ingest.serialize import managed_serialize
-from nucliadb.search.search import cache, paragraphs
-from nucliadb_models.common import FieldTypeName
-from nucliadb_models.resource import ExtractedDataTypeName, Resource
-from nucliadb_models.search import (
-    FindParagraph,
-    ResourceProperties,
-)
-from nucliadb_telemetry.metrics import Observer
-from nucliadb_utils import const
-from nucliadb_utils.asyncio_utils import ConcurrentRunner
-from nucliadb_utils.utilities import has_feature
-
-logger = logging.getLogger(__name__)
-
-hydrator_observer = Observer("hydrator", labels={"type": ""})
-
-
-class ResourceHydrationOptions(BaseModel):
-    """
-    Options for hydrating resources.
-    """
-
-    show: list[ResourceProperties] = []
-    extracted: list[ExtractedDataTypeName] = []
-    field_type_filter: list[FieldTypeName] = []
-
-
-class TextBlockHydrationOptions(BaseModel):
-    """
-    Options for hydrating text blocks (aka paragraphs).
-    """
-
-    # whether to highlight the text block with `<mark>...</mark>` tags or not
-    highlight: bool = False
-
-    # list of exact matches to highlight
-    ematches: Optional[list[str]] = None
-
-    # If true, only hydrate the text block if its text is not already populated
-    only_hydrate_empty: bool = False
-
-
-@hydrator_observer.wrap({"type": "resource_text"})
-async def hydrate_resource_text(
-    kbid: str, rid: str, *, max_concurrent_tasks: int
-) -> list[tuple[FieldId, str]]:
-    resource = await cache.get_resource(kbid, rid)
-    if resource is None:  # pragma: no cover
-        return []
-
-    # Schedule the extraction of the text of each field in the resource
-    async with get_driver().ro_transaction() as txn:
-        resource.txn = txn
-        runner = ConcurrentRunner(max_tasks=max_concurrent_tasks)
-        for field_type, field_key in await resource.get_fields(force=True):
-            field_id = FieldId.from_pb(rid, field_type, field_key)
-            runner.schedule(hydrate_field_text(kbid, field_id))
-
-        # Include the summary aswell
-        runner.schedule(hydrate_field_text(kbid, FieldId(rid=rid, type="a", key="summary")))
-
-        # Wait for the results
-        field_extracted_texts = await runner.wait()
-
-    return [text for text in field_extracted_texts if text is not None]
-
-
-@hydrator_observer.wrap({"type": "resource_metadata"})
-async def hydrate_resource_metadata(
-    kbid: str,
-    resource_id: str,
-    options: ResourceHydrationOptions,
-    *,
-    concurrency_control: Optional[asyncio.Semaphore] = None,
-    service_name: Optional[str] = None,
-) -> Optional[Resource]:
-    """Fetch resource metadata and return it serialized."""
-    show = options.show
-    extracted = options.extracted
-
-    if ResourceProperties.EXTRACTED in show and has_feature(
-        const.Features.IGNORE_EXTRACTED_IN_SEARCH, context={"kbid": kbid}, default=False
-    ):
-        # Returning extracted metadata in search results is deprecated and this flag
-        # will be set to True for all KBs in the future.
-        show.remove(ResourceProperties.EXTRACTED)
-        extracted = []
-
-    async with AsyncExitStack() as stack:
-        if concurrency_control is not None:
-            await stack.enter_async_context(concurrency_control)
-
-        async with get_driver().ro_transaction() as ro_txn:
-            serialized_resource = await managed_serialize(
-                txn=ro_txn,
-                kbid=kbid,
-                rid=resource_id,
-                show=show,
-                field_type_filter=options.field_type_filter,
-                extracted=extracted,
-                service_name=service_name,
-            )
-            if serialized_resource is None:
-                logger.warning(
-                    "Resource not found in database", extra={"kbid": kbid, "rid": resource_id}
-                )
-    return serialized_resource
-
-
-@hydrator_observer.wrap({"type": "field_text"})
-async def hydrate_field_text(
-    kbid: str,
-    field_id: FieldId,
-) -> Optional[tuple[FieldId, str]]:
-    extracted_text_pb = await cache.get_extracted_text_from_field_id(kbid, field_id)
-    if extracted_text_pb is None:  # pragma: no cover
-        return None
-
-    if field_id.subfield_id:
-        return field_id, extracted_text_pb.split_text[field_id.subfield_id]
-    else:
-        return field_id, extracted_text_pb.text
-
-
-@hydrator_observer.wrap({"type": "text_block"})
-async def hydrate_text_block(
-    kbid: str,
-    text_block: TextBlockMatch,
-    options: TextBlockHydrationOptions,
-    *,
-    concurrency_control: Optional[asyncio.Semaphore] = None,
-) -> TextBlockMatch:
-    """Given a `text_block`, fetch its corresponding text, modify and return the
-    `text_block` object.
-
-    """
-    if options.only_hydrate_empty and text_block.text:
-        return text_block
-    async with AsyncExitStack() as stack:
-        if concurrency_control is not None:
-            await stack.enter_async_context(concurrency_control)
-
-        text_block.text = await paragraphs.get_paragraph_text(
-            kbid=kbid,
-            paragraph_id=text_block.paragraph_id,
-            highlight=options.highlight,
-            matches=[],  # TODO: this was never implemented
-            ematches=options.ematches,
-        )
-    return text_block
-
-
-def text_block_to_find_paragraph(text_block: TextBlockMatch) -> FindParagraph:
-    return FindParagraph(
-        id=text_block.paragraph_id.full(),
-        text=text_block.text or "",
-        score=text_block.score,
-        score_type=text_block.score_type,
-        order=text_block.order,
-        labels=text_block.paragraph_labels,
-        fuzzy_result=text_block.fuzzy_search,
-        is_a_table=text_block.is_a_table,
-        reference=text_block.representation_file,
-        page_with_visual=text_block.page_with_visual,
-        position=text_block.position,
-        relevant_relations=text_block.relevant_relations,
-    )