nucliadb 6.3.7.post4091__py3-none-any.whl → 6.3.7.post4116__py3-none-any.whl

nucliadb/search/search/query_parser/parsers/find.py

@@ -53,7 +53,8 @@ from .common import (
     parse_keyword_query,
     parse_semantic_query,
     parse_top_k,
-    validate_base_request,
+    should_disable_vector_search,
+    validate_query_syntax,
 )
 
 
@@ -93,7 +94,7 @@ class _FindParser:
         self._top_k: Optional[int] = None
 
     async def parse(self) -> UnitRetrieval:
-        validate_base_request(self.item)
+        self._validate_request()
 
         self._top_k = parse_top_k(self.item)
 
@@ -101,13 +102,13 @@ class _FindParser:
 
         self._query = Query()
 
-        if search_models.SearchOptions.KEYWORD in self.item.features:
+        if search_models.FindOptions.KEYWORD in self.item.features:
             self._query.keyword = await parse_keyword_query(self.item, fetcher=self.fetcher)
 
-        if search_models.SearchOptions.SEMANTIC in self.item.features:
+        if search_models.FindOptions.SEMANTIC in self.item.features:
             self._query.semantic = await parse_semantic_query(self.item, fetcher=self.fetcher)
 
-        if search_models.SearchOptions.RELATIONS in self.item.features:
+        if search_models.FindOptions.RELATIONS in self.item.features:
             self._query.relation = await self._parse_relation_query()
 
         # TODO: graph search
@@ -130,13 +131,35 @@ class _FindParser:
         if isinstance(reranker, PredictReranker):
             rank_fusion.window = max(rank_fusion.window, reranker.window)
 
-        return UnitRetrieval(
+        retrieval = UnitRetrieval(
             query=self._query,
             top_k=self._top_k,
             filters=filters,
             rank_fusion=rank_fusion,
             reranker=reranker,
         )
+        return retrieval
+
+    def _validate_request(self):
+        validate_query_syntax(self.item.query)
+
+        # synonyms are not compatible with vector/graph search
+        if (
+            self.item.with_synonyms
+            and self.item.query
+            and (
+                search_models.FindOptions.SEMANTIC in self.item.features
+                or search_models.FindOptions.RELATIONS in self.item.features
+            )
+        ):
+            raise InvalidQueryError(
+                "synonyms",
+                "Search with custom synonyms is only supported on paragraph and document search",
+            )
+
+        if search_models.FindOptions.SEMANTIC in self.item.features:
+            if should_disable_vector_search(self.item):
+                self.item.features.remove(search_models.FindOptions.SEMANTIC)
 
     async def _parse_relation_query(self) -> RelationQuery:
         detected_entities = await self._get_detected_entities()
@@ -147,7 +170,7 @@ class _FindParser:
         deleted_entities = meta_cache.deleted_entities
 
         return RelationQuery(
-            detected_entities=detected_entities,
+            entry_points=detected_entities,
             deleted_entity_groups=deleted_entity_groups,
             deleted_entities=deleted_entities,
         )
@@ -220,7 +243,7 @@ class _FindParser:
         autofilter = None
         if self.item.autofilter:
             if self._query.relation is not None:
-                autofilter = self._query.relation.detected_entities
+                autofilter = self._query.relation.entry_points
             else:
                 autofilter = await self._get_detected_entities()
 

nucliadb/search/search/query_parser/parsers/search.py

@@ -26,10 +26,8 @@ from nucliadb.search.search.query_parser.fetcher import Fetcher
 from nucliadb.search.search.query_parser.filter_expression import parse_expression
 from nucliadb.search.search.query_parser.models import (
     Filters,
-    NoopReranker,
     ParsedQuery,
     Query,
-    RankFusion,
     RelationQuery,
     UnitRetrieval,
     _TextQuery,
@@ -46,7 +44,13 @@ from nucliadb_models.search import (
 )
 from nucliadb_protos import nodereader_pb2, utils_pb2
 
-from .common import parse_keyword_query, parse_semantic_query, parse_top_k, validate_base_request
+from .common import (
+    parse_keyword_query,
+    parse_semantic_query,
+    parse_top_k,
+    should_disable_vector_search,
+    validate_query_syntax,
+)
 
 INDEX_SORTABLE_FIELDS = [
     SortField.CREATED,
@@ -87,7 +91,7 @@ class _SearchParser:
         self._top_k: Optional[int] = None
 
     async def parse(self) -> UnitRetrieval:
-        validate_base_request(self.item)
+        self._validate_request()
 
         self._top_k = parse_top_k(self.item)
 
@@ -113,14 +117,33 @@ class _SearchParser:
 
         filters = await self._parse_filters()
 
-        return UnitRetrieval(
+        retrieval = UnitRetrieval(
             query=self._query,
             top_k=self._top_k,
             filters=filters,
-            # TODO: this should be in a post retrieval step
-            rank_fusion=RankFusion(window=self._top_k),
-            reranker=NoopReranker(),
         )
+        return retrieval
+
+    def _validate_request(self):
+        validate_query_syntax(self.item.query)
+
+        # synonyms are not compatible with vector/graph search
+        if (
+            self.item.with_synonyms
+            and self.item.query
+            and (
+                search_models.SearchOptions.SEMANTIC in self.item.features
+                or search_models.SearchOptions.RELATIONS in self.item.features
+            )
+        ):
+            raise InvalidQueryError(
+                "synonyms",
+                "Search with custom synonyms is only supported on paragraph and document search",
+            )
+
+        if search_models.SearchOptions.SEMANTIC in self.item.features:
+            if should_disable_vector_search(self.item):
+                self.item.features.remove(search_models.SearchOptions.SEMANTIC)
 
     async def _parse_text_query(self) -> _TextQuery:
         assert self._top_k is not None, "top_k must be parsed before text query"
@@ -140,7 +163,7 @@ class _SearchParser:
         meta_cache = await self.fetcher.get_entities_meta_cache()
         deleted_entities = meta_cache.deleted_entities
         return RelationQuery(
-            detected_entities=detected_entities,
+            entry_points=detected_entities,
             deleted_entity_groups=deleted_entity_groups,
             deleted_entities=deleted_entities,
         )
@@ -231,7 +254,7 @@ class _SearchParser:
         autofilter = None
         if self.item.autofilter:
             if self._query.relation is not None:
-                autofilter = self._query.relation.detected_entities
+                autofilter = self._query.relation.entry_points
             else:
                 autofilter = await self._get_detected_entities()
 

nucliadb/search/search/query_parser/parsers/unit_retrieval.py

@@ -19,154 +19,246 @@
 #
 from typing import Optional
 
-from nucliadb.search.search.filters import (
-    translate_label,
-)
-from nucliadb.search.search.metrics import (
-    node_features,
-    query_parser_observer,
-)
-from nucliadb.search.search.query import (
-    apply_entities_filter,
-    get_sort_field_proto,
-)
+from nucliadb.search.search.filters import translate_label
+from nucliadb.search.search.metrics import node_features, query_parser_observer
+from nucliadb.search.search.query import apply_entities_filter, get_sort_field_proto
 from nucliadb.search.search.query_parser.filter_expression import add_and_expression
 from nucliadb.search.search.query_parser.models import ParsedQuery, PredictReranker, UnitRetrieval
 from nucliadb_models.labels import LABEL_HIDDEN, translate_system_to_alias_label
-from nucliadb_models.search import (
-    SortOrderMap,
-)
+from nucliadb_models.search import SortOrderMap
 from nucliadb_protos import nodereader_pb2, utils_pb2
 from nucliadb_protos.nodereader_pb2 import SearchRequest
 
 
 @query_parser_observer.wrap({"type": "convert_retrieval_to_proto"})
-async def convert_retrieval_to_proto(
+async def legacy_convert_retrieval_to_proto(
     parsed: ParsedQuery,
 ) -> tuple[SearchRequest, bool, list[str], Optional[str]]:
-    request = SearchRequest()
+    converter = _Converter(parsed.retrieval)
+    request = converter.into_search_request()
 
-    ## queries
+    # XXX: legacy values that were returned by QueryParser but not always
+    # needed. We should find a better abstraction
 
-    if parsed.retrieval.query.keyword and parsed.retrieval.query.fulltext:
-        assert parsed.retrieval.query.keyword == parsed.retrieval.query.fulltext, (
-            "search proto doesn't support different queries for fulltext and keyword search"
-        )
+    incomplete = is_incomplete(parsed.retrieval)
+    autofilter = converter._autofilter
+
+    rephrased_query = None
+    if parsed.retrieval.query.semantic:
+        rephrased_query = await parsed.fetcher.get_rephrased_query()
 
-    if parsed.retrieval.query.fulltext:
-        request.document = True
-        node_features.inc({"type": "documents"})
-    if parsed.retrieval.query.keyword:
-        request.paragraph = True
-        node_features.inc({"type": "paragraphs"})
+    return request, incomplete, autofilter, rephrased_query
 
-    text_query = parsed.retrieval.query.keyword or parsed.retrieval.query.fulltext
-    if text_query is not None:
-        request.min_score_bm25 = text_query.min_score
+
+@query_parser_observer.wrap({"type": "convert_retrieval_to_proto"})
+def convert_retrieval_to_proto(retrieval: UnitRetrieval) -> SearchRequest:
+    converter = _Converter(retrieval)
+    request = converter.into_search_request()
+    return request
+
+
+class _Converter:
+    def __init__(self, retrieval: UnitRetrieval):
+        self.req = nodereader_pb2.SearchRequest()
+        self.retrieval = retrieval
+
+        self._autofilter: list[str] = []
+
+    def into_search_request(self) -> nodereader_pb2.SearchRequest:
+        """Generate a SearchRequest proto from a retrieval operation."""
+        self._apply_text_queries()
+        self._apply_semantic_query()
+        self._apply_relation_query()
+        self._apply_filters()
+        self._apply_top_k()
+        return self.req
+
+    def _apply_text_queries(self):
+        text_query = self.retrieval.query.keyword or self.retrieval.query.fulltext
+        if text_query is None:
+            return
+
+        if self.retrieval.query.keyword and self.retrieval.query.fulltext:
+            assert self.retrieval.query.keyword == self.retrieval.query.fulltext, (
+                "search proto doesn't support different queries for fulltext and keyword search"
+            )
+
+        if self.retrieval.query.fulltext:
+            self.req.document = True
+            node_features.inc({"type": "documents"})
+        if self.retrieval.query.keyword:
+            self.req.paragraph = True
+            node_features.inc({"type": "paragraphs"})
+
+        self.req.min_score_bm25 = text_query.min_score
 
         if text_query.is_synonyms_query:
-            request.advanced_query = text_query.query
+            self.req.advanced_query = text_query.query
         else:
-            request.body = text_query.query
+            self.req.body = text_query.query
 
         # sort order
         sort_field = get_sort_field_proto(text_query.order_by)
         if sort_field is not None:
-            request.order.sort_by = sort_field
-            request.order.type = SortOrderMap[text_query.sort]  # type: ignore
+            self.req.order.sort_by = sort_field
+            self.req.order.type = SortOrderMap[text_query.sort]  # type: ignore
+
+    def _apply_semantic_query(self):
+        if self.retrieval.query.semantic is None:
+            return
 
-    if parsed.retrieval.query.semantic:
         node_features.inc({"type": "vectors"})
 
-        request.min_score_semantic = parsed.retrieval.query.semantic.min_score
+        self.req.min_score_semantic = self.retrieval.query.semantic.min_score
 
-        query_vector = parsed.retrieval.query.semantic.query
+        query_vector = self.retrieval.query.semantic.query
         if query_vector is not None:
-            request.vectorset = parsed.retrieval.query.semantic.vectorset
-            request.vector.extend(query_vector)
+            self.req.vectorset = self.retrieval.query.semantic.vectorset
+            self.req.vector.extend(query_vector)
 
-    if parsed.retrieval.query.relation:
-        node_features.inc({"type": "relations"})
+    def _apply_relation_query(self):
+        """Relation queries are the legacy way to query the knowledge graph.
+        Given a set of entry points and some subtypes and entities to exclude
+        from search, it'd find the distance 1 neighbours (BFS)."""
 
-        request.relation_subgraph.entry_points.extend(parsed.retrieval.query.relation.detected_entities)
-        request.relation_subgraph.depth = 1
-        request.relation_subgraph.deleted_groups.extend(
-            parsed.retrieval.query.relation.deleted_entity_groups
-        )
-        for group_id, deleted_entities in parsed.retrieval.query.relation.deleted_entities.items():
-            request.relation_subgraph.deleted_entities.append(
-                nodereader_pb2.EntitiesSubgraphRequest.DeletedEntities(
-                    node_subtype=group_id, node_values=deleted_entities
-                )
-            )
+        if self.retrieval.query.relation is None:
+            return
 
-    # filters
-
-    request.with_duplicates = parsed.retrieval.filters.with_duplicates
-
-    request.faceted.labels.extend([translate_label(facet) for facet in parsed.retrieval.filters.facets])
-
-    if (
-        parsed.retrieval.filters.security is not None
-        and len(parsed.retrieval.filters.security.groups) > 0
-    ):
-        security_pb = utils_pb2.Security()
-        for group_id in parsed.retrieval.filters.security.groups:
-            if group_id not in security_pb.access_groups:
-                security_pb.access_groups.append(group_id)
-        request.security.CopyFrom(security_pb)
-
-    if parsed.retrieval.filters.field_expression:
-        request.field_filter.CopyFrom(parsed.retrieval.filters.field_expression)
-    if parsed.retrieval.filters.paragraph_expression:
-        request.paragraph_filter.CopyFrom(parsed.retrieval.filters.paragraph_expression)
-    request.filter_operator = parsed.retrieval.filters.filter_expression_operator
-
-    autofilter = []
-    if parsed.retrieval.filters.autofilter:
-        entity_filters = apply_entities_filter(request, parsed.retrieval.filters.autofilter)
-        autofilter.extend([translate_system_to_alias_label(e) for e in entity_filters])
-
-    if parsed.retrieval.filters.hidden is not None:
-        expr = nodereader_pb2.FilterExpression()
-        if parsed.retrieval.filters.hidden:
-            expr.facet.facet = LABEL_HIDDEN
-        else:
-            expr.bool_not.facet.facet = LABEL_HIDDEN
-
-        add_and_expression(request.field_filter, expr)
-
-    # top_k
-
-    # Adjust requested page size depending on rank fusion and reranking algorithms.
-    #
-    # Some rerankers want more results than the requested by the user so
-    # reranking can have more choices.
-
-    rank_fusion_window = 0
-    if parsed.retrieval.rank_fusion is not None:
-        rank_fusion_window = parsed.retrieval.rank_fusion.window
-
-    reranker_window = 0
-    if parsed.retrieval.reranker is not None and isinstance(parsed.retrieval.reranker, PredictReranker):
-        reranker_window = parsed.retrieval.reranker.window
-
-    request.result_per_page = max(
-        request.result_per_page,
-        rank_fusion_window,
-        reranker_window,
-    )
+        node_features.inc({"type": "relations"})
 
-    # XXX: legacy values that were returned by QueryParser but not always
-    # needed. We should find a better abstraction
+        # Entry points are source or target nodes we want to search for. We want
+        # any undirected path containing any entry point
+        entry_points_queries = []
+        for entry_point in self.retrieval.query.relation.entry_points:
+            q = nodereader_pb2.GraphQuery.PathQuery()
+            if entry_point.value:
+                q.path.source.value = entry_point.value
+            q.path.source.node_type = entry_point.ntype
+            if entry_point.subtype:
+                q.path.source.node_subtype = entry_point.subtype
+            q.path.undirected = True
+            entry_points_queries.append(q)
+
+        # A query can specifiy nodes marked as deleted in the db (but not
+        # removed from the index). We want to exclude any path containing any of
+        # those nodes.
+        #
+        # The request groups values per subtype (to optimize request size) but,
+        # as we don't support OR at node value level, we'll split them.
+        deleted_nodes_queries = []
+        for subtype, deleted_entities in self.retrieval.query.relation.deleted_entities.items():
+            if len(deleted_entities) == 0:
+                continue
+            for deleted_entity_value in deleted_entities:
+                q = nodereader_pb2.GraphQuery.PathQuery()
+                q.path.source.value = deleted_entity_value
+                q.path.source.node_subtype = subtype
+                q.path.undirected = True
+                deleted_nodes_queries.append(q)
+
+        # Subtypes can also be marked as deleted in the db (but kept in the
+        # index). We also want to exclude any triplet containg a node with such
+        # subtypes
+        excluded_subtypes_queries = []
+        for deleted_subtype in self.retrieval.query.relation.deleted_entity_groups:
+            q = nodereader_pb2.GraphQuery.PathQuery()
+            q.path.source.node_subtype = deleted_subtype
+            q.path.undirected = True
+            excluded_subtypes_queries.append(q)
+
+        subqueries = []
+
+        if len(entry_points_queries) > 0:
+            if len(entry_points_queries) == 1:
+                q = entry_points_queries[0]
+            else:
+                q = nodereader_pb2.GraphQuery.PathQuery()
+                q.bool_or.operands.extend(entry_points_queries)
+            subqueries.append(q)
+
+        if len(deleted_nodes_queries) > 0:
+            q = nodereader_pb2.GraphQuery.PathQuery()
+            if len(deleted_nodes_queries) == 1:
+                q.bool_not.CopyFrom(deleted_nodes_queries[0])
+            else:
+                q.bool_not.bool_or.operands.extend(deleted_nodes_queries)
+            subqueries.append(q)
+
+        if len(excluded_subtypes_queries) > 0:
+            q = nodereader_pb2.GraphQuery.PathQuery()
+            if len(excluded_subtypes_queries) == 1:
+                q.bool_not.CopyFrom(excluded_subtypes_queries[0])
+            else:
+                q.bool_not.bool_or.operands.extend(excluded_subtypes_queries)
+            subqueries.append(q)
+
+        if len(subqueries) == 0:
+            # don't set anything, no graph query
+            pass
+        elif len(subqueries) == 1:
+            q = subqueries[0]
+            self.req.graph_search.query.path.CopyFrom(q)
+        else:
+            self.req.graph_search.query.path.bool_and.operands.extend(subqueries)
 
-    incomplete = is_incomplete(parsed.retrieval)
+    def _apply_filters(self):
+        self.req.with_duplicates = self.retrieval.filters.with_duplicates
 
-    rephrased_query = None
-    if parsed.retrieval.query.semantic:
-        rephrased_query = await parsed.fetcher.get_rephrased_query()
+        self.req.faceted.labels.extend(
+            [translate_label(facet) for facet in self.retrieval.filters.facets]
+        )
 
-    return request, incomplete, autofilter, rephrased_query
+        if (
+            self.retrieval.filters.security is not None
+            and len(self.retrieval.filters.security.groups) > 0
+        ):
+            security_pb = utils_pb2.Security()
+            for group_id in self.retrieval.filters.security.groups:
+                if group_id not in security_pb.access_groups:
+                    security_pb.access_groups.append(group_id)
+            self.req.security.CopyFrom(security_pb)
+
+        if self.retrieval.filters.field_expression:
+            self.req.field_filter.CopyFrom(self.retrieval.filters.field_expression)
+        if self.retrieval.filters.paragraph_expression:
+            self.req.paragraph_filter.CopyFrom(self.retrieval.filters.paragraph_expression)
+        self.req.filter_operator = self.retrieval.filters.filter_expression_operator
+
+        if self.retrieval.filters.autofilter:
+            entity_filters = apply_entities_filter(self.req, self.retrieval.filters.autofilter)
+            self._autofilter.extend([translate_system_to_alias_label(e) for e in entity_filters])
+
+        if self.retrieval.filters.hidden is not None:
+            expr = nodereader_pb2.FilterExpression()
+            if self.retrieval.filters.hidden:
+                expr.facet.facet = LABEL_HIDDEN
+            else:
+                expr.bool_not.facet.facet = LABEL_HIDDEN
+
+            add_and_expression(self.req.field_filter, expr)
+
+    def _apply_top_k(self):
+        """Adjust requested page size depending on rank fusion and reranking
+        algorithms.
+
+        Some rerankers want more results than the requested by the user so
+        reranking can have more choices.
+        """
+        top_k = self.retrieval.top_k
+
+        rank_fusion_window = 0
+        if self.retrieval.rank_fusion is not None:
+            rank_fusion_window = self.retrieval.rank_fusion.window
+
+        reranker_window = 0
+        if self.retrieval.reranker is not None and isinstance(self.retrieval.reranker, PredictReranker):
+            reranker_window = self.retrieval.reranker.window
+
+        self.req.result_per_page = max(
+            top_k,
+            rank_fusion_window,
+            reranker_window,
+        )
 
 
 def is_incomplete(retrieval: UnitRetrieval) -> bool:

nucliadb/search/search/utils.py

@@ -18,12 +18,12 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import logging
-from typing import Optional, Union
+from typing import Optional
 
 from pydantic import BaseModel
 
 from nucliadb.common.datamanagers.atomic import kb
-from nucliadb_models.search import BaseSearchRequest, MinScore
+from nucliadb_models.search import MinScore
 from nucliadb_utils import const
 from nucliadb_utils.utilities import has_feature
 
@@ -39,36 +39,6 @@ async def filter_hidden_resources(kbid: str, show_hidden: bool) -> Optional[bool
         return None  # None = No filtering, show all resources
 
 
-def is_empty_query(request: BaseSearchRequest) -> bool:
-    return len(request.query) == 0
-
-
-def has_user_vectors(request: BaseSearchRequest) -> bool:
-    return request.vector is not None and len(request.vector) > 0
-
-
-def is_exact_match_only_query(request: BaseSearchRequest) -> bool:
-    """
-    '"something"' -> True
-    'foo "something" else' -> False
-    """
-    query = request.query.strip()
-    return len(query) > 0 and query.startswith('"') and query.endswith('"')
-
-
-def should_disable_vector_search(request: BaseSearchRequest) -> bool:
-    if has_user_vectors(request):
-        return False
-
-    if is_exact_match_only_query(request):
-        return True
-
-    if is_empty_query(request):
-        return True
-
-    return False
-
-
 def min_score_from_query_params(
     min_score_bm25: float,
     min_score_semantic: Optional[float],
@@ -79,16 +49,6 @@ def min_score_from_query_params(
     return MinScore(bm25=min_score_bm25, semantic=semantic)
 
 
-def min_score_from_payload(min_score: Optional[Union[float, MinScore]]) -> MinScore:
-    # Keep backward compatibility with the deprecated
-    # min_score payload parameter being a float
-    if min_score is None:
-        return MinScore(bm25=0, semantic=None)
-    elif isinstance(min_score, float):
-        return MinScore(bm25=0, semantic=min_score)
-    return min_score
-
-
 def maybe_log_request_payload(kbid: str, endpoint: str, item: BaseModel):
     if has_feature(const.Features.LOG_REQUEST_PAYLOADS, context={"kbid": kbid}, default=False):
         logger.info(

{nucliadb-6.3.7.post4091.dist-info → nucliadb-6.3.7.post4116.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.3.7.post4091
+Version: 6.3.7.post4116
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.7.post4091
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.7.post4091
-Requires-Dist: nucliadb-protos>=6.3.7.post4091
-Requires-Dist: nucliadb-models>=6.3.7.post4091
-Requires-Dist: nidx-protos>=6.3.7.post4091
+Requires-Dist: nucliadb-telemetry[all]>=6.3.7.post4116
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.7.post4116
+Requires-Dist: nucliadb-protos>=6.3.7.post4116
+Requires-Dist: nucliadb-models>=6.3.7.post4116
+Requires-Dist: nidx-protos>=6.3.7.post4116
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]