nucliadb 6.3.1.post3457__py3-none-any.whl → 6.3.1.post3470__py3-none-any.whl

nucliadb/common/cluster/rebalance.py

@@ -101,15 +101,15 @@ async def move_set_of_kb_resources(
     to_shard = [s for s in kb_shards.shards if s.shard == to_shard_id][0]
 
     from_node, from_shard_replica_id = choose_node(from_shard)
-    search_response: nodereader_pb2.SearchResponse = await from_node.reader.Search(  # type: ignore
-        nodereader_pb2.SearchRequest(
-            shard=from_shard_replica_id,
-            paragraph=False,
-            document=True,
-            result_per_page=count,
-            fields=["a/title"],
-        )
+    request = nodereader_pb2.SearchRequest(
+        shard=from_shard_replica_id,
+        paragraph=False,
+        document=True,
+        result_per_page=count,
     )
+    request.field_filter.field.field_type = "a"
+    request.field_filter.field.field_id = "title"
+    search_response: nodereader_pb2.SearchResponse = await from_node.reader.Search(request)  # type: ignore
 
     for result in search_response.document.results:
         resource_id = result.uuid

nucliadb/common/external_index_providers/pinecone.py

@@ -18,7 +18,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import asyncio
-import json
 import logging
 from copy import deepcopy
 from typing import Any, Iterator, Optional
@@ -41,7 +40,7 @@ from nucliadb.common.ids import FieldId, ParagraphId, VectorId
 from nucliadb_models.search import SCORE_TYPE, TextPosition
 from nucliadb_protos import knowledgebox_pb2 as kb_pb2
 from nucliadb_protos import utils_pb2
-from nucliadb_protos.nodereader_pb2 import SearchRequest, Timestamps
+from nucliadb_protos.nodereader_pb2 import FilterExpression, SearchRequest
 from nucliadb_protos.noderesources_pb2 import IndexParagraph, Resource, VectorSentence
 from nucliadb_telemetry.metrics import Observer
 from nucliadb_utils.aiopynecone.client import DataPlane, FilterOperator, LogicalOperator
@@ -762,24 +761,10 @@ def convert_to_pinecone_filter(request: SearchRequest) -> Optional[dict[str, Any
     can be used on Pinecone indexes.
     """
     and_terms = []
-    if request.HasField("filter"):
-        # Label filtering
-        if len(request.filter.paragraph_labels) > 0 and len(request.filter.field_labels) > 0:
-            raise ValueError("Cannot filter by paragraph and field labels at the same request")
-
-        decoded_expression: dict[str, Any] = json.loads(request.filter.labels_expression)
-        if len(request.filter.paragraph_labels) > 0:
-            and_terms.append(convert_label_filter_expression("paragraph_labels", decoded_expression))
-        else:
-            and_terms.append(convert_label_filter_expression("field_labels", decoded_expression))
-
-    if request.HasField("timestamps"):
-        # Date range filtering
-        and_terms.extend(convert_timestamp_filter(request.timestamps))
-
-    if len(request.key_filters) > 0:
-        # Filter by resource_id
-        and_terms.append({"rid": {FilterOperator.IN: list(set(request.key_filters))}})
+    if request.HasField("field_filter"):
+        and_terms.append(convert_filter_expression("field_labels", request.field_filter))
+    if request.HasField("paragraph_filter"):
+        and_terms.append(convert_filter_expression("paragraph_labels", request.field_filter))
 
     if len(request.security.access_groups):
         # Security filtering
@@ -795,12 +780,6 @@ def convert_to_pinecone_filter(request: SearchRequest) -> Optional[dict[str, Any
         }
         and_terms.append(security_term)
 
-    if len(request.fields) > 0:
-        # Filter by field_id
-        fields_term = {
-            "field_id": {FilterOperator.IN: list({field_id.strip("/") for field_id in request.fields})}
-        }
-        and_terms.append(fields_term)
     if len(and_terms) == 0:
         return None
     if len(and_terms) == 1:
@@ -808,8 +787,8 @@ def convert_to_pinecone_filter(request: SearchRequest) -> Optional[dict[str, Any
     return {LogicalOperator.AND: and_terms}
 
 
-def convert_label_filter_expression(
-    field: str, expression: dict[str, Any], negative: bool = False
+def convert_filter_expression(
+    field: str, expression: FilterExpression, negative: bool = False
 ) -> dict[str, Any]:
     """
     Converts internal label filter expressions to Pinecone's metadata query language.
@@ -817,77 +796,99 @@ def convert_label_filter_expression(
     Note: Since Pinecone does not support negation of expressions, we need to use De Morgan's laws to
     convert the expression to a positive one.
     """
-    if "literal" in expression:
-        if negative:
-            return {field: {FilterOperator.NOT_IN: [expression["literal"]]}}
-        else:
-            return {field: {FilterOperator.IN: [expression["literal"]]}}
 
-    if "and" in expression:
+    kind = expression.WhichOneof("expr")
+    if kind == "bool_and":
         if negative:
             return {
                 LogicalOperator.OR: [
-                    convert_label_filter_expression(field, sub_expression, negative=True)
-                    for sub_expression in expression["and"]
+                    convert_filter_expression(field, sub_expression, negative=True)
+                    for sub_expression in expression.bool_and.operands
                 ]
             }
         else:
             return {
                 LogicalOperator.AND: [
-                    convert_label_filter_expression(field, sub_expression)
-                    for sub_expression in expression["and"]
+                    convert_filter_expression(field, sub_expression)
+                    for sub_expression in expression.bool_and.operands
                 ]
             }
-
-    if "or" in expression:
+    elif kind == "bool_or":
         if negative:
             return {
                 LogicalOperator.AND: [
-                    convert_label_filter_expression(field, sub_expression, negative=True)
-                    for sub_expression in expression["or"]
+                    convert_filter_expression(field, sub_expression, negative=True)
+                    for sub_expression in expression.bool_or.operands
                 ]
             }
         else:
             return {
                 LogicalOperator.OR: [
-                    convert_label_filter_expression(field, sub_expression)
-                    for sub_expression in expression["or"]
+                    convert_filter_expression(field, sub_expression)
+                    for sub_expression in expression.bool_or.operands
                 ]
             }
 
-    if "not" in expression:
-        return convert_label_filter_expression(field, expression["not"], negative=True)
+    elif kind == "bool_not":
+        return convert_filter_expression(field, expression.bool_not, negative=not negative)
 
-    raise ValueError(f"Invalid label filter expression: {expression}")
+    elif kind == "resource":
+        operator = FilterOperator.NOT_EQUALS if negative else FilterOperator.EQUALS
+        return {"rid": {operator: expression.resource.resource_id}}
 
+    elif kind == "field":
+        field_id = expression.field.field_type
+        if expression.field.HasField("field_id"):
+            field_id += f"/{expression.field.field_id}"
+        operator = FilterOperator.NOT_EQUALS if negative else FilterOperator.EQUALS
+        return {"field_id": {operator: field_id}}
 
-def convert_timestamp_filter(timestamps: Timestamps) -> list[dict[str, Any]]:
-    """
-    Allows to filter by date_created and date_modified fields in Pinecone.
-    Powers date range filtering at NucliaDB.
-    """
-    and_terms = []
-    if timestamps.HasField("from_modified"):
-        and_terms.append(
-            {
-                "date_modified": {
-                    FilterOperator.GREATER_THAN_OR_EQUAL: timestamps.from_modified.ToSeconds()
-                }
-            }
-        )
-    if timestamps.HasField("to_modified"):
-        and_terms.append(
-            {"date_modified": {FilterOperator.LESS_THAN_OR_EQUAL: timestamps.to_modified.ToSeconds()}}
-        )
-    if timestamps.HasField("from_created"):
-        and_terms.append(
-            {"date_created": {FilterOperator.GREATER_THAN_OR_EQUAL: timestamps.from_created.ToSeconds()}}
-        )
-    if timestamps.HasField("to_created"):
-        and_terms.append(
-            {"date_created": {FilterOperator.LESS_THAN_OR_EQUAL: timestamps.to_created.ToSeconds()}}
+    elif kind == "keyword":
+        raise ValueError("Cannot filter by keywords")
+
+    elif kind == "date":
+        date_field = (
+            "date_created"
+            if expression.date.field == FilterExpression.DateRangeFilter.DateField.CREATED
+            else "date_modified"
         )
-    return and_terms
+        if negative:
+            terms = []
+            if expression.date.HasField("since"):
+                operator = FilterOperator.LESS_THAN
+                terms.append({date_field: {operator: expression.date.since.ToSeconds()}})
+            if expression.date.HasField("until"):
+                operator = FilterOperator.GREATER_THAN
+                terms.append({date_field: {operator: expression.date.until.ToSeconds()}})
+
+            if len(terms) == 2:
+                return {LogicalOperator.OR: terms}
+            elif len(terms) == 1:
+                return terms[0]
+            else:
+                raise ValueError(f"Invalid filter expression: {expression}")
+        else:
+            terms = []
+            if expression.date.HasField("since"):
+                operator = FilterOperator.GREATER_THAN_OR_EQUAL
+                terms.append({date_field: {operator: expression.date.since.ToSeconds()}})
+            if expression.date.HasField("until"):
+                operator = FilterOperator.LESS_THAN_OR_EQUAL
+                terms.append({date_field: {operator: expression.date.until.ToSeconds()}})
+
+            if len(terms) == 2:
+                return {LogicalOperator.AND: terms}
+            elif len(terms) == 1:
+                return terms[0]
+            else:
+                raise ValueError(f"Invalid filter expression: {expression}")
+
+    elif kind == "facet":
+        operator = FilterOperator.NOT_IN if negative else FilterOperator.IN
+        return {field: {operator: [expression.facet.facet]}}
+
+    else:
+        raise ValueError(f"Invalid filter expression: {expression}")
 
 
 def iter_paragraphs(resource: Resource) -> Iterator[tuple[str, IndexParagraph]]:

nucliadb/search/search/chat/ask.py

@@ -437,14 +437,32 @@ class NotEnoughContextAskResult(AskResult):
         context in the corpus to answer.
         """
         yield self._ndjson_encode(RetrievalAskResponseItem(results=self.main_results))
+        if self.prequeries_results:
+            yield self._ndjson_encode(
+                PrequeriesAskResponseItem(
+                    results={
+                        prequery.id or f"prequery_{index}": prequery_result
+                        for index, (prequery, prequery_result) in enumerate(self.prequeries_results)
+                    }
+                )
+            )
         yield self._ndjson_encode(AnswerAskResponseItem(text=NOT_ENOUGH_CONTEXT_ANSWER))
         status = AnswerStatusCode.NO_RETRIEVAL_DATA
         yield self._ndjson_encode(StatusAskResponseItem(code=status.value, status=status.prettify()))
 
     async def json(self) -> str:
+        prequeries = (
+            {
+                prequery.id or f"prequery_{index}": prequery_result
+                for index, (prequery, prequery_result) in enumerate(self.prequeries_results)
+            }
+            if self.prequeries_results
+            else None
+        )
         return SyncAskResponse(
             answer=NOT_ENOUGH_CONTEXT_ANSWER,
             retrieval_results=self.main_results,
+            prequeries=prequeries,
             status=AnswerStatusCode.NO_RETRIEVAL_DATA.prettify(),
         ).model_dump_json()
 

nucliadb/search/search/query.py

@@ -225,9 +225,6 @@ class QueryParser:
                     security_pb.access_groups.append(group_id)
             request.security.CopyFrom(security_pb)
 
-        if self.with_status is not None:
-            request.with_status = PROCESSING_STATUS_TO_PB_MAP[self.with_status]
-
         has_old_filters = False
         if self.old_filters:
             field_expr, paragraph_expr = await parse_old_filters(self.old_filters, self.fetcher)

nucliadb/train/generators/image_classifier.py

@@ -60,7 +60,6 @@ async def generate_image_classification_payloads(
 ) -> AsyncGenerator[ImageClassification, None]:
     request = StreamRequest()
     request.shard_id.id = shard_replica_id
-    request.reload = True
     async for item in node.stream_get_fields(request):
         rid = item.uuid
         resource = await get_resource_from_cache_or_db(kbid, rid)

{nucliadb-6.3.1.post3457.dist-info → nucliadb-6.3.1.post3470.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.3.1.post3457
+Version: 6.3.1.post3470
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3457
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3457
-Requires-Dist: nucliadb-protos>=6.3.1.post3457
-Requires-Dist: nucliadb-models>=6.3.1.post3457
-Requires-Dist: nidx-protos>=6.3.1.post3457
+Requires-Dist: nucliadb-telemetry[all]>=6.3.1.post3470
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.1.post3470
+Requires-Dist: nucliadb-protos>=6.3.1.post3470
+Requires-Dist: nucliadb-models>=6.3.1.post3470
+Requires-Dist: nidx-protos>=6.3.1.post3470
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.1.post3457.dist-info → nucliadb-6.3.1.post3470.dist-info}/RECORD

@@ -51,7 +51,7 @@ nucliadb/common/cluster/exceptions.py,sha256=V3c_fgH00GyJ-a5CaGLhwTuhwhUNR9YAGvS
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=L85wBnfab7Rev0CfsfUjPxQC6DiHPsETKrZAOLx9XHg,3510
 nucliadb/common/cluster/index_node.py,sha256=g38H1kiAliF3Y6et_CWYInpn_xPxf7THAFJ7RtgLNZo,3246
 nucliadb/common/cluster/manager.py,sha256=-y5PCUzxrvvUCBopmi8JztPiSBYYbUBeLer0XtO4Ukk,15383
-nucliadb/common/cluster/rebalance.py,sha256=cGrqc6GQ6Y2X1moW7NzIdLLsij0cbKySiu-qUa3fA-o,8700
+nucliadb/common/cluster/rebalance.py,sha256=cLUlR08SsqmnoA_9GDflV6k2tXmkAPpyFxZErzp45vo,8754
 nucliadb/common/cluster/rollover.py,sha256=iTJ9EQmHbzXL34foNFto-hqdC0Kq1pF1mNxqv0jqhBs,25362
 nucliadb/common/cluster/settings.py,sha256=TMoym-cZsQ2soWfLAce0moSa2XncttQyhahL43LrWTo,3384
 nucliadb/common/cluster/utils.py,sha256=7nQvnVFxM4XV7J560R8hUA-GPzrgD19UlQxHrl4mZUc,4687
@@ -77,7 +77,7 @@ nucliadb/common/external_index_providers/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A
 nucliadb/common/external_index_providers/base.py,sha256=yfPkCigT4unXFvAyzy1tXSy2UgWC481GcZAS9bdE4NI,8871
 nucliadb/common/external_index_providers/exceptions.py,sha256=nDhhOIkb66hjCrBk4Spvl2vN1SuW5gbwrMCDmrdjHHE,1209
 nucliadb/common/external_index_providers/manager.py,sha256=aFSrrKKYG1ydpTSyq4zYD0LOxFS7P-CO6rcKC0hiF4I,4267
-nucliadb/common/external_index_providers/pinecone.py,sha256=SZKOkc-F6vdSmkQsEBK3VrV3omfEfMuhkkxhEsvd6_A,39799
+nucliadb/common/external_index_providers/pinecone.py,sha256=19DIpgaItaWOwrcXsm3HnkbQrAjq-kG9Wehh38CcVbQ,40027
 nucliadb/common/external_index_providers/settings.py,sha256=EGHnIkwxqe6aypwKegXTlKO3AgUxNa-6GeAZG25Njis,2002
 nucliadb/common/http_clients/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/http_clients/auth.py,sha256=srfpgAbs2wmqA9u_l-HxsV4YoO77Tse4y3gm3q2YvYM,2112
@@ -225,14 +225,14 @@ nucliadb/search/search/metrics.py,sha256=81X-tahGW4n2CLvUzCPdNxNClmZqUWZjcVOGCUH
 nucliadb/search/search/paragraphs.py,sha256=pNAEiYqJGGUVcEf7xf-PFMVqz0PX4Qb-WNG-_zPGN2o,7799
 nucliadb/search/search/pgcatalog.py,sha256=V1NYLEUSXHpWmgcPIo1HS2riK_HDXSi-uykJjSoOOrE,9033
 nucliadb/search/search/predict_proxy.py,sha256=IFI3v_ODz2_UU1XZnyaD391fE7-2C0npSmj_HmDvzS4,3123
-nucliadb/search/search/query.py,sha256=S6kvvTNIgOaqsPEisasgbcfsI7ax5E1G5RJwp7-a9jA,29981
+nucliadb/search/search/query.py,sha256=CCz58Cs_VKxbwGO6VdhOzl9HXwKny2hfH2llJVR9emM,29859
 nucliadb/search/search/rank_fusion.py,sha256=tRGo_KlsFsVx1CQEy1iqQ6f0T1Dq1kf0axDXHuuzvvM,6946
 nucliadb/search/search/rerankers.py,sha256=3vep4EOVNeDJGsMdx-1g6Ar4ZGJG3IHym3HkxnbwtAQ,7321
 nucliadb/search/search/shards.py,sha256=JSRSrHgHcF4sXyuZZoJdMfK0v_LHpoSRf1lCr5-K5ko,2742
 nucliadb/search/search/summarize.py,sha256=ksmYPubEQvAQgfPdZHfzB_rR19B2ci4IYZ6jLdHxZo8,4996
 nucliadb/search/search/utils.py,sha256=iF2tbBA56gRMJH1TlE2hMrqeXqjoeOPt4KgRdp2m9Ek,3313
 nucliadb/search/search/chat/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/search/search/chat/ask.py,sha256=HPHM97s1dxbHvugoWZj6aP8vL4gzHjjtwoDvzUMaLfw,36547
+nucliadb/search/search/chat/ask.py,sha256=olZT08JVo3ZGDsDXkjvI2JTlqQln_o91HJzv0TKFQ7I,37244
 nucliadb/search/search/chat/exceptions.py,sha256=Siy4GXW2L7oPhIR86H3WHBhE9lkV4A4YaAszuGGUf54,1356
 nucliadb/search/search/chat/images.py,sha256=PA8VWxT5_HUGfW1ULhKTK46UBsVyINtWWqEM1ulzX1E,3095
 nucliadb/search/search/chat/prompt.py,sha256=Jnja-Ss7skgnnDY8BymVfdeYsFPnIQFL8tEvcRXTKUE,47356
@@ -294,7 +294,7 @@ nucliadb/train/api/v1/trainset.py,sha256=kpnpDgiMWr1FKHZJgwH7hue5kzilA8-i9X0YHlN
 nucliadb/train/generators/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/train/generators/field_classifier.py,sha256=yatj7U-LHRN5xTR6XsYz_3acIAUKTcpkNZcZaSY8MtE,3482
 nucliadb/train/generators/field_streaming.py,sha256=kjwg4VNiROVqVDN--mRd4ylLw55Sg2VYxKRDdbmpYSM,5398
-nucliadb/train/generators/image_classifier.py,sha256=NZ993L1Fxjc7kB_yEMhCpdlMaNQbPrFz2LOUiF9eSYs,6730
+nucliadb/train/generators/image_classifier.py,sha256=yqdBKIHiumeDSTggbxLFiUPanjMtbWGEaWo4mg2OcqA,6704
 nucliadb/train/generators/paragraph_classifier.py,sha256=0pOZYcT1cAmG7gjSD1HIUaMM5T3Ag-96iUTXRhiV8MI,2761
 nucliadb/train/generators/paragraph_streaming.py,sha256=dsM7a5hBd2iokvFuxnZhQeko4Jad6djyP2p3tevku8A,3586
 nucliadb/train/generators/question_answer_streaming.py,sha256=P7-de4W4yW2mgEQ82fF2OZVyx6QJHXezY52qDciDcmw,5680
@@ -340,8 +340,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.1.post3457.dist-info/METADATA,sha256=Q7ES2awV8PZOanfl6y5nQZebnPb0AzQhXJvQipxzQAA,4291
-nucliadb-6.3.1.post3457.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-nucliadb-6.3.1.post3457.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.1.post3457.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.1.post3457.dist-info/RECORD,,
+nucliadb-6.3.1.post3470.dist-info/METADATA,sha256=D70V8N1xQhfe8pD1j1CJs2JaOAn8Yf10reTomEznVFs,4291
+nucliadb-6.3.1.post3470.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+nucliadb-6.3.1.post3470.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.1.post3470.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.1.post3470.dist-info/RECORD,,