nucliadb 6.2.1.post3395__py3-none-any.whl → 6.3.0.post3402__py3-none-any.whl

nucliadb/search/api/v1/resource/search.py

@@ -17,10 +17,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import json
 from typing import Optional, Union, cast
 
 from fastapi import Header, Request, Response
 from fastapi_versioning import version
+from pydantic import ValidationError
 
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.search.api.v1.router import KB_PREFIX, RESOURCE_PREFIX, api
@@ -30,6 +32,7 @@ from nucliadb.search.search import cache
 from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb.search.search.merge import merge_paragraphs_results
 from nucliadb.search.search.query import paragraph_query_to_pb
+from nucliadb_models.filter import FilterExpression
 from nucliadb_models.resource import NucliaDBRoles
 from nucliadb_models.search import (
     NucliaDBClientType,
@@ -59,6 +62,9 @@ async def resource_search(
     kbid: str,
     query: str,
     rid: str,
+    filter_expression: Optional[str] = fastapi_query(
+        SearchParamDefaults.filter_expression, include_in_schema=False
+    ),
     fields: list[str] = fastapi_query(SearchParamDefaults.fields),
     filters: list[str] = fastapi_query(SearchParamDefaults.filters),
     faceted: list[str] = fastapi_query(SearchParamDefaults.faceted),
@@ -82,10 +88,13 @@ async def resource_search(
 
     with cache.request_caches():
         try:
+            expr = FilterExpression.model_validate_json(filter_expression) if filter_expression else None
+
             pb_query = await paragraph_query_to_pb(
                 kbid,
                 rid,
                 query,
+                expr,
                 fields,
                 filters,
                 faceted,
@@ -99,6 +108,9 @@ async def resource_search(
             )
         except InvalidQueryError as exc:
             return HTTPClientError(status_code=412, detail=str(exc))
+        except ValidationError as exc:
+            detail = json.loads(exc.json())
+            return HTTPClientError(status_code=422, detail=detail)
 
         results, incomplete_results, queried_nodes = await node_query(kbid, Method.SEARCH, pb_query)
 

nucliadb/search/api/v1/suggest.py

@@ -17,11 +17,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import json
 from datetime import datetime
 from typing import Optional, Union
 
 from fastapi import Header, Request, Response
 from fastapi_versioning import version
+from pydantic import ValidationError
 
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.search.api.v1.router import KB_PREFIX, api
@@ -115,6 +117,9 @@ async def suggest_knowledgebox(
         )
     except InvalidQueryError as exc:
         return HTTPClientError(status_code=412, detail=str(exc))
+    except ValidationError as exc:
+        detail = json.loads(exc.json())
+        return HTTPClientError(status_code=422, detail=detail)
 
 
 async def suggest(

nucliadb/search/search/chat/ask.py

@@ -47,6 +47,7 @@ from nucliadb.search.search.chat.prompt import PromptContextBuilder
 from nucliadb.search.search.chat.query import (
     NOT_ENOUGH_CONTEXT_ANSWER,
     ChatAuditor,
+    add_resource_filter,
     get_find_results,
     get_relations_results,
     maybe_audit_chat,
@@ -804,7 +805,7 @@ async def retrieval_in_resource(
         prequeries = calculate_prequeries_for_json_schema(ask_request)
 
     # Make sure the retrieval is scoped to the resource if provided
-    ask_request.resource_filters = [resource]
+    add_resource_filter(ask_request, [resource])
     if prequeries is not None:
         for prequery in prequeries.queries:
             if prequery.prefilter is True:
@@ -812,7 +813,7 @@ async def retrieval_in_resource(
                     "rag_strategies",
                     "Prequeries with prefilter are not supported when asking on a resource",
                 )
-            prequery.request.resource_filters = [resource]
+            add_resource_filter(prequery.request, [resource])
 
     with metrics.time("retrieval"):
         main_results, prequeries_results, query_parser = await get_find_results(

nucliadb/search/search/chat/query.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import asyncio
-from typing import Iterable, Optional
+from typing import Iterable, Optional, Union
 
 from nucliadb.common.models_utils import to_proto
 from nucliadb.search import logger
@@ -32,6 +32,7 @@ from nucliadb.search.search.metrics import RAGMetrics
 from nucliadb.search.search.query import QueryParser
 from nucliadb.search.settings import settings
 from nucliadb.search.utilities import get_predict
+from nucliadb_models import filter
 from nucliadb_models.search import (
     AskRequest,
     ChatContextMessage,
@@ -119,9 +120,10 @@ async def get_find_results(
                     raise NoRetrievalResultsError()
                 # Make sure the main query and prequeries use the same resource filters.
                 # This is important to avoid returning results that don't match the prefilter.
-                item.resource_filters = list(prefilter_matching_resources)
+                matching_resources = list(prefilter_matching_resources)
+                add_resource_filter(item, matching_resources)
                 for prequery in prequeries:
-                    prequery.request.resource_filters = list(prefilter_matching_resources)
+                    add_resource_filter(prequery.request, matching_resources)
                     prequery.request.show_hidden = item.show_hidden
 
         if prequeries:
@@ -151,6 +153,30 @@ async def get_find_results(
     return main_results, prequeries_results, query_parser
 
 
+def add_resource_filter(request: Union[FindRequest, AskRequest], resources: list[str]):
+    if len(resources) == 0:
+        return
+
+    if request.filter_expression is not None:
+        if len(resources) > 1:
+            resource_filter: filter.FieldFilterExpression = filter.Or.model_validate(
+                {"or": [filter.Resource(prop="resource", id=rid) for rid in resources]}
+            )
+        else:
+            resource_filter = filter.Resource(prop="resource", id=resources[0])
+
+        # Add to filter expression if set
+        if request.filter_expression.field is None:
+            request.filter_expression.field = resource_filter
+        else:
+            request.filter_expression.field = filter.And.model_validate(
+                {"and": [request.filter_expression.field, resource_filter]}
+            )
+    else:
+        # Add to old key filters instead
+        request.resource_filters = resources
+
+
 def find_request_from_ask_request(item: AskRequest, query: str) -> FindRequest:
     find_request = FindRequest()
     find_request.resource_filters = item.resource_filters

nucliadb/search/search/query.py

@@ -252,7 +252,10 @@ class QueryParser:
                 if expr:
                     request.paragraph_filter.CopyFrom(expr)
 
-            # TODO: Pass operator to PB
+            if self.filter_expression.operator == FilterExpression.Operator.OR:
+                request.filter_operator = nodereader_pb2.FilterOperator.OR
+            else:
+                request.filter_operator = nodereader_pb2.FilterOperator.AND
 
         if self.hidden is not None:
             expr = nodereader_pb2.FilterExpression()
@@ -486,6 +489,7 @@ async def paragraph_query_to_pb(
     kbid: str,
     rid: str,
     query: str,
+    filter_expression: Optional[FilterExpression],
     fields: list[str],
     filters: list[str],
     faceted: list[str],
@@ -514,7 +518,6 @@ async def paragraph_query_to_pb(
         range_creation_end=range_creation_end,
         range_modification_start=range_modification_start,
         range_modification_end=range_modification_end,
-        key_filters=[rid],
         fields=fields,
     )
     fetcher = Fetcher(
@@ -532,6 +535,29 @@ async def paragraph_query_to_pb(
     if paragraph_expr is not None:
         request.paragraph_filter.CopyFrom(paragraph_expr)
 
+    if (field_expr is not None or paragraph_expr is not None) and filter_expression is not None:
+        raise InvalidQueryError("filter_expression", "Cannot mix old filters with filter_expression")
+
+    if filter_expression:
+        if filter_expression.field:
+            expr = await parse_expression(filter_expression.field, kbid)
+            if expr:
+                request.field_filter.CopyFrom(expr)
+
+        if filter_expression.paragraph:
+            expr = await parse_expression(filter_expression.paragraph, kbid)
+            if expr:
+                request.paragraph_filter.CopyFrom(expr)
+
+        if filter_expression.operator == FilterExpression.Operator.OR:
+            request.filter_operator = nodereader_pb2.FilterOperator.OR
+        else:
+            request.filter_operator = nodereader_pb2.FilterOperator.AND
+
+    key_filter = nodereader_pb2.FilterExpression()
+    key_filter.resource.resource_id = rid
+    add_and_expression(request.field_filter, key_filter)
+
     return request
 
 
@@ -657,15 +683,14 @@ async def suggest_query_to_pb(
                 request.field_filter.CopyFrom(expr)
 
         if filter_expression.paragraph:
-            raise InvalidQueryError(
-                "filter_expression", "paragraph filters not yet available in suggest"
-            )
-            # TODO
-            # expr = await parse_expression(filter_expression.paragraph, kbid)
-            # if expr:
-            #     request.paragraph_filter.CopyFrom(expr)
+            expr = await parse_expression(filter_expression.paragraph, kbid)
+            if expr:
+                request.paragraph_filter.CopyFrom(expr)
 
-        # TODO: Pass operator to PB
+        if filter_expression.operator == FilterExpression.Operator.OR:
+            request.filter_operator = nodereader_pb2.FilterOperator.OR
+        else:
+            request.filter_operator = nodereader_pb2.FilterOperator.AND
 
     if hidden is not None:
         expr = nodereader_pb2.FilterExpression()

{nucliadb-6.2.1.post3395.dist-info → nucliadb-6.3.0.post3402.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post3395
+Version: 6.3.0.post3402
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3395
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3395
-Requires-Dist: nucliadb-protos>=6.2.1.post3395
-Requires-Dist: nucliadb-models>=6.2.1.post3395
-Requires-Dist: nidx-protos>=6.2.1.post3395
+Requires-Dist: nucliadb-telemetry[all]>=6.3.0.post3402
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.0.post3402
+Requires-Dist: nucliadb-protos>=6.3.0.post3402
+Requires-Dist: nucliadb-models>=6.3.0.post3402
+Requires-Dist: nidx-protos>=6.3.0.post3402
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.2.1.post3395.dist-info → nucliadb-6.3.0.post3402.dist-info}/RECORD

@@ -198,13 +198,13 @@ nucliadb/search/api/v1/knowledgebox.py,sha256=rWhx3PYWryingu19qwwFDbVvVYynq5Ky23
 nucliadb/search/api/v1/predict_proxy.py,sha256=QrGzo0hKjtmyGZ6pjlJHYAh4hxwVUIOTcVcerRCw7eE,3047
 nucliadb/search/api/v1/router.py,sha256=mtT07rBZcVfpa49doaw9b1tj3sdi3qLH0gn9Io6NYM0,988
 nucliadb/search/api/v1/search.py,sha256=DLXxh2FRXmLnZIIXaSLT7XaNoY2GZJTkpcduLTDyVW4,14023
-nucliadb/search/api/v1/suggest.py,sha256=tQX7rvPRjE_epk6qN8JB_Xue8JL02uUeVUNpbaJECQE,6318
+nucliadb/search/api/v1/suggest.py,sha256=urnA8rXLTcT2-Yulw2_43Ow8cAPWQEe5bxgDI9gNNZM,6505
 nucliadb/search/api/v1/summarize.py,sha256=VAHJvE6V3xUgEBfqNKhgoxmDqCvh30RnrEIBVhMcNLU,2499
 nucliadb/search/api/v1/utils.py,sha256=5Ve-frn7LAE2jqAgB85F8RSeqxDlyA08--gS-AdOLS4,1434
 nucliadb/search/api/v1/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/api/v1/resource/ask.py,sha256=nsVzBSanSSlf0Ody6LSTjdEy75Vg283_YhbkAtWEjh8,3637
 nucliadb/search/api/v1/resource/ingestion_agents.py,sha256=fqqRCd8Wc9GciS5P98lcnihvTKStsZYYtOU-T1bc-6E,4771
-nucliadb/search/api/v1/resource/search.py,sha256=oSU5lwG7XRnD7oBFct31JaECGjTjX5R8mxNF1mskINc,4715
+nucliadb/search/api/v1/resource/search.py,sha256=-GY84XiiGd2vyJtunWvZMdBhZU0O0giO-ISo-HMntBo,5227
 nucliadb/search/api/v1/resource/utils.py,sha256=-NjZqAQtFEXKpIh8ui5S26ItnJ5rzmmG0BHxGSS9QPw,1141
 nucliadb/search/requesters/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/search/requesters/utils.py,sha256=qL81UVPNgBftUMLpcxIYVr7ILsMqpKCo-9SY2EvAaXw,6681
@@ -224,18 +224,18 @@ nucliadb/search/search/metrics.py,sha256=81X-tahGW4n2CLvUzCPdNxNClmZqUWZjcVOGCUH
 nucliadb/search/search/paragraphs.py,sha256=pNAEiYqJGGUVcEf7xf-PFMVqz0PX4Qb-WNG-_zPGN2o,7799
 nucliadb/search/search/pgcatalog.py,sha256=IaNK4dAxdXs38PoIkTdgqMDuZDjeiOtcXn3LeaT-OMw,8855
 nucliadb/search/search/predict_proxy.py,sha256=IFI3v_ODz2_UU1XZnyaD391fE7-2C0npSmj_HmDvzS4,3123
-nucliadb/search/search/query.py,sha256=A1HnP7FzBtn3G4-oId_x-x0MHYlQPCTzrtP73LxoEwo,28733
+nucliadb/search/search/query.py,sha256=j7NdAXQXSytWvf_tyqpG-Fz4AhVGI-Y5i7NPm3s7-PI,29980
 nucliadb/search/search/rank_fusion.py,sha256=tRGo_KlsFsVx1CQEy1iqQ6f0T1Dq1kf0axDXHuuzvvM,6946
 nucliadb/search/search/rerankers.py,sha256=3vep4EOVNeDJGsMdx-1g6Ar4ZGJG3IHym3HkxnbwtAQ,7321
 nucliadb/search/search/shards.py,sha256=JSRSrHgHcF4sXyuZZoJdMfK0v_LHpoSRf1lCr5-K5ko,2742
 nucliadb/search/search/summarize.py,sha256=ksmYPubEQvAQgfPdZHfzB_rR19B2ci4IYZ6jLdHxZo8,4996
 nucliadb/search/search/utils.py,sha256=iF2tbBA56gRMJH1TlE2hMrqeXqjoeOPt4KgRdp2m9Ek,3313
 nucliadb/search/search/chat/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/search/search/chat/ask.py,sha256=xmHAO9rmw1hvutGeYc0kxmioxHPpYEjlB-sTC4ArX-k,36516
+nucliadb/search/search/chat/ask.py,sha256=HPHM97s1dxbHvugoWZj6aP8vL4gzHjjtwoDvzUMaLfw,36547
 nucliadb/search/search/chat/exceptions.py,sha256=Siy4GXW2L7oPhIR86H3WHBhE9lkV4A4YaAszuGGUf54,1356
 nucliadb/search/search/chat/images.py,sha256=PA8VWxT5_HUGfW1ULhKTK46UBsVyINtWWqEM1ulzX1E,3095
 nucliadb/search/search/chat/prompt.py,sha256=Jnja-Ss7skgnnDY8BymVfdeYsFPnIQFL8tEvcRXTKUE,47356
-nucliadb/search/search/chat/query.py,sha256=sFRJag80ahpXA7q3oP0XfIsUyRMNz0Y6K6nz8q4wP2A,15371
+nucliadb/search/search/chat/query.py,sha256=kMhtUOuVBkWbOaZnpzAUi32oRc5QnXKmlo2LFoozKWc,16382
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=szAOXUZ27oNY-OSa9t2hQ5HHkQQC0EX1FZz_LluJHJE,1224
 nucliadb/search/search/query_parser/fetcher.py,sha256=jhr__J0KmAzjdsTTadWQmD9qf6lZvqlKAfZdYjZH_UY,15742
@@ -338,8 +338,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post3395.dist-info/METADATA,sha256=dK45Sc6ekVyUZED7nXwb3j74ycmcL-hyQexGDV2hqxA,4291
-nucliadb-6.2.1.post3395.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post3395.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post3395.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post3395.dist-info/RECORD,,
+nucliadb-6.3.0.post3402.dist-info/METADATA,sha256=ffldOKngG5BDfwbQ3N430iCHKoBEh8KkL2iM2nAr7_s,4291
+nucliadb-6.3.0.post3402.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.3.0.post3402.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.0.post3402.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.0.post3402.dist-info/RECORD,,