nucliadb 6.3.4.post3705__py3-none-any.whl → 6.3.4.post3721__py3-none-any.whl

nucliadb/export_import/utils.py

@@ -20,6 +20,7 @@
 import functools
 from typing import AsyncGenerator, AsyncIterator, Callable, Optional
 
+import backoff
 from google.protobuf.message import DecodeError as ProtobufDecodeError
 
 from nucliadb import learning_proxy
@@ -38,6 +39,7 @@ from nucliadb_models.configuration import SearchConfiguration
 from nucliadb_models.export_import import Status
 from nucliadb_protos import knowledgebox_pb2 as kb_pb2
 from nucliadb_protos import resources_pb2, writer_pb2
+from nucliadb_protos.writer_pb2_grpc import WriterStub
 from nucliadb_utils.const import Streams
 from nucliadb_utils.transaction import MaxTransactionSizeExceededError
 from nucliadb_utils.utilities import get_ingest
@@ -114,18 +116,24 @@ async def import_broker_message(
 async def restore_broker_message(
     context: ApplicationContext, kbid: str, bm: writer_pb2.BrokerMessage
 ) -> None:
-    bm.kbid = kbid
+    await send_writer_bm(context, bm)
+    await send_processor_bm(context, bm)
+
 
-    # First, ingest the broker message writer part synchronously
-    async def ingest_request_stream() -> AsyncIterator[writer_pb2.BrokerMessage]:
+@backoff.on_exception(backoff.expo, (Exception,), jitter=backoff.random_jitter, max_tries=8)
+async def send_writer_bm(context: ApplicationContext, bm: writer_pb2.BrokerMessage) -> None:
+    async def _iterator() -> AsyncIterator[writer_pb2.BrokerMessage]:
         yield get_writer_bm(bm)
 
-    response = await get_ingest().ProcessMessage(ingest_request_stream())  # type: ignore
+    ingest_grpc: WriterStub = get_ingest()
+    response: writer_pb2.OpStatusWriter = await ingest_grpc.ProcessMessage(_iterator())  # type: ignore
     assert response.status == writer_pb2.OpStatusWriter.Status.OK, "Failed to process broker message"
 
+
+async def send_processor_bm(context: ApplicationContext, bm: writer_pb2.BrokerMessage) -> None:
     # Then enqueue the processor part asynchronously
     processor_bm = get_processor_bm(bm)
-    partition = context.partitioning.generate_partition(kbid, bm.uuid)
+    partition = context.partitioning.generate_partition(bm.kbid, bm.uuid)
     await transaction_commit(context, processor_bm, partition)
 
 

nucliadb/search/api/v1/graph.py

@@ -69,7 +69,7 @@ async def graph_search_knowledgebox(
     x_nucliadb_user: str = Header(""),
     x_forwarded_for: str = Header(""),
 ) -> GraphSearchResponse:
-    pb_query = parse_graph_search(item)
+    pb_query = await parse_graph_search(kbid, item)
 
     results, _, _ = await node_query(kbid, Method.GRAPH, pb_query)
 
@@ -96,7 +96,7 @@ async def graph_nodes_search_knowledgebox(
     x_nucliadb_user: str = Header(""),
     x_forwarded_for: str = Header(""),
 ) -> GraphNodesSearchResponse:
-    pb_query = parse_graph_node_search(item)
+    pb_query = await parse_graph_node_search(kbid, item)
 
     results, _, _ = await node_query(kbid, Method.GRAPH, pb_query)
 
@@ -123,7 +123,7 @@ async def graph_relations_search_knowledgebox(
     x_nucliadb_user: str = Header(""),
     x_forwarded_for: str = Header(""),
 ) -> GraphRelationsSearchResponse:
-    pb_query = parse_graph_relation_search(item)
+    pb_query = await parse_graph_relation_search(kbid, item)
 
     results, _, _ = await node_query(kbid, Method.GRAPH, pb_query)
 

nucliadb/search/search/query_parser/parsers/graph.py

@@ -18,37 +18,97 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
+from typing import Optional, Union
 
 from nucliadb.common.models_utils.from_proto import RelationNodeTypeMap, RelationTypeMap
+from nucliadb.search.search.query_parser.filter_expression import add_and_expression, parse_expression
 from nucliadb.search.search.query_parser.models import GraphRetrieval
+from nucliadb.search.search.utils import filter_hidden_resources
 from nucliadb_models.graph import requests as graph_requests
-from nucliadb_protos import nodereader_pb2
+from nucliadb_models.labels import LABEL_HIDDEN
+from nucliadb_protos import nodereader_pb2, utils_pb2
 
 
-def parse_graph_search(item: graph_requests.GraphSearchRequest) -> GraphRetrieval:
-    pb = nodereader_pb2.GraphSearchRequest()
+async def parse_graph_search(kbid: str, item: graph_requests.GraphSearchRequest) -> GraphRetrieval:
+    pb = await _parse_common(kbid, item)
     pb.query.path.CopyFrom(_parse_path_query(item.query))
-    pb.top_k = item.top_k
     pb.kind = nodereader_pb2.GraphSearchRequest.QueryKind.PATH
     return pb
 
 
-def parse_graph_node_search(item: graph_requests.GraphNodesSearchRequest) -> GraphRetrieval:
-    pb = nodereader_pb2.GraphSearchRequest()
+async def parse_graph_node_search(
+    kbid: str, item: graph_requests.GraphNodesSearchRequest
+) -> GraphRetrieval:
+    pb = await _parse_common(kbid, item)
     pb.query.path.CopyFrom(_parse_node_query(item.query))
-    pb.top_k = item.top_k
     pb.kind = nodereader_pb2.GraphSearchRequest.QueryKind.NODES
     return pb
 
 
-def parse_graph_relation_search(item: graph_requests.GraphRelationsSearchRequest) -> GraphRetrieval:
-    pb = nodereader_pb2.GraphSearchRequest()
+async def parse_graph_relation_search(
+    kbid: str, item: graph_requests.GraphRelationsSearchRequest
+) -> GraphRetrieval:
+    pb = await _parse_common(kbid, item)
     pb.query.path.CopyFrom(_parse_relation_query(item.query))
-    pb.top_k = item.top_k
     pb.kind = nodereader_pb2.GraphSearchRequest.QueryKind.RELATIONS
     return pb
 
 
+AnyGraphRequest = Union[
+    graph_requests.GraphSearchRequest,
+    graph_requests.GraphNodesSearchRequest,
+    graph_requests.GraphRelationsSearchRequest,
+]
+
+
+async def _parse_common(kbid: str, item: AnyGraphRequest) -> nodereader_pb2.GraphSearchRequest:
+    pb = nodereader_pb2.GraphSearchRequest()
+    pb.top_k = item.top_k
+
+    filter_expr = await _parse_filters(kbid, item)
+    if filter_expr is not None:
+        pb.field_filter.CopyFrom(filter_expr)
+
+    security = _parse_security(kbid, item)
+    if security is not None:
+        pb.security.CopyFrom(security)
+
+    return pb
+
+
+async def _parse_filters(kbid: str, item: AnyGraphRequest) -> Optional[nodereader_pb2.FilterExpression]:
+    filter_expr = nodereader_pb2.FilterExpression()
+    if item.filter_expression:
+        if item.filter_expression.field:
+            filter_expr = await parse_expression(item.filter_expression.field, kbid)
+
+    hidden = await filter_hidden_resources(kbid, item.show_hidden)
+    if hidden is not None:
+        expr = nodereader_pb2.FilterExpression()
+        if hidden:
+            expr.facet.facet = LABEL_HIDDEN
+        else:
+            expr.bool_not.facet.facet = LABEL_HIDDEN
+
+        add_and_expression(filter_expr, expr)
+
+    if filter_expr.HasField("expr"):
+        return filter_expr
+    else:
+        return None
+
+
+def _parse_security(kbid: str, item: AnyGraphRequest) -> Optional[utils_pb2.Security]:
+    if item.security is not None and len(item.security.groups) > 0:
+        security_pb = utils_pb2.Security()
+        for group_id in item.security.groups:
+            if group_id not in security_pb.access_groups:
+                security_pb.access_groups.append(group_id)
+        return security_pb
+    else:
+        return None
+
+
 def _parse_path_query(expr: graph_requests.GraphPathQuery) -> nodereader_pb2.GraphQuery.PathQuery:
     pb = nodereader_pb2.GraphQuery.PathQuery()
 

{nucliadb-6.3.4.post3705.dist-info → nucliadb-6.3.4.post3721.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.3.4.post3705
+Version: 6.3.4.post3721
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.4.post3705
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.4.post3705
-Requires-Dist: nucliadb-protos>=6.3.4.post3705
-Requires-Dist: nucliadb-models>=6.3.4.post3705
-Requires-Dist: nidx-protos>=6.3.4.post3705
+Requires-Dist: nucliadb-telemetry[all]>=6.3.4.post3721
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.4.post3721
+Requires-Dist: nucliadb-protos>=6.3.4.post3721
+Requires-Dist: nucliadb-models>=6.3.4.post3721
+Requires-Dist: nidx-protos>=6.3.4.post3721
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.4.post3705.dist-info → nucliadb-6.3.4.post3721.dist-info}/RECORD

@@ -111,7 +111,7 @@ nucliadb/export_import/exporter.py,sha256=k2QVx1EjqFlDYiggriWiEJzwtMXzHbldsqWdpG
 nucliadb/export_import/importer.py,sha256=v5cq9Nn8c2zrY_K_00mydR52f8mdFxR7tLdtNLQ0qvk,4229
 nucliadb/export_import/models.py,sha256=dbjScNkiMRv4X3Ktudy1JRliD25bfoDTy3JmEZgQSCc,2121
 nucliadb/export_import/tasks.py,sha256=DWbdqY97ffoyfipelGXz3Jqz1iam6JCjQSh367Fc3NA,2947
-nucliadb/export_import/utils.py,sha256=9M81mrFV7FyBcWjsq4KrqfLIXFYbt9_vD1mkY31z4gs,22819
+nucliadb/export_import/utils.py,sha256=8XOVMYXXw8b4ikojG7RjQ4tKN3Xu7nfu2yCUOqD50sk,23216
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
 nucliadb/ingest/app.py,sha256=TaVgh5B2riFVmcsrbPb7a5YCzmnybjx-NK0BXgTwGAY,7535
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
@@ -204,7 +204,7 @@ nucliadb/search/api/v1/ask.py,sha256=SFNjvhY3xAVniMChitILdtzwn7HmgV_k_ecn71egca0
 nucliadb/search/api/v1/catalog.py,sha256=Nw4wIj4AjGp-p64FFVQFN4v2LFcV3A0UJIxfo3_XGmY,7670
 nucliadb/search/api/v1/feedback.py,sha256=kNLc4dHz2SXHzV0PwC1WiRAwY88fDptPcP-kO0q-FrQ,2620
 nucliadb/search/api/v1/find.py,sha256=OOeqCVE88o4t1XzWgcknTWR0e3WcwzLDcnUhCeA1Pcc,10495
-nucliadb/search/api/v1/graph.py,sha256=5APs0-W-jNQfH-mRLrdRkk6B_mnsJxLb68t0NE_KZUk,4238
+nucliadb/search/api/v1/graph.py,sha256=ItVpzJbqfDLjoIo2fTb2mKGCM1Z34sx7CBb3gNmj6IQ,4274
 nucliadb/search/api/v1/knowledgebox.py,sha256=rWhx3PYWryingu19qwwFDbVvVYynq5Ky23FSlzmTutQ,8721
 nucliadb/search/api/v1/predict_proxy.py,sha256=QrGzo0hKjtmyGZ6pjlJHYAh4hxwVUIOTcVcerRCw7eE,3047
 nucliadb/search/api/v1/router.py,sha256=mtT07rBZcVfpa49doaw9b1tj3sdi3qLH0gn9Io6NYM0,988
@@ -257,7 +257,7 @@ nucliadb/search/search/query_parser/old_filters.py,sha256=-zbfN-RsXoj_DRjh3Lfp-w
 nucliadb/search/search/query_parser/parsers/__init__.py,sha256=ySCNSdbesLXGZyR88919njulA6UE10_3PhqMG_Yj1o4,1034
 nucliadb/search/search/query_parser/parsers/catalog.py,sha256=XdBiTweGTQkj8m_V_i2xbwp7P5pPO8K1Tud692XKhMw,7149
 nucliadb/search/search/query_parser/parsers/find.py,sha256=q3wH_i0DGceeKckYEH3c5MqM5EvRiMCL7r-6nCAId9Q,4666
-nucliadb/search/search/query_parser/parsers/graph.py,sha256=fdzKzwnwhSkMXgt8bKXsvBFR8zWjb12lKAgtPzGr37c,6396
+nucliadb/search/search/query_parser/parsers/graph.py,sha256=SfbMV-H-oCqZraQIwD8n5Hqi9dwOlvqzh_I3aZAKatg,8369
 nucliadb/standalone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/standalone/api_router.py,sha256=hgq9FXpihzgjHkwcVGfGCSwyXy67fqXTfLFHuINzIi0,5567
 nucliadb/standalone/app.py,sha256=mAApNK_iVsQgJyd-mtwCeZq5csSimwnXmlQGH9a70pE,5586
@@ -353,8 +353,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.4.post3705.dist-info/METADATA,sha256=BUOECm6cOmDYEdufJ2_6TfQX3ib7DgUIltnFx2y6X9k,4291
-nucliadb-6.3.4.post3705.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-nucliadb-6.3.4.post3705.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.4.post3705.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.4.post3705.dist-info/RECORD,,
+nucliadb-6.3.4.post3721.dist-info/METADATA,sha256=dRb4oW-OP03oqZIpufFKnLhTusmOZtwCdgvZTTLq_NQ,4291
+nucliadb-6.3.4.post3721.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+nucliadb-6.3.4.post3721.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.4.post3721.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.4.post3721.dist-info/RECORD,,

{nucliadb-6.3.4.post3705.dist-info → nucliadb-6.3.4.post3721.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.1.0)
+Generator: setuptools (77.0.3)
 Root-Is-Purelib: true
 Tag: py3-none-any