nucliadb 6.1.0.post2567__py3-none-any.whl → 6.1.0.post2572__py3-none-any.whl

nucliadb/export_import/importer.py

@@ -37,6 +37,7 @@ from nucliadb.export_import.utils import (
 )
 from nucliadb_protos import knowledgebox_pb2 as kb_pb2
 from nucliadb_protos import resources_pb2, writer_pb2
+from nucliadb_utils.utilities import _create_storage
 
 BinaryStream = AsyncGenerator[bytes, None]
 BinaryStreamGenerator = Callable[[int], BinaryStream]
@@ -97,17 +98,28 @@ async def import_kb_from_blob_storage(context: ApplicationContext, msg: NatsTask
     """
     Imports to a knowledgebox from an export stored in the blob storage service.
     """
-    kbid, import_id = msg.kbid, msg.id
-    dm = ExportImportDataManager(context.kv_driver, context.blob_storage)
-    metadata = await dm.get_metadata(type="import", kbid=kbid, id=import_id)
-
-    retry_handler = TaskRetryHandler("import", dm, metadata)
-
-    @retry_handler.wrap
-    async def import_kb_retried(context, kbid, metadata):
-        stream = dm.download_import(kbid, import_id)
-        await import_kb(context, kbid, stream, metadata)
-
-    await import_kb_retried(context, kbid, metadata)
-
-    await dm.try_delete_from_storage("import", kbid, import_id)
+    try:
+        # There is an issue when using the same aiobotocore client to download and upload files concurrently.
+        # https://github.com/aio-libs/aiobotocore/issues/1235
+        #
+        # As a workaround, we use a newly created client for the download of the import file, while using
+        # the normally configured storage for the uploads:
+        # - The `ExportImportDataManager` is used for the downloads and gets the new storage
+        # - `import_kb` gets the `context` with the default blob storage to use in uploads
+        download_storage = await _create_storage()
+        kbid, import_id = msg.kbid, msg.id
+        dm = ExportImportDataManager(context.kv_driver, download_storage)
+        metadata = await dm.get_metadata(type="import", kbid=kbid, id=import_id)
+
+        retry_handler = TaskRetryHandler("import", dm, metadata)
+
+        @retry_handler.wrap
+        async def import_kb_retried(context, kbid, metadata):
+            stream = dm.download_import(kbid, import_id)
+            await import_kb(context, kbid, stream, metadata)
+
+        await import_kb_retried(context, kbid, metadata)
+
+        await dm.try_delete_from_storage("import", kbid, import_id)
+    finally:
+        await download_storage.finalize()

nucliadb/ingest/orm/resource.py

@@ -1160,12 +1160,20 @@ def add_field_classifications(basic: PBBasic, fcmw: FieldComputedMetadataWrapper
     """
     Returns whether some new field classifications were added
     """
-    if len(fcmw.metadata.metadata.classifications) == 0:
+    if len(fcmw.metadata.metadata.classifications) == 0 and all(
+        len(split.classifications) == 0 for split in fcmw.metadata.split_metadata.values()
+    ):
         return False
+
     remove_field_classifications(basic, [fcmw.field])
     fcfs = FieldClassifications()
     fcfs.field.CopyFrom(fcmw.field)
     fcfs.classifications.extend(fcmw.metadata.metadata.classifications)
+
+    for split_id, split in fcmw.metadata.split_metadata.items():
+        if split_id not in fcmw.metadata.deleted_splits:
+            fcfs.classifications.extend(split.classifications)
+
     basic.computedmetadata.field_classifications.append(fcfs)
     return True
 

{nucliadb-6.1.0.post2567.dist-info → nucliadb-6.1.0.post2572.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nucliadb
-Version: 6.1.0.post2567
+Version: 6.1.0.post2572
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.1.0.post2567
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.1.0.post2567
-Requires-Dist: nucliadb-protos>=6.1.0.post2567
-Requires-Dist: nucliadb-models>=6.1.0.post2567
+Requires-Dist: nucliadb-telemetry[all]>=6.1.0.post2572
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.1.0.post2572
+Requires-Dist: nucliadb-protos>=6.1.0.post2572
+Requires-Dist: nucliadb-models>=6.1.0.post2572
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: uvicorn

{nucliadb-6.1.0.post2567.dist-info → nucliadb-6.1.0.post2572.dist-info}/RECORD

@@ -100,7 +100,7 @@ nucliadb/export_import/__init__.py,sha256=y-Is0Bxa8TMV6UiOW0deC_D3U465P65CQ5RjBj
 nucliadb/export_import/datamanager.py,sha256=b9Vhf-WqJ8HosTdNpKXlGj-Vi7MHyMoPxL0VpfPlYlE,6720
 nucliadb/export_import/exceptions.py,sha256=Dw8WqfG4r6MPJc5TPfbjMvCgXXWTcTOecGHRVU1h3kM,1949
 nucliadb/export_import/exporter.py,sha256=kgbW-B7FNW7mlc9rBVEfwkkFTqD58TWSTDe9zkmEnBc,7098
-nucliadb/export_import/importer.py,sha256=v5cq9Nn8c2zrY_K_00mydR52f8mdFxR7tLdtNLQ0qvk,4229
+nucliadb/export_import/importer.py,sha256=_tzoE2dUfLPivjp8CPE5BMMRCCr_1heOrWmqIe89_LQ,4964
 nucliadb/export_import/models.py,sha256=VHetyHlofHH5yzcV5q6hN15zGKJTRAu7GOTNveN-tGA,2063
 nucliadb/export_import/tasks.py,sha256=fpCBeFYPReyLIdk38LDM9Tpnw_VczeMrobT4n1RAIp4,2507
 nucliadb/export_import/utils.py,sha256=zrNrkkc9i3uT-R6Ju4J_0WNrzayln3KuQFCz-_qIaIA,19613
@@ -137,7 +137,7 @@ nucliadb/ingest/orm/entities.py,sha256=xWhp_JXxHjUJ-m3I08tLqkpIrThHH0LJDvIVgK4q7
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
 nucliadb/ingest/orm/knowledgebox.py,sha256=hKKqvZUG4tdPO3Z29_qvz5-dcX5_X2x5pkNQhDGnR7Q,22746
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
-nucliadb/ingest/orm/resource.py,sha256=yCSpJyFYZkBCVNO9Opwaz_LXLnsPvDJuDY3cbm9E6Ws,53478
+nucliadb/ingest/orm/resource.py,sha256=cW_19P2grUOY255pZXvgF13SEyPCy_e8JMogIRqWXxA,53772
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
@@ -331,9 +331,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.1.0.post2567.dist-info/METADATA,sha256=XMUKY9sqMxtxKBbcjyY3nFz3CeHhcSeR-yQa_lsY-9I,4390
-nucliadb-6.1.0.post2567.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-nucliadb-6.1.0.post2567.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.1.0.post2567.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.1.0.post2567.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.1.0.post2567.dist-info/RECORD,,
+nucliadb-6.1.0.post2572.dist-info/METADATA,sha256=Xgkb6zYf-w9Ys2nS1-c4wPB-fhPrlrsoH9KXB44Vfto,4390
+nucliadb-6.1.0.post2572.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+nucliadb-6.1.0.post2572.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.1.0.post2572.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.1.0.post2572.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.1.0.post2572.dist-info/RECORD,,