nucliadb 6.2.1.post3385__py3-none-any.whl → 6.2.1.post3395__py3-none-any.whl

migrations/0031_languages_deduplication.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""Migration #31
+
+At some point we had a bug that allowed to have repeated languages in the metadata basic.
+This migration aims to fix that by removing the duplicates.
+
+"""
+
+import logging
+
+from nucliadb.common import datamanagers
+from nucliadb.migrator.context import ExecutionContext
+
+logger = logging.getLogger(__name__)
+
+
+async def migrate(context: ExecutionContext) -> None: ...
+
+
+async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
+    async with datamanagers.with_ro_transaction() as rs_txn:
+        async for rid in datamanagers.resources.iterate_resource_ids(kbid=kbid):
+            basic = await datamanagers.resources.get_basic(rs_txn, kbid=kbid, rid=rid)
+            if basic is None:
+                continue
+            unique_langs = set(basic.metadata.languages)
+            if len(unique_langs) != len(basic.metadata.languages):
+                await fix_resource(kbid=kbid, rid=rid)
+
+
+async def fix_resource(kbid: str, rid: str):
+    async with datamanagers.with_rw_transaction() as txn:
+        basic = await datamanagers.resources.get_basic(txn, kbid=kbid, rid=rid)
+        if basic is None:
+            return
+        logger.info(f"Fixing duplicate languages", extra={"kbid": kbid, "rid": rid})
+        unique_langs = set(basic.metadata.languages)
+        basic.metadata.ClearField("languages")
+        basic.metadata.languages.extend(list(unique_langs))
+        await datamanagers.resources.set_basic(txn, kbid=kbid, rid=rid, basic=basic)
+        await txn.commit()

nucliadb/ingest/orm/resource.py

@@ -176,6 +176,16 @@ class Resource:
 
             self.basic.MergeFrom(payload)
 
+            # Prevent duplicated languages
+            unique_languages = set(self.basic.metadata.languages)
+            self.basic.metadata.ClearField("languages")
+            self.basic.metadata.languages.extend(unique_languages)
+
+            # Prevent duplicated labels
+            unique_labels = set(self.basic.labels)
+            self.basic.ClearField("labels")
+            self.basic.labels.extend(unique_labels)
+
             self.set_processing_status(self.basic, payload)
 
             # We force the usermetadata classification to be the one defined

{nucliadb-6.2.1.post3385.dist-info → nucliadb-6.2.1.post3395.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post3385
+Version: 6.2.1.post3395
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3385
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3385
-Requires-Dist: nucliadb-protos>=6.2.1.post3385
-Requires-Dist: nucliadb-models>=6.2.1.post3385
-Requires-Dist: nidx-protos>=6.2.1.post3385
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3395
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3395
+Requires-Dist: nucliadb-protos>=6.2.1.post3395
+Requires-Dist: nucliadb-models>=6.2.1.post3395
+Requires-Dist: nidx-protos>=6.2.1.post3395
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.2.1.post3385.dist-info → nucliadb-6.2.1.post3395.dist-info}/RECORD

@@ -26,6 +26,7 @@ migrations/0027_rollover_texts3.py,sha256=UQDaMOayVuqDisf82NDrPStoEVveHvdjkSmzbI
 migrations/0028_extracted_vectors_reference.py,sha256=49DHCIlBpjofU8cYVHTdWv0EBIlnPTWV2WCezf0rJUo,2392
 migrations/0029_backfill_field_status.py,sha256=QWF69n1da9lpRnbEpgbqPjSQ-Wfn6rMC7Enz6bBYGt4,5663
 migrations/0030_label_deduplication.py,sha256=y14TxtCMi3-TBMz_eZoyyPDHNlZb29taJujlDuHumsA,2008
+migrations/0031_languages_deduplication.py,sha256=o6va6lP3oTRT1uSzp5MIhHHBFbhCxSZ-oNlXXpiAdUo,2340
 migrations/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 migrations/pg/0001_bootstrap.py,sha256=Fsqkeof50m7fKiJN05kmNEMwiKDlOrAgcAS5sLLkutA,1256
 migrations/pg/0002_catalog.py,sha256=Rsleecu351Ty19kYZgOpqX5G3MEAY8nMxCJrAeuS2Mw,1690
@@ -133,7 +134,7 @@ nucliadb/ingest/orm/entities.py,sha256=3_n6lKhBy2GsdmNmkh0_mvxP8md20OZsbtTNEmfJ8
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
 nucliadb/ingest/orm/knowledgebox.py,sha256=IGOPvBR1qXqDxE5DeiOdYCLdPgjzOVVpsASJ2zYvWwQ,23651
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
-nucliadb/ingest/orm/resource.py,sha256=9J1O9VW2ZNTEbJ9Fv0J3yIGR5Gf_FFJJjiBzO0SeRXE,44219
+nucliadb/ingest/orm/resource.py,sha256=YEhTn3VB5IHOx6eOrEIMx0wlBBDO1WZOb-fhA494YSM,44640
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
@@ -337,8 +338,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post3385.dist-info/METADATA,sha256=9yWSfqPtCZGTZuIyyp0ieqekuoXpw7xCqq4TejYH29M,4291
-nucliadb-6.2.1.post3385.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post3385.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post3385.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post3385.dist-info/RECORD,,
+nucliadb-6.2.1.post3395.dist-info/METADATA,sha256=dK45Sc6ekVyUZED7nXwb3j74ycmcL-hyQexGDV2hqxA,4291
+nucliadb-6.2.1.post3395.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post3395.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post3395.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post3395.dist-info/RECORD,,