nucliadb 6.6.1.post4596__py3-none-any.whl → 6.6.1.post4601__py3-none-any.whl

migrations/0038_backfill_catalog_field_labels.py

@@ -0,0 +1,90 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""Migration #38
+
+Backfill the catalog with labels from fields metadata
+
+"""
+
+import logging
+from typing import cast
+
+from nucliadb.common import datamanagers
+from nucliadb.common.maindb.pg import PGDriver, PGTransaction
+from nucliadb.ingest.orm.index_message import get_resource_index_message
+from nucliadb.ingest.orm.processor.pgcatalog import pgcatalog_update
+from nucliadb.migrator.context import ExecutionContext
+from nucliadb_protos import resources_pb2
+
+logger = logging.getLogger(__name__)
+
+
+async def migrate(context: ExecutionContext) -> None: ...
+
+
+async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
+    if not isinstance(context.kv_driver, PGDriver):
+        return
+
+    BATCH_SIZE = 100
+    async with context.kv_driver.transaction() as txn:
+        txn = cast(PGTransaction, txn)
+        start = ""
+        while True:
+            async with txn.connection.cursor() as cur:
+                # Get list of resources except those already in the catalog
+                await cur.execute(
+                    """
+                    SELECT key, value FROM resources
+                    WHERE key ~ ('^/kbs/' || %s || '/r/[^/]*$')
+                    AND key > %s
+                    ORDER BY key
+                    LIMIT %s""",
+                    (kbid, start, BATCH_SIZE),
+                )
+
+                to_index = []
+                rows = await cur.fetchall()
+                if len(rows) == 0:
+                    return
+                for key, basic_pb in rows:
+                    start = key
+
+                    # Only reindex resources with labels in field computed metadata
+                    basic = resources_pb2.Basic()
+                    basic.ParseFromString(basic_pb)
+                    if basic.computedmetadata.field_classifications:
+                        to_index.append(key)
+
+                logger.info(f"Reindexing {len(to_index)} catalog entries from {start}")
+                # Index each resource
+                for key in to_index:
+                    rid = key.split("/")[4]
+                    resource = await datamanagers.resources.get_resource(txn, kbid=kbid, rid=rid)
+                    if resource is None:
+                        logger.warning(f"Could not load resource {rid} for kbid {kbid}")
+                        continue
+
+                    index_message = await get_resource_index_message(resource, reindex=False)
+                    await pgcatalog_update(txn, kbid, resource, index_message)
+
+                if to_index:
+                    await txn.commit()

nucliadb/ingest/orm/processor/pgcatalog.py

@@ -65,12 +65,20 @@ async def pgcatalog_update(txn: Transaction, kbid: str, resource: Resource, inde
         modified_at = created_at
 
     async with _pg_transaction(txn).connection.cursor() as cur:
+        # Do not index canceled labels
+        cancelled_labels = {
+            f"/l/{clf.labelset}/{clf.label}"
+            for clf in resource.basic.usermetadata.classifications
+            if clf.cancelled_by_user
+        }
+
         # Labels from the resource and classification labels from each field
         labels = [label for label in index_message.labels]
-        for field in (await resource.get_fields()).values():
-            meta = await field.get_field_metadata()
-            if meta:
-                labels += [f"/l/{c.labelset}/{c.label}" for c in meta.metadata.classifications]
+        for classification in resource.basic.computedmetadata.field_classifications:
+            for clf in classification.classifications:
+                label = f"/l/{clf.labelset}/{clf.label}"
+                if label not in cancelled_labels:
+                    labels.append(label)
 
         await cur.execute(
             """

{nucliadb-6.6.1.post4596.dist-info → nucliadb-6.6.1.post4601.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.6.1.post4596
+Version: 6.6.1.post4601
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.6.1.post4596
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.6.1.post4596
-Requires-Dist: nucliadb-protos>=6.6.1.post4596
-Requires-Dist: nucliadb-models>=6.6.1.post4596
-Requires-Dist: nidx-protos>=6.6.1.post4596
+Requires-Dist: nucliadb-telemetry[all]>=6.6.1.post4601
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.6.1.post4601
+Requires-Dist: nucliadb-protos>=6.6.1.post4601
+Requires-Dist: nucliadb-models>=6.6.1.post4601
+Requires-Dist: nidx-protos>=6.6.1.post4601
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.43.0
 Requires-Dist: uvicorn[standard]

{nucliadb-6.6.1.post4596.dist-info → nucliadb-6.6.1.post4601.dist-info}/RECORD

@@ -33,6 +33,7 @@ migrations/0034_rollover_nidx_texts_3.py,sha256=t19QtWUgHxmTaBPoR1DooAby2IYmkLTQ
 migrations/0035_rollover_nidx_texts_4.py,sha256=W0_AUd01pjMpYMDC3yqF6HzDLgcnnPprL80kfyb1WZI,1187
 migrations/0036_backfill_catalog_slug.py,sha256=mizRM-HfPswKq4iEmqofu4kIT6Gd97ruT3qhb257vZk,2954
 migrations/0037_backfill_catalog_facets.py,sha256=KAf3VKbKePw7ykDnJi47LyJ7pK1JwYkwMxrsXUnbt9g,2788
+migrations/0038_backfill_catalog_field_labels.py,sha256=EKJwJfU0p1nDq7s71CpGhaX4t1iD2d1ZCzTmLcUAhDs,3382
 migrations/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 migrations/pg/0001_bootstrap.py,sha256=3O_P17l0d0h48nebN6VQLXzM_B7S7zvDpaLR0koVgWE,1274
 migrations/pg/0002_catalog.py,sha256=Rsleecu351Ty19kYZgOpqX5G3MEAY8nMxCJrAeuS2Mw,1690
@@ -167,7 +168,7 @@ nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
-nucliadb/ingest/orm/processor/pgcatalog.py,sha256=f0_bV_5qMGXMG7iLK1DnVwi_pXwFvezfdt3aJ0fZfaQ,4347
+nucliadb/ingest/orm/processor/pgcatalog.py,sha256=VPQ_Evme7xmmGoQ45zt0Am0yPkaD4hxN1r5rEaVt6s8,4633
 nucliadb/ingest/orm/processor/processor.py,sha256=jaEBwbv--WyoC8zcdxWAyF0dAzVA5crVDJl56Bqv1eI,31444
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
@@ -375,8 +376,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.6.1.post4596.dist-info/METADATA,sha256=BGQonZKHKd6s_8MEWZ2NMWLh1cw9lLxHES6QWnoVQww,4158
-nucliadb-6.6.1.post4596.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.6.1.post4596.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.6.1.post4596.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.6.1.post4596.dist-info/RECORD,,
+nucliadb-6.6.1.post4601.dist-info/METADATA,sha256=sIQKpJ7uabOsctChT2-wqcLap668DGEKKG6v_PVYAJ4,4158
+nucliadb-6.6.1.post4601.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.6.1.post4601.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.6.1.post4601.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.6.1.post4601.dist-info/RECORD,,