nucliadb 6.5.0.post4415__py3-none-any.whl → 6.5.0.post4426__py3-none-any.whl

migrations/0036_backfill_catalog_slug.py

@@ -0,0 +1,81 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""Migration #36
+
+Backfill catalog slug field
+
+"""
+
+import logging
+from typing import cast
+
+from nucliadb.common.maindb.pg import PGDriver, PGTransaction
+from nucliadb.migrator.context import ExecutionContext
+
+logger = logging.getLogger(__name__)
+
+
+async def migrate(context: ExecutionContext) -> None:
+    driver = cast(PGDriver, context.kv_driver)
+
+    BATCH_SIZE = 10_000
+    async with driver.transaction() as txn:
+        txn = cast(PGTransaction, txn)
+        start_key = ""
+        while True:
+            async with txn.connection.cursor() as cur:
+                logger.info(f"Filling {BATCH_SIZE} catalog slugs from {start_key}")
+                # Get a batch of slugs from the resource table
+                await cur.execute(
+                    """
+                        CREATE TEMPORARY TABLE tmp_0036_backfill_catalog ON COMMIT DROP AS
+                        SELECT
+                            key,
+                            SPLIT_PART(key, '/', 3)::UUID AS kbid,
+                            SPLIT_PART(key, '/', 5) AS slug,
+                            ENCODE(value, 'escape')::UUID AS rid
+                        FROM resources
+                        WHERE key ~ '^/kbs/[^/]+/s/.*'
+                        AND key > %s
+                        ORDER BY key
+                        LIMIT %s
+                    """,
+                    (start_key, BATCH_SIZE),
+                )
+
+                # Set the key for next iteration
+                await cur.execute("SELECT MAX(key) FROM tmp_0036_backfill_catalog")
+                start_key = (await cur.fetchone())[0]  # type: ignore
+                if start_key is None:
+                    break
+
+                # Update the catalog with the slugs
+                await cur.execute(
+                    """
+                        UPDATE catalog c SET slug = tmp.slug
+                        FROM tmp_0036_backfill_catalog tmp
+                        WHERE c.kbid = tmp.kbid AND c.rid = tmp.rid
+                    """
+                )
+                await txn.commit()
+
+
+async def migrate_kb(context: ExecutionContext, kbid: str) -> None: ...

migrations/pg/0007_catalog_slug.py

@@ -0,0 +1,31 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from nucliadb.common.maindb.pg import PGTransaction
+
+
+async def migrate(txn: PGTransaction) -> None:
+    async with txn.connection.cursor() as cur:
+        await cur.execute(
+            """
+            ALTER TABLE catalog ADD COLUMN slug TEXT;
+            CREATE INDEX ON catalog(slug);
+            """
+        )

nucliadb/ingest/orm/processor/pgcatalog.py

@@ -57,14 +57,15 @@ async def pgcatalog_update(txn: Transaction, kbid: str, resource: Resource, inde
         await cur.execute(
             """
             INSERT INTO catalog
-            (kbid, rid, title, created_at, modified_at, labels)
+            (kbid, rid, title, created_at, modified_at, labels, slug)
             VALUES
-            (%(kbid)s, %(rid)s, %(title)s, %(created_at)s, %(modified_at)s, %(labels)s)
+            (%(kbid)s, %(rid)s, %(title)s, %(created_at)s, %(modified_at)s, %(labels)s, %(slug)s)
             ON CONFLICT (kbid, rid) DO UPDATE SET
             title = excluded.title,
             created_at = excluded.created_at,
             modified_at = excluded.modified_at,
-            labels = excluded.labels""",
+            labels = excluded.labels,
+            slug = excluded.slug""",
             {
                 "kbid": resource.kb.kbid,
                 "rid": resource.uuid,
@@ -72,6 +73,7 @@ async def pgcatalog_update(txn: Transaction, kbid: str, resource: Resource, inde
                 "created_at": created_at,
                 "modified_at": modified_at,
                 "labels": list(index_message.labels),
+                "slug": resource.basic.slug,
             },
         )
 

nucliadb/search/search/predict_proxy.py

@@ -187,7 +187,7 @@ async def chat_streaming_generator(
 
             if is_json:
                 try:
-                    parsed_chunk = GenerativeChunk.model_validate(chunk)
+                    parsed_chunk = GenerativeChunk.model_validate_json(chunk).chunk
                     if isinstance(parsed_chunk, TextGenerativeResponse):
                         text_answer += parsed_chunk.text
                     elif isinstance(parsed_chunk, JSONGenerativeResponse):
@@ -195,7 +195,7 @@ async def chat_streaming_generator(
                     elif isinstance(parsed_chunk, StatusGenerativeResponse):
                         status_code = parsed_chunk.code
                 except ValidationError:
-                    logger.warning(
+                    logger.exception(
                         f"Unexpected item in predict answer stream: {chunk.decode()}",
                         extra={"kbid": kbid},
                     )

{nucliadb-6.5.0.post4415.dist-info → nucliadb-6.5.0.post4426.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.5.0.post4415
+Version: 6.5.0.post4426
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.5.0.post4415
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.5.0.post4415
-Requires-Dist: nucliadb-protos>=6.5.0.post4415
-Requires-Dist: nucliadb-models>=6.5.0.post4415
-Requires-Dist: nidx-protos>=6.5.0.post4415
+Requires-Dist: nucliadb-telemetry[all]>=6.5.0.post4426
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.5.0.post4426
+Requires-Dist: nucliadb-protos>=6.5.0.post4426
+Requires-Dist: nucliadb-models>=6.5.0.post4426
+Requires-Dist: nidx-protos>=6.5.0.post4426
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.5.0.post4415.dist-info → nucliadb-6.5.0.post4426.dist-info}/RECORD

@@ -31,6 +31,7 @@ migrations/0032_remove_old_relations.py,sha256=ZaswhmRRsLgw6DVYVdT7cP-gdBf4X3PL9
 migrations/0033_rollover_nidx_relation_2.py,sha256=9etpqNLVS3PA14qIdsdhorReZxenDaBl-IJNN2AK_Fg,1340
 migrations/0034_rollover_nidx_texts_3.py,sha256=t19QtWUgHxmTaBPoR1DooAby2IYmkLTQj8qu1z2XkFc,1452
 migrations/0035_rollover_nidx_texts_4.py,sha256=W0_AUd01pjMpYMDC3yqF6HzDLgcnnPprL80kfyb1WZI,1187
+migrations/0036_backfill_catalog_slug.py,sha256=mizRM-HfPswKq4iEmqofu4kIT6Gd97ruT3qhb257vZk,2954
 migrations/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 migrations/pg/0001_bootstrap.py,sha256=Fsqkeof50m7fKiJN05kmNEMwiKDlOrAgcAS5sLLkutA,1256
 migrations/pg/0002_catalog.py,sha256=Rsleecu351Ty19kYZgOpqX5G3MEAY8nMxCJrAeuS2Mw,1690
@@ -38,6 +39,7 @@ migrations/pg/0003_catalog_kbid_index.py,sha256=uKq_vtnuf73GVf0mtl2rhzdk_czAoEU1
 migrations/pg/0004_catalog_facets.py,sha256=FJFASHjfEHG3sNve9BP2HnnLO4xr7dnR6Qpctnmt4LE,2180
 migrations/pg/0005_purge_tasks_index.py,sha256=3mtyFgpcK0QQ_NONYay7V9xICijCLNkyTPuoc0PBjRg,1139
 migrations/pg/0006_catalog_title_indexes.py,sha256=n2OGxwE4oeCwHAYaxBkja4t10BmwTjZ2IoCyOdjEBSc,1710
+migrations/pg/0007_catalog_slug.py,sha256=mArzZCBO-RD5DkWxRIyDKgEzrnAcis1TOGvSNUe7Kgg,1150
 migrations/pg/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/__init__.py,sha256=_abCmDJ_0ku483Os4UAjPX7Nywm39cQgAV_DiyjsKeQ,891
 nucliadb/health.py,sha256=UIxxA4oms4HIsCRZM_SZsdkIZIlgzmOxw-qSHLlWuak,3465
@@ -162,7 +164,7 @@ nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
-nucliadb/ingest/orm/processor/pgcatalog.py,sha256=ht9_I5WlPc6sSFTY8PsxHlpjN-EsaBaChwqsLlMXwUk,3100
+nucliadb/ingest/orm/processor/pgcatalog.py,sha256=Zh6s0gj_bwDKPBXSs61jlMKJ6XP-dLnPGbrMGD6RHcM,3195
 nucliadb/ingest/orm/processor/processor.py,sha256=jaEBwbv--WyoC8zcdxWAyF0dAzVA5crVDJl56Bqv1eI,31444
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
@@ -250,7 +252,7 @@ nucliadb/search/search/merge.py,sha256=XiRBsxhYPshPV7lZXD-9E259KZOPIf4I2tKosY0lP
 nucliadb/search/search/metrics.py,sha256=3I6IN0qDSmqIvUaWJmT3rt-Jyjs6LcvnKI8ZqCiuJPY,3501
 nucliadb/search/search/paragraphs.py,sha256=pNAEiYqJGGUVcEf7xf-PFMVqz0PX4Qb-WNG-_zPGN2o,7799
 nucliadb/search/search/pgcatalog.py,sha256=QtgArjoM-dW_B1oO0aXqp5au7GlLG8jAct9jevUHatw,10997
-nucliadb/search/search/predict_proxy.py,sha256=2451FR2DJLedA4eV8KbMbY4la5WZCgjBUHO_iQVaA8I,8634
+nucliadb/search/search/predict_proxy.py,sha256=cuD_sfM3RLdEoQaanRz0CflO6nKVGGKPzoFA17shb_w,8647
 nucliadb/search/search/query.py,sha256=0qIQdt548L3jtKOyKo06aGJ73SLBxAW3N38_Hc1M3Uw,11528
 nucliadb/search/search/rank_fusion.py,sha256=xZtXhbmKb_56gs73u6KkFm2efvTATOSMmpOV2wrAIqE,9613
 nucliadb/search/search/rerankers.py,sha256=E2J1QdKAojqbhHM3KAyaOXKf6tJyETUxKs4tf_BEyqk,7472
@@ -370,8 +372,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.5.0.post4415.dist-info/METADATA,sha256=lSbgbK7GgTJXjm5IIzYAeFt7-W8LdME3CKarZjmQwbg,4152
-nucliadb-6.5.0.post4415.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.5.0.post4415.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.5.0.post4415.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.5.0.post4415.dist-info/RECORD,,
+nucliadb-6.5.0.post4426.dist-info/METADATA,sha256=fRo_rQ3D5zAGctuqOfk22MzKACI4nZ8mijFy-JSGaT0,4152
+nucliadb-6.5.0.post4426.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.5.0.post4426.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.5.0.post4426.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.5.0.post4426.dist-info/RECORD,,