nucliadb 6.7.2.post4868__py3-none-any.whl → 6.7.2.post4876__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of nucliadb might be problematic. Click here for more details.
- migrations/0016_upgrade_to_paragraphs_v2.py +1 -1
- migrations/0017_multiple_writable_shards.py +1 -1
- migrations/0018_purge_orphan_kbslugs.py +1 -1
- migrations/0019_upgrade_to_paragraphs_v3.py +1 -1
- migrations/0021_overwrite_vectorsets_key.py +1 -1
- migrations/0023_backfill_pg_catalog.py +1 -1
- migrations/0025_assign_models_to_kbs_v2.py +3 -3
- migrations/0027_rollover_texts3.py +1 -1
- migrations/0029_backfill_field_status.py +1 -1
- migrations/0032_remove_old_relations.py +1 -1
- migrations/0036_backfill_catalog_slug.py +1 -1
- migrations/0037_backfill_catalog_facets.py +1 -1
- migrations/0038_backfill_catalog_field_labels.py +1 -1
- nucliadb/backups/create.py +3 -3
- nucliadb/backups/restore.py +3 -3
- nucliadb/common/cache.py +1 -1
- nucliadb/common/datamanagers/utils.py +2 -2
- nucliadb/common/locking.py +4 -4
- nucliadb/common/maindb/driver.py +11 -1
- nucliadb/common/maindb/local.py +1 -1
- nucliadb/common/maindb/pg.py +1 -1
- nucliadb/export_import/datamanager.py +3 -3
- nucliadb/ingest/consumer/shard_creator.py +1 -1
- nucliadb/ingest/orm/knowledgebox.py +4 -4
- nucliadb/ingest/orm/processor/auditing.py +1 -1
- nucliadb/ingest/orm/processor/processor.py +4 -4
- nucliadb/ingest/orm/processor/sequence_manager.py +1 -1
- nucliadb/ingest/serialize.py +2 -2
- nucliadb/ingest/service/writer.py +15 -15
- nucliadb/metrics_exporter.py +1 -1
- nucliadb/migrator/datamanager.py +11 -11
- nucliadb/migrator/migrator.py +2 -2
- nucliadb/purge/__init__.py +10 -10
- nucliadb/purge/orphan_shards.py +2 -2
- nucliadb/reader/api/v1/knowledgebox.py +3 -3
- nucliadb/reader/api/v1/resource.py +2 -2
- nucliadb/reader/api/v1/services.py +1 -1
- nucliadb/reader/reader/notifications.py +1 -1
- nucliadb/search/search/cache.py +1 -1
- nucliadb/search/search/chat/prompt.py +2 -2
- nucliadb/search/search/fetch.py +1 -1
- nucliadb/search/search/hydrator.py +2 -2
- nucliadb/search/search/query_parser/fetcher.py +5 -5
- nucliadb/search/search/summarize.py +1 -1
- nucliadb/tasks/retries.py +4 -4
- nucliadb/train/generators/utils.py +1 -1
- nucliadb/train/nodes.py +4 -4
- nucliadb/train/servicer.py +1 -1
- nucliadb/train/uploader.py +1 -1
- nucliadb/writer/api/v1/field.py +1 -1
- nucliadb/writer/api/v1/knowledgebox.py +1 -1
- nucliadb/writer/api/v1/resource.py +2 -2
- {nucliadb-6.7.2.post4868.dist-info → nucliadb-6.7.2.post4876.dist-info}/METADATA +6 -6
- {nucliadb-6.7.2.post4868.dist-info → nucliadb-6.7.2.post4876.dist-info}/RECORD +57 -57
- {nucliadb-6.7.2.post4868.dist-info → nucliadb-6.7.2.post4876.dist-info}/WHEEL +0 -0
- {nucliadb-6.7.2.post4868.dist-info → nucliadb-6.7.2.post4876.dist-info}/entry_points.txt +0 -0
- {nucliadb-6.7.2.post4868.dist-info → nucliadb-6.7.2.post4876.dist-info}/top_level.txt +0 -0
|
@@ -55,7 +55,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
# async def has_old_paragraphs_index(context: ExecutionContext, kbid: str) -> bool:
|
|
58
|
-
# async with context.kv_driver.
|
|
58
|
+
# async with context.kv_driver.ro_transaction() as txn:
|
|
59
59
|
# shards_object = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid, for_update=False)
|
|
60
60
|
# if not shards_object:
|
|
61
61
|
# raise ShardsObjectNotFound()
|
|
@@ -44,7 +44,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
44
44
|
|
|
45
45
|
# No longer relevant with nidx
|
|
46
46
|
|
|
47
|
-
# async with context.kv_driver.
|
|
47
|
+
# async with context.kv_driver.rw_transaction() as txn:
|
|
48
48
|
# shards = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid, for_update=True)
|
|
49
49
|
# if shards is None:
|
|
50
50
|
# logger.error("KB without shards", extra={"kbid": kbid})
|
|
@@ -35,7 +35,7 @@ logger = logging.getLogger(__name__)
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
async def migrate(context: ExecutionContext) -> None:
|
|
38
|
-
async with context.kv_driver.
|
|
38
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
39
39
|
async for key in txn.keys(KB_SLUGS_BASE):
|
|
40
40
|
slug = key.replace(KB_SLUGS_BASE, "")
|
|
41
41
|
value = await txn.get(key, for_update=False)
|
|
@@ -55,7 +55,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
# async def has_old_paragraphs_index(context: ExecutionContext, kbid: str) -> bool:
|
|
58
|
-
# async with context.kv_driver.
|
|
58
|
+
# async with context.kv_driver.ro_transaction() as txn:
|
|
59
59
|
# shards_object = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid)
|
|
60
60
|
# if not shards_object:
|
|
61
61
|
# raise ShardsObjectNotFound()
|
|
@@ -38,7 +38,7 @@ async def migrate(context: ExecutionContext) -> None: ...
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
41
|
-
async with context.kv_driver.
|
|
41
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
42
42
|
logger.info(f"Overwriting vectorsets key", extra={"kbid": kbid})
|
|
43
43
|
await datamanagers.vectorsets.initialize(txn, kbid=kbid)
|
|
44
44
|
await txn.commit()
|
|
@@ -44,7 +44,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
44
44
|
return
|
|
45
45
|
|
|
46
46
|
BATCH_SIZE = 100
|
|
47
|
-
async with context.kv_driver.
|
|
47
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
48
48
|
txn = cast(PGTransaction, txn)
|
|
49
49
|
continue_sql = ""
|
|
50
50
|
while True:
|
|
@@ -47,7 +47,7 @@ async def migrate(context: ExecutionContext) -> None: ...
|
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
50
|
-
async with context.kv_driver.
|
|
50
|
+
async with context.kv_driver.ro_transaction() as txn:
|
|
51
51
|
vectorsets_count = len([vs async for vs in datamanagers.vectorsets.iter(txn, kbid=kbid)])
|
|
52
52
|
if vectorsets_count > 0:
|
|
53
53
|
logger.info("Skipping KB with vectorsets already populated", extra={"kbid": kbid})
|
|
@@ -65,7 +65,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
65
65
|
learning_matryoshka_dimensions = learning_model_metadata.matryoshka_dimensions
|
|
66
66
|
learning_normalize_vectors = len(learning_matryoshka_dimensions) > 0
|
|
67
67
|
|
|
68
|
-
async with context.kv_driver.
|
|
68
|
+
async with context.kv_driver.ro_transaction() as txn:
|
|
69
69
|
semantic_model = await datamanagers.kb.get_model_metadata(txn, kbid=kbid)
|
|
70
70
|
|
|
71
71
|
maindb_similarity = semantic_model.similarity_function
|
|
@@ -103,7 +103,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
103
103
|
matryoshka_dimensions=maindb_matryoshka_dimensions,
|
|
104
104
|
)
|
|
105
105
|
|
|
106
|
-
async with context.kv_driver.
|
|
106
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
107
107
|
# Populate KB vectorsets with data from learning. We are skipping KBs
|
|
108
108
|
# with this key already set, so we can set here safely
|
|
109
109
|
await datamanagers.vectorsets.set(txn, kbid=kbid, config=default_vectorset)
|
|
@@ -49,7 +49,7 @@ async def maybe_fix_vector_dimensions(context: ExecutionContext, kbid: str) -> N
|
|
|
49
49
|
logger.warning(f"KB has no learning config", extra={"kbid": kbid})
|
|
50
50
|
return
|
|
51
51
|
|
|
52
|
-
async with context.kv_driver.
|
|
52
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
53
53
|
vectorsets = [vs async for vs in datamanagers.vectorsets.iter(txn, kbid=kbid)]
|
|
54
54
|
if len(vectorsets) != 1:
|
|
55
55
|
# If multiple vectorsets, they are new shards created correctly, we can safely skip it
|
|
@@ -45,7 +45,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None: ...
|
|
|
45
45
|
|
|
46
46
|
async def do_batch(context: ExecutionContext, start: str) -> Optional[str]:
|
|
47
47
|
logger.info(f"Running batch from {start}")
|
|
48
|
-
async with context.kv_driver.
|
|
48
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
49
49
|
async with txn.connection.cursor() as cur: # type: ignore
|
|
50
50
|
# Retrieve a batch of fields
|
|
51
51
|
await cur.execute(
|
|
@@ -47,7 +47,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
47
47
|
|
|
48
48
|
async def do_batch(context: ExecutionContext, start: str) -> Optional[str]:
|
|
49
49
|
logger.info(f"Running batch from {start}")
|
|
50
|
-
async with context.kv_driver.
|
|
50
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
51
51
|
async with txn.connection.cursor() as cur: # type: ignore
|
|
52
52
|
# Retrieve a batch of fields
|
|
53
53
|
await cur.execute(
|
|
@@ -37,7 +37,7 @@ async def migrate(context: ExecutionContext) -> None:
|
|
|
37
37
|
driver = cast(PGDriver, context.kv_driver)
|
|
38
38
|
|
|
39
39
|
BATCH_SIZE = 10_000
|
|
40
|
-
async with driver.
|
|
40
|
+
async with driver.rw_transaction() as txn:
|
|
41
41
|
txn = cast(PGTransaction, txn)
|
|
42
42
|
start_key = ""
|
|
43
43
|
while True:
|
|
@@ -37,7 +37,7 @@ async def migrate(context: ExecutionContext) -> None:
|
|
|
37
37
|
driver = cast(PGDriver, context.kv_driver)
|
|
38
38
|
|
|
39
39
|
BATCH_SIZE = 1_000
|
|
40
|
-
async with driver.
|
|
40
|
+
async with driver.rw_transaction() as txn:
|
|
41
41
|
txn = cast(PGTransaction, txn)
|
|
42
42
|
start_kbid = "00000000000000000000000000000000"
|
|
43
43
|
start_rid = "00000000000000000000000000000000"
|
|
@@ -45,7 +45,7 @@ async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
|
|
|
45
45
|
return
|
|
46
46
|
|
|
47
47
|
BATCH_SIZE = 100
|
|
48
|
-
async with context.kv_driver.
|
|
48
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
49
49
|
txn = cast(PGTransaction, txn)
|
|
50
50
|
start = ""
|
|
51
51
|
while True:
|
nucliadb/backups/create.py
CHANGED
|
@@ -269,7 +269,7 @@ async def backup_search_configurations(context: ApplicationContext, kbid: str, b
|
|
|
269
269
|
async def get_metadata(
|
|
270
270
|
context: ApplicationContext, kbid: str, backup_id: str
|
|
271
271
|
) -> Optional[BackupMetadata]:
|
|
272
|
-
async with context.kv_driver.
|
|
272
|
+
async with context.kv_driver.ro_transaction() as txn:
|
|
273
273
|
metadata_raw = await txn.get(MaindbKeys.METADATA.format(kbid=kbid, backup_id=backup_id))
|
|
274
274
|
if metadata_raw is None:
|
|
275
275
|
return None
|
|
@@ -277,7 +277,7 @@ async def get_metadata(
|
|
|
277
277
|
|
|
278
278
|
|
|
279
279
|
async def set_metadata(context: ApplicationContext, kbid: str, backup_id: str, metadata: BackupMetadata):
|
|
280
|
-
async with context.kv_driver.
|
|
280
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
281
281
|
await txn.set(
|
|
282
282
|
MaindbKeys.METADATA.format(kbid=kbid, backup_id=backup_id),
|
|
283
283
|
metadata.model_dump_json().encode(),
|
|
@@ -286,7 +286,7 @@ async def set_metadata(context: ApplicationContext, kbid: str, backup_id: str, m
|
|
|
286
286
|
|
|
287
287
|
|
|
288
288
|
async def delete_metadata(context: ApplicationContext, kbid: str, backup_id: str):
|
|
289
|
-
async with context.kv_driver.
|
|
289
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
290
290
|
await txn.delete(MaindbKeys.METADATA.format(kbid=kbid, backup_id=backup_id))
|
|
291
291
|
await txn.commit()
|
|
292
292
|
|
nucliadb/backups/restore.py
CHANGED
|
@@ -103,7 +103,7 @@ async def restore_resources(context: ApplicationContext, kbid: str, backup_id: s
|
|
|
103
103
|
|
|
104
104
|
async def get_last_restored(context: ApplicationContext, kbid: str, backup_id: str) -> Optional[str]:
|
|
105
105
|
key = MaindbKeys.LAST_RESTORED.format(kbid=kbid, backup_id=backup_id)
|
|
106
|
-
async with context.kv_driver.
|
|
106
|
+
async with context.kv_driver.ro_transaction() as txn:
|
|
107
107
|
raw = await txn.get(key)
|
|
108
108
|
if raw is None:
|
|
109
109
|
return None
|
|
@@ -112,14 +112,14 @@ async def get_last_restored(context: ApplicationContext, kbid: str, backup_id: s
|
|
|
112
112
|
|
|
113
113
|
async def set_last_restored(context: ApplicationContext, kbid: str, backup_id: str, resource_id: str):
|
|
114
114
|
key = MaindbKeys.LAST_RESTORED.format(kbid=kbid, backup_id=backup_id)
|
|
115
|
-
async with context.kv_driver.
|
|
115
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
116
116
|
await txn.set(key, resource_id.encode())
|
|
117
117
|
await txn.commit()
|
|
118
118
|
|
|
119
119
|
|
|
120
120
|
async def delete_last_restored(context: ApplicationContext, kbid: str, backup_id: str):
|
|
121
121
|
key = MaindbKeys.LAST_RESTORED.format(kbid=kbid, backup_id=backup_id)
|
|
122
|
-
async with context.kv_driver.
|
|
122
|
+
async with context.kv_driver.rw_transaction() as txn:
|
|
123
123
|
await txn.delete(key)
|
|
124
124
|
await txn.commit()
|
|
125
125
|
|
nucliadb/common/cache.py
CHANGED
|
@@ -90,7 +90,7 @@ class ResourceCache(Cache[[str, str], ResourceORM]):
|
|
|
90
90
|
@alru_cache(maxsize=cache_size)
|
|
91
91
|
async def _get_resource(kbid: str, rid: str) -> Optional[ResourceORM]:
|
|
92
92
|
storage = await get_storage()
|
|
93
|
-
async with get_driver().
|
|
93
|
+
async with get_driver().ro_transaction() as txn:
|
|
94
94
|
kb = KnowledgeBoxORM(txn, storage, kbid)
|
|
95
95
|
return await kb.get(rid)
|
|
96
96
|
|
|
@@ -42,7 +42,7 @@ async def get_kv_pb(
|
|
|
42
42
|
@contextlib.asynccontextmanager
|
|
43
43
|
async def with_rw_transaction():
|
|
44
44
|
driver = get_driver()
|
|
45
|
-
async with driver.
|
|
45
|
+
async with driver.rw_transaction() as txn:
|
|
46
46
|
yield txn
|
|
47
47
|
|
|
48
48
|
|
|
@@ -53,5 +53,5 @@ with_transaction = with_rw_transaction
|
|
|
53
53
|
@contextlib.asynccontextmanager
|
|
54
54
|
async def with_ro_transaction():
|
|
55
55
|
driver = get_driver()
|
|
56
|
-
async with driver.
|
|
56
|
+
async with driver.ro_transaction() as ro_txn:
|
|
57
57
|
yield ro_txn
|
nucliadb/common/locking.py
CHANGED
|
@@ -75,7 +75,7 @@ class _Lock:
|
|
|
75
75
|
start = time.time()
|
|
76
76
|
while True:
|
|
77
77
|
try:
|
|
78
|
-
async with self.driver.
|
|
78
|
+
async with self.driver.rw_transaction() as txn:
|
|
79
79
|
lock_data = await self.get_lock_data(txn)
|
|
80
80
|
if lock_data is None:
|
|
81
81
|
await self._set_lock_value(txn)
|
|
@@ -128,7 +128,7 @@ class _Lock:
|
|
|
128
128
|
while True:
|
|
129
129
|
try:
|
|
130
130
|
await asyncio.sleep(self.refresh_timeout)
|
|
131
|
-
async with self.driver.
|
|
131
|
+
async with self.driver.rw_transaction() as txn:
|
|
132
132
|
await self._update_lock_value(txn)
|
|
133
133
|
await txn.commit()
|
|
134
134
|
except (asyncio.CancelledError, RuntimeError):
|
|
@@ -138,12 +138,12 @@ class _Lock:
|
|
|
138
138
|
|
|
139
139
|
async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
|
|
140
140
|
self.task.cancel()
|
|
141
|
-
async with self.driver.
|
|
141
|
+
async with self.driver.rw_transaction() as txn:
|
|
142
142
|
await txn.delete(self.key)
|
|
143
143
|
await txn.commit()
|
|
144
144
|
|
|
145
145
|
async def is_locked(self) -> bool:
|
|
146
|
-
async with get_driver().
|
|
146
|
+
async with get_driver().ro_transaction() as txn:
|
|
147
147
|
lock_data = await self.get_lock_data(txn)
|
|
148
148
|
return lock_data is not None and time.time() < lock_data.expires_at
|
|
149
149
|
|
nucliadb/common/maindb/driver.py
CHANGED
|
@@ -81,5 +81,15 @@ class Driver:
|
|
|
81
81
|
pass
|
|
82
82
|
|
|
83
83
|
@asynccontextmanager
|
|
84
|
-
async def
|
|
84
|
+
async def _transaction(self, *, read_only: bool) -> AsyncGenerator[Transaction, None]:
|
|
85
85
|
yield Transaction()
|
|
86
|
+
|
|
87
|
+
@asynccontextmanager
|
|
88
|
+
async def ro_transaction(self) -> AsyncGenerator[Transaction, None]:
|
|
89
|
+
async with self._transaction(read_only=True) as txn:
|
|
90
|
+
yield txn
|
|
91
|
+
|
|
92
|
+
@asynccontextmanager
|
|
93
|
+
async def rw_transaction(self) -> AsyncGenerator[Transaction, None]:
|
|
94
|
+
async with self._transaction(read_only=False) as txn:
|
|
95
|
+
yield txn
|
nucliadb/common/maindb/local.py
CHANGED
|
@@ -222,7 +222,7 @@ class LocalDriver(Driver):
|
|
|
222
222
|
pass
|
|
223
223
|
|
|
224
224
|
@asynccontextmanager
|
|
225
|
-
async def
|
|
225
|
+
async def _transaction(self, *, read_only: bool) -> AsyncGenerator[Transaction, None]:
|
|
226
226
|
if self.url is None:
|
|
227
227
|
raise AttributeError("Invalid url")
|
|
228
228
|
txn = LocalTransaction(self.url, self)
|
nucliadb/common/maindb/pg.py
CHANGED
|
@@ -330,7 +330,7 @@ class PGDriver(Driver):
|
|
|
330
330
|
metric.set(value)
|
|
331
331
|
|
|
332
332
|
@asynccontextmanager
|
|
333
|
-
async def
|
|
333
|
+
async def _transaction(self, *, read_only: bool) -> AsyncGenerator[Transaction, None]:
|
|
334
334
|
if read_only:
|
|
335
335
|
yield ReadOnlyPGTransaction(self)
|
|
336
336
|
else:
|
|
@@ -54,7 +54,7 @@ class ExportImportDataManager:
|
|
|
54
54
|
|
|
55
55
|
async def get_metadata(self, type: str, kbid: str, id: str) -> Metadata:
|
|
56
56
|
key = self._get_maindb_metadata_key(type, kbid, id)
|
|
57
|
-
async with self.driver.
|
|
57
|
+
async with self.driver.ro_transaction() as txn:
|
|
58
58
|
data = await txn.get(key)
|
|
59
59
|
if data is None or data == b"":
|
|
60
60
|
raise MetadataNotFound()
|
|
@@ -89,13 +89,13 @@ class ExportImportDataManager:
|
|
|
89
89
|
metadata.modified = datetime.now(timezone.utc)
|
|
90
90
|
key = self._get_maindb_metadata_key(type, metadata.kbid, metadata.id)
|
|
91
91
|
data = metadata.model_dump_json().encode("utf-8")
|
|
92
|
-
async with self.driver.
|
|
92
|
+
async with self.driver.rw_transaction() as txn:
|
|
93
93
|
await txn.set(key, data)
|
|
94
94
|
await txn.commit()
|
|
95
95
|
|
|
96
96
|
async def delete_metadata(self, type: str, metadata: Metadata):
|
|
97
97
|
key = self._get_maindb_metadata_key(type, metadata.kbid, metadata.id)
|
|
98
|
-
async with self.driver.
|
|
98
|
+
async with self.driver.rw_transaction() as txn:
|
|
99
99
|
await txn.delete(key)
|
|
100
100
|
await txn.commit()
|
|
101
101
|
|
|
@@ -91,7 +91,7 @@ class ShardCreatorHandler:
|
|
|
91
91
|
@metrics.handler_histo.wrap({"type": "shard_creator"})
|
|
92
92
|
async def process_kb(self, kbid: str) -> None:
|
|
93
93
|
logger.info({"message": "Processing notification for kbid", "kbid": kbid})
|
|
94
|
-
async with self.driver.
|
|
94
|
+
async with self.driver.ro_transaction() as txn:
|
|
95
95
|
current_shard = await self.shard_manager.get_current_active_shard(txn, kbid)
|
|
96
96
|
|
|
97
97
|
if current_shard is None:
|
|
@@ -127,7 +127,7 @@ class KnowledgeBox:
|
|
|
127
127
|
rollback_ops: list[Callable[[], Coroutine[Any, Any, Any]]] = []
|
|
128
128
|
|
|
129
129
|
try:
|
|
130
|
-
async with driver.
|
|
130
|
+
async with driver.rw_transaction() as txn:
|
|
131
131
|
exists = await datamanagers.kb.get_kb_uuid(
|
|
132
132
|
txn, slug=slug
|
|
133
133
|
) or await datamanagers.kb.exists_kb(txn, kbid=kbid)
|
|
@@ -281,7 +281,7 @@ class KnowledgeBox:
|
|
|
281
281
|
|
|
282
282
|
@classmethod
|
|
283
283
|
async def delete(cls, driver: Driver, kbid: str):
|
|
284
|
-
async with driver.
|
|
284
|
+
async with driver.rw_transaction() as txn:
|
|
285
285
|
exists = await datamanagers.kb.exists_kb(txn, kbid=kbid)
|
|
286
286
|
if not exists:
|
|
287
287
|
return
|
|
@@ -347,7 +347,7 @@ class KnowledgeBox:
|
|
|
347
347
|
|
|
348
348
|
nidx_api = get_nidx_api_client()
|
|
349
349
|
|
|
350
|
-
async with driver.
|
|
350
|
+
async with driver.rw_transaction() as txn:
|
|
351
351
|
storage_to_delete = KB_TO_DELETE_STORAGE.format(kbid=kbid)
|
|
352
352
|
await txn.set(storage_to_delete, b"")
|
|
353
353
|
|
|
@@ -377,7 +377,7 @@ class KnowledgeBox:
|
|
|
377
377
|
@classmethod
|
|
378
378
|
async def delete_all_kb_keys(cls, driver: Driver, kbid: str, chunk_size: int = 1_000):
|
|
379
379
|
prefix = KB_KEYS.format(kbid=kbid)
|
|
380
|
-
async with driver.
|
|
380
|
+
async with driver.rw_transaction() as txn:
|
|
381
381
|
await txn.delete_by_prefix(prefix)
|
|
382
382
|
await txn.commit()
|
|
383
383
|
|
|
@@ -34,7 +34,7 @@ async def collect_audit_fields(
|
|
|
34
34
|
return []
|
|
35
35
|
|
|
36
36
|
audit_storage_fields: list[audit_pb2.AuditField] = []
|
|
37
|
-
async with driver.
|
|
37
|
+
async with driver.ro_transaction() as txn:
|
|
38
38
|
kb = KnowledgeBox(txn, storage, message.kbid)
|
|
39
39
|
resource = Resource(txn, storage, kb, message.uuid)
|
|
40
40
|
field_keys = await resource.get_fields_ids()
|
|
@@ -208,7 +208,7 @@ class Processor:
|
|
|
208
208
|
partition: str,
|
|
209
209
|
transaction_check: bool = True,
|
|
210
210
|
) -> None:
|
|
211
|
-
async with self.driver.
|
|
211
|
+
async with self.driver.rw_transaction() as txn:
|
|
212
212
|
try:
|
|
213
213
|
kb = KnowledgeBox(txn, self.storage, message.kbid)
|
|
214
214
|
|
|
@@ -267,7 +267,7 @@ class Processor:
|
|
|
267
267
|
# so we commit it in a different transaction to make it as short as possible
|
|
268
268
|
prev_txn = resource.txn
|
|
269
269
|
try:
|
|
270
|
-
async with self.driver.
|
|
270
|
+
async with self.driver.rw_transaction() as txn:
|
|
271
271
|
resource.txn = txn
|
|
272
272
|
await resource.set_slug()
|
|
273
273
|
await txn.commit()
|
|
@@ -294,7 +294,7 @@ class Processor:
|
|
|
294
294
|
await txn.commit()
|
|
295
295
|
return None
|
|
296
296
|
|
|
297
|
-
async with self.driver.
|
|
297
|
+
async with self.driver.rw_transaction() as txn:
|
|
298
298
|
try:
|
|
299
299
|
multi = messages[0].multiid
|
|
300
300
|
kb = KnowledgeBox(txn, self.storage, kbid)
|
|
@@ -731,7 +731,7 @@ class Processor:
|
|
|
731
731
|
logger.info(f"Skip when resource does not even have basic metadata: {resource}")
|
|
732
732
|
return
|
|
733
733
|
try:
|
|
734
|
-
async with self.driver.
|
|
734
|
+
async with self.driver.rw_transaction() as txn:
|
|
735
735
|
kb.txn = resource.txn = txn
|
|
736
736
|
resource.basic.metadata.status = resources_pb2.Metadata.Status.ERROR
|
|
737
737
|
await resource.set_basic(resource.basic)
|
|
@@ -31,7 +31,7 @@ async def get_last_seqid(driver: Driver, worker: str) -> Optional[int]:
|
|
|
31
31
|
This is oriented towards the ingest consumer and processor,
|
|
32
32
|
which is the only one that should be writing to this key.
|
|
33
33
|
"""
|
|
34
|
-
async with driver.
|
|
34
|
+
async with driver.ro_transaction() as txn:
|
|
35
35
|
key = TXNID.format(worker=worker)
|
|
36
36
|
last_seq = await txn.get(key)
|
|
37
37
|
if not last_seq:
|
nucliadb/ingest/serialize.py
CHANGED
|
@@ -135,7 +135,7 @@ async def serialize(
|
|
|
135
135
|
slug: Optional[str] = None,
|
|
136
136
|
) -> Optional[Resource]:
|
|
137
137
|
driver = get_driver()
|
|
138
|
-
async with driver.
|
|
138
|
+
async with driver.ro_transaction() as txn:
|
|
139
139
|
return await managed_serialize(
|
|
140
140
|
txn,
|
|
141
141
|
kbid,
|
|
@@ -392,6 +392,6 @@ async def get_resource_uuid_by_slug(
|
|
|
392
392
|
) -> Optional[str]:
|
|
393
393
|
storage = await get_storage(service_name=service_name)
|
|
394
394
|
driver = get_driver()
|
|
395
|
-
async with driver.
|
|
395
|
+
async with driver.ro_transaction() as txn:
|
|
396
396
|
kb = KnowledgeBox(txn, storage, kbid)
|
|
397
397
|
return await kb.get_resource_uuid_by_slug(slug)
|
|
@@ -167,7 +167,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
167
167
|
)
|
|
168
168
|
|
|
169
169
|
try:
|
|
170
|
-
async with self.driver.
|
|
170
|
+
async with self.driver.rw_transaction() as txn:
|
|
171
171
|
kbid = await KnowledgeBoxORM.update(
|
|
172
172
|
txn, uuid=request.uuid, slug=request.slug, config=request.config
|
|
173
173
|
)
|
|
@@ -221,13 +221,13 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
221
221
|
self, request: NewEntitiesGroupRequest, context=None
|
|
222
222
|
) -> NewEntitiesGroupResponse:
|
|
223
223
|
response = NewEntitiesGroupResponse()
|
|
224
|
-
async with self.driver.
|
|
224
|
+
async with self.driver.ro_transaction() as ro_txn:
|
|
225
225
|
kbobj = await self.proc.get_kb_obj(ro_txn, request.kb)
|
|
226
226
|
if kbobj is None:
|
|
227
227
|
response.status = NewEntitiesGroupResponse.Status.KB_NOT_FOUND
|
|
228
228
|
return response
|
|
229
229
|
|
|
230
|
-
async with self.driver.
|
|
230
|
+
async with self.driver.rw_transaction() as txn:
|
|
231
231
|
kbobj.txn = txn
|
|
232
232
|
entities_manager = EntitiesManager(kbobj, txn)
|
|
233
233
|
try:
|
|
@@ -244,7 +244,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
244
244
|
self, request: GetEntitiesRequest, context=None
|
|
245
245
|
) -> GetEntitiesResponse:
|
|
246
246
|
response = GetEntitiesResponse()
|
|
247
|
-
async with self.driver.
|
|
247
|
+
async with self.driver.ro_transaction() as txn:
|
|
248
248
|
kbobj = await self.proc.get_kb_obj(txn, request.kb)
|
|
249
249
|
if kbobj is None:
|
|
250
250
|
response.status = GetEntitiesResponse.Status.NOTFOUND
|
|
@@ -266,7 +266,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
266
266
|
self, request: ListEntitiesGroupsRequest, context=None
|
|
267
267
|
) -> ListEntitiesGroupsResponse:
|
|
268
268
|
response = ListEntitiesGroupsResponse()
|
|
269
|
-
async with self.driver.
|
|
269
|
+
async with self.driver.ro_transaction() as txn:
|
|
270
270
|
kbobj = await self.proc.get_kb_obj(txn, request.kb)
|
|
271
271
|
if kbobj is None:
|
|
272
272
|
response.status = ListEntitiesGroupsResponse.Status.NOTFOUND
|
|
@@ -290,7 +290,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
290
290
|
self, request: GetEntitiesGroupRequest, context=None
|
|
291
291
|
) -> GetEntitiesGroupResponse:
|
|
292
292
|
response = GetEntitiesGroupResponse()
|
|
293
|
-
async with self.driver.
|
|
293
|
+
async with self.driver.ro_transaction() as txn:
|
|
294
294
|
kbobj = await self.proc.get_kb_obj(txn, request.kb)
|
|
295
295
|
if kbobj is None:
|
|
296
296
|
response.status = GetEntitiesGroupResponse.Status.KB_NOT_FOUND
|
|
@@ -315,13 +315,13 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
315
315
|
|
|
316
316
|
async def SetEntities(self, request: SetEntitiesRequest, context=None) -> OpStatusWriter: # type: ignore
|
|
317
317
|
response = OpStatusWriter()
|
|
318
|
-
async with self.driver.
|
|
318
|
+
async with self.driver.ro_transaction() as ro_txn:
|
|
319
319
|
kbobj = await self.proc.get_kb_obj(ro_txn, request.kb)
|
|
320
320
|
if kbobj is None:
|
|
321
321
|
response.status = OpStatusWriter.Status.NOTFOUND
|
|
322
322
|
return response
|
|
323
323
|
|
|
324
|
-
async with self.driver.
|
|
324
|
+
async with self.driver.rw_transaction() as txn:
|
|
325
325
|
kbobj.txn = txn
|
|
326
326
|
entities_manager = EntitiesManager(kbobj, txn)
|
|
327
327
|
try:
|
|
@@ -339,13 +339,13 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
339
339
|
self, request: UpdateEntitiesGroupRequest, context=None
|
|
340
340
|
) -> UpdateEntitiesGroupResponse:
|
|
341
341
|
response = UpdateEntitiesGroupResponse()
|
|
342
|
-
async with self.driver.
|
|
342
|
+
async with self.driver.ro_transaction() as ro_txn:
|
|
343
343
|
kbobj = await self.proc.get_kb_obj(ro_txn, request.kb)
|
|
344
344
|
if kbobj is None:
|
|
345
345
|
response.status = UpdateEntitiesGroupResponse.Status.KB_NOT_FOUND
|
|
346
346
|
return response
|
|
347
347
|
|
|
348
|
-
async with self.driver.
|
|
348
|
+
async with self.driver.rw_transaction() as txn:
|
|
349
349
|
kbobj.txn = txn
|
|
350
350
|
entities_manager = EntitiesManager(kbobj, txn)
|
|
351
351
|
try:
|
|
@@ -368,13 +368,13 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
368
368
|
async def DelEntities(self, request: DelEntitiesRequest, context=None) -> OpStatusWriter: # type: ignore
|
|
369
369
|
response = OpStatusWriter()
|
|
370
370
|
|
|
371
|
-
async with self.driver.
|
|
371
|
+
async with self.driver.ro_transaction() as ro_txn:
|
|
372
372
|
kbobj = await self.proc.get_kb_obj(ro_txn, request.kb)
|
|
373
373
|
if kbobj is None:
|
|
374
374
|
response.status = OpStatusWriter.Status.NOTFOUND
|
|
375
375
|
return response
|
|
376
376
|
|
|
377
|
-
async with self.driver.
|
|
377
|
+
async with self.driver.rw_transaction() as txn:
|
|
378
378
|
kbobj.txn = txn
|
|
379
379
|
entities_manager = EntitiesManager(kbobj, txn)
|
|
380
380
|
try:
|
|
@@ -393,7 +393,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
393
393
|
) -> WriterStatusResponse:
|
|
394
394
|
logger.info("Status Call")
|
|
395
395
|
response = WriterStatusResponse()
|
|
396
|
-
async with self.driver.
|
|
396
|
+
async with self.driver.ro_transaction() as txn:
|
|
397
397
|
async for _, slug in datamanagers.kb.get_kbs(txn):
|
|
398
398
|
response.knowledgeboxes.append(slug)
|
|
399
399
|
|
|
@@ -405,7 +405,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
405
405
|
return response
|
|
406
406
|
|
|
407
407
|
async def Index(self, request: IndexResource, context=None) -> IndexStatus: # type: ignore
|
|
408
|
-
async with self.driver.
|
|
408
|
+
async with self.driver.ro_transaction() as txn:
|
|
409
409
|
kbobj = KnowledgeBoxORM(txn, self.storage, request.kbid)
|
|
410
410
|
resobj = ResourceORM(txn, self.storage, kbobj, request.rid)
|
|
411
411
|
bm = await generate_broker_message(resobj)
|
|
@@ -419,7 +419,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
|
|
|
419
419
|
|
|
420
420
|
async def ReIndex(self, request: IndexResource, context=None) -> IndexStatus: # type: ignore
|
|
421
421
|
try:
|
|
422
|
-
async with self.driver.
|
|
422
|
+
async with self.driver.rw_transaction() as txn:
|
|
423
423
|
kbobj = KnowledgeBoxORM(txn, self.storage, request.kbid)
|
|
424
424
|
resobj = ResourceORM(txn, self.storage, kbobj, request.rid)
|
|
425
425
|
resobj.disable_vectors = not request.reindex_vectors
|
nucliadb/metrics_exporter.py
CHANGED
|
@@ -42,7 +42,7 @@ async def iter_kbids(context: ApplicationContext) -> AsyncGenerator[str, None]:
|
|
|
42
42
|
"""
|
|
43
43
|
Return a list of all KB ids.
|
|
44
44
|
"""
|
|
45
|
-
async with context.kv_driver.
|
|
45
|
+
async with context.kv_driver.ro_transaction() as txn:
|
|
46
46
|
async for kbid, _ in datamanagers.kb.get_kbs(txn):
|
|
47
47
|
yield kbid
|
|
48
48
|
|