nucliadb 6.7.2.post4874__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- migrations/0023_backfill_pg_catalog.py +8 -4
- migrations/0028_extracted_vectors_reference.py +1 -1
- migrations/0029_backfill_field_status.py +3 -4
- migrations/0032_remove_old_relations.py +2 -3
- migrations/0038_backfill_catalog_field_labels.py +8 -4
- migrations/0039_backfill_converation_splits_metadata.py +106 -0
- migrations/0040_migrate_search_configurations.py +79 -0
- migrations/0041_reindex_conversations.py +137 -0
- migrations/pg/0010_shards_index.py +34 -0
- nucliadb/search/api/v1/resource/utils.py → migrations/pg/0011_catalog_statistics.py +5 -6
- migrations/pg/0012_catalog_statistics_undo.py +26 -0
- nucliadb/backups/create.py +2 -15
- nucliadb/backups/restore.py +4 -15
- nucliadb/backups/tasks.py +4 -1
- nucliadb/common/back_pressure/cache.py +2 -3
- nucliadb/common/back_pressure/materializer.py +7 -13
- nucliadb/common/back_pressure/settings.py +6 -6
- nucliadb/common/back_pressure/utils.py +1 -0
- nucliadb/common/cache.py +9 -9
- nucliadb/common/catalog/__init__.py +79 -0
- nucliadb/common/catalog/dummy.py +36 -0
- nucliadb/common/catalog/interface.py +85 -0
- nucliadb/{search/search/pgcatalog.py → common/catalog/pg.py} +330 -232
- nucliadb/common/catalog/utils.py +56 -0
- nucliadb/common/cluster/manager.py +8 -23
- nucliadb/common/cluster/rebalance.py +484 -112
- nucliadb/common/cluster/rollover.py +36 -9
- nucliadb/common/cluster/settings.py +4 -9
- nucliadb/common/cluster/utils.py +34 -8
- nucliadb/common/context/__init__.py +7 -8
- nucliadb/common/context/fastapi.py +1 -2
- nucliadb/common/datamanagers/__init__.py +2 -4
- nucliadb/common/datamanagers/atomic.py +9 -2
- nucliadb/common/datamanagers/cluster.py +1 -2
- nucliadb/common/datamanagers/fields.py +3 -4
- nucliadb/common/datamanagers/kb.py +6 -6
- nucliadb/common/datamanagers/labels.py +2 -3
- nucliadb/common/datamanagers/resources.py +10 -33
- nucliadb/common/datamanagers/rollover.py +5 -7
- nucliadb/common/datamanagers/search_configurations.py +1 -2
- nucliadb/common/datamanagers/synonyms.py +1 -2
- nucliadb/common/datamanagers/utils.py +4 -4
- nucliadb/common/datamanagers/vectorsets.py +4 -4
- nucliadb/common/external_index_providers/base.py +32 -5
- nucliadb/common/external_index_providers/manager.py +5 -34
- nucliadb/common/external_index_providers/settings.py +1 -27
- nucliadb/common/filter_expression.py +129 -41
- nucliadb/common/http_clients/exceptions.py +8 -0
- nucliadb/common/http_clients/processing.py +16 -23
- nucliadb/common/http_clients/utils.py +3 -0
- nucliadb/common/ids.py +82 -58
- nucliadb/common/locking.py +1 -2
- nucliadb/common/maindb/driver.py +9 -8
- nucliadb/common/maindb/local.py +5 -5
- nucliadb/common/maindb/pg.py +9 -8
- nucliadb/common/nidx.py +22 -5
- nucliadb/common/vector_index_config.py +1 -1
- nucliadb/export_import/datamanager.py +4 -3
- nucliadb/export_import/exporter.py +11 -19
- nucliadb/export_import/importer.py +13 -6
- nucliadb/export_import/tasks.py +2 -0
- nucliadb/export_import/utils.py +6 -18
- nucliadb/health.py +2 -2
- nucliadb/ingest/app.py +8 -8
- nucliadb/ingest/consumer/consumer.py +8 -10
- nucliadb/ingest/consumer/pull.py +10 -8
- nucliadb/ingest/consumer/service.py +5 -30
- nucliadb/ingest/consumer/shard_creator.py +16 -5
- nucliadb/ingest/consumer/utils.py +1 -1
- nucliadb/ingest/fields/base.py +37 -49
- nucliadb/ingest/fields/conversation.py +55 -9
- nucliadb/ingest/fields/exceptions.py +1 -2
- nucliadb/ingest/fields/file.py +22 -8
- nucliadb/ingest/fields/link.py +7 -7
- nucliadb/ingest/fields/text.py +2 -3
- nucliadb/ingest/orm/brain_v2.py +89 -57
- nucliadb/ingest/orm/broker_message.py +2 -4
- nucliadb/ingest/orm/entities.py +10 -209
- nucliadb/ingest/orm/index_message.py +128 -113
- nucliadb/ingest/orm/knowledgebox.py +91 -59
- nucliadb/ingest/orm/processor/auditing.py +1 -3
- nucliadb/ingest/orm/processor/data_augmentation.py +1 -2
- nucliadb/ingest/orm/processor/processor.py +98 -153
- nucliadb/ingest/orm/processor/sequence_manager.py +1 -2
- nucliadb/ingest/orm/resource.py +82 -71
- nucliadb/ingest/orm/utils.py +1 -1
- nucliadb/ingest/partitions.py +12 -1
- nucliadb/ingest/processing.py +17 -17
- nucliadb/ingest/serialize.py +202 -145
- nucliadb/ingest/service/writer.py +15 -114
- nucliadb/ingest/settings.py +36 -15
- nucliadb/ingest/utils.py +1 -2
- nucliadb/learning_proxy.py +23 -26
- nucliadb/metrics_exporter.py +20 -6
- nucliadb/middleware/__init__.py +82 -1
- nucliadb/migrator/datamanager.py +4 -11
- nucliadb/migrator/migrator.py +1 -2
- nucliadb/migrator/models.py +1 -2
- nucliadb/migrator/settings.py +1 -2
- nucliadb/models/internal/augment.py +614 -0
- nucliadb/models/internal/processing.py +19 -19
- nucliadb/openapi.py +2 -2
- nucliadb/purge/__init__.py +3 -8
- nucliadb/purge/orphan_shards.py +1 -2
- nucliadb/reader/__init__.py +5 -0
- nucliadb/reader/api/models.py +6 -13
- nucliadb/reader/api/v1/download.py +59 -38
- nucliadb/reader/api/v1/export_import.py +4 -4
- nucliadb/reader/api/v1/knowledgebox.py +37 -9
- nucliadb/reader/api/v1/learning_config.py +33 -14
- nucliadb/reader/api/v1/resource.py +61 -9
- nucliadb/reader/api/v1/services.py +18 -14
- nucliadb/reader/app.py +3 -1
- nucliadb/reader/reader/notifications.py +1 -2
- nucliadb/search/api/v1/__init__.py +3 -0
- nucliadb/search/api/v1/ask.py +3 -4
- nucliadb/search/api/v1/augment.py +585 -0
- nucliadb/search/api/v1/catalog.py +15 -19
- nucliadb/search/api/v1/find.py +16 -22
- nucliadb/search/api/v1/hydrate.py +328 -0
- nucliadb/search/api/v1/knowledgebox.py +1 -2
- nucliadb/search/api/v1/predict_proxy.py +1 -2
- nucliadb/search/api/v1/resource/ask.py +28 -8
- nucliadb/search/api/v1/resource/ingestion_agents.py +5 -6
- nucliadb/search/api/v1/resource/search.py +9 -11
- nucliadb/search/api/v1/retrieve.py +130 -0
- nucliadb/search/api/v1/search.py +28 -32
- nucliadb/search/api/v1/suggest.py +11 -14
- nucliadb/search/api/v1/summarize.py +1 -2
- nucliadb/search/api/v1/utils.py +2 -2
- nucliadb/search/app.py +3 -2
- nucliadb/search/augmentor/__init__.py +21 -0
- nucliadb/search/augmentor/augmentor.py +232 -0
- nucliadb/search/augmentor/fields.py +704 -0
- nucliadb/search/augmentor/metrics.py +24 -0
- nucliadb/search/augmentor/paragraphs.py +334 -0
- nucliadb/search/augmentor/resources.py +238 -0
- nucliadb/search/augmentor/utils.py +33 -0
- nucliadb/search/lifecycle.py +3 -1
- nucliadb/search/predict.py +33 -19
- nucliadb/search/predict_models.py +8 -9
- nucliadb/search/requesters/utils.py +11 -10
- nucliadb/search/search/cache.py +19 -42
- nucliadb/search/search/chat/ask.py +131 -59
- nucliadb/search/search/chat/exceptions.py +3 -5
- nucliadb/search/search/chat/fetcher.py +201 -0
- nucliadb/search/search/chat/images.py +6 -4
- nucliadb/search/search/chat/old_prompt.py +1375 -0
- nucliadb/search/search/chat/parser.py +510 -0
- nucliadb/search/search/chat/prompt.py +563 -615
- nucliadb/search/search/chat/query.py +453 -32
- nucliadb/search/search/chat/rpc.py +85 -0
- nucliadb/search/search/fetch.py +3 -4
- nucliadb/search/search/filters.py +8 -11
- nucliadb/search/search/find.py +33 -31
- nucliadb/search/search/find_merge.py +124 -331
- nucliadb/search/search/graph_strategy.py +14 -12
- nucliadb/search/search/hydrator/__init__.py +49 -0
- nucliadb/search/search/hydrator/fields.py +217 -0
- nucliadb/search/search/hydrator/images.py +130 -0
- nucliadb/search/search/hydrator/paragraphs.py +323 -0
- nucliadb/search/search/hydrator/resources.py +60 -0
- nucliadb/search/search/ingestion_agents.py +5 -5
- nucliadb/search/search/merge.py +90 -94
- nucliadb/search/search/metrics.py +24 -7
- nucliadb/search/search/paragraphs.py +7 -9
- nucliadb/search/search/predict_proxy.py +44 -18
- nucliadb/search/search/query.py +14 -86
- nucliadb/search/search/query_parser/fetcher.py +51 -82
- nucliadb/search/search/query_parser/models.py +19 -48
- nucliadb/search/search/query_parser/old_filters.py +20 -19
- nucliadb/search/search/query_parser/parsers/ask.py +5 -6
- nucliadb/search/search/query_parser/parsers/catalog.py +7 -11
- nucliadb/search/search/query_parser/parsers/common.py +21 -13
- nucliadb/search/search/query_parser/parsers/find.py +6 -29
- nucliadb/search/search/query_parser/parsers/graph.py +18 -28
- nucliadb/search/search/query_parser/parsers/retrieve.py +207 -0
- nucliadb/search/search/query_parser/parsers/search.py +15 -56
- nucliadb/search/search/query_parser/parsers/unit_retrieval.py +8 -29
- nucliadb/search/search/rank_fusion.py +18 -13
- nucliadb/search/search/rerankers.py +6 -7
- nucliadb/search/search/retrieval.py +300 -0
- nucliadb/search/search/summarize.py +5 -6
- nucliadb/search/search/utils.py +3 -4
- nucliadb/search/settings.py +1 -2
- nucliadb/standalone/api_router.py +1 -1
- nucliadb/standalone/app.py +4 -3
- nucliadb/standalone/auth.py +5 -6
- nucliadb/standalone/lifecycle.py +2 -2
- nucliadb/standalone/run.py +5 -4
- nucliadb/standalone/settings.py +5 -6
- nucliadb/standalone/versions.py +3 -4
- nucliadb/tasks/consumer.py +13 -8
- nucliadb/tasks/models.py +2 -1
- nucliadb/tasks/producer.py +3 -3
- nucliadb/tasks/retries.py +8 -7
- nucliadb/train/api/utils.py +1 -3
- nucliadb/train/api/v1/shards.py +1 -2
- nucliadb/train/api/v1/trainset.py +1 -2
- nucliadb/train/app.py +1 -1
- nucliadb/train/generator.py +4 -4
- nucliadb/train/generators/field_classifier.py +2 -2
- nucliadb/train/generators/field_streaming.py +6 -6
- nucliadb/train/generators/image_classifier.py +2 -2
- nucliadb/train/generators/paragraph_classifier.py +2 -2
- nucliadb/train/generators/paragraph_streaming.py +2 -2
- nucliadb/train/generators/question_answer_streaming.py +2 -2
- nucliadb/train/generators/sentence_classifier.py +4 -10
- nucliadb/train/generators/token_classifier.py +3 -2
- nucliadb/train/generators/utils.py +6 -5
- nucliadb/train/nodes.py +3 -3
- nucliadb/train/resource.py +6 -8
- nucliadb/train/settings.py +3 -4
- nucliadb/train/types.py +11 -11
- nucliadb/train/upload.py +3 -2
- nucliadb/train/uploader.py +1 -2
- nucliadb/train/utils.py +1 -2
- nucliadb/writer/api/v1/export_import.py +4 -1
- nucliadb/writer/api/v1/field.py +15 -14
- nucliadb/writer/api/v1/knowledgebox.py +18 -56
- nucliadb/writer/api/v1/learning_config.py +5 -4
- nucliadb/writer/api/v1/resource.py +9 -20
- nucliadb/writer/api/v1/services.py +10 -132
- nucliadb/writer/api/v1/upload.py +73 -72
- nucliadb/writer/app.py +8 -2
- nucliadb/writer/resource/basic.py +12 -15
- nucliadb/writer/resource/field.py +43 -5
- nucliadb/writer/resource/origin.py +7 -0
- nucliadb/writer/settings.py +2 -3
- nucliadb/writer/tus/__init__.py +2 -3
- nucliadb/writer/tus/azure.py +5 -7
- nucliadb/writer/tus/dm.py +3 -3
- nucliadb/writer/tus/exceptions.py +3 -4
- nucliadb/writer/tus/gcs.py +15 -22
- nucliadb/writer/tus/s3.py +2 -3
- nucliadb/writer/tus/storage.py +3 -3
- {nucliadb-6.7.2.post4874.dist-info → nucliadb-6.10.0.post5705.dist-info}/METADATA +10 -11
- nucliadb-6.10.0.post5705.dist-info/RECORD +410 -0
- nucliadb/common/datamanagers/entities.py +0 -139
- nucliadb/common/external_index_providers/pinecone.py +0 -894
- nucliadb/ingest/orm/processor/pgcatalog.py +0 -129
- nucliadb/search/search/hydrator.py +0 -197
- nucliadb-6.7.2.post4874.dist-info/RECORD +0 -383
- {nucliadb-6.7.2.post4874.dist-info → nucliadb-6.10.0.post5705.dist-info}/WHEEL +0 -0
- {nucliadb-6.7.2.post4874.dist-info → nucliadb-6.10.0.post5705.dist-info}/entry_points.txt +0 -0
- {nucliadb-6.7.2.post4874.dist-info → nucliadb-6.10.0.post5705.dist-info}/top_level.txt +0 -0
|
@@ -17,11 +17,12 @@
|
|
|
17
17
|
# You should have received a copy of the GNU Affero General Public License
|
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
19
|
#
|
|
20
|
-
from typing import
|
|
20
|
+
from typing import cast
|
|
21
21
|
|
|
22
22
|
from fastapi import Header, HTTPException, Query, Request, Response
|
|
23
23
|
from fastapi_versioning import version
|
|
24
24
|
|
|
25
|
+
from nucliadb.common import datamanagers
|
|
25
26
|
from nucliadb.common.datamanagers.resources import KB_RESOURCE_SLUG_BASE
|
|
26
27
|
from nucliadb.common.maindb.utils import get_driver
|
|
27
28
|
from nucliadb.common.models_utils import from_proto, to_proto
|
|
@@ -58,6 +59,57 @@ from nucliadb_utils.authentication import requires, requires_one
|
|
|
58
59
|
from nucliadb_utils.utilities import get_audit, get_storage
|
|
59
60
|
|
|
60
61
|
|
|
62
|
+
@api.head(
|
|
63
|
+
f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}",
|
|
64
|
+
status_code=200,
|
|
65
|
+
summary="Head Resource (by id)",
|
|
66
|
+
responses={404: {"description": "Resource does not exist"}},
|
|
67
|
+
tags=["Resources"],
|
|
68
|
+
)
|
|
69
|
+
@requires(NucliaDBRoles.READER)
|
|
70
|
+
@version(1)
|
|
71
|
+
async def head_resource_by_uuid(
|
|
72
|
+
request: Request,
|
|
73
|
+
kbid: str,
|
|
74
|
+
rid: str,
|
|
75
|
+
):
|
|
76
|
+
return await head_resource(kbid=kbid, rid=rid)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@api.head(
|
|
80
|
+
f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}",
|
|
81
|
+
status_code=200,
|
|
82
|
+
summary="Head Resource (by slug)",
|
|
83
|
+
responses={404: {"description": "Resource does not exist"}},
|
|
84
|
+
tags=["Resources"],
|
|
85
|
+
)
|
|
86
|
+
@requires(NucliaDBRoles.READER)
|
|
87
|
+
@version(1)
|
|
88
|
+
async def head_resource_by_slug(
|
|
89
|
+
request: Request,
|
|
90
|
+
kbid: str,
|
|
91
|
+
rslug: str,
|
|
92
|
+
):
|
|
93
|
+
return await head_resource(kbid=kbid, rslug=rslug)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def head_resource(
|
|
97
|
+
*,
|
|
98
|
+
kbid: str,
|
|
99
|
+
rslug: str | None = None,
|
|
100
|
+
rid: str | None = None,
|
|
101
|
+
) -> None:
|
|
102
|
+
if all([rslug, rid]) or not any([rslug, rid]):
|
|
103
|
+
raise ValueError("Either rid or rslug must be provided, but not both")
|
|
104
|
+
if rid is None:
|
|
105
|
+
rslug = cast(str, rslug)
|
|
106
|
+
rid = await datamanagers.atomic.resources.get_resource_uuid_from_slug(kbid=kbid, slug=rslug)
|
|
107
|
+
if rid is None:
|
|
108
|
+
raise HTTPException(status_code=404, detail="Resource does not exist")
|
|
109
|
+
if not await datamanagers.atomic.resources.resource_exists(kbid=kbid, rid=rid):
|
|
110
|
+
raise HTTPException(status_code=404, detail="Resource does not exist")
|
|
111
|
+
|
|
112
|
+
|
|
61
113
|
@api.get(
|
|
62
114
|
f"/{KB_PREFIX}/{{kbid}}/resources",
|
|
63
115
|
status_code=200,
|
|
@@ -211,8 +263,8 @@ async def get_resource_by_slug(
|
|
|
211
263
|
|
|
212
264
|
async def _get_resource(
|
|
213
265
|
*,
|
|
214
|
-
rslug:
|
|
215
|
-
rid:
|
|
266
|
+
rslug: str | None = None,
|
|
267
|
+
rid: str | None = None,
|
|
216
268
|
kbid: str,
|
|
217
269
|
show: list[ResourceProperties],
|
|
218
270
|
field_type_filter: list[FieldTypeName],
|
|
@@ -269,7 +321,7 @@ async def get_resource_field_rslug_prefix(
|
|
|
269
321
|
),
|
|
270
322
|
# not working with latest pydantic/fastapi
|
|
271
323
|
# page: Union[Literal["last", "first"], int] = Query("last"),
|
|
272
|
-
page:
|
|
324
|
+
page: str | int = Query("last"),
|
|
273
325
|
) -> Response:
|
|
274
326
|
return await _get_resource_field(
|
|
275
327
|
kbid,
|
|
@@ -309,7 +361,7 @@ async def get_resource_field_rid_prefix(
|
|
|
309
361
|
),
|
|
310
362
|
# not working with latest pydantic/fastapi
|
|
311
363
|
# page: Union[Literal["last", "first"], int] = Query("last"),
|
|
312
|
-
page:
|
|
364
|
+
page: str | int = Query("last"),
|
|
313
365
|
) -> Response:
|
|
314
366
|
return await _get_resource_field(
|
|
315
367
|
kbid,
|
|
@@ -328,9 +380,9 @@ async def _get_resource_field(
|
|
|
328
380
|
field_id: str,
|
|
329
381
|
show: list[ResourceFieldProperties],
|
|
330
382
|
extracted: list[ExtractedDataTypeName],
|
|
331
|
-
page:
|
|
332
|
-
rid:
|
|
333
|
-
rslug:
|
|
383
|
+
page: str | int,
|
|
384
|
+
rid: str | None = None,
|
|
385
|
+
rslug: str | None = None,
|
|
334
386
|
) -> Response:
|
|
335
387
|
storage = await get_storage(service_name=SERVICE_NAME)
|
|
336
388
|
driver = get_driver()
|
|
@@ -344,7 +396,7 @@ async def _get_resource_field(
|
|
|
344
396
|
if rid is None:
|
|
345
397
|
raise HTTPException(status_code=404, detail="Resource does not exist")
|
|
346
398
|
|
|
347
|
-
resource = ORMResource(txn, storage,
|
|
399
|
+
resource = ORMResource(txn, storage, kbid, rid)
|
|
348
400
|
field = await resource.get_field(field_id, pb_field_id, load=True)
|
|
349
401
|
if field is None:
|
|
350
402
|
raise HTTPException(status_code=404, detail="Knowledge Box does not exist")
|
|
@@ -18,9 +18,8 @@
|
|
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
19
|
#
|
|
20
20
|
import asyncio
|
|
21
|
-
from typing import Optional, Union
|
|
22
21
|
|
|
23
|
-
from fastapi import HTTPException
|
|
22
|
+
from fastapi import HTTPException, Path
|
|
24
23
|
from fastapi.responses import StreamingResponse
|
|
25
24
|
from fastapi_versioning import version
|
|
26
25
|
from google.protobuf.json_format import MessageToDict
|
|
@@ -69,7 +68,7 @@ from nucliadb_utils.utilities import get_ingest, get_storage
|
|
|
69
68
|
@version(1)
|
|
70
69
|
async def get_entities(
|
|
71
70
|
request: Request, kbid: str, show_entities: bool = False
|
|
72
|
-
) ->
|
|
71
|
+
) -> KnowledgeBoxEntities | HTTPClientError:
|
|
73
72
|
if show_entities:
|
|
74
73
|
return HTTPClientError(
|
|
75
74
|
status_code=400,
|
|
@@ -169,7 +168,14 @@ async def get_labelsets(kbid: str) -> KnowledgeBoxLabels:
|
|
|
169
168
|
)
|
|
170
169
|
@requires(NucliaDBRoles.READER)
|
|
171
170
|
@version(1)
|
|
172
|
-
async def get_labelset_endpoint(
|
|
171
|
+
async def get_labelset_endpoint(
|
|
172
|
+
request: Request,
|
|
173
|
+
kbid: str,
|
|
174
|
+
labelset: str = Path(
|
|
175
|
+
title="The ID of the labelset to get. This is a unique identifier that should be used at search time.",
|
|
176
|
+
examples=["categories", "movie-genres", "document-types"],
|
|
177
|
+
),
|
|
178
|
+
) -> LabelSet:
|
|
173
179
|
try:
|
|
174
180
|
return await get_labelset(kbid, labelset)
|
|
175
181
|
except KnowledgeBoxNotFound:
|
|
@@ -186,7 +192,7 @@ async def get_labelset(kbid: str, labelset_id: str) -> LabelSet:
|
|
|
186
192
|
kb_exists = await datamanagers.atomic.kb.exists_kb(kbid=kbid)
|
|
187
193
|
if not kb_exists:
|
|
188
194
|
raise KnowledgeBoxNotFound()
|
|
189
|
-
labelset:
|
|
195
|
+
labelset: writer_pb2.LabelSet | None = await datamanagers.atomic.labelset.get(
|
|
190
196
|
kbid=kbid, labelset_id=labelset_id
|
|
191
197
|
)
|
|
192
198
|
if labelset is None:
|
|
@@ -222,17 +228,15 @@ async def get_custom_synonyms(request: Request, kbid: str):
|
|
|
222
228
|
f"/{KB_PREFIX}/{{kbid}}/notifications",
|
|
223
229
|
status_code=200,
|
|
224
230
|
summary="Knowledge Box Notifications Stream",
|
|
225
|
-
description="Provides a stream of activity notifications for the given Knowledge Box. The stream will be automatically closed after 2 minutes.",
|
|
231
|
+
description="Provides a stream of activity notifications for the given Knowledge Box. The stream will be automatically closed after 2 minutes.",
|
|
226
232
|
tags=["Knowledge Box Services"],
|
|
227
|
-
response_description="Each line of the response is a Base64-encoded JSON object representing a notification. Refer to [the internal documentation](https://github.com/nuclia/nucliadb/blob/main/docs/tutorials/KB_NOTIFICATIONS.md) for a more detailed explanation of each notification type.",
|
|
233
|
+
response_description="Each line of the response is a Base64-encoded JSON object representing a notification. Refer to [the internal documentation](https://github.com/nuclia/nucliadb/blob/main/docs/tutorials/KB_NOTIFICATIONS.md) for a more detailed explanation of each notification type.",
|
|
228
234
|
response_model=None,
|
|
229
235
|
responses={"404": {"description": "Knowledge Box not found"}},
|
|
230
236
|
)
|
|
231
237
|
@requires(NucliaDBRoles.READER)
|
|
232
238
|
@version(1)
|
|
233
|
-
async def notifications_endpoint(
|
|
234
|
-
request: Request, kbid: str
|
|
235
|
-
) -> Union[StreamingResponse, HTTPClientError]:
|
|
239
|
+
async def notifications_endpoint(request: Request, kbid: str) -> StreamingResponse | HTTPClientError:
|
|
236
240
|
if in_standalone_mode():
|
|
237
241
|
return HTTPClientError(
|
|
238
242
|
status_code=404,
|
|
@@ -274,10 +278,10 @@ async def exists_kb(kbid: str) -> bool:
|
|
|
274
278
|
async def processing_status(
|
|
275
279
|
request: Request,
|
|
276
280
|
kbid: str,
|
|
277
|
-
cursor:
|
|
278
|
-
scheduled:
|
|
281
|
+
cursor: str | None = None,
|
|
282
|
+
scheduled: bool | None = None,
|
|
279
283
|
limit: int = 20,
|
|
280
|
-
) ->
|
|
284
|
+
) -> processing.RequestsResults | HTTPClientError:
|
|
281
285
|
if not await exists_kb(kbid=kbid):
|
|
282
286
|
return HTTPClientError(status_code=404, detail="Knowledge Box not found")
|
|
283
287
|
|
|
@@ -294,7 +298,7 @@ async def processing_status(
|
|
|
294
298
|
|
|
295
299
|
async def _composition(
|
|
296
300
|
result: processing.RequestsResult,
|
|
297
|
-
) ->
|
|
301
|
+
) -> processing.RequestsResult | None:
|
|
298
302
|
async with max_simultaneous:
|
|
299
303
|
resource = await kb.get(result.resource_id)
|
|
300
304
|
if resource is None:
|
nucliadb/reader/app.py
CHANGED
|
@@ -26,6 +26,7 @@ from starlette.middleware.authentication import AuthenticationMiddleware
|
|
|
26
26
|
from starlette.requests import ClientDisconnect, Request
|
|
27
27
|
from starlette.responses import HTMLResponse
|
|
28
28
|
|
|
29
|
+
from nucliadb.middleware import ClientErrorPayloadLoggerMiddleware
|
|
29
30
|
from nucliadb.reader import API_PREFIX
|
|
30
31
|
from nucliadb.reader.api.v1.router import api as api_v1
|
|
31
32
|
from nucliadb.reader.lifecycle import lifespan
|
|
@@ -49,6 +50,7 @@ middleware.extend(
|
|
|
49
50
|
backend=NucliaCloudAuthenticationBackend(),
|
|
50
51
|
),
|
|
51
52
|
Middleware(AuditMiddleware, audit_utility_getter=get_audit),
|
|
53
|
+
Middleware(ClientErrorPayloadLoggerMiddleware),
|
|
52
54
|
]
|
|
53
55
|
)
|
|
54
56
|
|
|
@@ -56,7 +58,6 @@ errors.setup_error_handling(importlib.metadata.distribution("nucliadb").version)
|
|
|
56
58
|
|
|
57
59
|
fastapi_settings = dict(
|
|
58
60
|
debug=running_settings.debug,
|
|
59
|
-
middleware=middleware,
|
|
60
61
|
lifespan=lifespan,
|
|
61
62
|
exception_handlers={
|
|
62
63
|
Exception: global_exception_handler,
|
|
@@ -78,6 +79,7 @@ def create_application() -> FastAPI:
|
|
|
78
79
|
prefix_format=f"/{API_PREFIX}/v{{major}}",
|
|
79
80
|
default_version=(1, 0),
|
|
80
81
|
enable_latest=False,
|
|
82
|
+
middleware=middleware,
|
|
81
83
|
kwargs=fastapi_settings,
|
|
82
84
|
)
|
|
83
85
|
|
|
@@ -21,7 +21,6 @@ import asyncio
|
|
|
21
21
|
import contextlib
|
|
22
22
|
import uuid
|
|
23
23
|
from collections.abc import AsyncGenerator
|
|
24
|
-
from typing import Optional
|
|
25
24
|
|
|
26
25
|
import async_timeout
|
|
27
26
|
from nats.aio.msg import Msg
|
|
@@ -200,7 +199,7 @@ async def get_resource_title_cached(
|
|
|
200
199
|
return resource_title
|
|
201
200
|
|
|
202
201
|
|
|
203
|
-
async def get_resource_title(kv_driver: Driver, kbid: str, resource_uuid: str) ->
|
|
202
|
+
async def get_resource_title(kv_driver: Driver, kbid: str, resource_uuid: str) -> str | None:
|
|
204
203
|
async with kv_driver.ro_transaction() as txn:
|
|
205
204
|
basic = await datamanagers.resources.get_basic(txn, kbid=kbid, rid=resource_uuid)
|
|
206
205
|
if basic is None:
|
nucliadb/search/api/v1/ask.py
CHANGED
|
@@ -18,7 +18,6 @@
|
|
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
19
|
#
|
|
20
20
|
import json
|
|
21
|
-
from typing import Optional, Union
|
|
22
21
|
|
|
23
22
|
from fastapi import Header, Request, Response
|
|
24
23
|
from fastapi_versioning import version
|
|
@@ -67,7 +66,7 @@ async def ask_knowledgebox_endpoint(
|
|
|
67
66
|
description="When set to true, outputs response as JSON in a non-streaming way. "
|
|
68
67
|
"This is slower and requires waiting for entire answer to be ready.",
|
|
69
68
|
),
|
|
70
|
-
) ->
|
|
69
|
+
) -> StreamingResponse | HTTPClientError | Response:
|
|
71
70
|
current_user: NucliaUser = request.user
|
|
72
71
|
# If present, security groups from AuthorizationBackend overrides any
|
|
73
72
|
# security group of the payload
|
|
@@ -116,8 +115,8 @@ async def create_ask_response(
|
|
|
116
115
|
client_type: NucliaDBClientType,
|
|
117
116
|
origin: str,
|
|
118
117
|
x_synchronous: bool,
|
|
119
|
-
resource:
|
|
120
|
-
extra_predict_headers:
|
|
118
|
+
resource: str | None = None,
|
|
119
|
+
extra_predict_headers: dict[str, str] | None = None,
|
|
121
120
|
) -> Response:
|
|
122
121
|
maybe_log_request_payload(kbid, "/ask", ask_request)
|
|
123
122
|
ask_request.max_tokens = parse_max_tokens(ask_request.max_tokens)
|