nucliadb 6.4.0.post4196__py3-none-any.whl → 6.4.0.post4204__py3-none-any.whl

nucliadb/writer/api/v1/upload.py

@@ -32,6 +32,7 @@ from fastapi_versioning import version
 from starlette.requests import Request as StarletteRequest
 
 from nucliadb.common import datamanagers
+from nucliadb.common.back_pressure import maybe_back_pressure
 from nucliadb.ingest.orm.utils import set_title
 from nucliadb.models.internal.processing import PushPayload, Source
 from nucliadb.models.responses import HTTPClientError
@@ -43,7 +44,6 @@ from nucliadb.writer.api.v1.resource import (
     validate_rid_exists_or_raise_error,
 )
 from nucliadb.writer.api.v1.slug import ensure_slug_uniqueness, noop_context_manager
-from nucliadb.writer.back_pressure import maybe_back_pressure
 from nucliadb.writer.resource.audit import parse_audit
 from nucliadb.writer.resource.basic import parse_basic_creation, parse_user_classifications
 from nucliadb.writer.resource.field import (
@@ -215,7 +215,7 @@ async def _tus_post(
             detail="Cannot hide a resource: the KB does not have hidden resources enabled",
         )
 
-    await maybe_back_pressure(request, kbid, resource_uuid=path_rid)
+    await maybe_back_pressure(kbid, resource_uuid=path_rid)
 
     dm = get_dm()
     storage_manager = get_storage_manager()
@@ -713,7 +713,7 @@ async def _upload(
     if path_rid is not None:
         await validate_rid_exists_or_raise_error(kbid, path_rid)
 
-    await maybe_back_pressure(request, kbid, resource_uuid=path_rid)
+    await maybe_back_pressure(kbid, resource_uuid=path_rid)
 
     md5_user = x_md5
     path, rid, valid_field = await validate_field_upload(kbid, path_rid, field, md5_user)

nucliadb/writer/lifecycle.py

@@ -21,12 +21,12 @@ from contextlib import asynccontextmanager
 
 from fastapi import FastAPI
 
+from nucliadb.common.back_pressure import start_materializer, stop_materializer
+from nucliadb.common.back_pressure.settings import settings as back_pressure_settings
 from nucliadb.common.context.fastapi import inject_app_context
 from nucliadb.ingest.processing import start_processing_engine, stop_processing_engine
 from nucliadb.ingest.utils import start_ingest, stop_ingest
 from nucliadb.writer import SERVICE_NAME
-from nucliadb.writer.back_pressure import start_materializer, stop_materializer
-from nucliadb.writer.settings import back_pressure_settings
 from nucliadb.writer.tus import finalize as storage_finalize
 from nucliadb.writer.tus import initialize as storage_initialize
 from nucliadb_telemetry.utils import clean_telemetry, setup_telemetry

nucliadb/writer/settings.py

@@ -19,7 +19,6 @@
 #
 from typing import Optional
 
-from pydantic import Field
 from pydantic_settings import BaseSettings
 
 
@@ -29,54 +28,4 @@ class Settings(BaseSettings):
     dm_redis_port: Optional[int] = None
 
 
-class BackPressureSettings(BaseSettings):
-    enabled: bool = Field(
-        default=False,
-        description="Enable or disable back pressure.",
-        alias="back_pressure_enabled",
-    )
-    indexing_rate: float = Field(
-        default=10,
-        description="Estimation of the indexing rate in messages per second. This is used to calculate the try again in time",  # noqa
-    )
-    ingest_rate: float = Field(
-        default=4,
-        description="Estimation of the ingest processed consumer rate in messages per second. This is used to calculate the try again in time",  # noqa
-    )
-    processing_rate: float = Field(
-        default=1,
-        description="Estimation of the processing rate in messages per second. This is used to calculate the try again in time",  # noqa
-    )
-    max_indexing_pending: int = Field(
-        default=1000,
-        description="Max number of messages pending to index in a node queue before rate limiting writes. Set to 0 to disable indexing back pressure checks",  # noqa
-        alias="back_pressure_max_indexing_pending",
-    )
-    max_ingest_pending: int = Field(
-        # Disabled by default
-        default=0,
-        description="Max number of messages pending to be ingested by processed consumers before rate limiting writes. Set to 0 to disable ingest back pressure checks",  # noqa
-        alias="back_pressure_max_ingest_pending",
-    )
-    max_processing_pending: int = Field(
-        default=1000,
-        description="Max number of messages pending to process per Knowledge Box before rate limiting writes. Set to 0 to disable processing back pressure checks",  # noqa
-        alias="back_pressure_max_processing_pending",
-    )
-    indexing_check_interval: int = Field(
-        default=30,
-        description="Interval in seconds to check the indexing pending messages",
-    )
-    ingest_check_interval: int = Field(
-        default=30,
-        description="Interval in seconds to check the ingest pending messages",
-    )
-    max_wait_time: int = Field(
-        default=60,
-        description="Max time in seconds to wait before trying again after back pressure",
-    )
-
-
 settings = Settings()
-
-back_pressure_settings = BackPressureSettings()

{nucliadb-6.4.0.post4196.dist-info → nucliadb-6.4.0.post4204.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.4.0.post4196
+Version: 6.4.0.post4204
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4196
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4196
-Requires-Dist: nucliadb-protos>=6.4.0.post4196
-Requires-Dist: nucliadb-models>=6.4.0.post4196
-Requires-Dist: nidx-protos>=6.4.0.post4196
+Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4204
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4204
+Requires-Dist: nucliadb-protos>=6.4.0.post4204
+Requires-Dist: nucliadb-models>=6.4.0.post4204
+Requires-Dist: nidx-protos>=6.4.0.post4204
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.4.0.post4196.dist-info → nucliadb-6.4.0.post4204.dist-info}/RECORD

@@ -60,6 +60,11 @@ nucliadb/common/ids.py,sha256=4QjoIofes_vtKj2HsFWZf8VVIVWXxdkYtLpx1n618Us,8239
 nucliadb/common/locking.py,sha256=RL0CabZVPzxHZyUjYeUyLvsJTm7W3J9o4fEgsY_ufNc,5896
 nucliadb/common/nidx.py,sha256=3EeQGjM_gxK0l_Rb54fspFWVNnzUiKF-_GMxTiiDC8Q,9116
 nucliadb/common/vector_index_config.py,sha256=LqGwhrDCp1q1vBow3scd1Chhr4GLYjYnGL72FKvOYYc,1552
+nucliadb/common/back_pressure/__init__.py,sha256=paAcAZcfGRTyURF9lnn3vX0vcwakTEVswG_xcdGBH-U,928
+nucliadb/common/back_pressure/cache.py,sha256=ANvXglWzI5naAD6N4E_fNi17qS6KNyAhjLeh6WlZZ84,2931
+nucliadb/common/back_pressure/materializer.py,sha256=YzYfN7xI5nlmSowbdLktWIkrJJb3Q2vEmoyz9O3eb2s,11667
+nucliadb/common/back_pressure/settings.py,sha256=3qNOzbI0KC6LMy-wMilXRSBfZu6CCpGHod26MTgAZ2o,3082
+nucliadb/common/back_pressure/utils.py,sha256=aZeP1XSkdgaRgZC76yR9Kje3511ZUCp7KB-XzcvhMYY,2018
 nucliadb/common/cluster/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/cluster/exceptions.py,sha256=t7v_l93t44l2tQpdQXgO_w-c4YZRcaayOz1A2i0w4RQ,1258
 nucliadb/common/cluster/grpc_node_dummy.py,sha256=JkufazWzMA4KFEU8EBkMbiiDW4C8lLcRhiiCxP7aCQY,2949
@@ -251,7 +256,7 @@ nucliadb/search/search/shards.py,sha256=mc5DK-MoCv9AFhlXlOFHbPvetcyNDzTFOJ5rimK8
 nucliadb/search/search/summarize.py,sha256=ksmYPubEQvAQgfPdZHfzB_rR19B2ci4IYZ6jLdHxZo8,4996
 nucliadb/search/search/utils.py,sha256=ajRIXfdTF67dBVahQCXW-rSv6gJpUMPt3QhJrWqArTQ,2175
 nucliadb/search/search/chat/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/search/search/chat/ask.py,sha256=xNdkdH2NzqfONj_qkdGGFvdGlz94PiyxE2ayMVk-MdQ,37471
+nucliadb/search/search/chat/ask.py,sha256=KO1YD1nGwblT4h2CfV7ccCpLP0OuoBCJQHBB5bDQatg,37476
 nucliadb/search/search/chat/exceptions.py,sha256=Siy4GXW2L7oPhIR86H3WHBhE9lkV4A4YaAszuGGUf54,1356
 nucliadb/search/search/chat/images.py,sha256=PA8VWxT5_HUGfW1ULhKTK46UBsVyINtWWqEM1ulzX1E,3095
 nucliadb/search/search/chat/prompt.py,sha256=Jnja-Ss7skgnnDY8BymVfdeYsFPnIQFL8tEvcRXTKUE,47356
@@ -328,28 +333,27 @@ nucliadb/train/generators/token_classifier.py,sha256=DdyMbrpxIVGWdTcz3SEN_3HwxKf
 nucliadb/train/generators/utils.py,sha256=ZNwvEVPZr-eP0MW3ABN7a11hPQKaa0NdVaRcgBcTp5w,3601
 nucliadb/writer/__init__.py,sha256=S298mrZL3vr62OrBqi97mdLxgR5cReMlRJgnaQHZV7s,1304
 nucliadb/writer/app.py,sha256=ABBO8-u4pDAa61b3mCdD0TFhuHAYcxMkgpZSGgWARuE,2736
-nucliadb/writer/back_pressure.py,sha256=4OwFGq9pvAbChB3WBZAY36lclfD-gD2ouC6YsKA4bIo,16892
 nucliadb/writer/exceptions.py,sha256=-Z7LW--eid7PNeKFuzo9kAlbLEBMUosxE-UVIgGD3SA,929
-nucliadb/writer/lifecycle.py,sha256=OYyhUZ1ejlybPzO-O_EsInjdifKiPiEzooy2d_2DW3k,2550
+nucliadb/writer/lifecycle.py,sha256=P1b_KoNkMTeF1IbyDCh_zhexWbeYe5LH6p2iFSJPiN4,2576
 nucliadb/writer/openapi.py,sha256=thqCO1ht_RJgOkXs-aIsv8aXJrU5z8wo2n05l2_LqMs,1032
 nucliadb/writer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/writer/run.py,sha256=euVZ_rtHDXs-O1kB-Pt1Id8eft9CYVpWH3zJzEoEqls,1448
-nucliadb/writer/settings.py,sha256=pA9aMAvY8H6zvsxAOdGY8SZLrThDvJ8KLhluGI0GxnQ,3288
+nucliadb/writer/settings.py,sha256=gKtCTDF2E1m6lYL0Iv4WwY4VZuvw1Dsa-uIBZxCHTdU,1071
 nucliadb/writer/utilities.py,sha256=AZ5qEny1Xm0IDsFtH13oJa2usvJZK8f0FdgF1LrnLCw,1036
 nucliadb/writer/api/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/api/constants.py,sha256=qWEDjFUycrEZnSJyLnNK4PQNodU2oVmkO4NycaEZtio,1738
 nucliadb/writer/api/utils.py,sha256=wIQHlU8RQiIGVLI72suvyVIKlCU44Unh0Ae0IiN6Qwo,1313
 nucliadb/writer/api/v1/__init__.py,sha256=akI9A_jloNLb0dU4T5zjfdyvmSAiDeIdjAlzNx74FlU,1128
-nucliadb/writer/api/v1/export_import.py,sha256=elf-EQY5DD3mhw8kWb9tQpDcbrF9sY6VFYqxQOjuVP0,8201
-nucliadb/writer/api/v1/field.py,sha256=KOOBqBJzwsNczn_isxl-YFBL-bmduz3rzSDWMbAJefc,18523
+nucliadb/writer/api/v1/export_import.py,sha256=v0sU55TtRSqDzwkDgcwv2uSaqKCuQTtGcMpYoHQYBQA,8192
+nucliadb/writer/api/v1/field.py,sha256=OicvLF1bnkJj1ixALFLuhvFX6NCMFpORROcFcS9nKpk,18505
 nucliadb/writer/api/v1/knowledgebox.py,sha256=PHEYDFa-sN5JrI8-EiVVg5FDOsRuCLT43kyAB4xt-xA,9530
 nucliadb/writer/api/v1/learning_config.py,sha256=CKBjqcbewkfPwGUPLDWzZSpro6XkmCaVppe5Qtpu5Go,3117
-nucliadb/writer/api/v1/resource.py,sha256=jxphiyeXJq342BR1R8pRQ81L0i3Tczf_Yarqx_DqvWs,19786
+nucliadb/writer/api/v1/resource.py,sha256=IaKHwP4M4Pm3xXj_xcnQCnTzKtXj_xj-r7YOHdH-89I,19750
 nucliadb/writer/api/v1/router.py,sha256=RjuoWLpZer6Kl2BW_wznpNo6XL3BOpdTGqXZCn3QrrQ,1034
 nucliadb/writer/api/v1/services.py,sha256=3AUjk-SmvqJx76v7y89DZx6oyasojPliGYeniRQjpcU,13337
 nucliadb/writer/api/v1/slug.py,sha256=xlVBDBpRi9bNulpBHZwhyftVvulfE0zFm1XZIWl-AKY,2389
 nucliadb/writer/api/v1/transaction.py,sha256=d2Vbgnkk_-FLGSTt3vfldwiJIUf0XoyD0wP1jQNz_DY,2430
-nucliadb/writer/api/v1/upload.py,sha256=fwWXA5BuLPuGKhOcuyf0CdutWJITjJ6fAvDzV_X9VsU,33809
+nucliadb/writer/api/v1/upload.py,sha256=vdKurdxRU7vYlcQIXf5RNTuX-G0waBSak2HnNRmAbLk,33791
 nucliadb/writer/api/v1/vectorsets.py,sha256=F3iMViL5G95_Tns4aO2SOA0DwAzxK2_P8MXxtd_XLRE,6973
 nucliadb/writer/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/resource/audit.py,sha256=FvxMZPzrNHtd31HgpZEvxzwAkbxJTZRhPLqRYYJi3tA,1426
@@ -365,8 +369,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.4.0.post4196.dist-info/METADATA,sha256=jtJRm9idqV9MmQAmslJDXhOsEzTsiLBHKsdTvCJ6pqM,4223
-nucliadb-6.4.0.post4196.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-nucliadb-6.4.0.post4196.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.4.0.post4196.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.4.0.post4196.dist-info/RECORD,,
+nucliadb-6.4.0.post4204.dist-info/METADATA,sha256=fAIY46KkkEIlOfObcPGceV3ZnO74SMottRW6kUPOFnU,4223
+nucliadb-6.4.0.post4204.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+nucliadb-6.4.0.post4204.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.4.0.post4204.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.4.0.post4204.dist-info/RECORD,,