nucliadb 6.7.2.post4884__py3-none-any.whl → 6.7.2.post4889__py3-none-any.whl

nucliadb/common/http_clients/exceptions.py

@@ -21,6 +21,10 @@ class ClientException(Exception):
     pass
 
 
+class ServerException(Exception):
+    pass
+
+
 class NotFoundException(ClientException):
     pass
 
@@ -35,3 +39,7 @@ class RateLimitException(ClientException):
 
 class AccountLimitException(ClientException):
     pass
+
+
+class ServiceUnavailableException(ServerException):
+    pass

nucliadb/common/http_clients/processing.py

@@ -209,6 +209,10 @@ class ProcessingHTTPClient:
     async def close(self):
         await self.session.close()
 
+    async def reset_session(self):
+        await self.close()
+        self.session = aiohttp.ClientSession()
+
     async def in_progress(self, ack_token: str):
         url = self.base_url_v2 + "/pull/in_progress"
         request = InProgressRequest(ack=[ack_token])

nucliadb/common/http_clients/utils.py

@@ -33,5 +33,8 @@ def check_status(resp: aiohttp.ClientResponse, resp_text: str) -> None:
         raise exceptions.AuthorizationException(f"Unauthorized to access: {resp.status}")
     elif resp.status == 429:
         raise exceptions.RateLimitException("Rate limited")
+    elif resp.status in (502, 503):
+        # Service unavailable, can be retried
+        raise exceptions.ServiceUnavailableException(f"Service unavailable: {resp.status} - {resp_text}")
     else:
         raise exceptions.ClientException(f"Unknown error: {resp.status} - {resp_text}")

nucliadb/ingest/consumer/pull.py

@@ -31,6 +31,7 @@ from opentelemetry.trace import (
     Link,
 )
 
+from nucliadb.common.http_clients.exceptions import ServiceUnavailableException
 from nucliadb.common.http_clients.processing import (
     ProcessingHTTPClient,
     ProcessingPullMessageProgressUpdater,
@@ -209,6 +210,12 @@ class PullV2Worker:
                             payload_length = len(base64.b64decode(data.payload))
                         logger.error(f"Message too big for transaction: {payload_length}")
                     raise e
+
+                except ServiceUnavailableException as ex:
+                    logger.warning(f"Processing api is unavailable, will retry shortly: {ex}")
+                    await processing_http_client.reset_session()
+                    await asyncio.sleep(self.pull_time_error_backoff)
+
                 except Exception:
                     logger.exception("Unhandled error pulling messages from processing")
                     await asyncio.sleep(self.pull_time_error_backoff)

nucliadb/ingest/partitions.py

@@ -25,12 +25,17 @@ from nucliadb.ingest.settings import Settings
 
 
 def assign_partitions(settings: Settings):
+    """
+    This function dynamically assigns the partitions to the current ingest sts
+    replica based on its hostname, typically (ingest-0, ingest-1, etc).
+    """
     # partitions start from 1, instead of 0
     all_partitions = [str(part + 1) for part in range(settings.nuclia_partitions)]
 
     # get replica number and total replicas from environment
     logger.info(f"PARTITIONS: Total Replicas = {settings.total_replicas}")
     if settings.replica_number == -1:
+        # Get replica number from hostname
         hostname = os.environ.get("HOSTNAME")
         if hostname is not None:
             sts_values = hostname.split("-")
@@ -39,10 +44,16 @@ def assign_partitions(settings: Settings):
                     settings.replica_number = int(sts_values[-1])
                 except Exception:
                     logger.error(f"Could not extract replica number from hostname: {hostname}")
-                    pass
+            else:
+                logger.warning(f"Could not determine replica number from hostname: {hostname}")
+        else:
+            logger.warning(f"Could not determine replica number from hostname.")
 
         if settings.replica_number == -1:
             settings.replica_number = 0
+    else:
+        # We assume that replica numbers are set manually via env variables
+        pass
     logger.info(f"PARTITIONS: Replica Number = {settings.replica_number}")
 
     # calculate assigned partitions based on total replicas and own replica number

nucliadb/ingest/settings.py

@@ -75,26 +75,40 @@ class ProcessingPullMode(Enum):
 
 
 class Settings(DriverSettings):
-    grpc_port: int = 8030
-
-    partitions: list[str] = ["1"]
-
+    # Pull worker settings
     pull_time_error_backoff: int = 30
     pull_api_timeout: int = 60
-    disable_pull_worker: bool = False
+    disable_pull_worker: bool = Field(
+        default=False, description="Set to true to disable the pull worker task"
+    )
 
-    # ingest consumer sts replica settings
-    replica_number: int = -1
-    total_replicas: int = 1  # number of ingest processor replicas in the cluster
-    nuclia_partitions: int = 50
+    # Ingest consumer sts replica settings
+    replica_number: int = Field(
+        default=-1,
+        description="The replica number of this ingest statefulset instance. Leave to -1 to auto-assign based on hostname.",
+    )
+    total_replicas: int = Field(default=1, description="Number of ingest statefulset replicas deployed")
+    nuclia_partitions: int = Field(
+        default=50, description="Total number of partitions of the nats stream."
+    )
+    partitions: list[str] = Field(
+        default=["1"],
+        description="List of partitions assigned to this ingest statefulset instance. This is automatically assigned based on the replica number and total replicas.",
+    )
+    max_concurrent_ingest_processing: int = Field(
+        default=5,
+        description="Controls the number of concurrent messages from different partitions that can be processed at the same time by ingest statefulset consumers.",
+    )
 
-    max_receive_message_length: int = 500  # In MB
+    # Grpc server settings
+    grpc_port: int = 8030
+    max_receive_message_length: int = Field(
+        default=500, description="Maximum receive grpc message length in MB."
+    )
 
     # Search query timeouts
     relation_search_timeout: float = 10.0
     relation_types_timeout: float = 10.0
 
-    max_concurrent_ingest_processing: int = 5
-
 
 settings = Settings()

{nucliadb-6.7.2.post4884.dist-info → nucliadb-6.7.2.post4889.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.7.2.post4884
+Version: 6.7.2.post4889
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4884
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4884
-Requires-Dist: nucliadb-protos>=6.7.2.post4884
-Requires-Dist: nucliadb-models>=6.7.2.post4884
-Requires-Dist: nidx-protos>=6.7.2.post4884
+Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4889
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4889
+Requires-Dist: nucliadb-protos>=6.7.2.post4889
+Requires-Dist: nucliadb-models>=6.7.2.post4889
+Requires-Dist: nidx-protos>=6.7.2.post4889
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.46.0
 Requires-Dist: uvicorn[standard]

{nucliadb-6.7.2.post4884.dist-info → nucliadb-6.7.2.post4889.dist-info}/RECORD

@@ -109,10 +109,10 @@ nucliadb/common/external_index_providers/pinecone.py,sha256=PB0lUBBZyI9qcyRxtoi9
 nucliadb/common/external_index_providers/settings.py,sha256=EGHnIkwxqe6aypwKegXTlKO3AgUxNa-6GeAZG25Njis,2002
 nucliadb/common/http_clients/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/http_clients/auth.py,sha256=srfpgAbs2wmqA9u_l-HxsV4YoO77Tse4y3gm3q2YvYM,2112
-nucliadb/common/http_clients/exceptions.py,sha256=47Y8OjkaGV_F18G07FpJhOzgWKUIexhlILyuVtICz8s,1100
-nucliadb/common/http_clients/processing.py,sha256=mKd9vRK-Wb71UG2LCoGu47wmnN5krqA0D1Z8vitsBPE,8976
+nucliadb/common/http_clients/exceptions.py,sha256=HniqLZEZN9BNfVv-AaBLpRyb8wpXzMpZNP5oANJYE6M,1208
+nucliadb/common/http_clients/processing.py,sha256=lIcR-Z9rqSUnTw0x8SjbIfyPWgV0nTQDr4o027GFmww,9086
 nucliadb/common/http_clients/pypi.py,sha256=VHIUjwJEJVntVUo_FRoXIo8sLmluy7sa9-iXSITcrMY,1540
-nucliadb/common/http_clients/utils.py,sha256=yGUkHNS41abHiBoHqo_Mg3QSqGsS7rUtbfGftbEC57U,1529
+nucliadb/common/http_clients/utils.py,sha256=j1jikzrqPzIWQnckvQ1ANM-PkAaQAao2P94p5-PvyGM,1717
 nucliadb/common/maindb/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/maindb/driver.py,sha256=y_puOqyZj-aq2indAVbFtMPnNNc2u2MShO4SVKr5FFE,2994
 nucliadb/common/maindb/exceptions.py,sha256=u6ZSQW6jk5QM_IL5XmQ_dF-vZ-JkuWEqZbNJ-S6FG_g,988
@@ -132,18 +132,18 @@ nucliadb/export_import/tasks.py,sha256=DWbdqY97ffoyfipelGXz3Jqz1iam6JCjQSh367Fc3
 nucliadb/export_import/utils.py,sha256=XV3tJJdhgnVJRSj8AxZjgeipONtB107M185HVJmHp2Q,21626
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
 nucliadb/ingest/app.py,sha256=qiPad2eWgudRdLq0tB0MQZOxOezXO7QBK_ZpPNKQZO0,7378
-nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
+nucliadb/ingest/partitions.py,sha256=c1OWrFWgadNtvghY3Fl-xlurdyV5hZpVJPEoRAsBt1k,2903
 nucliadb/ingest/processing.py,sha256=IKXMZXIPuuojKQiXR2T5-5NwMvmUnIQIhBXUGgzyFFo,21551
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/ingest/serialize.py,sha256=hiddxbV5gxVk8uY8-Q1AEq2DhJx5fOBP34zq5ONGgcs,16240
-nucliadb/ingest/settings.py,sha256=LskYx8Eefv5qdHkpcsMKHgkaVJuMhC9XnDHRS6s6BAc,3392
+nucliadb/ingest/settings.py,sha256=8OJMjVVbI3OWIbZLrXBqpB79zHbbLkCSb9VJA0IzRss,4269
 nucliadb/ingest/utils.py,sha256=l1myURu3r8oA11dx3GpHw-gNTUc1AFX8xdPm9Lgl2rA,2275
 nucliadb/ingest/consumer/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/consumer/auditing.py,sha256=xK21DIa_ZAiOJVVbnkmT4jgCRGshNGyPyxsqhE6kROE,7204
 nucliadb/ingest/consumer/consumer.py,sha256=1OetpJXp6glaAe4kKqUA_L46BS-ZyEccTkwt7TGf0Zw,11658
 nucliadb/ingest/consumer/materializer.py,sha256=tgD_rDI2twQzcz8kKNiW_L4YIth16IGh9mUfD5wiSD4,3858
 nucliadb/ingest/consumer/metrics.py,sha256=ji1l_4cKiHJthQd8YNem1ft4iMbw9KThmVvJmLcv3Xg,1075
-nucliadb/ingest/consumer/pull.py,sha256=vAOu2Zum-1e4RipoHvzzIha5PoNV28_C0nciQ2UFphc,8831
+nucliadb/ingest/consumer/pull.py,sha256=Ki_aHi72W83yD03lPt6Yz2l_uCeu62fd4upEMcOZcm4,9201
 nucliadb/ingest/consumer/service.py,sha256=8AD41mMN7EUeUtk4ZNy14zfvxzwmVjIX6Mwe05-bomA,6543
 nucliadb/ingest/consumer/shard_creator.py,sha256=UKIk0yaS_jC_nGQqymn9NGJWzwZEqhIr0gznJYorlAE,4348
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
@@ -376,8 +376,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vu1BGg4VqJ_x2P1u2BxqPKlSfw5orT_a3R-Ln5oPUpU,8483
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.7.2.post4884.dist-info/METADATA,sha256=IGL0gQWBXlU1qfCFDX83hrTIWGuXT3RN_j-yuWUishQ,4158
-nucliadb-6.7.2.post4884.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.7.2.post4884.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.7.2.post4884.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.7.2.post4884.dist-info/RECORD,,
+nucliadb-6.7.2.post4889.dist-info/METADATA,sha256=n5UTXqF3fg0n3X1orrwnWa9yaPykhsM2ntFhyJm-Z8E,4158
+nucliadb-6.7.2.post4889.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.7.2.post4889.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.7.2.post4889.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.7.2.post4889.dist-info/RECORD,,