nucliadb 6.7.2.post4882__py3-none-any.whl → 6.7.2.post4886__py3-none-any.whl

nucliadb/ingest/partitions.py

@@ -25,12 +25,17 @@ from nucliadb.ingest.settings import Settings
 
 
 def assign_partitions(settings: Settings):
+    """
+    This function dynamically assigns the partitions to the current ingest sts
+    replica based on its hostname, typically (ingest-0, ingest-1, etc).
+    """
     # partitions start from 1, instead of 0
     all_partitions = [str(part + 1) for part in range(settings.nuclia_partitions)]
 
     # get replica number and total replicas from environment
     logger.info(f"PARTITIONS: Total Replicas = {settings.total_replicas}")
     if settings.replica_number == -1:
+        # Get replica number from hostname
         hostname = os.environ.get("HOSTNAME")
         if hostname is not None:
             sts_values = hostname.split("-")
@@ -39,10 +44,16 @@ def assign_partitions(settings: Settings):
                     settings.replica_number = int(sts_values[-1])
                 except Exception:
                     logger.error(f"Could not extract replica number from hostname: {hostname}")
-                    pass
+            else:
+                logger.warning(f"Could not determine replica number from hostname: {hostname}")
+        else:
+            logger.warning(f"Could not determine replica number from hostname.")
 
         if settings.replica_number == -1:
             settings.replica_number = 0
+    else:
+        # We assume that replica numbers are set manually via env variables
+        pass
     logger.info(f"PARTITIONS: Replica Number = {settings.replica_number}")
 
     # calculate assigned partitions based on total replicas and own replica number

nucliadb/ingest/settings.py

@@ -75,26 +75,40 @@ class ProcessingPullMode(Enum):
 
 
 class Settings(DriverSettings):
-    grpc_port: int = 8030
-
-    partitions: list[str] = ["1"]
-
+    # Pull worker settings
     pull_time_error_backoff: int = 30
     pull_api_timeout: int = 60
-    disable_pull_worker: bool = False
+    disable_pull_worker: bool = Field(
+        default=False, description="Set to true to disable the pull worker task"
+    )
 
-    # ingest consumer sts replica settings
-    replica_number: int = -1
-    total_replicas: int = 1  # number of ingest processor replicas in the cluster
-    nuclia_partitions: int = 50
+    # Ingest consumer sts replica settings
+    replica_number: int = Field(
+        default=-1,
+        description="The replica number of this ingest statefulset instance. Leave to -1 to auto-assign based on hostname.",
+    )
+    total_replicas: int = Field(default=1, description="Number of ingest statefulset replicas deployed")
+    nuclia_partitions: int = Field(
+        default=50, description="Total number of partitions of the nats stream."
+    )
+    partitions: list[str] = Field(
+        default=["1"],
+        description="List of partitions assigned to this ingest statefulset instance. This is automatically assigned based on the replica number and total replicas.",
+    )
+    max_concurrent_ingest_processing: int = Field(
+        default=5,
+        description="Controls the number of concurrent messages from different partitions that can be processed at the same time by ingest statefulset consumers.",
+    )
 
-    max_receive_message_length: int = 500  # In MB
+    # Grpc server settings
+    grpc_port: int = 8030
+    max_receive_message_length: int = Field(
+        default=500, description="Maximum receive grpc message length in MB."
+    )
 
     # Search query timeouts
     relation_search_timeout: float = 10.0
     relation_types_timeout: float = 10.0
 
-    max_concurrent_ingest_processing: int = 5
-
 
 settings = Settings()

nucliadb/standalone/run.py

@@ -116,6 +116,9 @@ def run():
     if nuclia_settings.nuclia_service_account:
         settings_to_output["NUA API key"] = "Configured ✔"
         settings_to_output["NUA API zone"] = nuclia_settings.nuclia_zone
+        settings_to_output["NUA API url"] = (
+            nuclia_settings.nuclia_public_url.format(zone=nuclia_settings.nuclia_zone) + "/api"
+        )
 
     settings_to_output_fmted = "\n".join(
         [f"||      - {k}:{' ' * (27 - len(k))}{v}" for k, v in settings_to_output.items()]

{nucliadb-6.7.2.post4882.dist-info → nucliadb-6.7.2.post4886.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.7.2.post4882
+Version: 6.7.2.post4886
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4882
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4882
-Requires-Dist: nucliadb-protos>=6.7.2.post4882
-Requires-Dist: nucliadb-models>=6.7.2.post4882
-Requires-Dist: nidx-protos>=6.7.2.post4882
+Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4886
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4886
+Requires-Dist: nucliadb-protos>=6.7.2.post4886
+Requires-Dist: nucliadb-models>=6.7.2.post4886
+Requires-Dist: nidx-protos>=6.7.2.post4886
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.46.0
 Requires-Dist: uvicorn[standard]

{nucliadb-6.7.2.post4882.dist-info → nucliadb-6.7.2.post4886.dist-info}/RECORD

@@ -132,11 +132,11 @@ nucliadb/export_import/tasks.py,sha256=DWbdqY97ffoyfipelGXz3Jqz1iam6JCjQSh367Fc3
 nucliadb/export_import/utils.py,sha256=XV3tJJdhgnVJRSj8AxZjgeipONtB107M185HVJmHp2Q,21626
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
 nucliadb/ingest/app.py,sha256=qiPad2eWgudRdLq0tB0MQZOxOezXO7QBK_ZpPNKQZO0,7378
-nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
+nucliadb/ingest/partitions.py,sha256=c1OWrFWgadNtvghY3Fl-xlurdyV5hZpVJPEoRAsBt1k,2903
 nucliadb/ingest/processing.py,sha256=IKXMZXIPuuojKQiXR2T5-5NwMvmUnIQIhBXUGgzyFFo,21551
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/ingest/serialize.py,sha256=hiddxbV5gxVk8uY8-Q1AEq2DhJx5fOBP34zq5ONGgcs,16240
-nucliadb/ingest/settings.py,sha256=LskYx8Eefv5qdHkpcsMKHgkaVJuMhC9XnDHRS6s6BAc,3392
+nucliadb/ingest/settings.py,sha256=8OJMjVVbI3OWIbZLrXBqpB79zHbbLkCSb9VJA0IzRss,4269
 nucliadb/ingest/utils.py,sha256=l1myURu3r8oA11dx3GpHw-gNTUc1AFX8xdPm9Lgl2rA,2275
 nucliadb/ingest/consumer/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/consumer/auditing.py,sha256=xK21DIa_ZAiOJVVbnkmT4jgCRGshNGyPyxsqhE6kROE,7204
@@ -291,7 +291,7 @@ nucliadb/standalone/lifecycle.py,sha256=rdKLG-oOLN4rfd2VGG_2vlDUWYneWSCiuEhoeiFK
 nucliadb/standalone/migrations.py,sha256=s9-3RSZ-O3bjEw2TnBe_YWLUEKbub0bARUxi1gA3yuY,1950
 nucliadb/standalone/purge.py,sha256=ZY-cebb214FFiPG7OFmXZGg0G3CK5Amw0FLLm9WJhKE,1343
 nucliadb/standalone/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nucliadb/standalone/run.py,sha256=0QKEAT6pCaLvnuxTG3RltTlhE2g5-HI21KbOWfusBGE,5425
+nucliadb/standalone/run.py,sha256=gbJcociwaQVKHR5C0HEq0cpWuxgOLcmVjCzlE0S2ZWw,5577
 nucliadb/standalone/settings.py,sha256=fbgqVT37XB2cJHJARnR19MO_dz6NLbkuIC2okH7J80o,5714
 nucliadb/standalone/versions.py,sha256=8CxNMNt2NgWM8ct50UsR4d44-ae7wtQI-sV-yGiFqyI,3508
 nucliadb/standalone/static/favicon.ico,sha256=96pKGp6Sx457JkTfjy1dtApMhkitixfU6invCUGAYOU,2285
@@ -376,8 +376,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vu1BGg4VqJ_x2P1u2BxqPKlSfw5orT_a3R-Ln5oPUpU,8483
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.7.2.post4882.dist-info/METADATA,sha256=4iQdvdQQ2eCmmWGGgUH-zAT708gy9kMu8W6_DQnrw4Y,4158
-nucliadb-6.7.2.post4882.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.7.2.post4882.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.7.2.post4882.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.7.2.post4882.dist-info/RECORD,,
+nucliadb-6.7.2.post4886.dist-info/METADATA,sha256=OK7DQ2Ezub_EkiIyf-1Mu97pM8L1rbIzOlmSwwbdNa8,4158
+nucliadb-6.7.2.post4886.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.7.2.post4886.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.7.2.post4886.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.7.2.post4886.dist-info/RECORD,,