dayhoff-tools 1.1.33__py3-none-any.whl → 1.1.34__py3-none-any.whl

dayhoff_tools/deployment/deploy_gcp.py

@@ -50,9 +50,55 @@ def create_batch_job_config(config: dict, image_uri: str) -> dict:
 
     Returns:
         Dictionary containing GCP Batch job configuration
+
+    Raises:
+        ValueError: If the configuration contains unexpected keys.
     """
     gcp_config = config["gcp"]
 
+    # Validate top-level gcp_config keys used for Batch job JSON construction
+    EXPECTED_GCP_CONFIG_KEYS = {
+        "allocation_policy",  # Goes into batch_config.allocationPolicy
+        "logs_policy",  # Goes into batch_config.logsPolicy
+        "batch_job",  # Contains detailed task and resource specs
+        # Keys like job_name, region, registry_uri, repository are used by other functions
+        # or for other purposes, not directly for constructing the core batch_config JSON here.
+    }
+    actual_gcp_keys = set(gcp_config.keys())
+    # Filter out keys not relevant to this function's direct Batch config construction
+    # These keys are used by the calling context or other parts of the deployment.
+    keys_to_ignore_for_this_check = {"job_name", "region", "registry_uri", "repository"}
+    relevant_gcp_keys = {
+        key for key in actual_gcp_keys if key not in keys_to_ignore_for_this_check
+    }
+
+    unhandled_gcp_keys = relevant_gcp_keys - EXPECTED_GCP_CONFIG_KEYS
+    if unhandled_gcp_keys:
+        raise ValueError(
+            f"Unexpected keys in 'gcp' configuration section: {unhandled_gcp_keys}. "
+            f"Expected keys for Batch job JSON construction are: {EXPECTED_GCP_CONFIG_KEYS}"
+        )
+
+    # Validate keys within gcp_config["batch_job"]
+    if "batch_job" not in gcp_config:
+        raise ValueError("Missing 'batch_job' section in 'gcp' configuration.")
+
+    gcp_batch_job_config = gcp_config["batch_job"]
+    EXPECTED_GCP_BATCH_JOB_KEYS = {
+        "taskCount",
+        "parallelism",
+        "computeResource",
+        "instance",  # Contains machineType, accelerators
+        "volumes",
+    }
+    actual_batch_job_keys = set(gcp_batch_job_config.keys())
+    unhandled_batch_job_keys = actual_batch_job_keys - EXPECTED_GCP_BATCH_JOB_KEYS
+    if unhandled_batch_job_keys:
+        raise ValueError(
+            f"Unexpected keys in 'gcp.batch_job' configuration section: {unhandled_batch_job_keys}. "
+            f"Expected keys are: {EXPECTED_GCP_BATCH_JOB_KEYS}"
+        )
+
     # Start with the allocation and logs policies
     batch_config = {
         "allocationPolicy": gcp_config["allocation_policy"],
@@ -92,10 +138,10 @@ def create_batch_job_config(config: dict, image_uri: str) -> dict:
 
     # Build the task group configuration
     task_group = {
-        "taskCount": gcp_config["batch_job"]["taskCount"],
-        "parallelism": gcp_config["batch_job"]["parallelism"],
+        "taskCount": gcp_batch_job_config["taskCount"],
+        "parallelism": gcp_batch_job_config["parallelism"],
         "taskSpec": {
-            "computeResource": gcp_config["batch_job"]["computeResource"],
+            "computeResource": gcp_batch_job_config["computeResource"],
             "runnables": [{"container": container_config}],
         },
     }
@@ -107,8 +153,12 @@ def create_batch_job_config(config: dict, image_uri: str) -> dict:
     if env_vars:
         task_group["taskSpec"]["runnables"][0]["environment"] = {"variables": env_vars}
 
+    # Add volumes to the taskSpec if specified in the config
+    if "volumes" in gcp_batch_job_config and gcp_batch_job_config["volumes"]:
+        task_group["taskSpec"]["volumes"] = gcp_batch_job_config["volumes"]
+
     # Add machine type and optional accelerators from instance config
-    instance_config = gcp_config["batch_job"]["instance"]
+    instance_config = gcp_batch_job_config["instance"]
     if "machineType" in instance_config:
         # Add machine type to the allocation policy
         if "policy" not in batch_config["allocationPolicy"]["instances"]:

{dayhoff_tools-1.1.33.dist-info → dayhoff_tools-1.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dayhoff-tools
-Version: 1.1.33
+Version: 1.1.34
 Summary: Common tools for all the repos at Dayhoff Labs
 Author: Daniel Martin-Alarcon
 Author-email: dma@dayhofflabs.com

{dayhoff_tools-1.1.33.dist-info → dayhoff_tools-1.1.34.dist-info}/RECORD

@@ -8,7 +8,7 @@ dayhoff_tools/cli/swarm_commands.py,sha256=5EyKj8yietvT5lfoz8Zx0iQvVaNgc3SJX1z2z
 dayhoff_tools/cli/utility_commands.py,sha256=ER4VrJt4hu904MwrcltUXjwBWT4uFrP-aPXjdXyT3F8,24685
 dayhoff_tools/deployment/base.py,sha256=8tXwsPYvRo-zV-aNhHw1c7Rji-KWg8S5xoCCznFnVVI,17412
 dayhoff_tools/deployment/deploy_aws.py,sha256=O0gQxHioSU_sNU8T8MD4wSOPvWc--V8eRRZzlRu035I,16446
-dayhoff_tools/deployment/deploy_gcp.py,sha256=s3nt-VsF0NrEA-J9Ire8G9zsFgtp8A6dHWT7IYQL5uQ,6529
+dayhoff_tools/deployment/deploy_gcp.py,sha256=jiEE_tBVeSavAI8o_6qPDPpaoXKexcaNIa4uXcv3y0M,8839
 dayhoff_tools/deployment/deploy_utils.py,sha256=StFwbqnr2_FWiKVg3xnJF4kagTHzndqqDkpaIOaAn_4,26027
 dayhoff_tools/deployment/job_runner.py,sha256=hljvFpH2Bw96uYyUup5Ths72PZRL_X27KxlYzBMgguo,5086
 dayhoff_tools/deployment/processors.py,sha256=q2io07xO6f6twEH1iLz9GFoGaKh76qC4kcv519Q2G7g,34583
@@ -26,7 +26,7 @@ dayhoff_tools/intake/uniprot.py,sha256=BZYJQF63OtPcBBnQ7_P9gulxzJtqyorgyuDiPeOJq
 dayhoff_tools/logs.py,sha256=DKdeP0k0kliRcilwvX0mUB2eipO5BdWUeHwh-VnsICs,838
 dayhoff_tools/sqlite.py,sha256=jV55ikF8VpTfeQqqlHSbY8OgfyfHj8zgHNpZjBLos_E,18672
 dayhoff_tools/warehouse.py,sha256=TqV8nex1AluNaL4JuXH5zuu9P7qmE89lSo6f_oViy6U,14965
-dayhoff_tools-1.1.33.dist-info/METADATA,sha256=8v8Kj6e0GdUHhtDx3l1huf7t1YwcnA0vlbbaKWwpBAw,2843
-dayhoff_tools-1.1.33.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-dayhoff_tools-1.1.33.dist-info/entry_points.txt,sha256=iAf4jteNqW3cJm6CO6czLxjW3vxYKsyGLZ8WGmxamSc,49
-dayhoff_tools-1.1.33.dist-info/RECORD,,
+dayhoff_tools-1.1.34.dist-info/METADATA,sha256=-fobKYRniLRERxoTFy2_twRo1vgGMzlUabXPVNY8NcI,2843
+dayhoff_tools-1.1.34.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+dayhoff_tools-1.1.34.dist-info/entry_points.txt,sha256=iAf4jteNqW3cJm6CO6czLxjW3vxYKsyGLZ8WGmxamSc,49
+dayhoff_tools-1.1.34.dist-info/RECORD,,