dagster-k8s 0.24.10__tar.gz → 0.24.11__tar.gz

{dagster-k8s-0.24.10/dagster_k8s.egg-info → dagster-k8s-0.24.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.24.10
+Version: 0.24.11
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs

{dagster-k8s-0.24.10 → dagster-k8s-0.24.11}/dagster_k8s/container_context.py

@@ -51,7 +51,6 @@ class K8sContainerContext(
             ("server_k8s_config", UserDefinedDagsterK8sConfig),
             ("run_k8s_config", UserDefinedDagsterK8sConfig),
             ("namespace", Optional[str]),
-            ("labels", Mapping[str, str]),
         ],
     )
 ):
@@ -106,12 +105,19 @@ class K8sContainerContext(
         )
 
         run_k8s_config = K8sContainerContext._merge_k8s_config(
-            top_level_k8s_config,
+            top_level_k8s_config._replace(  # remove k8s service/deployment fields
+                deployment_metadata={},
+                service_metadata={},
+            ),
             run_k8s_config or UserDefinedDagsterK8sConfig.from_dict({}),
         )
 
         server_k8s_config = K8sContainerContext._merge_k8s_config(
-            top_level_k8s_config,
+            top_level_k8s_config._replace(  # remove k8s job fields
+                job_config={},
+                job_metadata={},
+                job_spec_config={},
+            ),
             server_k8s_config or UserDefinedDagsterK8sConfig.from_dict({}),
         )
 
@@ -120,7 +126,6 @@ class K8sContainerContext(
             run_k8s_config=run_k8s_config,
             server_k8s_config=server_k8s_config,
             namespace=namespace,
-            labels=check.opt_mapping_param(labels, "labels"),
         )
 
     @staticmethod
@@ -143,6 +148,8 @@ class K8sContainerContext(
         pod_spec_config = {}
         pod_template_spec_metadata = {}
         job_metadata = {}
+        deployment_metadata = {}
+        service_metadata = {}
 
         if volume_mounts:
             container_config["volume_mounts"] = volume_mounts
@@ -159,6 +166,8 @@ class K8sContainerContext(
         if labels:
             pod_template_spec_metadata["labels"] = labels
             job_metadata["labels"] = labels
+            deployment_metadata["labels"] = labels
+            service_metadata["labels"] = labels
 
         if image_pull_policy:
             container_config["image_pull_policy"] = image_pull_policy
@@ -196,6 +205,8 @@ class K8sContainerContext(
             pod_spec_config=pod_spec_config,
             pod_template_spec_metadata=pod_template_spec_metadata,
             job_metadata=job_metadata,
+            service_metadata=service_metadata,
+            deployment_metadata=deployment_metadata,
         )
 
     @staticmethod
@@ -260,7 +271,6 @@ class K8sContainerContext(
             ),
             run_k8s_config=self._merge_k8s_config(self.run_k8s_config, other.run_k8s_config),
             namespace=other.namespace if other.namespace else self.namespace,
-            labels={**self.labels, **other.labels},
         )
 
     def _snake_case_allowed_fields(
@@ -269,9 +279,18 @@ class K8sContainerContext(
         result = {}
 
         for key in only_allow_user_defined_k8s_config_fields:
+            if key == "namespace":
+                result[key] = only_allow_user_defined_k8s_config_fields[key]
+                continue
+
             if key == "container_config":
                 model_class = kubernetes.client.V1Container
-            elif key in {"job_metadata", "pod_template_spec_metadata"}:
+            elif key in {
+                "job_metadata",
+                "pod_template_spec_metadata",
+                "deployment_metadata",
+                "service_metadata",
+            }:
                 model_class = kubernetes.client.V1ObjectMeta
             elif key == "pod_spec_config":
                 model_class = kubernetes.client.V1PodSpec
@@ -284,12 +303,35 @@ class K8sContainerContext(
             )
         return result
 
-    def validate_user_k8s_config(
+    def validate_user_k8s_config_for_run(
         self,
         only_allow_user_defined_k8s_config_fields: Optional[Mapping[str, Any]],
         only_allow_user_defined_env_vars: Optional[Sequence[str]],
+    ):
+        return self._validate_user_k8s_config(
+            self.run_k8s_config,
+            only_allow_user_defined_k8s_config_fields,
+            only_allow_user_defined_env_vars,
+        )
+
+    def validate_user_k8s_config_for_code_server(
+        self,
+        only_allow_user_defined_k8s_config_fields: Optional[Mapping[str, Any]],
+        only_allow_user_defined_env_vars: Optional[Sequence[str]],
+    ):
+        return self._validate_user_k8s_config(
+            self.server_k8s_config,
+            only_allow_user_defined_k8s_config_fields,
+            only_allow_user_defined_env_vars,
+        )
+
+    def _validate_user_k8s_config(
+        self,
+        user_defined_k8s_config: UserDefinedDagsterK8sConfig,
+        only_allow_user_defined_k8s_config_fields: Optional[Mapping[str, Any]],
+        only_allow_user_defined_env_vars: Optional[Sequence[str]],
     ) -> "K8sContainerContext":
-        used_fields = self._get_used_k8s_config_fields()
+        used_fields = self._get_used_k8s_config_fields(user_defined_k8s_config)
 
         if only_allow_user_defined_k8s_config_fields is not None:
             snake_case_allowlist = self._snake_case_allowed_fields(
@@ -383,17 +425,11 @@ class K8sContainerContext(
             server_k8s_config=new_server_k8s_config,
         )
 
-    def _get_used_k8s_config_fields(self) -> Mapping[str, Mapping[str, Set[str]]]:
+    def _get_used_k8s_config_fields(
+        self, user_defined_k8s_config: UserDefinedDagsterK8sConfig
+    ) -> Mapping[str, Mapping[str, Set[str]]]:
         used_fields = {}
-        for key, fields in self.run_k8s_config.to_dict().items():
-            if key == "merge_behavior":
-                continue
-
-            used_fields[key] = used_fields.get(key, set()).union(
-                {field_key for field_key in fields}
-            )
-
-        for key, fields in self.server_k8s_config.to_dict().items():
+        for key, fields in user_defined_k8s_config.to_dict().items():
             if key == "merge_behavior":
                 continue
 
@@ -402,8 +438,7 @@ class K8sContainerContext(
             )
 
         if self.namespace:
-            used_fields["pod_template_spec_metadata"].add("namespace")
-            used_fields["job_metadata"].add("namespace")
+            used_fields["namespace"] = True
 
         return used_fields
 
@@ -457,7 +492,7 @@ class K8sContainerContext(
         # If there's an allowlist, make sure user_defined_container_context doesn't violate it
         if run_launcher:
             user_defined_container_context = (
-                user_defined_container_context.validate_user_k8s_config(
+                user_defined_container_context.validate_user_k8s_config_for_run(
                     run_launcher.only_allow_user_defined_k8s_config_fields,
                     run_launcher.only_allow_user_defined_env_vars,
                 )

{dagster-k8s-0.24.10 → dagster-k8s-0.24.11}/dagster_k8s/executor.py

@@ -27,7 +27,7 @@ from dagster._utils.merger import merge_dicts
 from dagster_k8s.client import DagsterKubernetesClient
 from dagster_k8s.container_context import K8sContainerContext
 from dagster_k8s.job import (
-    USER_DEFINED_K8S_CONFIG_SCHEMA,
+    USER_DEFINED_K8S_JOB_CONFIG_SCHEMA,
     DagsterK8sJobConfig,
     UserDefinedDagsterK8sConfig,
     construct_dagster_k8s_job,
@@ -69,7 +69,7 @@ _K8S_EXECUTOR_CONFIG_SCHEMA = merge_dicts(
         ),
         "tag_concurrency_limits": get_tag_concurrency_limits_config(),
         "step_k8s_config": Field(
-            USER_DEFINED_K8S_CONFIG_SCHEMA,
+            USER_DEFINED_K8S_JOB_CONFIG_SCHEMA,
             is_required=False,
             description="Raw Kubernetes configuration for each step launched by the executor.",
         ),

{dagster-k8s-0.24.10 → dagster-k8s-0.24.11}/dagster_k8s/job.py

@@ -63,7 +63,7 @@ class K8sConfigMergeBehavior(Enum):
 
 
 USER_DEFINED_K8S_CONFIG_KEY = "dagster-k8s/config"
-USER_DEFINED_K8S_CONFIG_SCHEMA = Shape(
+USER_DEFINED_K8S_JOB_CONFIG_SCHEMA = Shape(
     {
         "container_config": Permissive(),
         "pod_template_spec_metadata": Permissive(),
@@ -94,18 +94,23 @@ class UserDefinedDagsterK8sConfig(
             ("job_config", Mapping[str, Any]),
             ("job_metadata", Mapping[str, Any]),
             ("job_spec_config", Mapping[str, Any]),
+            ("deployment_metadata", Mapping[str, Any]),
+            ("service_metadata", Mapping[str, Any]),
             ("merge_behavior", K8sConfigMergeBehavior),
         ],
     )
 ):
     def __new__(
         cls,
+        *,
         container_config: Optional[Mapping[str, Any]] = None,
         pod_template_spec_metadata: Optional[Mapping[str, Any]] = None,
         pod_spec_config: Optional[Mapping[str, Any]] = None,
         job_config: Optional[Mapping[str, Any]] = None,
         job_metadata: Optional[Mapping[str, Any]] = None,
         job_spec_config: Optional[Mapping[str, Any]] = None,
+        deployment_metadata: Optional[Mapping[str, Any]] = None,
+        service_metadata: Optional[Mapping[str, Any]] = None,
         merge_behavior: K8sConfigMergeBehavior = K8sConfigMergeBehavior.DEEP,
     ):
         container_config = check.opt_mapping_param(
@@ -119,6 +124,13 @@ class UserDefinedDagsterK8sConfig(
         job_metadata = check.opt_mapping_param(job_metadata, "job_metadata", key_type=str)
         job_spec_config = check.opt_mapping_param(job_spec_config, "job_spec_config", key_type=str)
 
+        deployment_metadata = check.opt_mapping_param(
+            deployment_metadata, "deployment_metadata", key_type=str
+        )
+        service_metadata = check.opt_mapping_param(
+            service_metadata, "service_metadata", key_type=str
+        )
+
         if container_config:
             container_config = k8s_snake_case_dict(kubernetes.client.V1Container, container_config)
 
@@ -139,6 +151,14 @@ class UserDefinedDagsterK8sConfig(
         if job_spec_config:
             job_spec_config = k8s_snake_case_dict(kubernetes.client.V1JobSpec, job_spec_config)
 
+        if deployment_metadata:
+            deployment_metadata = k8s_snake_case_dict(
+                kubernetes.client.V1ObjectMeta, deployment_metadata
+            )
+
+        if service_metadata:
+            service_metadata = k8s_snake_case_dict(kubernetes.client.V1ObjectMeta, service_metadata)
+
         return super(UserDefinedDagsterK8sConfig, cls).__new__(
             cls,
             container_config=container_config,
@@ -147,6 +167,8 @@ class UserDefinedDagsterK8sConfig(
             job_config=job_config,
             job_metadata=job_metadata,
             job_spec_config=job_spec_config,
+            deployment_metadata=deployment_metadata,
+            service_metadata=service_metadata,
             merge_behavior=check.inst_param(
                 merge_behavior, "merge_behavior", K8sConfigMergeBehavior
             ),
@@ -160,6 +182,8 @@ class UserDefinedDagsterK8sConfig(
             "job_config": self.job_config,
             "job_metadata": self.job_metadata,
             "job_spec_config": self.job_spec_config,
+            "deployment_metadata": self.deployment_metadata,
+            "service_metadata": self.service_metadata,
             "merge_behavior": self.merge_behavior.value,
         }
 
@@ -172,6 +196,8 @@ class UserDefinedDagsterK8sConfig(
             job_config=config_dict.get("job_config"),
             job_metadata=config_dict.get("job_metadata"),
             job_spec_config=config_dict.get("job_spec_config"),
+            deployment_metadata=config_dict.get("deployment_metadata"),
+            service_metadata=config_dict.get("service_metadata"),
             merge_behavior=K8sConfigMergeBehavior(
                 config_dict.get("merge_behavior", K8sConfigMergeBehavior.DEEP.value)
             ),
@@ -205,7 +231,7 @@ def get_user_defined_k8s_config(tags: Mapping[str, str]):
 
     if USER_DEFINED_K8S_CONFIG_KEY in tags:
         user_defined_k8s_config_value = json.loads(tags[USER_DEFINED_K8S_CONFIG_KEY])
-        result = validate_config(USER_DEFINED_K8S_CONFIG_SCHEMA, user_defined_k8s_config_value)
+        result = validate_config(USER_DEFINED_K8S_JOB_CONFIG_SCHEMA, user_defined_k8s_config_value)
 
         if not result.success:
             raise DagsterInvalidConfigError(
@@ -232,6 +258,8 @@ def get_user_defined_k8s_config(tags: Mapping[str, str]):
         job_config=user_defined_k8s_config.get("job_config"),
         job_metadata=user_defined_k8s_config.get("job_metadata"),
         job_spec_config=user_defined_k8s_config.get("job_spec_config"),
+        deployment_metadata=user_defined_k8s_config.get("deployment_metadata"),
+        service_metadata=user_defined_k8s_config.get("service_metadata"),
         merge_behavior=K8sConfigMergeBehavior(
             user_defined_k8s_config.get("merge_behavior", K8sConfigMergeBehavior.DEEP.value)
         ),
@@ -465,6 +493,7 @@ class DagsterK8sJobConfig(
                             "job_spec_config": Field(
                                 Map(key_type=str, inner_type=bool), is_required=False
                             ),
+                            "namespace": Field(BoolSource, is_required=False),
                         }
                     ),
                     is_required=False,
@@ -647,7 +676,7 @@ class DagsterK8sJobConfig(
                     ),
                 ),
                 "run_k8s_config": Field(
-                    USER_DEFINED_K8S_CONFIG_SCHEMA,
+                    USER_DEFINED_K8S_JOB_CONFIG_SCHEMA,
                     is_required=False,
                     description="Raw Kubernetes configuration for launched runs.",
                 ),
@@ -661,6 +690,8 @@ class DagsterK8sJobConfig(
                                 DagsterEnum.from_python_enum(K8sConfigMergeBehavior),
                                 is_required=False,
                             ),
+                            "deployment_metadata": Permissive(),
+                            "service_metadata": Permissive(),
                         }
                     ),
                     is_required=False,

{dagster-k8s-0.24.10 → dagster-k8s-0.24.11}/dagster_k8s/ops/k8s_job_op.py

@@ -163,6 +163,8 @@ def execute_k8s_job(
     job_spec_config: Optional[Dict[str, Any]] = None,
     k8s_job_name: Optional[str] = None,
     merge_behavior: K8sConfigMergeBehavior = K8sConfigMergeBehavior.DEEP,
+    delete_failed_k8s_jobs: Optional[bool] = True,
+    _kubeconfig_file_context: Optional[str] = None,
 ):
     """This function is a utility for executing a Kubernetes job from within a Dagster op.
 
@@ -238,6 +240,11 @@ def execute_k8s_job(
             are recursively merged, appending list fields together and merging dictionary fields.
             Setting it to SHALLOW will make the dictionaries shallowly merged - any shared values
             in the dictionaries will be replaced by the values set on this op.
+        delete_failed_k8s_jobs (bool): Whether to immediately delete failed Kubernetes jobs. If False,
+            failed jobs will remain accessible through the Kubernetes API until deleted by a user or cleaned up by the
+            .spec.ttlSecondsAfterFinished parameter of the job.
+            (https://kubernetes.io/docs/concepts/workloads/controllers/ttlafterfinished/).
+            Defaults to True.
     """
     run_container_context = K8sContainerContext.create_for_run(
         context.dagster_run,
@@ -320,7 +327,7 @@ def execute_k8s_job(
     if load_incluster_config:
         kubernetes.config.load_incluster_config()
     else:
-        kubernetes.config.load_kube_config(kubeconfig_file)
+        kubernetes.config.load_kube_config(kubeconfig_file, context=_kubeconfig_file_context)
 
     # changing this to be able to be passed in will allow for unit testing
     api_client = DagsterKubernetesClient.production_client()
@@ -409,10 +416,11 @@ def execute_k8s_job(
             num_pods_to_wait_for=num_pods_to_wait_for,
         )
     except (DagsterExecutionInterruptedError, Exception) as e:
-        context.log.info(
-            f"Deleting Kubernetes job {job_name} in namespace {namespace} due to exception"
-        )
-        api_client.delete_job(job_name=job_name, namespace=namespace)
+        if delete_failed_k8s_jobs:
+            context.log.info(
+                f"Deleting Kubernetes job {job_name} in namespace {namespace} due to exception"
+            )
+            api_client.delete_job(job_name=job_name, namespace=namespace)
         raise e
 
 

dagster-k8s-0.24.11/dagster_k8s/version.py

@@ -0,0 +1 @@
+__version__ = "0.24.11"

{dagster-k8s-0.24.10 → dagster-k8s-0.24.11/dagster_k8s.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.24.10
+Version: 0.24.11
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs

{dagster-k8s-0.24.10 → dagster-k8s-0.24.11}/dagster_k8s.egg-info/requires.txt

@@ -1,3 +1,3 @@
-dagster==1.8.10
+dagster==1.8.11
 kubernetes<32
 google-auth!=2.23.1

{dagster-k8s-0.24.10 → dagster-k8s-0.24.11}/setup.py

@@ -44,7 +44,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.8,<3.13",
     install_requires=[
-        "dagster==1.8.10",
+        "dagster==1.8.11",
         f"kubernetes<{KUBERNETES_VERSION_UPPER_BOUND}",
         # exclude a google-auth release that added an overly restrictive urllib3 pin that confuses dependency resolvers
         "google-auth!=2.23.1",

dagster-k8s-0.24.10/dagster_k8s/version.py

@@ -1 +0,0 @@
-__version__ = "0.24.10"