dagster-k8s 0.24.9__tar.gz → 0.24.10__tar.gz

{dagster-k8s-0.24.9/dagster_k8s.egg-info → dagster-k8s-0.24.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.24.9
+Version: 0.24.10
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs

{dagster-k8s-0.24.9 → dagster-k8s-0.24.10}/dagster_k8s/client.py

@@ -6,11 +6,13 @@ from typing import Any, Callable, List, Optional, Set, TypeVar
 
 import kubernetes.client
 import kubernetes.client.rest
+import six
 from dagster import (
     DagsterInstance,
     _check as check,
 )
 from dagster._core.storage.dagster_run import DagsterRunStatus
+from kubernetes.client.api_client import ApiClient
 from kubernetes.client.models import V1Job, V1JobStatus
 
 try:
@@ -91,6 +93,39 @@ WHITELISTED_TRANSIENT_K8S_STATUS_CODES = [
 ]
 
 
+class PatchedApiClient(ApiClient):
+    # Forked from ApiClient implementation to pass configuration object down into created model
+    # objects, avoiding lock contention issues. See https://github.com/kubernetes-client/python/issues/2284
+    def __deserialize_model(self, data, klass):
+        """Deserializes list or dict to model.
+
+        :param data: dict, list.
+        :param klass: class literal.
+        :return: model object.
+        """
+        if not klass.openapi_types and not hasattr(klass, "get_real_child_model"):
+            return data
+
+        # Below is the only change from the base ApiClient implementation - pass through the
+        # Configuration object to each newly created model so that each one does not have to create
+        # one and acquire a lock
+        kwargs = {"local_vars_configuration": self.configuration}
+
+        if data is not None and klass.openapi_types is not None and isinstance(data, (list, dict)):
+            for attr, attr_type in six.iteritems(klass.openapi_types):
+                if klass.attribute_map[attr] in data:
+                    value = data[klass.attribute_map[attr]]
+                    kwargs[attr] = self.__deserialize(value, attr_type)
+
+        instance = klass(**kwargs)
+
+        if hasattr(instance, "get_real_child_model"):
+            klass_name = instance.get_real_child_model(data)
+            if klass_name:
+                instance = self.__deserialize(data, klass_name)
+        return instance
+
+
 def k8s_api_retry(
     fn: Callable[..., T],
     max_retries: int,
@@ -209,8 +244,12 @@ class DagsterKubernetesClient:
     @staticmethod
     def production_client(batch_api_override=None, core_api_override=None):
         return DagsterKubernetesClient(
-            batch_api=batch_api_override or kubernetes.client.BatchV1Api(),
-            core_api=core_api_override or kubernetes.client.CoreV1Api(),
+            batch_api=(
+                batch_api_override or kubernetes.client.BatchV1Api(api_client=PatchedApiClient())
+            ),
+            core_api=(
+                core_api_override or kubernetes.client.CoreV1Api(api_client=PatchedApiClient())
+            ),
             logger=logging.info,
             sleeper=time.sleep,
             timer=time.time,

dagster-k8s-0.24.10/dagster_k8s/kubernetes_version.py

@@ -0,0 +1 @@
+KUBERNETES_VERSION_UPPER_BOUND = "32"

{dagster-k8s-0.24.9 → dagster-k8s-0.24.10}/dagster_k8s/models.py

@@ -7,11 +7,20 @@ import kubernetes
 import kubernetes.client.models
 from dagster._vendored.dateutil.parser import parse
 from kubernetes.client.api_client import ApiClient
+from kubernetes.client.configuration import Configuration
 
 # Unclear what the correct type is to use for a bound here.
 T_KubernetesModel = TypeVar("T_KubernetesModel")
 
 
+# Create a single Configuration object to pass through to each model creation -
+# the default otherwise in the OpenAPI version currently in use by the k8s
+# client will create one on each model creation otherwise, which can cause
+# lock contention since it acquires the global python logger lock
+# see: https://github.com/kubernetes-client/python/issues/1921
+shared_k8s_model_configuration = Configuration()
+
+
 def _get_k8s_class(classname: str) -> Type[Any]:
     if classname in ApiClient.NATIVE_TYPES_MAPPING:
         return ApiClient.NATIVE_TYPES_MAPPING[classname]
@@ -149,7 +158,10 @@ def k8s_model_from_dict(
     if len(invalid_keys):
         raise Exception(f"Unexpected keys in model class {model_class.__name__}: {invalid_keys}")
 
-    kwargs = {}
+    # Pass through the configuration object since the default implementation creates a new one
+    # in the constructor, which can create lock contention if multiple threads are calling this
+    # simultaneously
+    kwargs = {"local_vars_configuration": shared_k8s_model_configuration}
     for attr, attr_type in model_class.openapi_types.items():  # type: ignore
         # e.g. config_map => configMap
         if attr in model_dict:

dagster-k8s-0.24.10/dagster_k8s/version.py

@@ -0,0 +1 @@
+__version__ = "0.24.10"

{dagster-k8s-0.24.9 → dagster-k8s-0.24.10/dagster_k8s.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dagster-k8s
-Version: 0.24.9
+Version: 0.24.10
 Summary: A Dagster integration for k8s
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-k8s
 Author: Dagster Labs

{dagster-k8s-0.24.9 → dagster-k8s-0.24.10}/dagster_k8s.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ dagster_k8s/client.py
 dagster_k8s/container_context.py
 dagster_k8s/executor.py
 dagster_k8s/job.py
+dagster_k8s/kubernetes_version.py
 dagster_k8s/launcher.py
 dagster_k8s/models.py
 dagster_k8s/pipes.py

dagster-k8s-0.24.10/dagster_k8s.egg-info/requires.txt

@@ -0,0 +1,3 @@
+dagster==1.8.10
+kubernetes<32
+google-auth!=2.23.1

{dagster-k8s-0.24.9 → dagster-k8s-0.24.10}/setup.py

@@ -1,18 +1,26 @@
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Tuple
 
 from setuptools import find_packages, setup
 
 
-def get_version() -> str:
+def get_version() -> Tuple[str, str]:
     version: Dict[str, str] = {}
+    kubernetes_version: Dict[str, str] = {}
+
     with open(Path(__file__).parent / "dagster_k8s/version.py", encoding="utf8") as fp:
         exec(fp.read(), version)
 
-    return version["__version__"]
+    with open(Path(__file__).parent / "dagster_k8s/kubernetes_version.py", encoding="utf8") as fp:
+        exec(fp.read(), kubernetes_version)
+
+    return version["__version__"], kubernetes_version["KUBERNETES_VERSION_UPPER_BOUND"]
 
 
-ver = get_version()
+(
+    ver,
+    KUBERNETES_VERSION_UPPER_BOUND,
+) = get_version()
 # dont pin dev installs to avoid pip dep resolver issues
 pin = "" if ver == "1!0+dev" else f"=={ver}"
 setup(
@@ -36,8 +44,8 @@ setup(
     include_package_data=True,
     python_requires=">=3.8,<3.13",
     install_requires=[
-        "dagster==1.8.9",
-        "kubernetes",
+        "dagster==1.8.10",
+        f"kubernetes<{KUBERNETES_VERSION_UPPER_BOUND}",
         # exclude a google-auth release that added an overly restrictive urllib3 pin that confuses dependency resolvers
         "google-auth!=2.23.1",
     ],

dagster-k8s-0.24.9/dagster_k8s/version.py

@@ -1 +0,0 @@
-__version__ = "0.24.9"

dagster-k8s-0.24.9/dagster_k8s.egg-info/requires.txt

@@ -1,3 +0,0 @@
-dagster==1.8.9
-kubernetes
-google-auth!=2.23.1