ob-metaflow 2.14.0.1__tar.gz → 2.14.0.2rc0__tar.gz

{ob_metaflow-2.14.0.1/ob_metaflow.egg-info → ob_metaflow-2.14.0.2rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ob-metaflow
-Version: 2.14.0.1
+Version: 2.14.0.2rc0
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.14.0.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.14.0.2rc0; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0}/metaflow/plugins/argo/argo_workflows.py

@@ -1966,6 +1966,11 @@ class ArgoWorkflows(object):
                 resources["disk"],
             )
 
+            extended_resources = resources.get("extended_resources", {})
+
+            qos_requests = {**qos_requests, **extended_resources}
+            qos_limits = {**qos_limits, **extended_resources}
+
             # Create a ContainerTemplate for this node. Ideally, we would have
             # liked to inline this ContainerTemplate and avoid scanning the workflow
             # twice, but due to issues with variable substitution, we will have to
@@ -2022,6 +2027,7 @@ class ArgoWorkflows(object):
                     shared_memory=shared_memory,
                     port=port,
                     qos=resources["qos"],
+                    extended_resources=extended_resources,
                 )
 
                 for k, v in env.items():

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0}/metaflow/plugins/kubernetes/kubernetes.py

@@ -194,6 +194,7 @@ class Kubernetes(object):
         port=None,
         num_parallel=None,
         qos=None,
+        extended_resources=None,
     ):
         name = "js-%s" % str(uuid4())[:6]
         jobset = (
@@ -227,6 +228,7 @@ class Kubernetes(object):
                 port=port,
                 num_parallel=num_parallel,
                 qos=qos,
+                extended_resources=extended_resources,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)
@@ -504,6 +506,7 @@ class Kubernetes(object):
         name_pattern=None,
         qos=None,
         annotations=None,
+        extended_resources=None,
     ):
         if env is None:
             env = {}
@@ -546,6 +549,7 @@ class Kubernetes(object):
                 shared_memory=shared_memory,
                 port=port,
                 qos=qos,
+                extended_resources=extended_resources,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0}/metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -145,6 +145,12 @@ def kubernetes():
     type=JSONTypeClass(),
     multiple=False,
 )
+@click.option(
+    "--extended-resources",
+    default=None,
+    type=JSONTypeClass(),
+    multiple=False,
+)
 @click.pass_context
 def step(
     ctx,
@@ -176,6 +182,7 @@ def step(
     qos=None,
     labels=None,
     annotations=None,
+    extended_resources=None,
     **kwargs
 ):
     def echo(msg, stream="stderr", job_id=None, **kwargs):
@@ -319,6 +326,7 @@ def step(
                 qos=qos,
                 labels=labels,
                 annotations=annotations,
+                extended_resources=extended_resources,
             )
     except Exception:
         traceback.print_exc(chain=False)

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -123,6 +123,9 @@ class KubernetesDecorator(StepDecorator):
         Only applicable when @parallel is used.
     qos: str, default: Burstable
         Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
+    extended_resources: Dict[str, str], optional, default None
+        Extended resources to be requested for the pod.
+        https://kubernetes.io/docs/tasks/administer-cluster/extended-resource-node/
     """
 
     name = "kubernetes"
@@ -153,6 +156,7 @@ class KubernetesDecorator(StepDecorator):
         "executable": None,
         "hostname_resolution_timeout": 10 * 60,
         "qos": KUBERNETES_QOS,
+        "extended_resources": {},
     }
     package_url = None
     package_sha = None
@@ -488,6 +492,7 @@ class KubernetesDecorator(StepDecorator):
                     "persistent_volume_claims",
                     "labels",
                     "annotations",
+                    "extended_resources",
                 ]:
                     cli_args.command_options[k] = json.dumps(v)
                 else:

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0}/metaflow/plugins/kubernetes/kubernetes_job.py

@@ -89,10 +89,11 @@ class KubernetesJob(object):
 
         additional_obp_configs = {
             "OBP_PERIMETER": initial_configs["OBP_PERIMETER"],
-            "OBP_INTEGRATIONS_URL": initial_configs[
-                "OBP_INTEGRATIONS_URL"
-            ],
+            "OBP_INTEGRATIONS_URL": initial_configs["OBP_INTEGRATIONS_URL"],
         }
+        extended_resources = self._kwargs.get("extended_resources", {})
+        qos_requests = {**qos_requests, **extended_resources}
+        qos_limits = {**qos_limits, **extended_resources}
 
         return client.V1JobSpec(
             # Retries are handled by Metaflow when it is responsible for
@@ -118,6 +119,7 @@ class KubernetesJob(object):
                     containers=[
                         client.V1Container(
                             command=self._kwargs["command"],
+                            security_context=client.V1SecurityContext(privileged=True),
                             termination_message_policy="FallbackToLogsOnError",
                             ports=(
                                 []

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0}/metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -560,6 +560,11 @@ class JobSetSpec(object):
             self._kwargs["memory"],
             self._kwargs["disk"],
         )
+
+        extended_resources = self._kwargs.get("extended_resources", {})
+        qos_requests = {**qos_requests, **extended_resources}
+        qos_limits = {**qos_limits, **extended_resources}
+
         return dict(
             name=self.name,
             template=client.api_client.ApiClient().sanitize_for_serialization(
@@ -611,6 +616,9 @@ class JobSetSpec(object):
                                                 )
                                             ]
                                         ),
+                                        security_context=client.V1SecurityContext(
+                                            privileged=True
+                                        ),
                                         env=[
                                             client.V1EnvVar(name=k, value=str(v))
                                             for k, v in self._kwargs.get(

ob_metaflow-2.14.0.2rc0/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.14.0.2rc0"

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ob-metaflow
-Version: 2.14.0.1
+Version: 2.14.0.2rc0
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.14.0.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.14.0.2rc0; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.14.0.1 → ob_metaflow-2.14.0.2rc0}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-metaflow-stubs==2.14.0.1
+metaflow-stubs==2.14.0.2rc0

ob_metaflow-2.14.0.1/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.14.0.1"