metaflow 2.15.8__tar.gz → 2.15.10__tar.gz

{metaflow-2.15.8/metaflow.egg-info → metaflow-2.15.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.15.8
+Version: 2.15.10
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.8; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.10; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/argo/argo_workflows.py

@@ -1978,6 +1978,15 @@ class ArgoWorkflows(object):
                 resources["disk"],
             )
 
+            security_context = resources.get("security_context", None)
+            _security_context = {}
+            if security_context is not None and len(security_context) > 0:
+                _security_context = {
+                    "security_context": kubernetes_sdk.V1SecurityContext(
+                        **security_context
+                    )
+                }
+
             # Create a ContainerTemplate for this node. Ideally, we would have
             # liked to inline this ContainerTemplate and avoid scanning the workflow
             # twice, but due to issues with variable substitution, we will have to
@@ -2034,6 +2043,7 @@ class ArgoWorkflows(object):
                     shared_memory=shared_memory,
                     port=port,
                     qos=resources["qos"],
+                    security_context=security_context,
                 )
 
                 for k, v in env.items():
@@ -2310,6 +2320,7 @@ class ArgoWorkflows(object):
                                     is not None
                                     else []
                                 ),
+                                **_security_context,
                             ).to_dict()
                         )
                     )

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -40,6 +40,7 @@ unsupported_decorators = {
     "snowpark": "Step *%s* is marked for execution on Snowpark with Argo Workflows which isn't currently supported.",
     "slurm": "Step *%s* is marked for execution on Slurm with Argo Workflows which isn't currently supported.",
     "nvidia": "Step *%s* is marked for execution on Nvidia with Argo Workflows which isn't currently supported.",
+    "nvct": "Step *%s* is marked for execution on Nvct with Argo Workflows which isn't currently supported.",
 }
 
 

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/datatools/s3/s3.py

@@ -1,3 +1,4 @@
+import errno
 import json
 import os
 import re
@@ -137,6 +138,10 @@ class MetaflowS3InvalidRange(MetaflowException):
     headline = "S3 invalid range"
 
 
+class MetaflowS3InsufficientDiskSpace(MetaflowException):
+    headline = "Insufficient disk space"
+
+
 class S3Object(object):
     """
     This object represents a path or an object in S3,
@@ -1377,8 +1382,10 @@ class S3(object):
                 elif error_code == "NoSuchBucket":
                     raise MetaflowS3URLException("Specified S3 bucket doesn't exist.")
                 error = str(err)
+            except OSError as e:
+                if e.errno == errno.ENOSPC:
+                    raise MetaflowS3InsufficientDiskSpace(str(e))
             except Exception as ex:
-                # TODO specific error message for out of disk space
                 error = str(ex)
             if tmp:
                 os.unlink(tmp.name)

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/datatools/s3/s3op.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import errno
 import json
 import time
 import math
@@ -108,6 +109,7 @@ ERROR_VERIFY_FAILED = 9
 ERROR_LOCAL_FILE_NOT_FOUND = 10
 ERROR_INVALID_RANGE = 11
 ERROR_TRANSIENT = 12
+ERROR_OUT_OF_DISK_SPACE = 13
 
 
 def format_result_line(idx, prefix, url="", local=""):
@@ -277,6 +279,17 @@ def worker(result_file_name, queue, mode, s3config):
                             err = convert_to_client_error(e)
                             handle_client_error(err, idx, result_file)
                             continue
+                        except OSError as e:
+                            tmp.close()
+                            os.unlink(tmp.name)
+                            if e.errno == errno.ENOSPC:
+                                result_file.write(
+                                    "%d %d\n" % (idx, -ERROR_OUT_OF_DISK_SPACE)
+                                )
+                            else:
+                                result_file.write("%d %d\n" % (idx, -ERROR_TRANSIENT))
+                            result_file.flush()
+                            continue
                     except (SSLError, Exception) as e:
                         tmp.close()
                         os.unlink(tmp.name)
@@ -643,6 +656,8 @@ def exit(exit_code, url):
         msg = "Local file not found: %s" % url
     elif exit_code == ERROR_TRANSIENT:
         msg = "Transient error for url: %s" % url
+    elif exit_code == ERROR_OUT_OF_DISK_SPACE:
+        msg = "Out of disk space when downloading URL: %s" % url
     else:
         msg = "Unknown error"
     print("s3op failed:\n%s" % msg, file=sys.stderr)
@@ -1173,6 +1188,8 @@ def get(
             )
             if verify:
                 verify_info.append((url, sz))
+        elif sz == -ERROR_OUT_OF_DISK_SPACE:
+            exit(ERROR_OUT_OF_DISK_SPACE, url)
         elif sz == -ERROR_URL_ACCESS_DENIED:
             denied_url = url
             break

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/kubernetes/kubernetes.py

@@ -194,6 +194,7 @@ class Kubernetes(object):
         port=None,
         num_parallel=None,
         qos=None,
+        security_context=None,
     ):
         name = "js-%s" % str(uuid4())[:6]
         jobset = (
@@ -227,6 +228,7 @@ class Kubernetes(object):
                 port=port,
                 num_parallel=num_parallel,
                 qos=qos,
+                security_context=security_context,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)
@@ -488,6 +490,7 @@ class Kubernetes(object):
         name_pattern=None,
         qos=None,
         annotations=None,
+        security_context=None,
     ):
         if env is None:
             env = {}
@@ -530,6 +533,7 @@ class Kubernetes(object):
                 shared_memory=shared_memory,
                 port=port,
                 qos=qos,
+                security_context=security_context,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -145,6 +145,12 @@ def kubernetes():
     type=JSONTypeClass(),
     multiple=False,
 )
+@click.option(
+    "--security-context",
+    default=None,
+    type=JSONTypeClass(),
+    multiple=False,
+)
 @click.pass_context
 def step(
     ctx,
@@ -176,6 +182,7 @@ def step(
     qos=None,
     labels=None,
     annotations=None,
+    security_context=None,
     **kwargs
 ):
     def echo(msg, stream="stderr", job_id=None, **kwargs):
@@ -319,6 +326,7 @@ def step(
                 qos=qos,
                 labels=labels,
                 annotations=annotations,
+                security_context=security_context,
             )
     except Exception:
         traceback.print_exc(chain=False)

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -122,6 +122,14 @@ class KubernetesDecorator(StepDecorator):
         Only applicable when @parallel is used.
     qos: str, default: Burstable
         Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
+
+    security_context: Dict[str, Any], optional, default None
+        Container security context. Applies to the task container. Allows the following keys:
+        - privileged: bool, optional, default None
+        - allow_privilege_escalation: bool, optional, default None
+        - run_as_user: int, optional, default None
+        - run_as_group: int, optional, default None
+        - run_as_non_root: bool, optional, default None
     """
 
     name = "kubernetes"
@@ -152,6 +160,7 @@ class KubernetesDecorator(StepDecorator):
         "executable": None,
         "hostname_resolution_timeout": 10 * 60,
         "qos": KUBERNETES_QOS,
+        "security_context": None,
     }
     package_url = None
     package_sha = None
@@ -474,6 +483,7 @@ class KubernetesDecorator(StepDecorator):
                     "persistent_volume_claims",
                     "labels",
                     "annotations",
+                    "security_context",
                 ]:
                     cli_args.command_options[k] = json.dumps(v)
                 else:

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/kubernetes/kubernetes_job.py

@@ -80,6 +80,13 @@ class KubernetesJob(object):
             self._kwargs["disk"],
         )
 
+        security_context = self._kwargs.get("security_context", {})
+        _security_context = {}
+        if security_context is not None and len(security_context) > 0:
+            _security_context = {
+                "security_context": client.V1SecurityContext(**security_context)
+            }
+
         return client.V1JobSpec(
             # Retries are handled by Metaflow when it is responsible for
             # executing the flow. The responsibility is moved to Kubernetes
@@ -203,6 +210,7 @@ class KubernetesJob(object):
                                 if self._kwargs["persistent_volume_claims"] is not None
                                 else []
                             ),
+                            **_security_context,
                         )
                     ],
                     node_selector=self._kwargs.get("node_selector"),

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -562,6 +562,12 @@ class JobSetSpec(object):
             self._kwargs["memory"],
             self._kwargs["disk"],
         )
+        security_context = self._kwargs.get("security_context", {})
+        _security_context = {}
+        if security_context is not None and len(security_context) > 0:
+            _security_context = {
+                "security_context": client.V1SecurityContext(**security_context)
+            }
         return dict(
             name=self.name,
             template=client.api_client.ApiClient().sanitize_for_serialization(
@@ -708,6 +714,7 @@ class JobSetSpec(object):
                                             is not None
                                             else []
                                         ),
+                                        **_security_context,
                                     )
                                 ],
                                 node_selector=self._kwargs.get("node_selector"),

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/pypi/conda_decorator.py

@@ -227,7 +227,8 @@ class CondaStepDecorator(StepDecorator):
         self.interpreter = (
             self.environment.interpreter(self.step)
             if not any(
-                decorator.name in ["batch", "kubernetes", "nvidia", "snowpark", "slurm"]
+                decorator.name
+                in ["batch", "kubernetes", "nvidia", "snowpark", "slurm", "nvct"]
                 for decorator in next(
                     step for step in self.flow if step.name == self.step
                 ).decorators

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/plugins/pypi/conda_environment.py

@@ -326,6 +326,7 @@ class CondaEnvironment(MetaflowEnvironment):
                 "nvidia",
                 "snowpark",
                 "slurm",
+                "nvct",
             ]:
                 target_platform = getattr(decorator, "target_platform", "linux-64")
                 break

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/runner/deployer.py

@@ -1,3 +1,4 @@
+import os
 import json
 import time
 
@@ -52,7 +53,7 @@ class Deployer(metaclass=DeployerMeta):
     Parameters
     ----------
     flow_file : str
-        Path to the flow file to deploy.
+        Path to the flow file to deploy, relative to current directory.
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default.
     profile : Optional[str], default None
@@ -80,7 +81,12 @@ class Deployer(metaclass=DeployerMeta):
         file_read_timeout: int = 3600,
         **kwargs,
     ):
-        self.flow_file = flow_file
+        # Convert flow_file to absolute path if it's relative
+        if not os.path.isabs(flow_file):
+            self.flow_file = os.path.abspath(flow_file)
+        else:
+            self.flow_file = flow_file
+
         self.show_output = show_output
         self.profile = profile
         self.env = env

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/runner/deployer_impl.py

@@ -25,7 +25,7 @@ class DeployerImpl(object):
     Parameters
     ----------
     flow_file : str
-        Path to the flow file to deploy.
+        Path to the flow file to deploy, relative to current directory.
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default.
     profile : Optional[str], default None
@@ -80,7 +80,11 @@ class DeployerImpl(object):
         from metaflow.cli import start
         from metaflow.runner.click_api import MetaflowAPI
 
-        self.flow_file = flow_file
+        # Convert flow_file to absolute path if it's relative
+        if not os.path.isabs(flow_file):
+            self.flow_file = os.path.abspath(flow_file)
+        else:
+            self.flow_file = flow_file
         self.show_output = show_output
         self.profile = profile
         self.env = env

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow/runner/metaflow_runner.py

@@ -229,7 +229,7 @@ class Runner(metaclass=RunnerMeta):
     Parameters
     ----------
     flow_file : str
-        Path to the flow file to run
+        Path to the flow file to run, relative to current directory.
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default,
         Only applicable for synchronous 'run' and 'resume' functions.
@@ -286,7 +286,12 @@ class Runner(metaclass=RunnerMeta):
         from metaflow.cli import start
         from metaflow.runner.click_api import MetaflowAPI
 
-        self.flow_file = flow_file
+        # Convert flow_file to absolute path if it's relative
+        if not os.path.isabs(flow_file):
+            self.flow_file = os.path.abspath(flow_file)
+        else:
+            self.flow_file = flow_file
+
         self.show_output = show_output
 
         self.env_vars = os.environ.copy()

metaflow-2.15.10/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.15.10"

{metaflow-2.15.8 → metaflow-2.15.10/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.15.8
+Version: 2.15.10
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.8; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.10; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.15.8 → metaflow-2.15.10}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.15.8
+metaflow-stubs==2.15.10

metaflow-2.15.8/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.15.8"