ob-metaflow 2.15.14.1__py2.py3-none-any.whl → 2.15.17.1__py2.py3-none-any.whl

metaflow/plugins/cards/card_modules/test_cards.py

@@ -34,7 +34,7 @@ class TestPathSpecCard(MetaflowCard):
 class TestEditableCard(MetaflowCard):
     type = "test_editable_card"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     ALLOW_USER_COMPONENTS = True
 
@@ -42,13 +42,13 @@ class TestEditableCard(MetaflowCard):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestEditableCard2(MetaflowCard):
     type = "test_editable_card_2"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     ALLOW_USER_COMPONENTS = True
 
@@ -56,19 +56,19 @@ class TestEditableCard2(MetaflowCard):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestNonEditableCard(MetaflowCard):
     type = "test_non_editable_card"
 
-    seperator = "$&#!!@*"
+    separator = "$&#!!@*"
 
     def __init__(self, components=[], **kwargs):
         self._components = components
 
     def render(self, task):
-        return self.seperator.join([str(comp) for comp in self._components])
+        return self.separator.join([str(comp) for comp in self._components])
 
 
 class TestMockCard(MetaflowCard):

metaflow/plugins/cards/component_serializer.py

@@ -57,15 +57,8 @@ class ComponentStore:
         The `_component_map` attribute is supposed to be a dictionary so that we can access the components by their ids.
         But we also want to maintain order in which components are inserted since all of these components are going to be visible on a UI.
         Since python3.6 dictionaries are ordered by default so we can use the default python `dict`.
-        For python3.5 and below we need to use an OrderedDict since `dict`'s are not ordered by default.
         """
-        python_version = int(platform.python_version_tuple()[0]) * 10 + int(
-            platform.python_version_tuple()[1]
-        )
-        if python_version < 36:
-            self._component_map = OrderedDict()
-        else:
-            self._component_map = {}
+        self._component_map = {}
 
     def __init__(self, logger, card_type=None, components=None, user_set_id=None):
         self._logger = logger

metaflow/plugins/datatools/s3/s3op.py

@@ -131,7 +131,7 @@ def normalize_client_error(err):
     except ValueError:
         if error_code in ("AccessDenied", "AllAccessDisabled", "InvalidAccessKeyId"):
             return 403
-        if error_code == "NoSuchKey":
+        if error_code in ("NoSuchKey", "NoSuchBucket"):
             return 404
         if error_code == "InvalidRange":
             return 416

metaflow/plugins/kubernetes/kubernetes.py

@@ -170,6 +170,7 @@ class Kubernetes(object):
         code_package_ds,
         docker_image,
         docker_image_pull_policy,
+        image_pull_secrets=None,
         step_cli=None,
         service_account=None,
         secrets=None,
@@ -206,6 +207,7 @@ class Kubernetes(object):
                 node_selector=node_selector,
                 image=docker_image,
                 image_pull_policy=docker_image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 cpu=cpu,
                 memory=memory,
                 disk=disk,
@@ -483,6 +485,7 @@ class Kubernetes(object):
         step_cli,
         docker_image,
         docker_image_pull_policy,
+        image_pull_secrets=None,
         service_account=None,
         secrets=None,
         node_selector=None,
@@ -529,6 +532,7 @@ class Kubernetes(object):
                 ),
                 image=docker_image,
                 image_pull_policy=docker_image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 cpu=cpu,
                 memory=memory,
                 disk=disk,

metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -53,6 +53,12 @@ def kubernetes():
     default=None,
     help="Optional Docker Image Pull Policy for Kubernetes pod.",
 )
+@click.option(
+    "--image-pull-secrets",
+    default=None,
+    type=JSONTypeClass(),
+    multiple=False,
+)
 @click.option(
     "--service-account",
     help="IRSA requirement for Kubernetes pod.",
@@ -160,6 +166,7 @@ def step(
     executable=None,
     image=None,
     image_pull_policy=None,
+    image_pull_secrets=None,
     service_account=None,
     secrets=None,
     node_selector=None,
@@ -303,6 +310,7 @@ def step(
                 step_cli=step_cli,
                 docker_image=image,
                 docker_image_pull_policy=image_pull_policy,
+                image_pull_secrets=image_pull_secrets,
                 service_account=service_account,
                 secrets=secrets,
                 node_selector=node_selector,

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -18,6 +18,7 @@ from metaflow.metaflow_config import (
     KUBERNETES_FETCH_EC2_METADATA,
     KUBERNETES_GPU_VENDOR,
     KUBERNETES_IMAGE_PULL_POLICY,
+    KUBERNETES_IMAGE_PULL_SECRETS,
     KUBERNETES_MEMORY,
     KUBERNETES_LABELS,
     KUBERNETES_ANNOTATIONS,
@@ -74,6 +75,10 @@ class KubernetesDecorator(StepDecorator):
         not, a default Docker image mapping to the current version of Python is used.
     image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
         If given, the imagePullPolicy to be applied to the Docker image of the step.
+    image_pull_secrets: List[str], default []
+        The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
+        Kubernetes image pull secrets to use when pulling container images
+        in Kubernetes.
     service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
         Kubernetes service account to use when launching pod in Kubernetes.
     secrets : List[str], optional, default None
@@ -141,6 +146,7 @@ class KubernetesDecorator(StepDecorator):
         "disk": "10240",
         "image": None,
         "image_pull_policy": None,
+        "image_pull_secrets": None,  # e.g., ["regcred"]
         "service_account": None,
         "secrets": None,  # e.g., mysecret
         "node_selector": None,  # e.g., kubernetes.io/os=linux
@@ -194,6 +200,10 @@ class KubernetesDecorator(StepDecorator):
             )
         if not self.attributes["image_pull_policy"] and KUBERNETES_IMAGE_PULL_POLICY:
             self.attributes["image_pull_policy"] = KUBERNETES_IMAGE_PULL_POLICY
+        if not self.attributes["image_pull_secrets"] and KUBERNETES_IMAGE_PULL_SECRETS:
+            self.attributes["image_pull_secrets"] = json.loads(
+                KUBERNETES_IMAGE_PULL_SECRETS
+            )
 
         if isinstance(self.attributes["node_selector"], str):
             self.attributes["node_selector"] = parse_kube_keyvalue_list(
@@ -494,6 +504,7 @@ class KubernetesDecorator(StepDecorator):
                         for key, val in v.items()
                     ]
                 elif k in [
+                    "image_pull_secrets",
                     "tolerations",
                     "persistent_volume_claims",
                     "labels",

metaflow/plugins/kubernetes/kubernetes_job.py

@@ -235,8 +235,10 @@ class KubernetesJob(object):
                         )
                     ],
                     node_selector=self._kwargs.get("node_selector"),
-                    # TODO (savin): Support image_pull_secrets
-                    # image_pull_secrets=?,
+                    image_pull_secrets=[
+                        client.V1LocalObjectReference(secret)
+                        for secret in self._kwargs.get("image_pull_secrets") or []
+                    ],
                     # TODO (savin): Support preemption policies
                     # preemption_policy=?,
                     #

metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -718,8 +718,11 @@ class JobSetSpec(object):
                                     )
                                 ],
                                 node_selector=self._kwargs.get("node_selector"),
-                                # TODO (savin): Support image_pull_secrets
-                                # image_pull_secrets=?,
+                                image_pull_secrets=[
+                                    client.V1LocalObjectReference(secret)
+                                    for secret in self._kwargs.get("image_pull_secrets")
+                                    or []
+                                ],
                                 # TODO (savin): Support preemption policies
                                 # preemption_policy=?,
                                 #

metaflow/plugins/metadata_providers/service.py

@@ -72,14 +72,18 @@ class ServiceMetadataProvider(MetadataProvider):
     @classmethod
     def compute_info(cls, val):
         v = val.rstrip("/")
-        try:
-            resp = cls._session.get(
-                os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
-            )
-            resp.raise_for_status()
-        except:  # noqa E722
-            raise ValueError("Metaflow service [%s] unreachable." % v)
-        return v
+        for i in range(SERVICE_RETRY_COUNT):
+            try:
+                resp = cls._session.get(
+                    os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
+                )
+                resp.raise_for_status()
+            except:  # noqa E722
+                time.sleep(2 ** (i - 1))
+            else:
+                return v
+
+        raise ValueError("Metaflow service [%s] unreachable." % v)
 
     @classmethod
     def default_info(cls):

metaflow/plugins/package_cli.py

@@ -38,14 +38,24 @@ def info(obj):
 
 
 @package.command(help="List files included in the code package.")
+@click.option(
+    "--archive/--no-archive",
+    default=False,
+    help="If True, lists the file paths as present in the code package archive; "
+    "otherwise, lists the files on your filesystem included in the code package",
+    show_default=True,
+)
 @click.pass_obj
-def list(obj):
+def list(obj, archive=False):
     obj.echo(
         "Files included in the code package " "(change with --package-suffixes):",
         fg="magenta",
         bold=False,
     )
-    obj.echo_always("\n".join(path for path, _ in obj.package.path_tuples()))
+    if archive:
+        obj.echo_always("\n".join(path for _, path in obj.package.path_tuples()))
+    else:
+        obj.echo_always("\n".join(path for path, _ in obj.package.path_tuples()))
 
 
 @package.command(help="Save the current code package in a tar file")

metaflow/plugins/pypi/bootstrap.py

@@ -14,7 +14,7 @@ from urllib.request import urlopen
 from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, CONDA_USE_FAST_INIT
 from metaflow.plugins import DATASTORES
 from metaflow.plugins.pypi.utils import MICROMAMBA_MIRROR_URL, MICROMAMBA_URL
-from metaflow.util import which
+from metaflow.util import which, get_metaflow_root
 from urllib.request import Request
 import warnings
 
@@ -366,7 +366,7 @@ if __name__ == "__main__":
         # Move MAGIC_FILE inside local datastore.
         os.makedirs(manifest_dir, exist_ok=True)
         shutil.move(
-            os.path.join(os.getcwd(), MAGIC_FILE),
+            os.path.join(get_metaflow_root(), MAGIC_FILE),
             os.path.join(manifest_dir, MAGIC_FILE),
         )
         with open(os.path.join(manifest_dir, MAGIC_FILE)) as f:

metaflow/plugins/uv/bootstrap.py

@@ -4,6 +4,7 @@ import sys
 import time
 
 from metaflow.util import which
+from metaflow.info_file import read_info_file
 from metaflow.metaflow_config import get_pinned_conda_libs
 from urllib.request import Request, urlopen
 from urllib.error import URLError
@@ -78,11 +79,27 @@ if __name__ == "__main__":
         # return only dependency names instead of pinned versions
         return pinned.keys()
 
+    def skip_metaflow_dependencies():
+        skip_pkgs = ["metaflow", "ob-metaflow"]
+        info = read_info_file()
+        if info is not None:
+            try:
+                skip_pkgs.extend([ext_name for ext_name in info["ext_info"][0].keys()])
+            except Exception:
+                print(
+                    "Failed to read INFO. Metaflow-related packages might get installed during runtime."
+                )
+
+        return skip_pkgs
+
     def sync_uv_project(datastore_type):
         print("Syncing uv project...")
         dependencies = " ".join(get_dependencies(datastore_type))
+        skip_pkgs = " ".join(
+            [f"--no-install-package {dep}" for dep in skip_metaflow_dependencies()]
+        )
         cmd = f"""set -e;
-            uv sync --frozen --no-install-package metaflow;
+            uv sync --frozen {skip_pkgs};
             uv pip install {dependencies} --strict
             """
         run_cmd(cmd)

metaflow/plugins/uv/uv_environment.py

@@ -22,7 +22,7 @@ class UVEnvironment(MetaflowEnvironment):
         self.logger("Bootstrapping uv...")
 
     def executable(self, step_name, default=None):
-        return "uv run python"
+        return "uv run --no-sync python"
 
     def add_to_package(self):
         # NOTE: We treat uv.lock and pyproject.toml as regular project assets and ship these along user code as part of the code package

metaflow/runner/click_api.py

@@ -467,9 +467,14 @@ class MetaflowAPI(object):
                     config_file = defaults.get("config")
 
                 if config_file:
-                    config_file = map(
-                        lambda x: (x[0], ConvertPath.convert_value(x[1], is_default)),
-                        config_file,
+                    config_file = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertPath.convert_value(x[1], is_default),
+                            ),
+                            config_file,
+                        )
                     )
 
                 is_default = False
@@ -479,12 +484,14 @@ class MetaflowAPI(object):
                     config_value = defaults.get("config_value")
 
                 if config_value:
-                    config_value = map(
-                        lambda x: (
-                            x[0],
-                            ConvertDictOrStr.convert_value(x[1], is_default),
-                        ),
-                        config_value,
+                    config_value = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertDictOrStr.convert_value(x[1], is_default),
+                            ),
+                            config_value,
+                        )
                     )
 
                 if (config_file is None) ^ (config_value is None):

metaflow/runner/deployer.py

@@ -7,6 +7,55 @@ from typing import ClassVar, Dict, Optional, TYPE_CHECKING
 from metaflow.exception import MetaflowNotFound
 from metaflow.metaflow_config import DEFAULT_FROM_DEPLOYMENT_IMPL
 
+
+def generate_fake_flow_file_contents(
+    flow_name: str, param_info: dict, project_name: Optional[str] = None
+):
+    params_code = ""
+    for _, param_details in param_info.items():
+        param_python_var_name = param_details["python_var_name"]
+        param_name = param_details["name"]
+        param_type = param_details["type"]
+        param_help = param_details["description"]
+        param_required = param_details["is_required"]
+
+        if param_type == "JSON":
+            params_code += (
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type=JSONType, help='''{param_help}''', required={param_required})\n"
+            )
+        elif param_type == "FilePath":
+            is_text = param_details.get("is_text", True)
+            encoding = param_details.get("encoding", "utf-8")
+            params_code += (
+                f"    {param_python_var_name} = IncludeFile('{param_name}', "
+                f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
+                f"required={param_required})\n"
+            )
+        else:
+            params_code += (
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type={param_type}, help='''{param_help}''', required={param_required})\n"
+            )
+
+    project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
+
+    contents = f"""\
+from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
+{project_decorator}class {flow_name}(FlowSpec):
+{params_code}
+    @step
+    def start(self):
+        self.next(self.end)
+    @step
+    def end(self):
+        pass
+if __name__ == '__main__':
+    {flow_name}()
+"""
+    return contents
+
+
 if TYPE_CHECKING:
     import metaflow
     import metaflow.runner.deployer_impl

metaflow/runner/deployer_impl.py

@@ -5,8 +5,10 @@ import sys
 
 from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type
 
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
+
 from .subprocess_manager import SubprocessManager
-from .utils import get_lower_level_group, handle_timeout, temporary_fifo
+from .utils import get_lower_level_group, handle_timeout, temporary_fifo, with_dir
 
 if TYPE_CHECKING:
     import metaflow.runner.deployer
@@ -88,7 +90,7 @@ class DeployerImpl(object):
         self.show_output = show_output
         self.profile = profile
         self.env = env
-        self.cwd = cwd
+        self.cwd = cwd or os.getcwd()
         self.file_read_timeout = file_read_timeout
 
         self.env_vars = os.environ.copy()
@@ -140,9 +142,19 @@ class DeployerImpl(object):
     ) -> "metaflow.runner.deployer.DeployedFlow":
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
-            command = get_lower_level_group(
-                self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
-            ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            # TODO: Get rid of CLICK_API_PROCESS_CONFIG in the near future
+            if CLICK_API_PROCESS_CONFIG:
+                # We need to run this in the cwd because configs depend on files
+                # that may be located in paths relative to the directory the user
+                # wants to run in
+                with with_dir(self.cwd):
+                    command = get_lower_level_group(
+                        self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                    ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            else:
+                command = get_lower_level_group(
+                    self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                ).create(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.spm.run_command(
                 [sys.executable, *command],

metaflow/runner/metaflow_runner.py

@@ -7,12 +7,15 @@ from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run
 
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
+
 from metaflow.plugins import get_runner_cli
 
 from .utils import (
     temporary_fifo,
     handle_timeout,
     async_handle_timeout,
+    with_dir,
 )
 from .subprocess_manager import CommandManager, SubprocessManager
 
@@ -299,7 +302,7 @@ class Runner(metaclass=RunnerMeta):
         if profile:
             self.env_vars["METAFLOW_PROFILE"] = profile
 
-        self.cwd = cwd
+        self.cwd = cwd or os.getcwd()
         self.file_read_timeout = file_read_timeout
         self.spm = SubprocessManager()
         self.top_level_kwargs = kwargs
@@ -359,9 +362,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun containing the results of the run.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).run(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).run(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -390,9 +399,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun containing the results of the resumed run.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).resume(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).resume(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -423,9 +438,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun representing the run that was started.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).run(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).run(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = await self.spm.async_run_command(
                 [sys.executable, *command],
@@ -455,9 +476,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun representing the resumed run that was started.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).resume(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).resume(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = await self.spm.async_run_command(
                 [sys.executable, *command],

metaflow/runner/subprocess_manager.py

@@ -237,7 +237,7 @@ class CommandManager(object):
         self.command = command
 
         self.env = env if env is not None else os.environ.copy()
-        self.cwd = cwd if cwd is not None else os.getcwd()
+        self.cwd = cwd or os.getcwd()
 
         self.process = None
         self.stdout_thread = None

metaflow/runner/utils.py

@@ -322,3 +322,11 @@ def get_lower_level_group(
         raise ValueError(f"Sub-command '{sub_command}' not found in API '{api.name}'")
 
     return sub_command_obj(**sub_command_kwargs)
+
+
+@contextmanager
+def with_dir(new_dir):
+    current_dir = os.getcwd()
+    os.chdir(new_dir)
+    yield new_dir
+    os.chdir(current_dir)

metaflow/user_configs/config_decorators.py

@@ -200,7 +200,7 @@ class MutableFlow:
         for name, value in self._flow_cls._flow_state.get(
             _FlowState.CONFIGS, {}
         ).items():
-            yield name, ConfigValue(value)
+            yield name, ConfigValue(value) if value is not None else None
 
     @property
     def parameters(self) -> Generator[Tuple[str, Any], None, None]: