metaflow 2.15.14__py2.py3-none-any.whl → 2.15.16__py2.py3-none-any.whl

metaflow/plugins/uv/bootstrap.py

@@ -4,6 +4,7 @@ import sys
 import time
 
 from metaflow.util import which
+from metaflow.info_file import read_info_file
 from metaflow.metaflow_config import get_pinned_conda_libs
 from urllib.request import Request, urlopen
 from urllib.error import URLError
@@ -78,11 +79,27 @@ if __name__ == "__main__":
         # return only dependency names instead of pinned versions
         return pinned.keys()
 
+    def skip_metaflow_dependencies():
+        skip_pkgs = ["metaflow"]
+        info = read_info_file()
+        if info is not None:
+            try:
+                skip_pkgs.extend([ext_name for ext_name in info["ext_info"][0].keys()])
+            except Exception:
+                print(
+                    "Failed to read INFO. Metaflow-related packages might get installed during runtime."
+                )
+
+        return skip_pkgs
+
     def sync_uv_project(datastore_type):
         print("Syncing uv project...")
         dependencies = " ".join(get_dependencies(datastore_type))
+        skip_pkgs = " ".join(
+            [f"--no-install-package {dep}" for dep in skip_metaflow_dependencies()]
+        )
         cmd = f"""set -e;
-            uv sync --frozen --no-install-package metaflow;
+            uv sync --frozen {skip_pkgs};
             uv pip install {dependencies} --strict
             """
         run_cmd(cmd)

metaflow/plugins/uv/uv_environment.py

@@ -22,7 +22,7 @@ class UVEnvironment(MetaflowEnvironment):
         self.logger("Bootstrapping uv...")
 
     def executable(self, step_name, default=None):
-        return "uv run python"
+        return "uv run --no-sync python"
 
     def add_to_package(self):
         # NOTE: We treat uv.lock and pyproject.toml as regular project assets and ship these along user code as part of the code package

metaflow/runner/click_api.py

@@ -467,9 +467,14 @@ class MetaflowAPI(object):
                     config_file = defaults.get("config")
 
                 if config_file:
-                    config_file = map(
-                        lambda x: (x[0], ConvertPath.convert_value(x[1], is_default)),
-                        config_file,
+                    config_file = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertPath.convert_value(x[1], is_default),
+                            ),
+                            config_file,
+                        )
                     )
 
                 is_default = False
@@ -479,12 +484,14 @@ class MetaflowAPI(object):
                     config_value = defaults.get("config_value")
 
                 if config_value:
-                    config_value = map(
-                        lambda x: (
-                            x[0],
-                            ConvertDictOrStr.convert_value(x[1], is_default),
-                        ),
-                        config_value,
+                    config_value = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertDictOrStr.convert_value(x[1], is_default),
+                            ),
+                            config_value,
+                        )
                     )
 
                 if (config_file is None) ^ (config_value is None):

metaflow/runner/deployer.py

@@ -7,6 +7,55 @@ from typing import ClassVar, Dict, Optional, TYPE_CHECKING
 from metaflow.exception import MetaflowNotFound
 from metaflow.metaflow_config import DEFAULT_FROM_DEPLOYMENT_IMPL
 
+
+def generate_fake_flow_file_contents(
+    flow_name: str, param_info: dict, project_name: Optional[str] = None
+):
+    params_code = ""
+    for _, param_details in param_info.items():
+        param_python_var_name = param_details["python_var_name"]
+        param_name = param_details["name"]
+        param_type = param_details["type"]
+        param_help = param_details["description"]
+        param_required = param_details["is_required"]
+
+        if param_type == "JSON":
+            params_code += (
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type=JSONType, help='''{param_help}''', required={param_required})\n"
+            )
+        elif param_type == "FilePath":
+            is_text = param_details.get("is_text", True)
+            encoding = param_details.get("encoding", "utf-8")
+            params_code += (
+                f"    {param_python_var_name} = IncludeFile('{param_name}', "
+                f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
+                f"required={param_required})\n"
+            )
+        else:
+            params_code += (
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type={param_type}, help='''{param_help}''', required={param_required})\n"
+            )
+
+    project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
+
+    contents = f"""\
+from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
+{project_decorator}class {flow_name}(FlowSpec):
+{params_code}
+    @step
+    def start(self):
+        self.next(self.end)
+    @step
+    def end(self):
+        pass
+if __name__ == '__main__':
+    {flow_name}()
+"""
+    return contents
+
+
 if TYPE_CHECKING:
     import metaflow
     import metaflow.runner.deployer_impl

metaflow/runner/deployer_impl.py

@@ -5,8 +5,10 @@ import sys
 
 from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type
 
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
+
 from .subprocess_manager import SubprocessManager
-from .utils import get_lower_level_group, handle_timeout, temporary_fifo
+from .utils import get_lower_level_group, handle_timeout, temporary_fifo, with_dir
 
 if TYPE_CHECKING:
     import metaflow.runner.deployer
@@ -88,7 +90,7 @@ class DeployerImpl(object):
         self.show_output = show_output
         self.profile = profile
         self.env = env
-        self.cwd = cwd
+        self.cwd = cwd or os.getcwd()
         self.file_read_timeout = file_read_timeout
 
         self.env_vars = os.environ.copy()
@@ -140,9 +142,19 @@ class DeployerImpl(object):
     ) -> "metaflow.runner.deployer.DeployedFlow":
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
-            command = get_lower_level_group(
-                self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
-            ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            # TODO: Get rid of CLICK_API_PROCESS_CONFIG in the near future
+            if CLICK_API_PROCESS_CONFIG:
+                # We need to run this in the cwd because configs depend on files
+                # that may be located in paths relative to the directory the user
+                # wants to run in
+                with with_dir(self.cwd):
+                    command = get_lower_level_group(
+                        self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                    ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            else:
+                command = get_lower_level_group(
+                    self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                ).create(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.spm.run_command(
                 [sys.executable, *command],

metaflow/runner/metaflow_runner.py

@@ -7,12 +7,15 @@ from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run
 
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
+
 from metaflow.plugins import get_runner_cli
 
 from .utils import (
     temporary_fifo,
     handle_timeout,
     async_handle_timeout,
+    with_dir,
 )
 from .subprocess_manager import CommandManager, SubprocessManager
 
@@ -299,7 +302,7 @@ class Runner(metaclass=RunnerMeta):
         if profile:
             self.env_vars["METAFLOW_PROFILE"] = profile
 
-        self.cwd = cwd
+        self.cwd = cwd or os.getcwd()
         self.file_read_timeout = file_read_timeout
         self.spm = SubprocessManager()
         self.top_level_kwargs = kwargs
@@ -359,9 +362,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun containing the results of the run.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).run(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).run(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -390,9 +399,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun containing the results of the resumed run.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).resume(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).resume(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -423,9 +438,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun representing the run that was started.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).run(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).run(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = await self.spm.async_run_command(
                 [sys.executable, *command],
@@ -455,9 +476,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun representing the resumed run that was started.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).resume(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).resume(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = await self.spm.async_run_command(
                 [sys.executable, *command],

metaflow/runner/subprocess_manager.py

@@ -237,7 +237,7 @@ class CommandManager(object):
         self.command = command
 
         self.env = env if env is not None else os.environ.copy()
-        self.cwd = cwd if cwd is not None else os.getcwd()
+        self.cwd = cwd or os.getcwd()
 
         self.process = None
         self.stdout_thread = None

metaflow/runner/utils.py

@@ -322,3 +322,11 @@ def get_lower_level_group(
         raise ValueError(f"Sub-command '{sub_command}' not found in API '{api.name}'")
 
     return sub_command_obj(**sub_command_kwargs)
+
+
+@contextmanager
+def with_dir(new_dir):
+    current_dir = os.getcwd()
+    os.chdir(new_dir)
+    yield new_dir
+    os.chdir(current_dir)

metaflow/user_configs/config_options.py

@@ -221,13 +221,13 @@ class ConfigInput:
         if param_name == "config_value":
             self._value_values = {
                 k.lower(): v
-                for k, v in param_value
+                for k, v in param_value.items()
                 if v is not None and not v.startswith(_CONVERTED_DEFAULT)
             }
         else:
             self._path_values = {
                 k.lower(): v
-                for k, v in param_value
+                for k, v in param_value.items()
                 if v is not None and not v.startswith(_CONVERTED_DEFAULT)
             }
         if do_return:
@@ -329,12 +329,12 @@ class ConfigInput:
                 to_return[name] = None
                 flow_cls._flow_state[_FlowState.CONFIGS][name] = None
                 continue
-            if val.startswith(_CONVERTED_DEFAULT_NO_FILE):
-                no_default_file.append(name)
-                continue
             if val.startswith(_CONVERTED_NO_FILE):
                 no_file.append(name)
                 continue
+            if val.startswith(_CONVERTED_DEFAULT_NO_FILE):
+                no_default_file.append(name)
+                continue
 
             val = val[len(_CONVERT_PREFIX) :]  # Remove the _CONVERT_PREFIX
             if val.startswith(_DEFAULT_PREFIX):  # Remove the _DEFAULT_PREFIX if needed
@@ -398,7 +398,7 @@ class ConfigInput:
         return self.process_configs(
             ctx.obj.flow.name,
             param.name,
-            value,
+            dict(value),
             ctx.params["quiet"],
             ctx.params["datastore"],
             click_obj=ctx.obj,