metaflow 2.15.14__py2.py3-none-any.whl → 2.15.15__py2.py3-none-any.whl

metaflow/runner/metaflow_runner.py

@@ -7,12 +7,15 @@ from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run
 
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
+
 from metaflow.plugins import get_runner_cli
 
 from .utils import (
     temporary_fifo,
     handle_timeout,
     async_handle_timeout,
+    with_dir,
 )
 from .subprocess_manager import CommandManager, SubprocessManager
 
@@ -299,7 +302,7 @@ class Runner(metaclass=RunnerMeta):
         if profile:
             self.env_vars["METAFLOW_PROFILE"] = profile
 
-        self.cwd = cwd
+        self.cwd = cwd or os.getcwd()
         self.file_read_timeout = file_read_timeout
         self.spm = SubprocessManager()
         self.top_level_kwargs = kwargs
@@ -359,9 +362,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun containing the results of the run.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).run(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).run(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -390,9 +399,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun containing the results of the resumed run.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).resume(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).resume(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = self.spm.run_command(
                 [sys.executable, *command],
@@ -423,9 +438,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun representing the run that was started.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).run(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).run(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).run(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = await self.spm.async_run_command(
                 [sys.executable, *command],
@@ -455,9 +476,15 @@ class Runner(metaclass=RunnerMeta):
             ExecutingRun representing the resumed run that was started.
         """
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
-            command = self.api(**self.top_level_kwargs).resume(
-                runner_attribute_file=attribute_file_path, **kwargs
-            )
+            if CLICK_API_PROCESS_CONFIG:
+                with with_dir(self.cwd):
+                    command = self.api(**self.top_level_kwargs).resume(
+                        runner_attribute_file=attribute_file_path, **kwargs
+                    )
+            else:
+                command = self.api(**self.top_level_kwargs).resume(
+                    runner_attribute_file=attribute_file_path, **kwargs
+                )
 
             pid = await self.spm.async_run_command(
                 [sys.executable, *command],

metaflow/runner/subprocess_manager.py

@@ -237,7 +237,7 @@ class CommandManager(object):
         self.command = command
 
         self.env = env if env is not None else os.environ.copy()
-        self.cwd = cwd if cwd is not None else os.getcwd()
+        self.cwd = cwd or os.getcwd()
 
         self.process = None
         self.stdout_thread = None

metaflow/runner/utils.py

@@ -322,3 +322,11 @@ def get_lower_level_group(
         raise ValueError(f"Sub-command '{sub_command}' not found in API '{api.name}'")
 
     return sub_command_obj(**sub_command_kwargs)
+
+
+@contextmanager
+def with_dir(new_dir):
+    current_dir = os.getcwd()
+    os.chdir(new_dir)
+    yield new_dir
+    os.chdir(current_dir)

metaflow/user_configs/config_options.py

@@ -221,13 +221,13 @@ class ConfigInput:
         if param_name == "config_value":
             self._value_values = {
                 k.lower(): v
-                for k, v in param_value
+                for k, v in param_value.items()
                 if v is not None and not v.startswith(_CONVERTED_DEFAULT)
             }
         else:
             self._path_values = {
                 k.lower(): v
-                for k, v in param_value
+                for k, v in param_value.items()
                 if v is not None and not v.startswith(_CONVERTED_DEFAULT)
             }
         if do_return:
@@ -329,12 +329,12 @@ class ConfigInput:
                 to_return[name] = None
                 flow_cls._flow_state[_FlowState.CONFIGS][name] = None
                 continue
-            if val.startswith(_CONVERTED_DEFAULT_NO_FILE):
-                no_default_file.append(name)
-                continue
             if val.startswith(_CONVERTED_NO_FILE):
                 no_file.append(name)
                 continue
+            if val.startswith(_CONVERTED_DEFAULT_NO_FILE):
+                no_default_file.append(name)
+                continue
 
             val = val[len(_CONVERT_PREFIX) :]  # Remove the _CONVERT_PREFIX
             if val.startswith(_DEFAULT_PREFIX):  # Remove the _DEFAULT_PREFIX if needed
@@ -398,7 +398,7 @@ class ConfigInput:
         return self.process_configs(
             ctx.obj.flow.name,
             param.name,
-            value,
+            dict(value),
             ctx.params["quiet"],
             ctx.params["datastore"],
             click_obj=ctx.obj,

metaflow/user_configs/config_parameters.py

@@ -1,9 +1,22 @@
-import collections.abc
+import inspect
 import json
+import collections.abc
+import copy
 import os
 import re
 
-from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    TYPE_CHECKING,
+    Union,
+)
 
 
 from ..exception import MetaflowException
@@ -54,7 +67,7 @@ def dump_config_values(flow: "FlowSpec"):
     return {}
 
 
-class ConfigValue(collections.abc.Mapping):
+class ConfigValue(collections.abc.Mapping, dict):
     """
     ConfigValue is a thin wrapper around an arbitrarily nested dictionary-like
     configuration object. It allows you to access elements of this nested structure
@@ -69,8 +82,67 @@ class ConfigValue(collections.abc.Mapping):
     # Thin wrapper to allow configuration values to be accessed using a "." notation
     # as well as a [] notation.
 
-    def __init__(self, data: Dict[str, Any]):
-        self._data = data
+    # We inherit from dict to allow the isinstanceof check to work easily and also
+    # to provide a simple json dumps functionality.
+
+    def __init__(self, data: Union["ConfigValue", Dict[str, Any]]):
+        self._data = {k: self._construct(v) for k, v in data.items()}
+
+        # Enable json dumps
+        dict.__init__(self, self._data)
+
+    @classmethod
+    def fromkeys(cls, iterable: Iterable, value: Any = None) -> "ConfigValue":
+        """
+        Creates a new ConfigValue object from the given iterable and value.
+
+        Parameters
+        ----------
+        iterable : Iterable
+            Iterable to create the ConfigValue from.
+        value : Any, optional
+            Value to set for each key in the iterable.
+
+        Returns
+        -------
+        ConfigValue
+            A new ConfigValue object.
+        """
+        return cls(dict.fromkeys(iterable, value))
+
+    def to_dict(self) -> Dict[Any, Any]:
+        """
+        Returns a dictionary representation of this configuration object.
+
+        Returns
+        -------
+        Dict[Any, Any]
+            Dictionary equivalent of this configuration object.
+        """
+        return self._to_dict(self._data)
+
+    def copy(self) -> "ConfigValue":
+        return self.__copy__()
+
+    def clear(self) -> None:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
+
+    def update(self, *args, **kwargs) -> None:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
+
+    def setdefault(self, key: Any, default: Any = None) -> Any:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
+
+    def pop(self, key: Any, default: Any = None) -> Any:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
+
+    def popitem(self) -> Tuple[Any, Any]:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
 
     def __getattr__(self, key: str) -> Any:
         """
@@ -115,33 +187,93 @@ class ConfigValue(collections.abc.Mapping):
         Any
             Element of the configuration
         """
-        value = self._data[key]
-        if isinstance(value, dict):
-            value = ConfigValue(value)
-        return value
+        return self._data[key]
 
-    def __len__(self):
+    def __setitem__(self, key: Any, value: Any) -> None:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
+
+    def __delattr__(self, key) -> None:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
+
+    def __delitem__(self, key: Any) -> None:
+        # Prevent configuration modification
+        raise TypeError("ConfigValue is immutable")
+
+    def __len__(self) -> int:
         return len(self._data)
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator:
         return iter(self._data)
 
-    def __repr__(self):
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, ConfigValue):
+            return self._data == other._data
+        if isinstance(other, dict):
+            return self._data == other
+        return False
+
+    def __ne__(self, other: Any) -> bool:
+        return not self.__eq__(other)
+
+    def __copy__(self) -> "ConfigValue":
+        cls = self.__class__
+        result = cls.__new__(cls)
+        result.__dict__.update({k: copy.copy(v) for k, v in self.__dict__.items()})
+        return result
+
+    def __repr__(self) -> str:
         return repr(self._data)
 
-    def __str__(self):
-        return json.dumps(self._data)
+    def __str__(self) -> str:
+        return str(self._data)
 
-    def to_dict(self) -> Dict[Any, Any]:
+    def __dir__(self) -> Iterable[str]:
+        return dir(type(self)) + [k for k in self._data.keys() if ID_PATTERN.match(k)]
+
+    def __contains__(self, key: Any) -> bool:
+        try:
+            self[key]
+        except KeyError:
+            return False
+        return True
+
+    def keys(self):
         """
-        Returns a dictionary representation of this configuration object.
+        Returns the keys of this configuration object.
 
         Returns
         -------
-        Dict[Any, Any]
-            Dictionary equivalent of this configuration object.
+        Any
+            Keys of this configuration object.
         """
-        return dict(self._data)
+        return self._data.keys()
+
+    @classmethod
+    def _construct(cls, obj: Any) -> Any:
+        # Internal method to construct a ConfigValue so that all mappings internally
+        # are also converted to ConfigValue
+        if isinstance(obj, ConfigValue):
+            v = obj
+        elif isinstance(obj, collections.abc.Mapping):
+            v = ConfigValue({k: cls._construct(v) for k, v in obj.items()})
+        elif isinstance(obj, (list, tuple, set)):
+            v = type(obj)([cls._construct(x) for x in obj])
+        else:
+            v = obj
+        return v
+
+    @classmethod
+    def _to_dict(cls, obj: Any) -> Any:
+        # Internal method to convert all nested mappings to dicts
+        if isinstance(obj, collections.abc.Mapping):
+            v = {k: cls._to_dict(v) for k, v in obj.items()}
+        elif isinstance(obj, (list, tuple, set)):
+            v = type(obj)([cls._to_dict(x) for x in obj])
+        else:
+            v = obj
+        return v
 
 
 class DelayEvaluator(collections.abc.Mapping):
@@ -157,8 +289,9 @@ class DelayEvaluator(collections.abc.Mapping):
         of _unpacked_delayed_*
     """
 
-    def __init__(self, ex: str):
+    def __init__(self, ex: str, saved_globals: Optional[Dict[str, Any]] = None):
         self._config_expr = ex
+        self._globals = saved_globals
         if ID_PATTERN.match(self._config_expr):
             # This is a variable only so allow things like config_expr("config").var
             self._is_var_only = True
@@ -166,6 +299,21 @@ class DelayEvaluator(collections.abc.Mapping):
         else:
             self._is_var_only = False
             self._access = None
+        self._cached_expr = None
+
+    def __copy__(self):
+        c = DelayEvaluator(self._config_expr)
+        c._access = self._access.copy() if self._access is not None else None
+        # Globals are not copied -- always kept as a reference
+        return c
+
+    def __deepcopy__(self, memo):
+        c = DelayEvaluator(self._config_expr)
+        c._access = (
+            copy.deepcopy(self._access, memo) if self._access is not None else None
+        )
+        # Globals are not copied -- always kept as a reference
+        return c
 
     def __iter__(self):
         yield "%s%d" % (UNPACK_KEY, id(self))
@@ -175,8 +323,15 @@ class DelayEvaluator(collections.abc.Mapping):
             return self
         if self._access is None:
             raise KeyError(key)
-        self._access.append(key)
-        return self
+
+        # Make a copy so that we can support something like
+        # foo = delay_evaluator["blah"]
+        # bar = delay_evaluator["baz"]
+        # and don't end up with a access list that contains both "blah" and "baz"
+        c = self.__copy__()
+        c._access.append(key)
+        c._cached_expr = None
+        return c
 
     def __len__(self):
         return 1
@@ -184,8 +339,10 @@ class DelayEvaluator(collections.abc.Mapping):
     def __getattr__(self, name):
         if self._access is None:
             raise AttributeError(name)
-        self._access.append(name)
-        return self
+        c = self.__copy__()
+        c._access.append(name)
+        c._cached_expr = None
+        return c
 
     def __call__(self, ctx=None, deploy_time=False):
         from ..flowspec import _FlowState  # Prevent circular import
@@ -199,7 +356,9 @@ class DelayEvaluator(collections.abc.Mapping):
                 "Config object can only be used directly in the FlowSpec defining them "
                 "(or their flow decorators)."
             )
-        if self._access is not None:
+        if self._cached_expr is not None:
+            to_eval_expr = self._cached_expr
+        elif self._access is not None:
             # Build the final expression by adding all the fields in access as . fields
             access_list = [self._config_expr]
             for a in self._access:
@@ -212,27 +371,24 @@ class DelayEvaluator(collections.abc.Mapping):
                     raise MetaflowException(
                         "Field '%s' of type '%s' is not supported" % (str(a), type(a))
                     )
-            self._config_expr = ".".join(access_list)
+            to_eval_expr = self._cached_expr = ".".join(access_list)
+        else:
+            to_eval_expr = self._cached_expr = self._config_expr
         # Evaluate the expression setting the config values as local variables
         try:
             return eval(
-                self._config_expr,
-                globals(),
+                to_eval_expr,
+                self._globals or globals(),
                 {
                     k: ConfigValue(v)
                     for k, v in flow_cls._flow_state.get(_FlowState.CONFIGS, {}).items()
                 },
             )
         except NameError as e:
-            potential_config_name = self._config_expr.split(".")[0]
-            if potential_config_name not in flow_cls._flow_state.get(
-                _FlowState.CONFIGS, {}
-            ):
-                raise MetaflowException(
-                    "Config '%s' not found in the flow (maybe not required and not "
-                    "provided?)" % potential_config_name
-                ) from e
-            raise
+            raise MetaflowException(
+                "Config expression '%s' could not be evaluated: %s"
+                % (to_eval_expr, str(e))
+            ) from e
 
 
 def config_expr(expr: str) -> DelayEvaluator:
@@ -262,7 +418,10 @@ def config_expr(expr: str) -> DelayEvaluator:
     expr : str
         Expression using the config values.
     """
-    return DelayEvaluator(expr)
+    # Get globals where the expression is defined so that the user can use
+    # something like `config_expr("my_func()")` in the expression.
+    parent_globals = inspect.currentframe().f_back.f_globals
+    return DelayEvaluator(expr, saved_globals=parent_globals)
 
 
 class Config(Parameter, collections.abc.Mapping):
@@ -385,23 +544,30 @@ class Config(Parameter, collections.abc.Mapping):
         return DelayEvaluator(self.name.lower())[key]
 
 
-def resolve_delayed_evaluator(v: Any, ignore_errors: bool = False) -> Any:
+def resolve_delayed_evaluator(
+    v: Any, ignore_errors: bool = False, to_dict: bool = False
+) -> Any:
     # NOTE: We don't ignore errors in downstream calls because we want to have either
     # all or nothing for the top-level call by the user.
     try:
         if isinstance(v, DelayEvaluator):
-            return v()
+            to_return = v()
+            if to_dict and isinstance(to_return, ConfigValue):
+                to_return = to_return.to_dict()
+            return to_return
         if isinstance(v, dict):
             return {
-                resolve_delayed_evaluator(k): resolve_delayed_evaluator(v)
+                resolve_delayed_evaluator(
+                    k, to_dict=to_dict
+                ): resolve_delayed_evaluator(v, to_dict=to_dict)
                 for k, v in v.items()
             }
         if isinstance(v, list):
-            return [resolve_delayed_evaluator(x) for x in v]
+            return [resolve_delayed_evaluator(x, to_dict=to_dict) for x in v]
         if isinstance(v, tuple):
-            return tuple(resolve_delayed_evaluator(x) for x in v)
+            return tuple(resolve_delayed_evaluator(x, to_dict=to_dict) for x in v)
         if isinstance(v, set):
-            return {resolve_delayed_evaluator(x) for x in v}
+            return {resolve_delayed_evaluator(x, to_dict=to_dict) for x in v}
         return v
     except Exception as e:
         if ignore_errors:
@@ -426,7 +592,7 @@ def unpack_delayed_evaluator(
         else:
             # k.startswith(UNPACK_KEY)
             try:
-                new_vals = resolve_delayed_evaluator(v)
+                new_vals = resolve_delayed_evaluator(v, to_dict=True)
                 new_keys.extend(new_vals.keys())
                 result.update(new_vals)
             except Exception as e:

metaflow/util.py

@@ -418,7 +418,7 @@ def to_pascalcase(obj):
     if isinstance(obj, dict):
         res = obj.__class__()
         for k in obj:
-            res[re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), k, 1)] = (
+            res[re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), k, count=1)] = (
                 to_pascalcase(obj[k])
             )
     elif isinstance(obj, (list, set, tuple)):
@@ -467,7 +467,4 @@ def to_pod(value):
     return str(value)
 
 
-if sys.version_info[:2] > (3, 5):
-    from metaflow._vendor.packaging.version import parse as version_parse
-else:
-    from distutils.version import LooseVersion as version_parse
+from metaflow._vendor.packaging.version import parse as version_parse

metaflow/vendor.py

@@ -11,7 +11,6 @@ WHITELIST = {
     "README.txt",
     "__init__.py",
     "vendor_any.txt",
-    "vendor_v3_5.txt",
     "vendor_v3_6.txt",
     "vendor_v3_7.txt",
     "pip.LICENSE",

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.15.14"
+metaflow_version = "2.15.15"

{metaflow-2.15.14.dist-info → metaflow-2.15.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.15.14
+Version: 2.15.15
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.14; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.15; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier