metaflow 2.12.39__py2.py3-none-any.whl → 2.13.1__py2.py3-none-any.whl

metaflow/runner/click_api.py

@@ -18,6 +18,7 @@ import json
 from collections import OrderedDict
 from typing import Any, Callable, Dict, List, Optional
 from typing import OrderedDict as TOrderedDict
+from typing import Tuple as TTuple
 from typing import Union
 
 from metaflow import FlowSpec, Parameter
@@ -38,8 +39,16 @@ from metaflow._vendor.typeguard import TypeCheckError, check_type
 from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
-from metaflow.user_configs.config_options import LocalFileInput
+from metaflow.user_configs.config_options import (
+    ConfigValue,
+    ConvertDictOrStr,
+    ConvertPath,
+    LocalFileInput,
+    MultipleTuple,
+    config_options_with_config_input,
+)
 
 # Define a recursive type alias for JSON
 JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
@@ -58,6 +67,7 @@ click_to_python_types = {
     JSONTypeClass: JSON,
     FilePathClass: str,
     LocalFileInput: str,
+    MultipleTuple: TTuple[str, Union[JSON, ConfigValue]],
 }
 
 
@@ -68,7 +78,7 @@ def _method_sanity_check(
     defaults: TOrderedDict[str, Any],
     **kwargs
 ) -> Dict[str, Any]:
-    method_params = {"args": {}, "options": {}}
+    method_params = {"args": {}, "options": {}, "defaults": defaults}
 
     possible_params = OrderedDict()
     possible_params.update(possible_arg_params)
@@ -90,10 +100,26 @@ def _method_sanity_check(
                 % (supplied_k, annotations[supplied_k], defaults[supplied_k])
             )
 
-        # because Click expects stringified JSON..
-        supplied_v = (
-            json.dumps(supplied_v) if annotations[supplied_k] == JSON else supplied_v
-        )
+        # Clean up values to make them into what click expects
+        if annotations[supplied_k] == JSON:
+            # JSON should be a string (json dumps)
+            supplied_v = json.dumps(supplied_v)
+        elif supplied_k == "config_value":
+            # Special handling of config value because we need to go look in the tuple
+            new_list = []
+            for cfg_name, cfg_value in supplied_v:
+                if isinstance(cfg_value, ConfigValue):
+                    # ConfigValue should be JSONified and converted to a string
+                    new_list.append((cfg_name, json.dumps(cfg_value.to_dict())))
+                elif isinstance(cfg_value, dict):
+                    # ConfigValue passed as a dictionary
+                    new_list.append((cfg_name, json.dumps(cfg_value)))
+                else:
+                    raise TypeError(
+                        "Invalid type for a config-value, expected a ConfigValue or "
+                        "dict but got '%s'" % type(cfg_value)
+                    )
+            supplied_v = new_list
 
         if supplied_k in possible_arg_params:
             cli_name = possible_arg_params[supplied_k].opts[0].strip("-")
@@ -188,35 +214,56 @@ loaded_modules = {}
 def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
     if not os.path.exists(flow_file):
         raise FileNotFoundError("Flow file not present at '%s'" % flow_file)
-    # Check if the module has already been loaded
-    if flow_file in loaded_modules:
-        module = loaded_modules[flow_file]
-    else:
-        # Load the module if it's not already loaded
-        spec = importlib.util.spec_from_file_location("module", flow_file)
-        module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(module)
-        # Cache the loaded module
-        loaded_modules[flow_file] = module
-    classes = inspect.getmembers(module, inspect.isclass)
-
-    flow_cls = None
-    for _, kls in classes:
-        if kls != FlowSpec and issubclass(kls, FlowSpec):
-            if flow_cls is not None:
-                raise MetaflowException(
-                    "Multiple FlowSpec classes found in %s" % flow_file
-                )
-            flow_cls = kls
-
-    return flow_cls
+
+    flow_dir = os.path.dirname(os.path.abspath(flow_file))
+    path_was_added = False
+
+    # Only add to path if it's not already there
+    if flow_dir not in sys.path:
+        sys.path.insert(0, flow_dir)
+        path_was_added = True
+
+    try:
+        # Check if the module has already been loaded
+        if flow_file in loaded_modules:
+            module = loaded_modules[flow_file]
+        else:
+            # Load the module if it's not already loaded
+            spec = importlib.util.spec_from_file_location("module", flow_file)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            # Cache the loaded module
+            loaded_modules[flow_file] = module
+        classes = inspect.getmembers(module, inspect.isclass)
+
+        flow_cls = None
+        for _, kls in classes:
+            if kls != FlowSpec and issubclass(kls, FlowSpec):
+                if flow_cls is not None:
+                    raise MetaflowException(
+                        "Multiple FlowSpec classes found in %s" % flow_file
+                    )
+                flow_cls = kls
+
+        if flow_cls is None:
+            raise MetaflowException("No FlowSpec class found in %s" % flow_file)
+        return flow_cls
+    finally:
+        # Only remove from path if we added it
+        if path_was_added:
+            try:
+                sys.path.remove(flow_dir)
+            except ValueError:
+                # User's code might have removed it already
+                pass
 
 
 class MetaflowAPI(object):
-    def __init__(self, parent=None, flow_cls=None, **kwargs):
+    def __init__(self, parent=None, flow_cls=None, config_input=None, **kwargs):
         self._parent = parent
         self._chain = [{self._API_NAME: kwargs}]
         self._flow_cls = flow_cls
+        self._config_input = config_input
         self._cached_computed_parameters = None
 
     @property
@@ -238,11 +285,19 @@ class MetaflowAPI(object):
         flow_cls = extract_flow_class_from_file(flow_file)
 
         with flow_context(flow_cls) as _:
-            add_decorator_options(cli_collection)
+            cli_collection, config_input = config_options_with_config_input(
+                cli_collection
+            )
+            cli_collection = add_decorator_options(cli_collection)
 
         def getattr_wrapper(_self, name):
             # Functools.partial do not automatically bind self (no __get__)
-            return _self._internal_getattr(_self, name)
+            with flow_context(flow_cls) as _:
+                # We also wrap this in the proper flow context because since commands
+                # are loaded lazily, we need the proper flow context to compute things
+                # like parameters. If we do not do this, the outer flow's context will
+                # be used.
+                return _self._internal_getattr(_self, name)
 
         class_dict = {
             "__module__": "metaflow",
@@ -272,7 +327,12 @@ class MetaflowAPI(object):
                 defaults,
                 **kwargs,
             )
-            return to_return(parent=None, flow_cls=flow_cls, **method_params)
+            return to_return(
+                parent=None,
+                flow_cls=flow_cls,
+                config_input=config_input,
+                **method_params,
+            )
 
         m = _method
         m.__name__ = cli_collection.name
@@ -313,8 +373,12 @@ class MetaflowAPI(object):
                 for k, v in options.items():
                     if isinstance(v, list):
                         for i in v:
-                            components.append("--%s" % k)
-                            components.append(str(i))
+                            if isinstance(i, tuple):
+                                components.append("--%s" % k)
+                                components.extend(map(str, i))
+                            else:
+                                components.append("--%s" % k)
+                                components.append(str(i))
                     else:
                         components.append("--%s" % k)
                         if v != "flag":
@@ -323,21 +387,93 @@ class MetaflowAPI(object):
         return components
 
     def _compute_flow_parameters(self):
-        if self._flow_cls is None or self._parent is not None:
+        if (
+            self._flow_cls is None
+            or self._config_input is None
+            or self._parent is not None
+        ):
             raise RuntimeError(
                 "Computing flow-level parameters for a non start API. "
                 "Please report to the Metaflow team."
             )
-        # TODO: We need to actually compute the new parameters (based on configs) which
-        # would involve processing the options at least partially. We will do this
-        # before GA but for now making it work for regular parameters
+
         if self._cached_computed_parameters is not None:
             return self._cached_computed_parameters
         self._cached_computed_parameters = []
+
+        config_options = None
+        if CLICK_API_PROCESS_CONFIG:
+            with flow_context(self._flow_cls) as _:
+                # We are going to resolve the configs first and then get the parameters.
+                # Note that configs may update/add parameters so the order is important
+                # Since part of the processing of configs happens by click, we need to
+                # "fake" it.
+
+                # Extract any config options as well as datastore and quiet options
+                method_params = self._chain[0][self._API_NAME]
+                opts = method_params["options"]
+                defaults = method_params["defaults"]
+
+                ds = opts.get("datastore", defaults["datastore"])
+                quiet = opts.get("quiet", defaults["quiet"])
+                is_default = False
+                config_file = opts.get("config-file")
+                if config_file is None:
+                    is_default = True
+                    config_file = defaults.get("config_file")
+
+                if config_file:
+                    config_file = map(
+                        lambda x: (x[0], ConvertPath.convert_value(x[1], is_default)),
+                        config_file,
+                    )
+
+                is_default = False
+                config_value = opts.get("config-value")
+                if config_value is None:
+                    is_default = True
+                    config_value = defaults.get("config_value")
+
+                if config_value:
+                    config_value = map(
+                        lambda x: (
+                            x[0],
+                            ConvertDictOrStr.convert_value(x[1], is_default),
+                        ),
+                        config_value,
+                    )
+
+                if (config_file is None) ^ (config_value is None):
+                    # If we have one, we should have the other
+                    raise MetaflowException(
+                        "Options were not properly set -- this is an internal error."
+                    )
+
+                if config_file:
+                    # Process both configurations; the second one will return all the merged
+                    # configuration options properly processed.
+                    self._config_input.process_configs(
+                        self._flow_cls.__name__, "config_file", config_file, quiet, ds
+                    )
+                    config_options = self._config_input.process_configs(
+                        self._flow_cls.__name__, "config_value", config_value, quiet, ds
+                    )
+
+        # At this point, we are like in start() in cli.py -- we obtained the
+        # properly processed config_options which we can now use to process
+        # the config decorators (including CustomStep/FlowDecorators)
+        # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
+        # it will init all parameters (config_options will be None)
+        # We ignore any errors if we don't check the configs in the click API.
+        new_cls = self._flow_cls._process_config_decorators(
+            config_options, ignore_errors=not CLICK_API_PROCESS_CONFIG
+        )
+        if new_cls:
+            self._flow_cls = new_cls
+
         for _, param in self._flow_cls._get_parameters():
             if param.IS_CONFIG_PARAMETER:
                 continue
-            param.init()
             self._cached_computed_parameters.append(param)
         return self._cached_computed_parameters
 

metaflow/runner/deployer_impl.py

@@ -61,8 +61,13 @@ class DeployerImpl(object):
                 "of DeployerImpl."
             )
 
+        from metaflow.parameters import flow_context
+
         if "metaflow.cli" in sys.modules:
-            importlib.reload(sys.modules["metaflow.cli"])
+            # Reload the CLI with an "empty" flow -- this will remove any configuration
+            # options. They are re-added in from_cli (called below).
+            with flow_context(None) as _:
+                importlib.reload(sys.modules["metaflow.cli"])
         from metaflow.cli import start
         from metaflow.runner.click_api import MetaflowAPI
 

metaflow/runner/metaflow_runner.py

@@ -245,8 +245,13 @@ class Runner(object):
         # This ability is made possible by the statement:
         # 'from .metaflow_runner import Runner' in '__init__.py'
 
+        from metaflow.parameters import flow_context
+
         if "metaflow.cli" in sys.modules:
-            importlib.reload(sys.modules["metaflow.cli"])
+            # Reload the CLI with an "empty" flow -- this will remove any configuration
+            # options. They are re-added in from_cli (called below).
+            with flow_context(None) as _:
+                importlib.reload(sys.modules["metaflow.cli"])
         from metaflow.cli import start
         from metaflow.runner.click_api import MetaflowAPI
 

metaflow/runner/utils.py

@@ -111,6 +111,11 @@ def read_from_fifo_when_ready(
     poll.register(fifo_fd, select.POLLIN)
     max_timeout = 3  # Wait for 10 * 3 = 30 ms after last write
     while True:
+        if check_process_exited(command_obj) and command_obj.process.returncode != 0:
+            raise CalledProcessError(
+                command_obj.process.returncode, command_obj.command
+            )
+
         if timeout < 0:
             raise TimeoutError("Timeout while waiting for the file content")
 

metaflow/user_configs/config_options.py

@@ -15,7 +15,7 @@ from ..util import get_username
 
 _CONVERT_PREFIX = "@!c!@:"
 _DEFAULT_PREFIX = "@!d!@:"
-_NO_FILE = "@!n!@"
+_NO_FILE = "@!n!@:"
 
 _CONVERTED_DEFAULT = _CONVERT_PREFIX + _DEFAULT_PREFIX
 _CONVERTED_NO_FILE = _CONVERT_PREFIX + _NO_FILE
@@ -41,7 +41,8 @@ class ConvertPath(click.Path):
         is_default = False
         if value and value.startswith(_DEFAULT_PREFIX):
             is_default = True
-            value = super().convert(value[len(_DEFAULT_PREFIX) :], param, ctx)
+            value = value[len(_DEFAULT_PREFIX) :]
+        value = super().convert(value, param, ctx)
         return self.convert_value(value, is_default)
 
     @staticmethod
@@ -170,7 +171,15 @@ class ConfigInput:
             cls.loaded_configs = all_configs
         return cls.loaded_configs.get(config_name, None)
 
-    def process_configs(self, ctx, param, value):
+    def process_configs(
+        self,
+        flow_name: str,
+        param_name: str,
+        param_value: Dict[str, Optional[str]],
+        quiet: bool,
+        datastore: str,
+        click_obj: Optional[Any] = None,
+    ):
         from ..cli import echo_always, echo_dev_null  # Prevent circular import
         from ..flowspec import _FlowState  # Prevent circular import
 
@@ -198,31 +207,32 @@ class ConfigInput:
         #       - the default (including None if there is no default). If the default is
         #         not None, it will start with _CONVERTED_DEFAULT since Click will make
         #         the value go through ConvertPath or ConvertDictOrStr
-        #  - the actual value passed through prefixed with _CONVERT_PREFIX
+        #       - the actual value passed through prefixed with _CONVERT_PREFIX
 
         debug.userconf_exec(
             "Processing configs for %s -- incoming values: %s"
-            % (param.name, str(value))
+            % (param_name, str(param_value))
         )
 
         do_return = self._value_values is None and self._path_values is None
         # We only keep around non default values. We could simplify by checking just one
         # value and if it is default it means all are but this doesn't seem much more effort
         # and is clearer
-        if param.name == "config_value_options":
+        if param_name == "config_value":
             self._value_values = {
                 k.lower(): v
-                for k, v in value
+                for k, v in param_value
                 if v is not None and not v.startswith(_CONVERTED_DEFAULT)
             }
         else:
             self._path_values = {
                 k.lower(): v
-                for k, v in value
+                for k, v in param_value
                 if v is not None and not v.startswith(_CONVERTED_DEFAULT)
             }
         if do_return:
-            # One of config_value_options or config_file_options will be None
+            # One of values["value"] or values["path"] is None -- we are in the first
+            # go around
             debug.userconf_exec("Incomplete config options; waiting for more")
             return None
 
@@ -233,8 +243,13 @@ class ConfigInput:
         # down configurations (like METAFLOW_FLOW_CONFIG) could still be present and
         # would cause an issue -- we can ignore those as the kv. values should trump
         # everything else.
+        # NOTE: These are all *non default* keys
         all_keys = set(self._value_values).union(self._path_values)
-        # Make sure we have at least some keys (ie: some non default values)
+
+        if all_keys and click_obj:
+            click_obj.has_cl_config_options = True
+        # Make sure we have at least some non default keys (we need some if we have
+        # all kv)
         has_all_kv = all_keys and all(
             self._value_values.get(k, "").startswith(_CONVERT_PREFIX + "kv.")
             for k in all_keys
@@ -244,27 +259,35 @@ class ConfigInput:
         to_return = {}
 
         if not has_all_kv:
-            # Check that the user didn't provide *both* a path and a value.
+            # Check that the user didn't provide *both* a path and a value. Again, these
+            # are only user-provided (not defaults)
             common_keys = set(self._value_values or []).intersection(
                 [k for k, v in self._path_values.items()] or []
             )
             if common_keys:
-                raise click.UsageError(
+                exc = click.UsageError(
                     "Cannot provide both a value and a file for the same configuration. "
                     "Found such values for '%s'" % "', '".join(common_keys)
                 )
+                if click_obj:
+                    click_obj.delayed_config_exception = exc
+                    return None
+                raise exc
 
-            all_values = dict(self._path_values or {})
-            all_values.update(self._value_values or {})
+            all_values = dict(self._path_values)
+            all_values.update(self._value_values)
 
             debug.userconf_exec("All config values: %s" % str(all_values))
 
             merged_configs = {}
+            # Now look at everything (including defaults)
             for name, (val, is_path) in self._defaults.items():
                 n = name.lower()
                 if n in all_values:
+                    # We have the value provided by the user -- use that.
                     merged_configs[n] = all_values[n]
                 else:
+                    # No value provided by the user -- use the default
                     if isinstance(val, DeployTimeField):
                         # This supports a default value that is a deploy-time field (similar
                         # to Parameter).)
@@ -273,25 +296,27 @@ class ConfigInput:
                         # of circularity. Note also that quiet and datastore are *eager*
                         # options so are available here.
                         param_ctx = ParameterContext(
-                            flow_name=ctx.obj.flow.name,
+                            flow_name=flow_name,
                             user_name=get_username(),
                             parameter_name=n,
-                            logger=(
-                                echo_dev_null if ctx.params["quiet"] else echo_always
-                            ),
-                            ds_type=ctx.params["datastore"],
+                            logger=(echo_dev_null if quiet else echo_always),
+                            ds_type=datastore,
                             configs=None,
                         )
                         val = val.fun(param_ctx)
                     if is_path:
                         # This is a file path
-                        merged_configs[n] = ConvertPath.convert_value(val, False)
+                        merged_configs[n] = ConvertPath.convert_value(val, True)
                     else:
                         # This is a value
-                        merged_configs[n] = ConvertDictOrStr.convert_value(val, False)
+                        merged_configs[n] = ConvertDictOrStr.convert_value(val, True)
         else:
             debug.userconf_exec("Fast path due to pre-processed values")
             merged_configs = self._value_values
+
+        if click_obj:
+            click_obj.has_config_options = True
+
         debug.userconf_exec("Configs merged with defaults: %s" % str(merged_configs))
 
         missing_configs = set()
@@ -302,20 +327,27 @@ class ConfigInput:
             if val is None:
                 missing_configs.add(name)
                 continue
-            if val.startswith(_CONVERTED_NO_FILE):
-                no_file.append(name)
-                continue
             if val.startswith(_CONVERTED_DEFAULT_NO_FILE):
                 no_default_file.append(name)
                 continue
+            if val.startswith(_CONVERTED_NO_FILE):
+                no_file.append(name)
+                continue
+
             val = val[len(_CONVERT_PREFIX) :]  # Remove the _CONVERT_PREFIX
+            if val.startswith(_DEFAULT_PREFIX):  # Remove the _DEFAULT_PREFIX if needed
+                val = val[len(_DEFAULT_PREFIX) :]
             if val.startswith("kv."):
                 # This means to load it from a file
                 read_value = self.get_config(val[3:])
                 if read_value is None:
-                    raise click.UsageError(
+                    exc = click.UsageError(
                         "Could not find configuration '%s' in INFO file" % val
                     )
+                    if click_obj:
+                        click_obj.delayed_config_exception = exc
+                        return None
+                    raise exc
                 flow_cls._flow_state[_FlowState.CONFIGS][name] = read_value
                 to_return[name] = ConfigValue(read_value)
             else:
@@ -348,13 +380,27 @@ class ConfigInput:
                 % (merged_configs[missing][len(_CONVERTED_DEFAULT_NO_FILE) :], missing)
             )
         if msgs:
-            raise click.UsageError(
+            exc = click.UsageError(
                 "Bad values passed for configuration options: %s" % ", ".join(msgs)
             )
+            if click_obj:
+                click_obj.delayed_config_exception = exc
+                return None
+            raise exc
 
         debug.userconf_exec("Finalized configs: %s" % str(to_return))
         return to_return
 
+    def process_configs_click(self, ctx, param, value):
+        return self.process_configs(
+            ctx.obj.flow.name,
+            param.name,
+            value,
+            ctx.params["quiet"],
+            ctx.params["datastore"],
+            click_obj=ctx.obj,
+        )
+
     def __str__(self):
         return repr(self)
 
@@ -399,7 +445,7 @@ class LocalFileInput(click.Path):
         return "LocalFileInput"
 
 
-def config_options(cmd):
+def config_options_with_config_input(cmd):
     help_strs = []
     required_names = []
     defaults = {}
@@ -407,7 +453,7 @@ def config_options(cmd):
     parsers = {}
     flow_cls = getattr(current_flow, "flow_cls", None)
     if flow_cls is None:
-        return cmd
+        return cmd, None
 
     parameters = [p for _, p in flow_cls._get_parameters() if p.IS_CONFIG_PARAMETER]
     # List all the configuration options
@@ -432,20 +478,22 @@ def config_options(cmd):
         parsers[arg.name.lower()] = arg.parser
 
     if not config_seen:
-        # No configurations -- don't add anything
-        return cmd
+        # No configurations -- don't add anything; we set it to False so that it
+        # can be checked whether or not we called this.
+        return cmd, False
 
     help_str = (
         "Configuration options for the flow. "
-        "Multiple configurations can be specified."
+        "Multiple configurations can be specified. Cannot be used with resume."
     )
     help_str = "\n\n".join([help_str] + help_strs)
-    cb_func = ConfigInput(required_names, defaults, parsers).process_configs
+    config_input = ConfigInput(required_names, defaults, parsers)
+    cb_func = config_input.process_configs_click
 
     cmd.params.insert(
         0,
         click.Option(
-            ["--config-value", "config_value_options"],
+            ["--config-value", "config_value"],
             nargs=2,
             multiple=True,
             type=MultipleTuple([click.Choice(config_seen), ConvertDictOrStr()]),
@@ -470,7 +518,7 @@ def config_options(cmd):
     cmd.params.insert(
         0,
         click.Option(
-            ["--config", "config_file_options"],
+            ["--config", "config_file"],
             nargs=2,
             multiple=True,
             type=MultipleTuple([click.Choice(config_seen), ConvertPath()]),
@@ -492,4 +540,9 @@ def config_options(cmd):
             required=False,
         ),
     )
+    return cmd, config_input
+
+
+def config_options(cmd):
+    cmd, _ = config_options_with_config_input(cmd)
     return cmd