ob-metaflow 2.12.36.2__py2.py3-none-any.whl → 2.12.36.3__py2.py3-none-any.whl

metaflow/cli_args.py

@@ -12,14 +12,7 @@
 # well as the converting of options in runtime.py. We should make it so that we
 # can properly shlex things and un-shlex when using. Ideally this should all be
 # done in one place.
-#
-# NOTE: There is an important between these two as well:
-#  - this one will include local_config_file whereas the other one WILL NOT.
-#    This is because this is used when constructing the parallel UBF command which
-#    executes locally and therefore needs the local_config_file but the other (remote)
-#    commands do not.
 
-from .user_configs.config_options import ConfigInput
 from .util import to_unicode
 
 
@@ -72,16 +65,6 @@ class CLIArgs(object):
             # keyword in Python, so we call it 'decospecs' in click args
             if k == "decospecs":
                 k = "with"
-            if k in ("config_file_options", "config_value_options"):
-                # Special handling here since we gather them all in one option but actually
-                # need to send them one at a time using --config-value <name> kv.<name>.
-                # Note it can be either config_file_options or config_value_options depending
-                # on click processing order.
-                for config_name in v.keys():
-                    yield "--config-value"
-                    yield to_unicode(config_name)
-                    yield to_unicode(ConfigInput.make_key_name(config_name))
-                continue
             k = k.replace("_", "-")
             v = v if isinstance(v, (list, tuple, set)) else [v]
             for value in v:

metaflow/cmd/develop/stub_generator.py

@@ -1238,32 +1238,25 @@ class StubGenerator:
                 buff.write(indentation + deco + "\n")
             buff.write(indentation + "def " + name + "(")
             kw_only_param = False
-            has_var_args = False
             for i, (par_name, parameter) in enumerate(my_sign.parameters.items()):
                 annotation = self._exploit_annotation(parameter.annotation)
+
                 default = exploit_default(parameter.default)
 
-                if (
-                    kw_only_param
-                    and not has_var_args
-                    and parameter.kind != inspect.Parameter.KEYWORD_ONLY
-                ):
+                if kw_only_param and parameter.kind != inspect.Parameter.KEYWORD_ONLY:
                     raise RuntimeError(
                         "In function '%s': cannot have a positional parameter after a "
                         "keyword only parameter" % name
                     )
-
                 if (
                     parameter.kind == inspect.Parameter.KEYWORD_ONLY
                     and not kw_only_param
-                    and not has_var_args
                 ):
                     kw_only_param = True
                     buff.write("*, ")
                 if parameter.kind == inspect.Parameter.VAR_KEYWORD:
                     par_name = "**%s" % par_name
                 elif parameter.kind == inspect.Parameter.VAR_POSITIONAL:
-                    has_var_args = True
                     par_name = "*%s" % par_name
 
                 if default:

metaflow/decorators.py

@@ -12,12 +12,6 @@ from .exception import (
 )
 
 from .parameters import current_flow
-from .user_configs.config_decorators import CustomStepDecorator
-from .user_configs.config_parameters import (
-    UNPACK_KEY,
-    resolve_delayed_evaluator,
-    unpack_delayed_evaluator,
-)
 
 from metaflow._vendor import click
 
@@ -27,11 +21,6 @@ except NameError:
     unicode = str
     basestring = str
 
-# Contains the decorators on which _init was called. We want to ensure it is called
-# only once on each decorator and, as the _init() function below can be called in
-# several places, we need to track which decorator had their init function called
-_inited_decorators = set()
-
 
 class BadStepDecoratorException(MetaflowException):
     headline = "Syntax error"
@@ -126,36 +115,14 @@ class Decorator(object):
     def __init__(self, attributes=None, statically_defined=False):
         self.attributes = self.defaults.copy()
         self.statically_defined = statically_defined
-        self._user_defined_attributes = set()
-        self._ran_init = False
 
         if attributes:
             for k, v in attributes.items():
-                if k in self.defaults or k.startswith(UNPACK_KEY):
+                if k in self.defaults:
                     self.attributes[k] = v
-                    if not k.startswith(UNPACK_KEY):
-                        self._user_defined_attributes.add(k)
                 else:
                     raise InvalidDecoratorAttribute(self.name, k, self.defaults)
 
-    def init(self):
-        """
-        Initializes the decorator. In general, any operation you would do in __init__
-        should be done here.
-        """
-
-        # In some cases (specifically when using remove_decorator), we may need to call
-        # init multiple times. Short-circuit re-evaluating.
-        if self._ran_init:
-            return
-
-        # Note that by design, later values override previous ones.
-        self.attributes = unpack_delayed_evaluator(self.attributes)
-        self._user_defined_attributes.update(self.attributes.keys())
-        self.attributes = resolve_delayed_evaluator(self.attributes)
-
-        self._ran_init = True
-
     @classmethod
     def _parse_decorator_spec(cls, deco_spec):
         if len(deco_spec) == 0:
@@ -236,13 +203,10 @@ class FlowDecorator(Decorator):
 
 # compare this to parameters.add_custom_parameters
 def add_decorator_options(cmd):
+    seen = {}
     flow_cls = getattr(current_flow, "flow_cls", None)
     if flow_cls is None:
         return cmd
-
-    seen = {}
-    existing_params = set(p.name.lower() for p in cmd.params)
-    # Add decorator options
     for deco in flow_decorators(flow_cls):
         for option, kwargs in deco.options.items():
             if option in seen:
@@ -253,13 +217,7 @@ def add_decorator_options(cmd):
                     % (deco.name, option, seen[option])
                 )
                 raise MetaflowInternalError(msg)
-            elif deco.name.lower() in existing_params:
-                raise MetaflowInternalError(
-                    "Flow decorator '%s' uses an option '%s' which is a reserved "
-                    "keyword. Please use a different option name." % (deco.name, option)
-                )
             else:
-                kwargs["envvar"] = "METAFLOW_FLOW_%s" % option.upper()
                 seen[option] = deco.name
                 cmd.params.insert(0, click.Option(("--" + option,), **kwargs))
     return cmd
@@ -467,13 +425,10 @@ def _base_step_decorator(decotype, *args, **kwargs):
     Decorator prototype for all step decorators. This function gets specialized
     and imported for all decorators types by _import_plugin_decorators().
     """
-
     if args:
         # No keyword arguments specified for the decorator, e.g. @foobar.
         # The first argument is the function to be decorated.
         func = args[0]
-        if isinstance(func, CustomStepDecorator):
-            func = func._my_step
         if not hasattr(func, "is_step"):
             raise BadStepDecoratorException(decotype.name, func)
 
@@ -555,21 +510,6 @@ def _attach_decorators_to_step(step, decospecs):
             step.decorators.append(deco)
 
 
-def _init(flow, only_non_static=False):
-    for decorators in flow._flow_decorators.values():
-        for deco in decorators:
-            if deco in _inited_decorators:
-                continue
-            deco.init()
-            _inited_decorators.add(deco)
-    for flowstep in flow:
-        for deco in flowstep.decorators:
-            if deco in _inited_decorators:
-                continue
-            deco.init()
-            _inited_decorators.add(deco)
-
-
 def _init_flow_decorators(
     flow, graph, environment, flow_datastore, metadata, logger, echo, deco_options
 ):
@@ -680,7 +620,6 @@ def step(
     """
     f.is_step = True
     f.decorators = []
-    f.config_decorators = []
     try:
         # python 3
         f.name = f.__name__

metaflow/extension_support/plugins.py

@@ -93,32 +93,7 @@ def merge_lists(base, overrides, attr):
     base[:] = l[:]
 
 
-def get_plugin(category, class_path, name):
-    path, cls_name = class_path.rsplit(".", 1)
-    try:
-        plugin_module = importlib.import_module(path)
-    except ImportError as e:
-        raise ValueError(
-            "Cannot locate %s plugin '%s' at '%s'" % (category, name, path)
-        ) from e
-    cls = getattr(plugin_module, cls_name, None)
-    if cls is None:
-        raise ValueError(
-            "Cannot locate '%s' class for %s plugin at '%s'"
-            % (cls_name, category, path)
-        )
-    extracted_name = get_plugin_name(category, cls)
-    if extracted_name and extracted_name != name:
-        raise ValueError(
-            "Class '%s' at '%s' for %s plugin expected to be named '%s' but got '%s'"
-            % (cls_name, path, category, name, extracted_name)
-        )
-    globals()[cls_name] = cls
-    _ext_debug("        Added %s plugin '%s' from '%s'" % (category, name, class_path))
-    return cls
-
-
-def resolve_plugins(category, path_only=False):
+def resolve_plugins(category):
     # Called to return a list of classes that are the available plugins for 'category'
 
     # The ENABLED_<category> variable is set in process_plugins
@@ -139,7 +114,7 @@ def resolve_plugins(category, path_only=False):
 
     available_plugins = globals()[_dict_for_category(category)]
     name_extractor = _plugin_categories[category]
-    if path_only or not name_extractor:
+    if not name_extractor:
         # If we have no name function, it means we just use the name in the dictionary
         # and we return a dictionary. This is for sidecars mostly as they do not have
         # a field that indicates their name
@@ -157,14 +132,32 @@ def resolve_plugins(category, path_only=False):
                 "Configuration requested %s plugin '%s' but no such plugin is available"
                 % (category, name)
             )
-        if path_only:
-            to_return[name] = class_path
+        path, cls_name = class_path.rsplit(".", 1)
+        try:
+            plugin_module = importlib.import_module(path)
+        except ImportError:
+            raise ValueError(
+                "Cannot locate %s plugin '%s' at '%s'" % (category, name, path)
+            )
+        cls = getattr(plugin_module, cls_name, None)
+        if cls is None:
+            raise ValueError(
+                "Cannot locate '%s' class for %s plugin at '%s'"
+                % (cls_name, category, path)
+            )
+        if name_extractor and name_extractor(cls) != name:
+            raise ValueError(
+                "Class '%s' at '%s' for %s plugin expected to be named '%s' but got '%s'"
+                % (cls_name, path, category, name, name_extractor(cls))
+            )
+        globals()[cls_name] = cls
+        if name_extractor is not None:
+            to_return.append(cls)
         else:
-            if name_extractor is not None:
-                to_return.append(get_plugin(category, class_path, name))
-            else:
-                to_return[name] = get_plugin(category, class_path, name)
-
+            to_return[name] = cls
+        _ext_debug(
+            "        Added %s plugin '%s' from '%s'" % (category, name, class_path)
+        )
     return to_return
 
 
@@ -200,13 +193,6 @@ _plugin_categories = {
 }
 
 
-def get_plugin_name(category, plugin):
-    extractor = _plugin_categories[category]
-    if extractor:
-        return extractor(plugin)
-    return None
-
-
 def _list_for_category(category):
     # Convenience function to name the variable containing List[Tuple[str, str]] where
     # each tuple contains:

metaflow/flowspec.py

@@ -4,18 +4,15 @@ import sys
 import traceback
 import reprlib
 
-from enum import Enum
 from itertools import islice
 from types import FunctionType, MethodType
-from typing import TYPE_CHECKING, Any, Callable, Generator, List, Optional, Tuple
+from typing import Any, Callable, List, Optional, Tuple
 
 from . import cmd_with_io, parameters
-from .debug import debug
 from .parameters import DelayedEvaluationParameter, Parameter
 from .exception import (
     MetaflowException,
     MissingInMergeArtifactsException,
-    MetaflowInternalError,
     UnhandledInMergeArtifactsException,
 )
 
@@ -23,12 +20,6 @@ from .extension_support import extension_info
 
 from .graph import FlowGraph
 from .unbounded_foreach import UnboundedForeachInput
-from .user_configs.config_decorators import (
-    CustomFlowDecorator,
-    CustomStepDecorator,
-    MutableFlow,
-    MutableStep,
-)
 from .util import to_pod
 from .metaflow_config import INCLUDE_FOREACH_STACK, MAXIMUM_FOREACH_VALUE_CHARS
 
@@ -74,27 +65,14 @@ class ParallelUBF(UnboundedForeachInput):
         return item or 0  # item is None for the control task, but it is also split 0
 
 
-class _FlowState(Enum):
-    CONFIGS = 1
-    CONFIG_DECORATORS = 2
-    CACHED_PARAMETERS = 3
-
-
 class FlowSpecMeta(type):
     def __new__(cls, name, bases, dct):
         f = super().__new__(cls, name, bases, dct)
-        # We store some state in the flow class itself. This is primarily used to
-        # attach global state to a flow. It is *not* an actual global because of
-        # Runner/NBRunner. This is also created here in the meta class to avoid it being
-        # shared between different children classes.
-
-        # We should move _flow_decorators into this structure as well but keeping it
-        # out to limit the changes for now.
+        # This makes sure to give _flow_decorators to each
+        # child class (and not share it with the FlowSpec base
+        # class). This is important to not make a "global"
+        # _flow_decorators
         f._flow_decorators = {}
-
-        # Keys are _FlowState enum values
-        f._flow_state = {}
-
         return f
 
 
@@ -118,7 +96,6 @@ class FlowSpec(metaclass=FlowSpecMeta):
         "_cached_input",
         "_graph",
         "_flow_decorators",
-        "_flow_state",
         "_steps",
         "index",
         "input",
@@ -171,11 +148,16 @@ class FlowSpec(metaclass=FlowSpecMeta):
             fname = fname[:-1]
         return os.path.basename(fname)
 
-    def _check_parameters(self, config_parameters=False):
+    def _set_constants(self, graph, kwargs):
+        from metaflow.decorators import (
+            flow_decorators,
+        )  # To prevent circular dependency
+
+        # Persist values for parameters and other constants (class level variables)
+        # only once. This method is called before persist_constants is called to
+        # persist all values set using setattr
         seen = set()
-        for _, param in self._get_parameters():
-            if param.IS_CONFIG_PARAMETER != config_parameters:
-                continue
+        for var, param in self._get_parameters():
             norm = param.name.lower()
             if norm in seen:
                 raise MetaflowException(
@@ -184,127 +166,13 @@ class FlowSpec(metaclass=FlowSpecMeta):
                     "case-insensitive." % param.name
                 )
             seen.add(norm)
-
-    def _process_config_decorators(self, config_options):
-        current_cls = self.__class__
-
-        # Fast path for no user configurations
-        if not self._flow_state.get(_FlowState.CONFIG_DECORATORS):
-            # Process parameters to allow them to also use config values easily
-            for var, param in self._get_parameters():
-                if param.IS_CONFIG_PARAMETER:
-                    continue
-                param.init()
-            return self
-
-        debug.userconf_exec("Processing mutating step/flow decorators")
-        # We need to convert all the user configurations from DelayedEvaluationParameters
-        # to actual values so they can be used as is in the config decorators.
-
-        # We then reset them to be proper configs so they can be re-evaluated in
-        # _set_constants
-        to_reset_configs = []
-        self._check_parameters(config_parameters=True)
-        for var, param in self._get_parameters():
-            if not param.IS_CONFIG_PARAMETER:
-                continue
-            # Note that a config with no default and not required will be None
-            val = config_options.get(param.name.replace("-", "_").lower())
-            if isinstance(val, DelayedEvaluationParameter):
-                val = val()
-            # We store the value as well so that in _set_constants, we don't try
-            # to recompute (no guarantee that it is stable)
-            param._store_value(val)
-            to_reset_configs.append((var, param))
-            debug.userconf_exec("Setting config %s to %s" % (var, str(val)))
-            setattr(current_cls, var, val)
-
-        # Run all the decorators. Step decorators are directly in the step and
-        # we will run those first and *then* we run all the flow level decorators
-        for step in self._steps:
-            for deco in step.config_decorators:
-                if isinstance(deco, CustomStepDecorator):
-                    debug.userconf_exec(
-                        "Evaluating step level decorator %s for %s"
-                        % (deco.__class__.__name__, step.name)
-                    )
-                    deco.evaluate(MutableStep(current_cls, step))
-                else:
-                    raise MetaflowInternalError(
-                        "A non CustomFlowDecorator found in step custom decorators"
-                    )
-            if step.config_decorators:
-                # We remove all mention of the custom step decorator
-                setattr(current_cls, step.name, step)
-
-        mutable_flow = MutableFlow(current_cls)
-        for deco in self._flow_state[_FlowState.CONFIG_DECORATORS]:
-            if isinstance(deco, CustomFlowDecorator):
-                # Sanity check to make sure we are applying the decorator to the right
-                # class
-                if not deco._flow_cls == current_cls and not issubclass(
-                    current_cls, deco._flow_cls
-                ):
-                    raise MetaflowInternalError(
-                        "CustomFlowDecorator registered on the wrong flow -- "
-                        "expected %s but got %s"
-                        % (deco._flow_cls.__name__, current_cls.__name__)
-                    )
-                debug.userconf_exec(
-                    "Evaluating flow level decorator %s" % deco.__class__.__name__
-                )
-                deco.evaluate(mutable_flow)
-                # We reset cached_parameters on the very off chance that the user added
-                # more configurations based on the configuration
-                if _FlowState.CACHED_PARAMETERS in current_cls._flow_state:
-                    del current_cls._flow_state[_FlowState.CACHED_PARAMETERS]
-            else:
-                raise MetaflowInternalError(
-                    "A non CustomFlowDecorator found in flow custom decorators"
-                )
-
-        # Process parameters to allow them to also use config values easily
-        for var, param in self._get_parameters():
-            if param.IS_CONFIG_PARAMETER:
-                continue
-            param.init()
-        # Reset all configs that were already present in the class.
-        # TODO: This means that users can't override configs directly. Not sure if this
-        # is a pattern we want to support
-        for var, param in to_reset_configs:
-            setattr(current_cls, var, param)
-
-        # Reset cached parameters again since we added back the config parameters
-        if _FlowState.CACHED_PARAMETERS in current_cls._flow_state:
-            del current_cls._flow_state[_FlowState.CACHED_PARAMETERS]
-
-        # Set the current flow class we are in (the one we just created)
-        parameters.replace_flow_context(current_cls)
-        return current_cls(use_cli=False)
-
-    def _set_constants(self, graph, kwargs, config_options):
-        from metaflow.decorators import (
-            flow_decorators,
-        )  # To prevent circular dependency
-
-        # Persist values for parameters and other constants (class level variables)
-        # only once. This method is called before persist_constants is called to
-        # persist all values set using setattr
-        self._check_parameters(config_parameters=False)
-
-        seen = set()
+        seen.clear()
         self._success = True
 
         parameters_info = []
         for var, param in self._get_parameters():
             seen.add(var)
-            if param.IS_CONFIG_PARAMETER:
-                # Use computed value if already evaluated, else get from config_options
-                val = param._computed_value or config_options.get(
-                    param.name.replace("-", "_").lower()
-                )
-            else:
-                val = kwargs[param.name.replace("-", "_").lower()]
+            val = kwargs[param.name.replace("-", "_").lower()]
             # Support for delayed evaluation of parameters.
             if isinstance(val, DelayedEvaluationParameter):
                 val = val()
@@ -350,12 +218,6 @@ class FlowSpec(metaclass=FlowSpecMeta):
 
     @classmethod
     def _get_parameters(cls):
-        cached = cls._flow_state.get(_FlowState.CACHED_PARAMETERS)
-        if cached is not None:
-            for var in cached:
-                yield var, getattr(cls, var)
-            return
-        build_list = []
         for var in dir(cls):
             if var[0] == "_" or var in cls._NON_PARAMETERS:
                 continue
@@ -364,9 +226,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
             except:
                 continue
             if isinstance(val, Parameter):
-                build_list.append(var)
                 yield var, val
-        cls._flow_state[_FlowState.CACHED_PARAMETERS] = build_list
 
     def _set_datastore(self, datastore):
         self._datastore = datastore

metaflow/includefile.py

@@ -20,7 +20,6 @@ from .parameters import (
 )
 
 from .plugins import DATACLIENTS
-from .user_configs.config_parameters import ConfigValue
 from .util import get_username
 
 import functools
@@ -137,7 +136,6 @@ class FilePathClass(click.ParamType):
                 parameter_name=param.name,
                 logger=ctx.obj.echo,
                 ds_type=ctx.obj.datastore_impl.TYPE,
-                configs=None,
             )
 
         if len(value) > 0 and (value.startswith("{") or value.startswith('"{')):
@@ -245,63 +243,29 @@ class IncludeFile(Parameter):
     default : Union[str, Callable[ParameterContext, str]]
         Default path to a local file. A function
         implies that the parameter corresponds to a *deploy-time parameter*.
-    is_text : bool, optional, default None
+    is_text : bool, default True
         Convert the file contents to a string using the provided `encoding`.
-        If False, the artifact is stored in `bytes`. A value of None is equivalent to
-        True.
-    encoding : str, optional, default None
-        Use this encoding to decode the file contexts if `is_text=True`. A value of None
-        is equivalent to "utf-8".
-    required : bool, optional, default None
+        If False, the artifact is stored in `bytes`.
+    encoding : str, optional, default 'utf-8'
+        Use this encoding to decode the file contexts if `is_text=True`.
+    required : bool, default False
         Require that the user specified a value for the parameter.
-        `required=True` implies that the `default` is not used. A value of None is
-        equivalent to False
+        `required=True` implies that the `default` is not used.
     help : str, optional
         Help text to show in `run --help`.
     show_default : bool, default True
-        If True, show the default value in the help text. A value of None is equivalent
-        to True.
+        If True, show the default value in the help text.
     """
 
     def __init__(
         self,
         name: str,
-        required: Optional[bool] = None,
-        is_text: Optional[bool] = None,
-        encoding: Optional[str] = None,
+        required: bool = False,
+        is_text: bool = True,
+        encoding: str = "utf-8",
         help: Optional[str] = None,
         **kwargs: Dict[str, str]
     ):
-        self._includefile_overrides = {}
-        if is_text is not None:
-            self._includefile_overrides["is_text"] = is_text
-        if encoding is not None:
-            self._includefile_overrides["encoding"] = encoding
-        # NOTA: Right now, there is an issue where these can't be overridden by config
-        # in all circumstances. Ignoring for now.
-        super(IncludeFile, self).__init__(
-            name,
-            required=required,
-            help=help,
-            type=FilePathClass(
-                self._includefile_overrides.get("is_text", True),
-                self._includefile_overrides.get("encoding", "utf-8"),
-            ),
-            **kwargs,
-        )
-
-    def init(self):
-        super(IncludeFile, self).init()
-
-        # This will use the values set explicitly in the args if present, else will
-        # use and remove from kwargs else will use True/utf-8
-        is_text = self._includefile_overrides.get(
-            "is_text", self.kwargs.pop("is_text", True)
-        )
-        encoding = self._includefile_overrides.get(
-            "encoding", self.kwargs.pop("encoding", "utf-8")
-        )
-
         # If a default is specified, it needs to be uploaded when the flow is deployed
         # (for example when doing a `step-functions create`) so we make the default
         # be a DeployTimeField. This means that it will be evaluated in two cases:
@@ -311,7 +275,7 @@ class IncludeFile(Parameter):
         # In the first case, we will need to fully upload the file whereas in the
         # second case, we can just return the string as the FilePath.convert method
         # will take care of evaluating things.
-        v = self.kwargs.get("default")
+        v = kwargs.get("default")
         if v is not None:
             # If the default is a callable, we have two DeployTimeField:
             #  - the callable nature of the default will require us to "call" the default
@@ -324,15 +288,23 @@ class IncludeFile(Parameter):
             # (call the default)
             if callable(v) and not isinstance(v, DeployTimeField):
                 # If default is a callable, make it a DeployTimeField (the inner one)
-                v = DeployTimeField(self.name, str, "default", v, return_str=True)
-            self.kwargs["default"] = DeployTimeField(
-                self.name,
+                v = DeployTimeField(name, str, "default", v, return_str=True)
+            kwargs["default"] = DeployTimeField(
+                name,
                 str,
                 "default",
                 IncludeFile._eval_default(is_text, encoding, v),
                 print_representation=v,
             )
 
+        super(IncludeFile, self).__init__(
+            name,
+            required=required,
+            help=help,
+            type=FilePathClass(is_text, encoding),
+            **kwargs,
+        )
+
     def load_parameter(self, v):
         if v is None:
             return v

metaflow/metaflow_config.py

@@ -441,7 +441,7 @@ ESCAPE_HATCH_WARNING = from_conf("ESCAPE_HATCH_WARNING", True)
 ###
 # Debug configuration
 ###
-DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client", "tracing", "stubgen", "userconf"]
+DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client", "tracing", "stubgen"]
 
 for typ in DEBUG_OPTIONS:
     vars()["DEBUG_%s" % typ.upper()] = from_conf("DEBUG_%s" % typ.upper(), False)