ob-metaflow 2.15.14.1__py2.py3-none-any.whl → 2.15.17.1__py2.py3-none-any.whl

metaflow/__init__.py

@@ -128,8 +128,8 @@ _import_tl_plugins(globals())
 # this auto-generates decorator functions from Decorator objects
 # in the top-level metaflow namespace
 _import_plugin_decorators(globals())
-# Setting card import for only python 3.4
-if sys.version_info[0] >= 3 and sys.version_info[1] >= 4:
+# Setting card import for only python 3.6
+if sys.version_info[0] >= 3 and sys.version_info[1] >= 6:
     from . import cards
 
 # Client

metaflow/_vendor/click/core.py

@@ -719,7 +719,7 @@ class BaseCommand(object):
         prog_name=None,
         complete_var=None,
         standalone_mode=True,
-        **extra
+        **extra,
     ):
         """This is the way to invoke a script with all the bells and
         whistles as a command line application.  This will always terminate
@@ -1101,7 +1101,7 @@ class MultiCommand(Command):
         subcommand_metavar=None,
         chain=False,
         result_callback=None,
-        **attrs
+        **attrs,
     ):
         Command.__init__(self, name, **attrs)
         if no_args_is_help is None:
@@ -1463,6 +1463,7 @@ class Parameter(object):
         parameter. The old callback format will still work, but it will
         raise a warning to give you a chance to migrate the code easier.
     """
+
     param_type_name = "parameter"
 
     def __init__(
@@ -1708,7 +1709,7 @@ class Option(Parameter):
         hidden=False,
         show_choices=True,
         show_envvar=False,
-        **attrs
+        **attrs,
     ):
         default_is_missing = attrs.get("default", _missing) is _missing
         Parameter.__init__(self, param_decls, type=type, **attrs)

metaflow/cli.py

@@ -496,7 +496,7 @@ def start(
         ctx.obj.echo,
         ctx.obj.flow_datastore,
         {
-            k: ConfigValue(v)
+            k: ConfigValue(v) if v is not None else None
             for k, v in ctx.obj.flow.__class__._flow_state.get(
                 _FlowState.CONFIGS, {}
             ).items()
@@ -524,7 +524,7 @@ def start(
             decorators._attach_decorators(ctx.obj.flow, all_decospecs)
             decorators._init(ctx.obj.flow)
             # Regenerate graph if we attached more decorators
-            ctx.obj.flow.__class__._init_attrs()
+            ctx.obj.flow.__class__._init_graph()
             ctx.obj.graph = ctx.obj.flow._graph
 
         decorators._init_step_decorators(

metaflow/cli_components/run_cmds.py

@@ -45,7 +45,7 @@ def before_run(obj, tags, decospecs):
         decorators._attach_decorators(obj.flow, all_decospecs)
         decorators._init(obj.flow)
         # Regenerate graph if we attached more decorators
-        obj.flow.__class__._init_attrs()
+        obj.flow.__class__._init_graph()
         obj.graph = obj.flow._graph
 
     obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint)

metaflow/cmd/develop/stub_generator.py

@@ -488,9 +488,6 @@ class StubGenerator:
                 self._imports.add(name)
 
         def _add_to_typing_check(name, is_module=False):
-            # if name != self._current_module_name:
-            #    self._typing_imports.add(name)
-            #
             if name == "None":
                 return
             if is_module:
@@ -504,6 +501,24 @@ class StubGenerator:
                     # the current file
                     self._typing_imports.add(splits[0])
 
+        def _format_qualified_class_name(cls: type) -> str:
+            """Helper to format a class with its qualified module name"""
+            # Special case for NoneType - return None
+            if cls.__name__ == "NoneType":
+                return "None"
+
+            module = inspect.getmodule(cls)
+            if (
+                module
+                and module.__name__ != "builtins"
+                and module.__name__ != "__main__"
+            ):
+                module_name = self._get_module_name_alias(module.__name__)
+                _add_to_typing_check(module_name, is_module=True)
+                return f"{module_name}.{cls.__name__}"
+            else:
+                return cls.__name__
+
         if isinstance(element, str):
             # Special case for self referential things (particularly in a class)
             if element == self._current_name:
@@ -557,19 +572,15 @@ class StubGenerator:
                 return element.__name__
         elif isinstance(element, type(Ellipsis)):
             return "..."
-        # elif (
-        #    isinstance(element, typing._GenericAlias)
-        #    and hasattr(element, "_name")
-        #    and element._name in ("List", "Tuple", "Dict", "Set")
-        # ):
-        #    # 3.7 has these as _GenericAlias but they don't behave like the ones in 3.10
-        #    _add_to_import("typing")
-        #    return str(element)
         elif isinstance(element, typing._GenericAlias):
             # We need to check things recursively in __args__ if it exists
             args_str = []
             for arg in getattr(element, "__args__", []):
-                args_str.append(self._get_element_name_with_module(arg))
+                # Special handling for class objects in type arguments
+                if isinstance(arg, type):
+                    args_str.append(_format_qualified_class_name(arg))
+                else:
+                    args_str.append(self._get_element_name_with_module(arg))
 
             _add_to_import("typing")
             if element._name:
@@ -584,12 +595,15 @@ class StubGenerator:
                         args_str = [call_args, args_str[-1]]
                 return "typing.%s[%s]" % (element._name, ", ".join(args_str))
             else:
-                return "%s[%s]" % (element.__origin__, ", ".join(args_str))
+                # Handle the case where we have a generic type without a _name
+                origin = element.__origin__
+                if isinstance(origin, type):
+                    origin_str = _format_qualified_class_name(origin)
+                else:
+                    origin_str = str(origin)
+                return "%s[%s]" % (origin_str, ", ".join(args_str))
         elif isinstance(element, ForwardRef):
             f_arg = self._get_module_name_alias(element.__forward_arg__)
-            # if f_arg in ("Run", "Task"):  # HACK -- forward references in current.py
-            #    _add_to_import("metaflow")
-            #    f_arg = "metaflow.%s" % f_arg
             _add_to_typing_check(f_arg)
             return '"%s"' % f_arg
         elif inspect.getmodule(element) == inspect.getmodule(typing):

metaflow/cmd/develop/stubs.py

@@ -12,25 +12,13 @@ from . import develop
 from .stub_generator import StubGenerator
 
 _py_ver = sys.version_info[:2]
-_metadata_package = None
 
-
-def _check_stubs_supported():
-    global _metadata_package
-    if _metadata_package is not None:
-        return _metadata_package
-    else:
-        if _py_ver >= (3, 4):
-            if _py_ver >= (3, 8):
-                from importlib import metadata
-            elif _py_ver >= (3, 7):
-                from metaflow._vendor.v3_7 import importlib_metadata as metadata
-            elif _py_ver >= (3, 6):
-                from metaflow._vendor.v3_6 import importlib_metadata as metadata
-            else:
-                from metaflow._vendor.v3_5 import importlib_metadata as metadata
-            _metadata_package = metadata
-        return _metadata_package
+if _py_ver >= (3, 8):
+    from importlib import metadata
+elif _py_ver >= (3, 7):
+    from metaflow._vendor.v3_7 import importlib_metadata as metadata
+else:
+    from metaflow._vendor.v3_6 import importlib_metadata as metadata
 
 
 @develop.group(short_help="Stubs management")
@@ -45,12 +33,6 @@ def stubs(ctx: Any):
     This CLI provides utilities to check and generate stubs for your current Metaflow
     installation.
     """
-    if _check_stubs_supported() is None:
-        raise click.UsageError(
-            "Building and installing stubs are not supported on Python %d.%d "
-            "(3.4 minimum required)" % _py_ver,
-            ctx=ctx,
-        )
 
 
 @stubs.command(short_help="Check validity of stubs")
@@ -187,7 +169,7 @@ setup(
     packages=find_namespace_packages(),
     package_data={{"metaflow-stubs": ["generated_for.txt", "py.typed", "**/*.pyi"]}},
     install_requires=["metaflow=={mf_version}"],
-    python_requires=">=3.5.2",
+    python_requires=">=3.6.1",
 )
                 """
             )
@@ -330,14 +312,14 @@ def get_packages_for_stubs() -> Tuple[List[Tuple[str, str]], List[str]]:
     # some reason it shows up multiple times.
     interesting_dists = [
         d
-        for d in _metadata_package.distributions()
+        for d in metadata.distributions()
         if any(
             [
                 p == "metaflow-stubs"
                 for p in (d.read_text("top_level.txt") or "").split()
             ]
         )
-        and isinstance(d, _metadata_package.PathDistribution)
+        and isinstance(d, metadata.PathDistribution)
     ]
 
     for dist in interesting_dists:

metaflow/datastore/task_datastore.py

@@ -118,7 +118,7 @@ class TaskDataStore(object):
         # The GZIP encodings are for backward compatibility
         self._encodings = {"pickle-v2", "gzip+pickle-v2"}
         ver = sys.version_info[0] * 10 + sys.version_info[1]
-        if ver >= 34:
+        if ver >= 36:
             self._encodings.add("pickle-v4")
             self._encodings.add("gzip+pickle-v4")
 
@@ -289,7 +289,7 @@ class TaskDataStore(object):
                     except (SystemError, OverflowError) as e:
                         raise DataException(
                             "Artifact *%s* is very large (over 2GB). "
-                            "You need to use Python 3.4 or newer if you want to "
+                            "You need to use Python 3.6 or newer if you want to "
                             "serialize large objects." % name
                         ) from e
                     except TypeError as e:
@@ -352,7 +352,7 @@ class TaskDataStore(object):
                 encode_type = "gzip+pickle-v2"
             if encode_type not in self._encodings:
                 raise DataException(
-                    "Python 3.4 or later is required to load artifact '%s'" % name
+                    "Python 3.6 or later is required to load artifact '%s'" % name
                 )
             else:
                 to_load[self._objects[name]].append(name)

metaflow/decorators.py

@@ -152,7 +152,7 @@ class Decorator(object):
         # Note that by design, later values override previous ones.
         self.attributes, new_user_attributes = unpack_delayed_evaluator(self.attributes)
         self._user_defined_attributes.update(new_user_attributes)
-        self.attributes = resolve_delayed_evaluator(self.attributes)
+        self.attributes = resolve_delayed_evaluator(self.attributes, to_dict=True)
 
         self._ran_init = True
 
@@ -638,7 +638,7 @@ StepFlag = NewType("StepFlag", bool)
 
 @overload
 def step(
-    f: Callable[[FlowSpecDerived], None]
+    f: Callable[[FlowSpecDerived], None],
 ) -> Callable[[FlowSpecDerived, StepFlag], None]: ...
 
 
@@ -649,7 +649,7 @@ def step(
 
 
 def step(
-    f: Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]]
+    f: Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]],
 ):
     """
     Marks a method in a FlowSpec as a Metaflow Step. Note that this

metaflow/extension_support/__init__.py

@@ -103,9 +103,6 @@ def load_module(module_name):
 
 def get_modules(extension_point):
     modules_to_load = []
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return []
     if extension_point not in _extension_points:
         raise RuntimeError(
             "Metaflow extension point '%s' not supported" % extension_point
@@ -146,9 +143,6 @@ def dump_module_info(all_packages=None, pkgs_per_extension_point=None):
 
 
 def get_extensions_in_dir(d):
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return None, None
     return _get_extension_packages(ignore_info_file=True, restrict_to_directories=[d])
 
 
@@ -312,21 +306,16 @@ def multiload_all(modules, extension_point, dst_globals):
 
 
 _py_ver = sys.version_info[:2]
-_mfext_supported = False
 _aliased_modules = []
 
-if _py_ver >= (3, 4):
-    import importlib.util
+import importlib.util
 
-    if _py_ver >= (3, 8):
-        from importlib import metadata
-    elif _py_ver >= (3, 7):
-        from metaflow._vendor.v3_7 import importlib_metadata as metadata
-    elif _py_ver >= (3, 6):
-        from metaflow._vendor.v3_6 import importlib_metadata as metadata
-    else:
-        from metaflow._vendor.v3_5 import importlib_metadata as metadata
-    _mfext_supported = True
+if _py_ver >= (3, 8):
+    from importlib import metadata
+elif _py_ver >= (3, 7):
+    from metaflow._vendor.v3_7 import importlib_metadata as metadata
+else:
+    from metaflow._vendor.v3_6 import importlib_metadata as metadata
 
 # Extension points are the directories that can be present in a EXT_PKG to
 # contribute to that extension point. For example, if you have
@@ -355,10 +344,6 @@ def _ext_debug(*args, **kwargs):
 
 
 def _get_extension_packages(ignore_info_file=False, restrict_to_directories=None):
-    if not _mfext_supported:
-        _ext_debug("Not supported for your Python version -- 3.4+ is needed")
-        return {}, {}
-
     # If we have an INFO file with the appropriate information (if running from a saved
     # code package for example), we use that directly
     # Pre-compute on _extension_points
@@ -381,12 +366,11 @@ def _get_extension_packages(ignore_info_file=False, restrict_to_directories=None
     try:
         extensions_module = importlib.import_module(EXT_PKG)
     except ImportError as e:
-        if _py_ver >= (3, 6):
-            # e.name is set to the name of the package that fails to load
-            # so don't error ONLY IF the error is importing this module (but do
-            # error if there is a transitive import error)
-            if not (isinstance(e, ModuleNotFoundError) and e.name == EXT_PKG):
-                raise
+        # e.name is set to the name of the package that fails to load
+        # so don't error ONLY IF the error is importing this module (but do
+        # error if there is a transitive import error)
+        if not (isinstance(e, ModuleNotFoundError) and e.name == EXT_PKG):
+            raise
         return {}, {}
 
     if restrict_to_directories:
@@ -894,20 +878,19 @@ def _attempt_load_module(module_name):
     try:
         extension_module = importlib.import_module(module_name)
     except ImportError as e:
-        if _py_ver >= (3, 6):
-            # e.name is set to the name of the package that fails to load
-            # so don't error ONLY IF the error is importing this module (but do
-            # error if there is a transitive import error)
-            errored_names = [EXT_PKG]
-            parts = module_name.split(".")
-            for p in parts[1:]:
-                errored_names.append("%s.%s" % (errored_names[-1], p))
-            if not (isinstance(e, ModuleNotFoundError) and e.name in errored_names):
-                print(
-                    "The following exception occurred while trying to load '%s' ('%s')"
-                    % (EXT_PKG, module_name)
-                )
-                raise
+        # e.name is set to the name of the package that fails to load
+        # so don't error ONLY IF the error is importing this module (but do
+        # error if there is a transitive import error)
+        errored_names = [EXT_PKG]
+        parts = module_name.split(".")
+        for p in parts[1:]:
+            errored_names.append("%s.%s" % (errored_names[-1], p))
+        if not (isinstance(e, ModuleNotFoundError) and e.name in errored_names):
+            print(
+                "The following exception occurred while trying to load '%s' ('%s')"
+                % (EXT_PKG, module_name)
+            )
+            raise
         _ext_debug("        Unknown error when loading '%s': %s" % (module_name, e))
         return None
     else:

metaflow/flowspec.py

@@ -87,6 +87,9 @@ class FlowSpecMeta(type):
         if name == "FlowSpec":
             return
 
+        cls._init_attrs()
+
+    def _init_attrs(cls):
         from .decorators import (
             DuplicateFlowDecoratorException,
         )  # Prevent circular import
@@ -103,6 +106,12 @@ class FlowSpecMeta(type):
         # Keys are _FlowState enum values
         cls._flow_state = {}
 
+        # Keep track if configs have been processed -- this is particularly applicable
+        # for the Runner/Deployer where calling multiple APIs on the same flow could
+        # cause the configs to be processed multiple times. For a given flow, once
+        # the configs have been processed, we do not process them again.
+        cls._configs_processed = False
+
         # We inherit stuff from our parent classes as well -- we need to be careful
         # in terms of the order; we will follow the MRO with the following rules:
         #  - decorators (cls._flow_decorators) will cause an error if they do not
@@ -127,10 +136,9 @@ class FlowSpecMeta(type):
                     cls._flow_state.setdefault(_FlowState.CONFIG_DECORATORS, []).extend(
                         base_configs
                     )
+        cls._init_graph()
 
-        cls._init_attrs()
-
-    def _init_attrs(cls):
+    def _init_graph(cls):
         # Graph and steps are specific to the class -- store here so we can access
         # in class method _process_config_decorators
         cls._graph = FlowGraph(cls)
@@ -225,6 +233,10 @@ class FlowSpec(metaclass=FlowSpecMeta):
     @classmethod
     def _process_config_decorators(cls, config_options, process_configs=True):
 
+        if cls._configs_processed:
+            return None
+        cls._configs_processed = True
+
         # Fast path for no user configurations
         if not process_configs or (
             not cls._flow_state.get(_FlowState.CONFIG_DECORATORS)
@@ -325,7 +337,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
         parameters.replace_flow_context(cls)
 
         # Re-calculate class level attributes after modifying the class
-        cls._init_attrs()
+        cls._init_graph()
         return cls
 
     def _set_constants(self, graph, kwargs, config_options):

metaflow/metaflow_config.py

@@ -371,6 +371,8 @@ KUBERNETES_CONTAINER_IMAGE = from_conf(
 )
 # Image pull policy for container images
 KUBERNETES_IMAGE_PULL_POLICY = from_conf("KUBERNETES_IMAGE_PULL_POLICY", None)
+# Image pull secrets for container images
+KUBERNETES_IMAGE_PULL_SECRETS = from_conf("KUBERNETES_IMAGE_PULL_SECRETS", "")
 # Default container registry for K8S
 KUBERNETES_CONTAINER_REGISTRY = from_conf(
     "KUBERNETES_CONTAINER_REGISTRY", DEFAULT_CONTAINER_REGISTRY

metaflow/parameters.py

@@ -373,7 +373,7 @@ class Parameter(object):
         # Do it one item at a time so errors are ignored at that level (as opposed to
         # at the entire kwargs level)
         self.kwargs = {
-            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
+            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors, to_dict=True)
             for k, v in self.kwargs.items()
         }
 
@@ -382,7 +382,7 @@ class Parameter(object):
         for key, value in self._override_kwargs.items():
             if value is not None:
                 self.kwargs[key] = resolve_delayed_evaluator(
-                    value, ignore_errors=ignore_errors
+                    value, ignore_errors=ignore_errors, to_dict=True
                 )
         # Set two default values if no-one specified them
         self.kwargs.setdefault("required", False)

metaflow/plugins/argo/argo_workflows.py

@@ -2021,6 +2021,7 @@ class ArgoWorkflows(object):
                     namespace=resources["namespace"],
                     image=resources["image"],
                     image_pull_policy=resources["image_pull_policy"],
+                    image_pull_secrets=resources["image_pull_secrets"],
                     service_account=resources["service_account"],
                     secrets=(
                         [
@@ -2209,6 +2210,8 @@ class ArgoWorkflows(object):
                     .node_selectors(resources.get("node_selector"))
                     # Set tolerations
                     .tolerations(resources.get("tolerations"))
+                    # Set image pull secrets
+                    .image_pull_secrets(resources.get("image_pull_secrets"))
                     # Set container
                     .container(
                         # TODO: Unify the logic with kubernetes.py
@@ -3785,6 +3788,10 @@ class Template(object):
         self.payload["tolerations"] = tolerations
         return self
 
+    def image_pull_secrets(self, image_pull_secrets):
+        self.payload["image_pull_secrets"] = image_pull_secrets
+        return self
+
     def to_json(self):
         return self.payload
 

metaflow/plugins/argo/argo_workflows_cli.py

@@ -344,20 +344,20 @@ def create(
 
 
 def check_python_version(obj):
-    # argo-workflows integration for Metaflow isn't supported for Py versions below 3.5.
+    # argo-workflows integration for Metaflow isn't supported for Py versions below 3.6.
     # This constraint can very well be lifted if desired.
-    if sys.version_info < (3, 5):
+    if sys.version_info < (3, 6):
         obj.echo("")
         obj.echo(
             "Metaflow doesn't support Argo Workflows for Python %s right now."
             % platform.python_version()
         )
         obj.echo(
-            "Please upgrade your Python interpreter to version 3.5 (or higher) or "
+            "Please upgrade your Python interpreter to version 3.6 (or higher) or "
             "reach out to us at slack.outerbounds.co for more help."
         )
         raise UnsupportedPythonVersion(
-            "Try again with a more recent version of Python (>=3.5)."
+            "Try again with a more recent version of Python (>=3.6)."
         )
 
 

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -9,59 +9,16 @@ from metaflow.exception import MetaflowException
 from metaflow.plugins.argo.argo_client import ArgoClient
 from metaflow.metaflow_config import KUBERNETES_NAMESPACE
 from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
-from metaflow.runner.deployer import Deployer, DeployedFlow, TriggeredRun
+from metaflow.runner.deployer import (
+    Deployer,
+    DeployedFlow,
+    TriggeredRun,
+    generate_fake_flow_file_contents,
+)
 
 from metaflow.runner.utils import get_lower_level_group, handle_timeout, temporary_fifo
 
 
-def generate_fake_flow_file_contents(
-    flow_name: str, param_info: dict, project_name: Optional[str] = None
-):
-    params_code = ""
-    for _, param_details in param_info.items():
-        param_python_var_name = param_details["python_var_name"]
-        param_name = param_details["name"]
-        param_type = param_details["type"]
-        param_help = param_details["description"]
-        param_required = param_details["is_required"]
-
-        if param_type == "JSON":
-            params_code += (
-                f"    {param_python_var_name} = Parameter('{param_name}', "
-                f"type=JSONType, help='''{param_help}''', required={param_required})\n"
-            )
-        elif param_type == "FilePath":
-            is_text = param_details.get("is_text", True)
-            encoding = param_details.get("encoding", "utf-8")
-            params_code += (
-                f"    {param_python_var_name} = IncludeFile('{param_name}', "
-                f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
-                f"required={param_required})\n"
-            )
-        else:
-            params_code += (
-                f"    {param_python_var_name} = Parameter('{param_name}', "
-                f"type={param_type}, help='''{param_help}''', required={param_required})\n"
-            )
-
-    project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
-
-    contents = f"""\
-from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
-{project_decorator}class {flow_name}(FlowSpec):
-{params_code}
-    @step
-    def start(self):
-        self.next(self.end)
-    @step
-    def end(self):
-        pass
-if __name__ == '__main__':
-    {flow_name}()
-"""
-    return contents
-
-
 class ArgoWorkflowsTriggeredRun(TriggeredRun):
     """
     A class representing a triggered Argo Workflow execution.

metaflow/plugins/aws/aws_client.py

@@ -35,6 +35,12 @@ class Boto3ClientProvider(object):
                 "Could not import module 'boto3'. Install boto3 first."
             )
 
+        # Convert dictionary config to Config object if needed
+        if "config" in client_params and not isinstance(
+            client_params["config"], Config
+        ):
+            client_params["config"] = Config(**client_params["config"])
+
         if module == "s3" and (
             "config" not in client_params or client_params["config"].retries is None
         ):

metaflow/plugins/cards/card_modules/chevron/renderer.py

@@ -61,7 +61,7 @@ def _get_key(key, scopes, warn, keep, def_ldel, def_rdel):
     # Loop through the scopes
     for scope in scopes:
         try:
-            # For every dot seperated key
+            # For every dot separated key
             for child in key.split("."):
                 # Move into the scope
                 try: