metaflow 2.13.6__py2.py3-none-any.whl → 2.13.7__py2.py3-none-any.whl

metaflow/decorators.py

@@ -591,9 +591,13 @@ def _init_flow_decorators(
                 )
         else:
             # Each "non-multiple" flow decorator is only allowed to have one set of options
+            # Note that there may be no deco_options if a MutableFlow config injected
+            # the decorator.
             deco_flow_init_options = {
-                option: deco_options[option.replace("-", "_")]
-                for option in deco.options
+                option: deco_options.get(
+                    option.replace("-", "_"), option_info["default"]
+                )
+                for option, option_info in deco.options.items()
             }
         for deco in decorators:
             deco.flow_init(

metaflow/flowspec.py

@@ -86,6 +86,11 @@ class FlowSpecMeta(type):
         super().__init__(name, bases, attrs)
         if name == "FlowSpec":
             return
+
+        from .decorators import (
+            DuplicateFlowDecoratorException,
+        )  # Prevent circular import
+
         # We store some state in the flow class itself. This is primarily used to
         # attach global state to a flow. It is *not* an actual global because of
         # Runner/NBRunner. This is also created here in the meta class to avoid it being
@@ -98,6 +103,31 @@ class FlowSpecMeta(type):
         # Keys are _FlowState enum values
         cls._flow_state = {}
 
+        # We inherit stuff from our parent classes as well -- we need to be careful
+        # in terms of the order; we will follow the MRO with the following rules:
+        #  - decorators (cls._flow_decorators) will cause an error if they do not
+        #    support multiple and we see multiple instances of the same
+        #  - config decorators will be joined
+        #  - configs will be added later directly by the class; base class configs will
+        #    be taken into account as they would be inherited.
+
+        # We only need to do this for the base classes since the current class will
+        # get updated as decorators are parsed.
+        for base in cls.__mro__:
+            if base != cls and base != FlowSpec and issubclass(base, FlowSpec):
+                # Take care of decorators
+                for deco_name, deco in base._flow_decorators.items():
+                    if deco_name in cls._flow_decorators and not deco.allow_multiple:
+                        raise DuplicateFlowDecoratorException(deco_name)
+                    cls._flow_decorators.setdefault(deco_name, []).extend(deco)
+
+                # Take care of configs and config decorators
+                base_configs = base._flow_state.get(_FlowState.CONFIG_DECORATORS)
+                if base_configs:
+                    cls._flow_state.setdefault(_FlowState.CONFIG_DECORATORS, []).extend(
+                        base_configs
+                    )
+
         cls._init_attrs()
 
     def _init_attrs(cls):

metaflow/parameters.py

@@ -316,7 +316,7 @@ class Parameter(object):
     help : str, optional, default None
         Help text to show in `run --help`.
     required : bool, optional, default None
-        Require that the user specified a value for the parameter. Note that if
+        Require that the user specifies a value for the parameter. Note that if
         a default is provide, the required flag is ignored.
         A value of None is equivalent to False.
     show_default : bool, optional, default None

metaflow/runner/click_api.py

@@ -371,6 +371,8 @@ class MetaflowAPI(object):
                     else:
                         components.append(v)
                 for k, v in options.items():
+                    if v is None:
+                        continue
                     if isinstance(v, list):
                         for i in v:
                             if isinstance(i, tuple):

metaflow/user_configs/config_options.py

@@ -169,7 +169,7 @@ class ConfigInput:
                     "Please contact support."
                 )
             cls.loaded_configs = all_configs
-        return cls.loaded_configs.get(config_name, None)
+        return cls.loaded_configs[config_name]
 
     def process_configs(
         self,
@@ -326,6 +326,8 @@ class ConfigInput:
         for name, val in merged_configs.items():
             if val is None:
                 missing_configs.add(name)
+                to_return[name] = None
+                flow_cls._flow_state[_FlowState.CONFIGS][name] = None
                 continue
             if val.startswith(_CONVERTED_DEFAULT_NO_FILE):
                 no_default_file.append(name)
@@ -339,15 +341,16 @@ class ConfigInput:
                 val = val[len(_DEFAULT_PREFIX) :]
             if val.startswith("kv."):
                 # This means to load it from a file
-                read_value = self.get_config(val[3:])
-                if read_value is None:
+                try:
+                    read_value = self.get_config(val[3:])
+                except KeyError as e:
                     exc = click.UsageError(
                         "Could not find configuration '%s' in INFO file" % val
                     )
                     if click_obj:
                         click_obj.delayed_config_exception = exc
                         return None
-                    raise exc
+                    raise exc from e
                 flow_cls._flow_state[_FlowState.CONFIGS][name] = read_value
                 to_return[name] = ConfigValue(read_value)
             else:

metaflow/user_configs/config_parameters.py

@@ -290,17 +290,17 @@ class Config(Parameter, collections.abc.Mapping):
     default : Union[str, Callable[[ParameterContext], str], optional, default None
         Default path from where to read this configuration. A function implies that the
         value will be computed using that function.
-        You can only specify default or default_value.
+        You can only specify default or default_value, not both.
     default_value : Union[str, Dict[str, Any], Callable[[ParameterContext, Union[str, Dict[str, Any]]], Any], optional, default None
         Default value for the parameter. A function
         implies that the value will be computed using that function.
-        You can only specify default or default_value.
+        You can only specify default or default_value, not both.
     help : str, optional, default None
         Help text to show in `run --help`.
     required : bool, optional, default None
-        Require that the user specified a value for the configuration. Note that if
-        a default is provided, the required flag is ignored. A value of None is
-        equivalent to False.
+        Require that the user specifies a value for the configuration. Note that if
+        a default or default_value is provided, the required flag is ignored.
+        A value of None is equivalent to False.
     parser : Union[str, Callable[[str], Dict[Any, Any]]], optional, default None
         If a callable, it is a function that can parse the configuration string
         into an arbitrarily nested dictionary. If a string, the string should refer to
@@ -330,13 +330,13 @@ class Config(Parameter, collections.abc.Mapping):
         **kwargs: Dict[str, str]
     ):
 
-        if default and default_value:
+        if default is not None and default_value is not None:
             raise MetaflowException(
                 "For config '%s', you can only specify default or default_value, not both"
                 % name
             )
         self._default_is_file = default is not None
-        kwargs["default"] = default or default_value
+        kwargs["default"] = default if default is not None else default_value
         super(Config, self).__init__(
             name, required=required, help=help, type=str, **kwargs
         )

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.13.6"
+metaflow_version = "2.13.7"

{metaflow-2.13.6.dist-info → metaflow-2.13.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.13.6
+Version: 2.13.7
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.6; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.7; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.13.6.dist-info → metaflow-2.13.7.dist-info}/RECORD

@@ -6,11 +6,11 @@ metaflow/cli_args.py,sha256=muIh9pdVqMRG09uAYFKcAcUKFyDE4N3Wm6YahWRaUNI,3594
 metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
 metaflow/debug.py,sha256=HEmt_16tJtqHXQXsqD9pqOFe3CWR5GZ7VwpaYQgnRdU,1466
-metaflow/decorators.py,sha256=5xgIUuIcO52dKGUQ8fe-pmULdXIUwwMy86Uz2OZxOkw,23929
+metaflow/decorators.py,sha256=cbOCahmwVlnHklMN2O_j5DKvZA7m_Q72_6LBzzBZRhk,24131
 metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
 metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=_m9ZBJM0cooHRslDqfxCPQmkChqaTh6fGxp7HvISnYI,5161
-metaflow/flowspec.py,sha256=Ph4HrZPQ7t3Z7T3vAoNKlHCd7ejsOFkMrfXGpqttVtU,33952
+metaflow/flowspec.py,sha256=YtLlqg-MeH16rjsOU38NfPg_F-0cmzm2w4w-CPSfLxE,35510
 metaflow/graph.py,sha256=cdpnWr85aEj_rRn-7EjbndWjr_i8Dt3P7-oPUW0NNpI,12393
 metaflow/includefile.py,sha256=kWKDSlzVcRVNGG9PV5eB3o2ynrzqhVsfaLtkqjshn7Q,20948
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
@@ -25,7 +25,7 @@ metaflow/metaflow_version.py,sha256=duhIzfKZtcxMVMs2uiBqBvUarSHJqyWDwMhaBOQd_g0,
 metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
 metaflow/multicore_utils.py,sha256=yEo5T6Gemn4_vl8b6IOz7fsTUYtEyqa3AaKZgJY96Wc,4974
 metaflow/package.py,sha256=yfwVMVB1mD-Sw94KwXNK3N-26YHoKMn6btrcgd67Izs,7845
-metaflow/parameters.py,sha256=ycSTJzQc3rOburSl9prD__qgCxlPU0Cx8ntdEYuzoYU,18621
+metaflow/parameters.py,sha256=zyxDTkHXqVr7CUw509qsrBXGFpBMlqLL2-iwbIr0oiw,18621
 metaflow/procpoll.py,sha256=U2tE4iK_Mwj2WDyVTx_Uglh6xZ-jixQOo4wrM9OOhxg,2859
 metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/pylint_wrapper.py,sha256=zzBY9YaSUZOGH-ypDKAv2B_7XcoyMZj-zCoCrmYqNRc,2865
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=hKjHl6NYJkKBSU2tzdVbddfOX1zWK73T4GCO42A0XB4,14666
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=p2wlGBdWzsaPFEcG9sZp37O29SjTggTsTxrhb2gRZvc,28
+metaflow/version.py,sha256=I5W69AiJspPF2RE5ZwKwkBuaUE9t49qlX9xe1PtKOMI,28
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -312,7 +312,7 @@ metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcw
 metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpLZDBybXLergiDlN63E0u3x8,294
 metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
 metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/runner/click_api.py,sha256=13CM3RsH3dJ6YJTQnNafLEZ_teTnYMyfoR2Cf5baVbM,21772
+metaflow/runner/click_api.py,sha256=2uj3y-pZ2OF3J-mz4VbMQqcXFYy6NUeoN2OgjKZT5-c,21839
 metaflow/runner/deployer.py,sha256=Yas_SZCss3kfJw3hLC8_IyzgiytUFGoEGHz-l-rBBKk,8980
 metaflow/runner/deployer_impl.py,sha256=nzQJiJxjgZxewkkK5pHshfVeZOUUf5-FzS0pPJimktM,5930
 metaflow/runner/metaflow_runner.py,sha256=T41AWkuQq56ID90B7I-RFr9zexuZYtknsstSoqell7A,15861
@@ -358,11 +358,11 @@ metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIe
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
 metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/user_configs/config_decorators.py,sha256=Tj0H88UT8Q6pylXxHXgiA6cqnNlw4d3mR7M8J9g3ZUg,20139
-metaflow/user_configs/config_options.py,sha256=Knpiax_YGmYAdR3zKmaepN8puW1MyL9g6-eMGAkcylo,20942
-metaflow/user_configs/config_parameters.py,sha256=T0Zz18o9zKEV7mMcKotFWvXixhJpotLRBVrKx6ENErQ,15416
-metaflow-2.13.6.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.13.6.dist-info/METADATA,sha256=7bPykdKjdrtqd0D8QhWa-ItxZbzsh_JAOp4gpTvE6v4,6121
-metaflow-2.13.6.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-metaflow-2.13.6.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.13.6.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.13.6.dist-info/RECORD,,
+metaflow/user_configs/config_options.py,sha256=t6c9KNVGz9GNK55YAow74Lof4sDZqCbeeZSzldUBFmA,21072
+metaflow/user_configs/config_parameters.py,sha256=oeJGVKu1ao_YQX6Lg6P2FEv5k5-_F4sARLlVpTW9ezM,15502
+metaflow-2.13.7.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.13.7.dist-info/METADATA,sha256=Vaur5cylzQRJJ1ItdnUycx6lScVYa6cMFWbBsr0P-dY,6121
+metaflow-2.13.7.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+metaflow-2.13.7.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.13.7.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.13.7.dist-info/RECORD,,