metaflow 2.13.9__py2.py3-none-any.whl → 2.14.0__py2.py3-none-any.whl

metaflow/cli.py

@@ -206,15 +206,15 @@ def output_raw(obj, json):
     else:
         _graph = str(obj.graph)
         _msg = "Internal representation of the flow:"
-    echo(_msg, fg="magenta", bold=False)
+    echo_always(_msg, fg="magenta", bold=False)
     echo_always(_graph, err=False)
 
 
 @cli.command(help="Visualize the flow with Graphviz.")
 @click.pass_obj
 def output_dot(obj):
-    echo("Visualizing the flow as a GraphViz graph", fg="magenta", bold=False)
-    echo(
+    echo_always("Visualizing the flow as a GraphViz graph", fg="magenta", bold=False)
+    echo_always(
         "Try piping the output to 'dot -Tpng -o graph.png' to produce "
         "an actual image.",
         indent=True,
@@ -330,7 +330,7 @@ def start(
     event_logger=None,
     monitor=None,
     local_config_file=None,
-    config_file=None,
+    config=None,
     config_value=None,
     **deco_options
 ):
@@ -383,7 +383,7 @@ def start(
     # When we process the options, the first one processed will return None and the
     # second one processed will return the actual options. The order of processing
     # depends on what (and in what order) the user specifies on the command line.
-    config_options = config_file or config_value
+    config_options = config or config_value
 
     if (
         hasattr(ctx, "saved_args")
@@ -396,7 +396,7 @@ def start(
         # if we need to in the first place
         if getattr(ctx.obj, "has_cl_config_options", False):
             raise click.UsageError(
-                "Cannot specify --config-file or --config-value with 'resume'"
+                "Cannot specify --config or --config-value with 'resume'"
             )
         # We now load the config artifacts from the original run id
         run_id = None

metaflow/cli_args.py

@@ -72,10 +72,10 @@ class CLIArgs(object):
             # keyword in Python, so we call it 'decospecs' in click args
             if k == "decospecs":
                 k = "with"
-            if k in ("config_file", "config_value"):
+            if k in ("config", "config_value"):
                 # Special handling here since we gather them all in one option but actually
                 # need to send them one at a time using --config-value <name> kv.<name>.
-                # Note it can be either config_file or config_value depending
+                # Note it can be either config or config_value depending
                 # on click processing order.
                 for config_name in v.keys():
                     yield "--config-value"

metaflow/cmd/develop/stub_generator.py

@@ -1133,8 +1133,14 @@ class StubGenerator:
             result = result[1:]
         # Add doc to first and last overloads. Jedi uses the last one and pycharm
         # the first one. Go figure.
-        result[0] = (result[0][0], docs["func_doc"])
-        result[-1] = (result[-1][0], docs["func_doc"])
+        result[0] = (
+            result[0][0],
+            docs["func_doc"] + "\nParameters\n----------\n" + docs["param_doc"],
+        )
+        result[-1] = (
+            result[-1][0],
+            docs["func_doc"] + "\nParameters\n----------\n" + docs["param_doc"],
+        )
         return result
 
     def _generate_function_stub(

metaflow/datastore/task_datastore.py

@@ -253,7 +253,7 @@ class TaskDataStore(object):
 
     @only_if_not_done
     @require_mode("w")
-    def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0):
+    def save_artifacts(self, artifacts_iter, len_hint=0):
         """
         Saves Metaflow Artifacts (Python objects) to the datastore and stores
         any relevant metadata needed to retrieve them.
@@ -269,11 +269,6 @@ class TaskDataStore(object):
         artifacts : Iterator[(string, object)]
             Iterator over the human-readable name of the object to save
             and the object itself
-        force_v4 : boolean or Dict[string -> boolean]
-            Indicates whether the artifact should be pickled using the v4
-            version of pickle. If a single boolean, applies to all artifacts.
-            If a dictionary, applies to the object named only. Defaults to False
-            if not present or not specified
         len_hint: integer
             Estimated number of items in artifacts_iter
         """
@@ -281,40 +276,24 @@ class TaskDataStore(object):
 
         def pickle_iter():
             for name, obj in artifacts_iter:
-                do_v4 = (
-                    force_v4 and force_v4
-                    if isinstance(force_v4, bool)
-                    else force_v4.get(name, False)
-                )
-                if do_v4:
-                    encode_type = "gzip+pickle-v4"
-                    if encode_type not in self._encodings:
-                        raise DataException(
-                            "Artifact *%s* requires a serialization encoding that "
-                            "requires Python 3.4 or newer." % name
-                        )
+                encode_type = "gzip+pickle-v4"
+                if encode_type in self._encodings:
                     try:
                         blob = pickle.dumps(obj, protocol=4)
                     except TypeError as e:
-                        raise UnpicklableArtifactException(name)
+                        raise UnpicklableArtifactException(name) from e
                 else:
                     try:
                         blob = pickle.dumps(obj, protocol=2)
                         encode_type = "gzip+pickle-v2"
-                    except (SystemError, OverflowError):
-                        encode_type = "gzip+pickle-v4"
-                        if encode_type not in self._encodings:
-                            raise DataException(
-                                "Artifact *%s* is very large (over 2GB). "
-                                "You need to use Python 3.4 or newer if you want to "
-                                "serialize large objects." % name
-                            )
-                        try:
-                            blob = pickle.dumps(obj, protocol=4)
-                        except TypeError as e:
-                            raise UnpicklableArtifactException(name)
+                    except (SystemError, OverflowError) as e:
+                        raise DataException(
+                            "Artifact *%s* is very large (over 2GB). "
+                            "You need to use Python 3.4 or newer if you want to "
+                            "serialize large objects." % name
+                        ) from e
                     except TypeError as e:
-                        raise UnpicklableArtifactException(name)
+                        raise UnpicklableArtifactException(name) from e
 
                 self._info[name] = {
                     "size": len(blob),

metaflow/flowspec.py

@@ -223,17 +223,18 @@ class FlowSpec(metaclass=FlowSpecMeta):
             seen.add(norm)
 
     @classmethod
-    def _process_config_decorators(cls, config_options, ignore_errors=False):
+    def _process_config_decorators(cls, config_options, process_configs=True):
 
         # Fast path for no user configurations
-        if not cls._flow_state.get(_FlowState.CONFIG_DECORATORS) and all(
-            len(step.config_decorators) == 0 for step in cls._steps
+        if not process_configs or (
+            not cls._flow_state.get(_FlowState.CONFIG_DECORATORS)
+            and all(len(step.config_decorators) == 0 for step in cls._steps)
         ):
             # Process parameters to allow them to also use config values easily
             for var, param in cls._get_parameters():
                 if param.IS_CONFIG_PARAMETER:
                     continue
-                param.init(ignore_errors)
+                param.init(not process_configs)
             return None
 
         debug.userconf_exec("Processing mutating step/flow decorators")
@@ -258,6 +259,11 @@ class FlowSpec(metaclass=FlowSpecMeta):
             debug.userconf_exec("Setting config %s to %s" % (var, str(val)))
             setattr(cls, var, val)
 
+        # Reset cached parameters since we have replaced configs already with ConfigValue
+        # so they are not parameters anymore to be re-evaluated when we do _get_parameters
+        if _FlowState.CACHED_PARAMETERS in cls._flow_state:
+            del cls._flow_state[_FlowState.CACHED_PARAMETERS]
+
         # Run all the decorators. Step decorators are directly in the step and
         # we will run those first and *then* we run all the flow level decorators
         for step in cls._steps:
@@ -277,7 +283,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
                 setattr(cls, step.name, step)
 
         mutable_flow = MutableFlow(cls)
-        for deco in cls._flow_state[_FlowState.CONFIG_DECORATORS]:
+        for deco in cls._flow_state.get(_FlowState.CONFIG_DECORATORS, []):
             if isinstance(deco, CustomFlowDecorator):
                 # Sanity check to make sure we are applying the decorator to the right
                 # class

metaflow/metaflow_environment.py

@@ -125,7 +125,9 @@ class MetaflowEnvironment(object):
             )
 
     def _get_install_dependencies_cmd(self, datastore_type):
-        base_cmd = "{} -m pip install -qqq".format(self._python())
+        base_cmd = "{} -m pip install -qqq --no-compile --no-cache-dir --disable-pip-version-check".format(
+            self._python()
+        )
 
         datastore_packages = {
             "s3": ["boto3"],

metaflow/plugins/argo/argo_workflows.py

@@ -111,6 +111,9 @@ class ArgoWorkflows(object):
         notify_on_success=False,
         notify_slack_webhook_url=None,
         notify_pager_duty_integration_key=None,
+        notify_incident_io_api_key=None,
+        incident_io_success_severity_id=None,
+        incident_io_error_severity_id=None,
         enable_heartbeat_daemon=True,
         enable_error_msg_capture=False,
     ):
@@ -160,6 +163,9 @@ class ArgoWorkflows(object):
         self.notify_on_success = notify_on_success
         self.notify_slack_webhook_url = notify_slack_webhook_url
         self.notify_pager_duty_integration_key = notify_pager_duty_integration_key
+        self.notify_incident_io_api_key = notify_incident_io_api_key
+        self.incident_io_success_severity_id = incident_io_success_severity_id
+        self.incident_io_error_severity_id = incident_io_error_severity_id
         self.enable_heartbeat_daemon = enable_heartbeat_daemon
         self.enable_error_msg_capture = enable_error_msg_capture
         self.parameters = self._process_parameters()
@@ -891,6 +897,17 @@ class ArgoWorkflows(object):
                             and self.notify_pager_duty_integration_key
                             else {}
                         ),
+                        **(
+                            {
+                                # workflow status maps to Completed
+                                "notify-incident-io-on-success": LifecycleHook()
+                                .expression("workflow.status == 'Succeeded'")
+                                .template("notify-incident-io-on-success"),
+                            }
+                            if self.notify_on_success
+                            and self.notify_incident_io_api_key
+                            else {}
+                        ),
                         **(
                             {
                                 # workflow status maps to Failed or Error
@@ -918,6 +935,19 @@ class ArgoWorkflows(object):
                             and self.notify_pager_duty_integration_key
                             else {}
                         ),
+                        **(
+                            {
+                                # workflow status maps to Failed or Error
+                                "notify-incident-io-on-failure": LifecycleHook()
+                                .expression("workflow.status == 'Failed'")
+                                .template("notify-incident-io-on-error"),
+                                "notify-incident-io-on-error": LifecycleHook()
+                                .expression("workflow.status == 'Error'")
+                                .template("notify-incident-io-on-error"),
+                            }
+                            if self.notify_on_error and self.notify_incident_io_api_key
+                            else {}
+                        ),
                         # Warning: terrible hack to workaround a bug in Argo Workflow
                         #          where the hooks listed above do not execute unless
                         #          there is an explicit exit hook. as and when this
@@ -2270,9 +2300,11 @@ class ArgoWorkflows(object):
         if self.notify_on_error:
             templates.append(self._slack_error_template())
             templates.append(self._pager_duty_alert_template())
+            templates.append(self._incident_io_alert_template())
         if self.notify_on_success:
             templates.append(self._slack_success_template())
             templates.append(self._pager_duty_change_template())
+            templates.append(self._incident_io_change_template())
         if self.notify_on_error or self.notify_on_success:
             # Warning: terrible hack to workaround a bug in Argo Workflow where the
             #          templates listed above do not execute unless there is an
@@ -2466,6 +2498,82 @@ class ArgoWorkflows(object):
             )
         )
 
+    def _incident_io_alert_template(self):
+        if self.notify_incident_io_api_key is None:
+            return None
+        if self.incident_io_error_severity_id is None:
+            raise MetaflowException(
+                "Creating incidents for errors requires a severity id."
+            )
+        return Template("notify-incident-io-on-error").http(
+            Http("POST")
+            .url("https://api.incident.io/v2/incidents")
+            .header("Content-Type", "application/json")
+            .header("Authorization", "Bearer %s" % self.notify_incident_io_api_key)
+            .body(
+                json.dumps(
+                    {
+                        "idempotency_key": "argo-{{workflow.name}}",  # use run id to deduplicate alerts.
+                        "visibility": "public",
+                        "severity_id": self.incident_io_error_severity_id,
+                        "name": "Flow %s has failed." % self.flow.name,
+                        "summary": "Metaflow run %s/argo-{{workflow.name}} failed! %s"
+                        % (self.flow.name, self._incident_io_ui_urls_for_run()),
+                        # TODO: Add support for custom field entries.
+                    }
+                )
+            )
+        )
+
+    def _incident_io_change_template(self):
+        if self.notify_incident_io_api_key is None:
+            return None
+        if self.incident_io_success_severity_id is None:
+            raise MetaflowException(
+                "Creating incidents for successes requires a severity id."
+            )
+        return Template("notify-incident-io-on-success").http(
+            Http("POST")
+            .url("https://api.incident.io/v2/incidents")
+            .header("Content-Type", "application/json")
+            .header("Authorization", "Bearer %s" % self.notify_incident_io_api_key)
+            .body(
+                json.dumps(
+                    {
+                        "idempotency_key": "argo-{{workflow.name}}",  # use run id to deduplicate alerts.
+                        "visibility": "public",
+                        "severity_id": self.incident_io_success_severity_id,
+                        # TODO: Do we need to make incident type configurable for successes? otherwise they are created as 'investigating'
+                        # "incident_type_id": ""
+                        "name": "Flow %s has succeeded." % self.flow.name,
+                        "summary": "Metaflow run %s/argo-{{workflow.name}} succeeded!%s"
+                        % (self.flow.name, self._incident_io_ui_urls_for_run()),
+                        # TODO: Add support for custom field entries.
+                    }
+                )
+            )
+        )
+
+    def _incident_io_ui_urls_for_run(self):
+        links = []
+        if UI_URL:
+            url = "[Metaflow UI](%s/%s/%s)" % (
+                UI_URL.rstrip("/"),
+                self.flow.name,
+                "argo-{{workflow.name}}",
+            )
+            links.append(url)
+        if ARGO_WORKFLOWS_UI_URL:
+            url = "[Argo UI](%s/workflows/%s/%s)" % (
+                ARGO_WORKFLOWS_UI_URL.rstrip("/"),
+                "{{workflow.namespace}}",
+                "{{workflow.name}}",
+            )
+            links.append(url)
+        if links:
+            links = ["See details for the run at: ", *links]
+        return "\n\n".join(links)
+
     def _pager_duty_change_template(self):
         # https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgy-send-a-change-event
         if self.notify_pager_duty_integration_key is None:

metaflow/plugins/argo/argo_workflows_cli.py

@@ -176,6 +176,21 @@ def argo_workflows(obj, name=None):
     default="",
     help="PagerDuty Events API V2 Integration key for workflow success/failure notifications.",
 )
+@click.option(
+    "--notify-incident-io-api-key",
+    default=None,
+    help="Incident.io API V2 key for workflow success/failure notifications.",
+)
+@click.option(
+    "--incident-io-success-severity-id",
+    default=None,
+    help="Incident.io severity id for success alerts.",
+)
+@click.option(
+    "--incident-io-error-severity-id",
+    default=None,
+    help="Incident.io severity id for error alerts.",
+)
 @click.option(
     "--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
     default=False,
@@ -213,6 +228,9 @@ def create(
     notify_on_success=False,
     notify_slack_webhook_url=None,
     notify_pager_duty_integration_key=None,
+    notify_incident_io_api_key=None,
+    incident_io_success_severity_id=None,
+    incident_io_error_severity_id=None,
     enable_heartbeat_daemon=True,
     deployer_attribute_file=None,
     enable_error_msg_capture=False,
@@ -268,6 +286,9 @@ def create(
         notify_on_success,
         notify_slack_webhook_url,
         notify_pager_duty_integration_key,
+        notify_incident_io_api_key,
+        incident_io_success_severity_id,
+        incident_io_error_severity_id,
         enable_heartbeat_daemon,
         enable_error_msg_capture,
     )
@@ -442,6 +463,9 @@ def make_flow(
     notify_on_success,
     notify_slack_webhook_url,
     notify_pager_duty_integration_key,
+    notify_incident_io_api_key,
+    incident_io_success_severity_id,
+    incident_io_error_severity_id,
     enable_heartbeat_daemon,
     enable_error_msg_capture,
 ):
@@ -453,17 +477,30 @@ def make_flow(
         )
 
     if (notify_on_error or notify_on_success) and not (
-        notify_slack_webhook_url or notify_pager_duty_integration_key
+        notify_slack_webhook_url
+        or notify_pager_duty_integration_key
+        or notify_incident_io_api_key
     ):
         raise MetaflowException(
-            "Notifications require specifying an incoming Slack webhook url via --notify-slack-webhook-url or "
-            "PagerDuty events v2 integration key via --notify-pager-duty-integration-key.\n If you would like to set up "
-            "notifications for your Slack workspace, follow the instructions at "
-            "https://api.slack.com/messaging/webhooks to generate a webhook url.\n For notifications through PagerDuty, "
-            "generate an integration key by following the instructions at "
-            "https://support.pagerduty.com/docs/services-and-integrations#create-a-generic-events-api-integration"
+            "Notifications require specifying an incoming Slack webhook url via --notify-slack-webhook-url, PagerDuty events v2 integration key via --notify-pager-duty-integration-key or\n"
+            "Incident.io integration API key via --notify-incident-io-api-key.\n"
+            " If you would like to set up notifications for your Slack workspace, follow the instructions at "
+            "https://api.slack.com/messaging/webhooks to generate a webhook url.\n"
+            " For notifications through PagerDuty, generate an integration key by following the instructions at "
+            "https://support.pagerduty.com/docs/services-and-integrations#create-a-generic-events-api-integration\n"
+            " For notifications through Incident.io, generate an API key with a permission to create incidents."
         )
 
+    if notify_incident_io_api_key:
+        if notify_on_error and incident_io_error_severity_id is None:
+            raise MetaflowException(
+                "Incident.io error notifications require a severity id. Please set one with --incident-io-error-severity-id"
+            )
+
+        if notify_on_success and incident_io_success_severity_id is None:
+            raise MetaflowException(
+                "Incident.io success notifications require a severity id. Please set one with --incident-io-success-severity-id"
+            )
     # Attach @kubernetes and @environment decorator to the flow to
     # ensure that the related decorator hooks are invoked.
     decorators._attach_decorators(
@@ -507,6 +544,9 @@ def make_flow(
         notify_on_success=notify_on_success,
         notify_slack_webhook_url=notify_slack_webhook_url,
         notify_pager_duty_integration_key=notify_pager_duty_integration_key,
+        notify_incident_io_api_key=notify_incident_io_api_key,
+        incident_io_success_severity_id=incident_io_success_severity_id,
+        incident_io_error_severity_id=incident_io_error_severity_id,
         enable_heartbeat_daemon=enable_heartbeat_daemon,
         enable_error_msg_capture=enable_error_msg_capture,
     )

metaflow/plugins/metadata_providers/service.py

@@ -1,23 +1,18 @@
 import os
 import random
+import time
 
 import requests
-import time
 
 from metaflow.exception import (
     MetaflowException,
-    MetaflowTaggingError,
     MetaflowInternalError,
-)
-from metaflow.metaflow_config import (
-    SERVICE_RETRY_COUNT,
-    SERVICE_HEADERS,
-    SERVICE_URL,
+    MetaflowTaggingError,
 )
 from metaflow.metadata_provider import MetadataProvider
 from metaflow.metadata_provider.heartbeat import HB_URL_KEY
+from metaflow.metaflow_config import SERVICE_HEADERS, SERVICE_RETRY_COUNT, SERVICE_URL
 from metaflow.sidecar import Message, MessageTypes, Sidecar
-
 from metaflow.util import version_parse
 
 
@@ -39,6 +34,23 @@ class ServiceException(MetaflowException):
 class ServiceMetadataProvider(MetadataProvider):
     TYPE = "service"
 
+    _session = requests.Session()
+    _session.mount(
+        "http://",
+        requests.adapters.HTTPAdapter(
+            pool_connections=20,
+            pool_maxsize=20,
+            max_retries=0,  # Handle retries explicitly
+            pool_block=False,
+        ),
+    )
+    _session.mount(
+        "https://",
+        requests.adapters.HTTPAdapter(
+            pool_connections=20, pool_maxsize=20, max_retries=0, pool_block=False
+        ),
+    )
+
     _supports_attempt_gets = None
     _supports_tag_mutation = None
 
@@ -59,7 +71,9 @@ class ServiceMetadataProvider(MetadataProvider):
     def compute_info(cls, val):
         v = val.rstrip("/")
         try:
-            resp = requests.get(os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy())
+            resp = cls._session.get(
+                os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
+            )
             resp.raise_for_status()
         except:  # noqa E722
             raise ValueError("Metaflow service [%s] unreachable." % v)
@@ -412,27 +426,27 @@ class ServiceMetadataProvider(MetadataProvider):
                 if method == "GET":
                     if monitor:
                         with monitor.measure("metaflow.service_metadata.get"):
-                            resp = requests.get(url, headers=SERVICE_HEADERS.copy())
+                            resp = cls._session.get(url, headers=SERVICE_HEADERS.copy())
                     else:
-                        resp = requests.get(url, headers=SERVICE_HEADERS.copy())
+                        resp = cls._session.get(url, headers=SERVICE_HEADERS.copy())
                 elif method == "POST":
                     if monitor:
                         with monitor.measure("metaflow.service_metadata.post"):
-                            resp = requests.post(
+                            resp = cls._session.post(
                                 url, headers=SERVICE_HEADERS.copy(), json=data
                             )
                     else:
-                        resp = requests.post(
+                        resp = cls._session.post(
                             url, headers=SERVICE_HEADERS.copy(), json=data
                         )
                 elif method == "PATCH":
                     if monitor:
                         with monitor.measure("metaflow.service_metadata.patch"):
-                            resp = requests.patch(
+                            resp = cls._session.patch(
                                 url, headers=SERVICE_HEADERS.copy(), json=data
                             )
                     else:
-                        resp = requests.patch(
+                        resp = cls._session.patch(
                             url, headers=SERVICE_HEADERS.copy(), json=data
                         )
                 else:
@@ -475,7 +489,6 @@ class ServiceMetadataProvider(MetadataProvider):
                         resp.text,
                     )
             time.sleep(2**i)
-
         if resp:
             raise ServiceException(
                 "Metadata request (%s) failed (code %s): %s"
@@ -499,9 +512,9 @@ class ServiceMetadataProvider(MetadataProvider):
             try:
                 if monitor:
                     with monitor.measure("metaflow.service_metadata.get"):
-                        resp = requests.get(url, headers=SERVICE_HEADERS.copy())
+                        resp = cls._session.get(url, headers=SERVICE_HEADERS.copy())
                 else:
-                    resp = requests.get(url, headers=SERVICE_HEADERS.copy())
+                    resp = cls._session.get(url, headers=SERVICE_HEADERS.copy())
             except:
                 if monitor:
                     with monitor.count("metaflow.service_metadata.failed_request"):

metaflow/plugins/project_decorator.py

@@ -26,6 +26,24 @@ class ProjectDecorator(FlowDecorator):
         projects that use the same production scheduler. The name may
         contain only lowercase alphanumeric characters and underscores.
 
+    branch : Optional[str], default None
+        The branch to use. If not specified, the branch is set to
+        `user.<username>` unless `production` is set to `True`. This can
+        also be set on the command line using `--branch` as a top-level option.
+        It is an error to specify `branch` in the decorator and on the command line.
+
+    production : bool, default False
+        Whether or not the branch is the production branch. This can also be set on the
+        command line using `--production` as a top-level option. It is an error to specify
+        `production` in the decorator and on the command line.
+        The project branch name will be:
+          - if `branch` is specified:
+            - if `production` is True: `prod.<branch>`
+            - if `production` is False: `test.<branch>`
+          - if `branch` is not specified:
+            - if `production` is True: `prod`
+            - if `production` is False: `user.<username>`
+
     MF Add To Current
     -----------------
     project_name -> str

metaflow/runner/click_api.py

@@ -41,6 +41,7 @@ from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
 from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
+from metaflow.user_configs.config_decorators import CustomFlowDecorator
 from metaflow.user_configs.config_options import (
     ConfigValue,
     ConvertDictOrStr,
@@ -252,10 +253,14 @@ def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
             # Cache the loaded module
             loaded_modules[flow_file] = module
 
-        classes = inspect.getmembers(module, inspect.isclass)
+        classes = inspect.getmembers(
+            module, lambda x: inspect.isclass(x) or isinstance(x, CustomFlowDecorator)
+        )
         flow_cls = None
 
         for _, kls in classes:
+            if isinstance(kls, CustomFlowDecorator):
+                kls = kls._flow_cls
             if (
                 kls is not FlowSpec
                 and kls.__module__ == module_name
@@ -444,10 +449,10 @@ class MetaflowAPI(object):
                 ds = opts.get("datastore", defaults["datastore"])
                 quiet = opts.get("quiet", defaults["quiet"])
                 is_default = False
-                config_file = opts.get("config-file")
+                config_file = opts.get("config")
                 if config_file is None:
                     is_default = True
-                    config_file = defaults.get("config_file")
+                    config_file = defaults.get("config")
 
                 if config_file:
                     config_file = map(
@@ -480,7 +485,7 @@ class MetaflowAPI(object):
                     # Process both configurations; the second one will return all the merged
                     # configuration options properly processed.
                     self._config_input.process_configs(
-                        self._flow_cls.__name__, "config_file", config_file, quiet, ds
+                        self._flow_cls.__name__, "config", config_file, quiet, ds
                     )
                     config_options = self._config_input.process_configs(
                         self._flow_cls.__name__, "config_value", config_value, quiet, ds
@@ -493,7 +498,7 @@ class MetaflowAPI(object):
         # it will init all parameters (config_options will be None)
         # We ignore any errors if we don't check the configs in the click API.
         new_cls = self._flow_cls._process_config_decorators(
-            config_options, ignore_errors=not CLICK_API_PROCESS_CONFIG
+            config_options, process_configs=CLICK_API_PROCESS_CONFIG
         )
         if new_cls:
             self._flow_cls = new_cls

metaflow/user_configs/config_decorators.py

@@ -406,19 +406,6 @@ class CustomFlowDecorator:
             self._args = args
             self._kwargs = kwargs
 
-    def __get__(self, instance, owner):
-        # Required so that we "present" as a FlowSpec when the flow decorator is
-        # of the form
-        # @MyFlowDecorator
-        # class MyFlow(FlowSpec):
-        #     pass
-        #
-        # In that case, if we don't have __get__, the object is a CustomFlowDecorator
-        # and not a FlowSpec. This is more critical for steps (and CustomStepDecorator)
-        # because other parts of the code rely on steps having is_step. There are
-        # other ways to solve this but this allowed for minimal changes going forward.
-        return self()
-
     def __call__(
         self, flow_spec: Optional["metaflow.flowspec.FlowSpecMeta"] = None
     ) -> "metaflow.flowspec.FlowSpecMeta":
@@ -447,6 +434,15 @@ class CustomFlowDecorator:
             raise MetaflowException(
                 "A CustomFlowDecorator can only be applied to a FlowSpec"
             )
+        # NOTA: This returns self._flow_cls() because the object in the case of
+        # @FlowDecorator
+        # class MyFlow(FlowSpec):
+        #     pass
+        # the object is a FlowDecorator and when the main function calls it, we end up
+        # here and need to actually call the FlowSpec. This is not the case when using
+        # a decorator with arguments because in the line above, we will have returned a
+        # FlowSpec object. Previous solution was to use __get__ but this does not seem
+        # to work properly.
         return self._flow_cls()
 
     def _set_flow_cls(
@@ -500,7 +496,16 @@ class CustomStepDecorator:
             self._kwargs = kwargs
 
     def __get__(self, instance, owner):
-        # See explanation in CustomFlowDecorator.__get__
+        # Required so that we "present" as a step when the step decorator is
+        # of the form
+        # @MyStepDecorator
+        # @step
+        # def my_step(self):
+        #     pass
+        #
+        # In that case, if we don't have __get__, the object is a CustomStepDecorator
+        # and not a step. Other parts of the code rely on steps having is_step. There are
+        # other ways to solve this but this allowed for minimal changes going forward.
         return self()
 
     def __call__(

metaflow/user_configs/config_options.py

@@ -521,7 +521,7 @@ def config_options_with_config_input(cmd):
     cmd.params.insert(
         0,
         click.Option(
-            ["--config", "config_file"],
+            ["--config", "config"],
             nargs=2,
             multiple=True,
             type=MultipleTuple([click.Choice(config_seen), ConvertPath()]),

metaflow/util.py

@@ -307,10 +307,10 @@ def dict_to_cli_options(params):
             # keyword in Python, so we call it 'decospecs' in click args
             if k == "decospecs":
                 k = "with"
-            if k in ("config_file", "config_value"):
+            if k in ("config", "config_value"):
                 # Special handling here since we gather them all in one option but actually
                 # need to send them one at a time using --config-value <name> kv.<name>
-                # Note it can be either config_file or config_value depending
+                # Note it can be either config or config_value depending
                 # on click processing order.
                 for config_name in v.keys():
                     yield "--config-value"

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.13.9"
+metaflow_version = "2.14.0"

{metaflow-2.13.9.dist-info → metaflow-2.14.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: metaflow
-Version: 2.13.9
+Version: 2.14.0
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.9; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.14.0; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.13.9.dist-info → metaflow-2.14.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
 metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
 metaflow/__init__.py,sha256=fbhdWiWnEoAX4KnzRHMY_iQcT-uYlMWhzrXPKvK0i5g,5832
 metaflow/cards.py,sha256=IbRmredvmFEU0V6JL7DR8wCESwVmmZJubr6x24bo7U4,442
-metaflow/cli.py,sha256=zTXaVpfAa0NBvAbBe7trS1kMRkzOIdt7_Vi1q6J1FGQ,21877
-metaflow/cli_args.py,sha256=muIh9pdVqMRG09uAYFKcAcUKFyDE4N3Wm6YahWRaUNI,3594
+metaflow/cli.py,sha256=ASW0FwUa4QjF7qQ7XmUTiDrA9d5WMRS3VggSrpG5OLQ,21883
+metaflow/cli_args.py,sha256=hDsdWdRmfXYifVGq6b6FDfgoWxtIG2nr_lU6EBV0Pnk,3584
 metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
 metaflow/debug.py,sha256=HEmt_16tJtqHXQXsqD9pqOFe3CWR5GZ7VwpaYQgnRdU,1466
@@ -10,7 +10,7 @@ metaflow/decorators.py,sha256=cbOCahmwVlnHklMN2O_j5DKvZA7m_Q72_6LBzzBZRhk,24131
 metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
 metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=_m9ZBJM0cooHRslDqfxCPQmkChqaTh6fGxp7HvISnYI,5161
-metaflow/flowspec.py,sha256=XxgE0sJ7cpnkoIRNvuN8Z3V2ob1VHzEE8XSDph_GUCg,35597
+metaflow/flowspec.py,sha256=GgbTeUBtG3AmZwIF-prRFMsZqFYGVysd5xBS9IPIPBs,35953
 metaflow/graph.py,sha256=cdpnWr85aEj_rRn-7EjbndWjr_i8Dt3P7-oPUW0NNpI,12393
 metaflow/includefile.py,sha256=kWKDSlzVcRVNGG9PV5eB3o2ynrzqhVsfaLtkqjshn7Q,20948
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
@@ -19,7 +19,7 @@ metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
 metaflow/metaflow_config.py,sha256=Uu5GmiVLlAu1-VT6MTprnQ_kPC5WrRS_C2UJ-qgNi9I,23477
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
 metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
-metaflow/metaflow_environment.py,sha256=rojFyGdyY56sN1HaEb1-0XX53Q3XPNnl0SaH-8xXZ8w,7987
+metaflow/metaflow_environment.py,sha256=gs_UpYpuOKEEfFM0z0tnwje7zAVsQ5Ck7Dp2M9_1utQ,8065
 metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
 metaflow/metaflow_version.py,sha256=duhIzfKZtcxMVMs2uiBqBvUarSHJqyWDwMhaBOQd_g0,7491
 metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
@@ -34,9 +34,9 @@ metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
 metaflow/task.py,sha256=xVVLWy8NH16OlLu2VoOb1OfiFzcOVVCdQldlmb1Zb_w,29691
 metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
-metaflow/util.py,sha256=hKjHl6NYJkKBSU2tzdVbddfOX1zWK73T4GCO42A0XB4,14666
+metaflow/util.py,sha256=mJBkV5tShIyCsLDeM1zygQGeciQVMrVPm_qI8Oi33G0,14656
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=xeHffcr9LL_YPy5rUqIu5yORGDOAVqaxh4ieE2_npAY,28
+metaflow/version.py,sha256=EaSK29SvOJFXVcYk1a_Ie2OHqYgRDs6H9YMj74CUnmU,28
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -125,7 +125,7 @@ metaflow/cmd/main_cli.py,sha256=E546zT_jYQKysmjwfpEgzZd5QMsyirs28M2s0OPU93E,2966
 metaflow/cmd/tutorials_cmd.py,sha256=8FdlKkicTOhCIDKcBR5b0Oz6giDvS-EMY3o9skIrRqw,5156
 metaflow/cmd/util.py,sha256=jS_0rUjOnGGzPT65fzRLdGjrYAOOLA4jU2S0HJLV0oc,406
 metaflow/cmd/develop/__init__.py,sha256=p1Sy8yU1MEKSrH5ttOWOZvNcI1qYu6J6jghdTHwPgOw,689
-metaflow/cmd/develop/stub_generator.py,sha256=7xm36_sli0nlOwnOD4pPCIbqunaj1yrC_BS4YAD4zj0,64992
+metaflow/cmd/develop/stub_generator.py,sha256=vhhEAPkHN-9wZYW4rqnMjVsjwqm4Avn_inVx-wC6DKE,65164
 metaflow/cmd/develop/stubs.py,sha256=JX2qNZDvG0upvPueAcLhoR_zyLtRranZMwY05tLdpRQ,11884
 metaflow/datastore/__init__.py,sha256=VxP6ddJt3rwiCkpiSfAhyVkUCOe1pgZZsytVEJzFmSQ,155
 metaflow/datastore/content_addressed_store.py,sha256=6T7tNqL29kpmecyMLHF35RhoSBOb-OZcExnsB65AvnI,7641
@@ -134,7 +134,7 @@ metaflow/datastore/datastore_storage.py,sha256=7V43QuiWDQ_Q4oHw9y7Z7X9lYj3GI-LV1
 metaflow/datastore/exceptions.py,sha256=r7Ab5FvHIzyFh6kwiptA1lO5nLqWg0xRBoeYGefvapA,373
 metaflow/datastore/flow_datastore.py,sha256=rDMEHdYwub1PwLp2uaK-8CHdd8hiwxqeELXzsUfuqZs,10250
 metaflow/datastore/inputs.py,sha256=i43dXr2xvgtsgKMO9allgCR18bk80GeayeQFyUTH36w,449
-metaflow/datastore/task_datastore.py,sha256=oIlA4COEghSNVPqsw9BA7og4nyiZy7375BBRQ09Sw4E,36176
+metaflow/datastore/task_datastore.py,sha256=VCV-5v3gYbzhmrBr9zcrrzQWJ-L7N-ZbRGbAiVE2aH4,35060
 metaflow/extension_support/__init__.py,sha256=2z0c4R8zsVmEFOMGT2Jujsl6xveDVa9KLll7moL58NE,52984
 metaflow/extension_support/_empty_file.py,sha256=HENjnM4uAfeNygxMB_feCCWORFoSat9n_QwzSx2oXPw,109
 metaflow/extension_support/cmd.py,sha256=hk8iBUUINqvKCDxInKgWpum8ThiRZtHSJP7qBASHzl8,5711
@@ -158,7 +158,7 @@ metaflow/plugins/events_decorator.py,sha256=T_YSK-DlgZhd3ge9PlpTRNaMi15GK0tKZMZl
 metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
 metaflow/plugins/package_cli.py,sha256=-J6D4cupHfWSZ4GEFo2yy9Je9oL3owRWm5pEJwaiqd4,1649
 metaflow/plugins/parallel_decorator.py,sha256=GR6LKIW7_S7AoU50Ar2_0nndVtO2epdn3LuthE0vKMQ,9127
-metaflow/plugins/project_decorator.py,sha256=_hMh1-wEGQxckBYw3020auPccGkIaCXvqdT8kGrEJLU,6630
+metaflow/plugins/project_decorator.py,sha256=uhwsguEj7OM_E2OnY1ap3MoGocQHeywuJSa-qPuWn-U,7592
 metaflow/plugins/resources_decorator.py,sha256=AtoOwg4mHYHYthg-CAfbfam-QiT0ViuDLDoukoDvF6Q,1347
 metaflow/plugins/retry_decorator.py,sha256=tz_2Tq6GLg3vjDBZp0KKVTk3ADlCvqaWTSf7blmFdUw,1548
 metaflow/plugins/storage_executor.py,sha256=FqAgR0-L9MuqN8fRtTe4jjUfJL9lqt6fQkYaglAjRbk,6137
@@ -181,8 +181,8 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=PS_cYGnPw9h4X7TP_plObDH3clMw4reOsBLkkGPTd0Y,16282
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=72zuqG23uBHDhoCJl52jHmbLE_w1lvZV5EnrkVggBJg,176389
-metaflow/plugins/argo/argo_workflows_cli.py,sha256=11_8l4IrtkwviKsijInTZPt7YK5TZzClREnw_Cf4D5o,36706
+metaflow/plugins/argo/argo_workflows.py,sha256=gAzznOrNQT8SQ2zDqo__lEd1ye5gYqaResrlzzCgKY4,181585
+metaflow/plugins/argo/argo_workflows_cli.py,sha256=kzZF2XRr5If9t135wMXXDTpIRupnWCK7mtDpvZ6K8P0,38424
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
 metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=bs0E8WJGQYXuwi6u0OiwTn_jkfeKb5DywCmuJHeRl8I,13949
@@ -298,7 +298,7 @@ metaflow/plugins/kubernetes/spot_metadata_cli.py,sha256=an0nWCxgflmqIPBCBrlb4m3D
 metaflow/plugins/kubernetes/spot_monitor_sidecar.py,sha256=zrWU-smQwPnL6MBHmzTxWyEA00R6iKKQbhhy50xFwQ8,3832
 metaflow/plugins/metadata_providers/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/plugins/metadata_providers/local.py,sha256=9UAxe9caN6kU1lkSlIoJbRGgTqsMa62cBTnyMwhqiaA,22446
-metaflow/plugins/metadata_providers/service.py,sha256=NKZfFMamx6upP6aFRJfXlfYIhySgFNzz6kbp1yPD7LA,20222
+metaflow/plugins/metadata_providers/service.py,sha256=XhXarANcgkQ4D5q8qhywvd8BjPN274DLGLVx1KtxWF0,20725
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=oRfJkAp99R338nYX2wq4FrV0Ax5h4QqqifNmAXrM3CY,14892
 metaflow/plugins/pypi/conda_decorator.py,sha256=piFcE4uGmWhhbGlxMK0GHd7BGEyqy6r9BFy8Mjoi80Q,15937
@@ -312,7 +312,7 @@ metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcw
 metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpLZDBybXLergiDlN63E0u3x8,294
 metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
 metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/runner/click_api.py,sha256=hCwIxeehI4XtgIrQRjr10k6oFmTdx_60lih5fXUc6wU,22894
+metaflow/runner/click_api.py,sha256=truFw2mbtnEKWVhfY5haIxJEyyoB6-cipoY5-ZWXFXo,23111
 metaflow/runner/deployer.py,sha256=Yas_SZCss3kfJw3hLC8_IyzgiytUFGoEGHz-l-rBBKk,8980
 metaflow/runner/deployer_impl.py,sha256=Kab9rLoA3EiBJDtTTulhPCeKzqiljW366nx2Tm0LYy0,6143
 metaflow/runner/metaflow_runner.py,sha256=L302ew_BPBPs-NnW8n92dqqbqmHwrwGL5D6kTZvl5vY,16074
@@ -357,12 +357,12 @@ metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuV
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
 metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/user_configs/config_decorators.py,sha256=Tj0H88UT8Q6pylXxHXgiA6cqnNlw4d3mR7M8J9g3ZUg,20139
-metaflow/user_configs/config_options.py,sha256=t6c9KNVGz9GNK55YAow74Lof4sDZqCbeeZSzldUBFmA,21072
+metaflow/user_configs/config_decorators.py,sha256=qCKVAvd0NKgaCxQ2OThes5-DYHXq6A1HqURubYNeFdw,20481
+metaflow/user_configs/config_options.py,sha256=m6jccSpzI4qUJ7vyYkYBIf8G3V0Caunxg_k7zg4Zlqg,21067
 metaflow/user_configs/config_parameters.py,sha256=oeJGVKu1ao_YQX6Lg6P2FEv5k5-_F4sARLlVpTW9ezM,15502
-metaflow-2.13.9.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.13.9.dist-info/METADATA,sha256=7wOsTSpzHy1tnEuiaSz8qvAk4A5vE_zd4m98Hk-rOZI,6121
-metaflow-2.13.9.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-metaflow-2.13.9.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.13.9.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.13.9.dist-info/RECORD,,
+metaflow-2.14.0.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.14.0.dist-info/METADATA,sha256=qxM8LtLppUZz43NgvqAQ8H2D0pRnlYoTwkGfwEdYXow,6121
+metaflow-2.14.0.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+metaflow-2.14.0.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.14.0.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.14.0.dist-info/RECORD,,