metaflow 2.15.6__py2.py3-none-any.whl → 2.15.8__py2.py3-none-any.whl

metaflow/cli.py

@@ -17,6 +17,7 @@ from .flowspec import _FlowState
 from .graph import FlowGraph
 from .metaflow_config import (
     DEFAULT_DATASTORE,
+    DEFAULT_DECOSPECS,
     DEFAULT_ENVIRONMENT,
     DEFAULT_EVENT_LOGGER,
     DEFAULT_METADATA,
@@ -509,9 +510,16 @@ def start(
     ):
         # run/resume are special cases because they can add more decorators with --with,
         # so they have to take care of themselves.
+
         all_decospecs = ctx.obj.tl_decospecs + list(
             ctx.obj.environment.decospecs() or []
         )
+
+        # We add the default decospecs for everything except init and step since in those
+        # cases, the decospecs will already have been handled by either a run/resume
+        # or a scheduler setting them up in their own way.
+        if ctx.saved_args[0] not in ("step", "init"):
+            all_decospecs += DEFAULT_DECOSPECS.split()
         if all_decospecs:
             decorators._attach_decorators(ctx.obj.flow, all_decospecs)
             decorators._init(ctx.obj.flow)

metaflow/cli_components/run_cmds.py

@@ -71,7 +71,7 @@ def write_file(file_path, content):
             f.write(str(content))
 
 
-def config_merge_cb(ctx, param, value):
+def config_callback(ctx, param, value):
     # Callback to:
     #  - read  the Click auto_envvar variable from both the
     #    environment AND the configuration
@@ -127,7 +127,7 @@ def common_run_options(func):
         help="Add a decorator to all steps. You can specify this "
         "option multiple times to attach multiple decorators "
         "in steps.",
-        callback=config_merge_cb,
+        callback=config_callback,
     )
     @click.option(
         "--run-id-file",

metaflow/cmd/main_cli.py

@@ -94,7 +94,7 @@ def start(ctx):
         echo("(%s)\n" % version, fg="magenta", bold=False)
 
     if ctx.invoked_subcommand is None:
-        echo("More data science, less engineering\n", fg="magenta")
+        echo("More AI, less engineering\n", fg="magenta")
 
         lnk_sz = max(len(lnk) for lnk in CONTACT_INFO.values()) + 1
         for what, lnk in CONTACT_INFO.items():

metaflow/metadata_provider/metadata.py

@@ -630,6 +630,20 @@ class MetadataProvider(object):
             sys_info["r_version"] = env["r_version_code"]
         return sys_info
 
+    def _get_git_info_as_dict(self):
+        git_info = {}
+        env = self._environment.get_environment_info()
+        for key in [
+            "repo_url",
+            "branch_name",
+            "commit_sha",
+            "has_uncommitted_changes",
+        ]:
+            if key in env and env[key]:
+                git_info[key] = env[key]
+
+        return git_info
+
     def _get_system_tags(self):
         """Convert system info dictionary into a list of system tags"""
         return [
@@ -670,6 +684,27 @@ class MetadataProvider(object):
                     tags=["attempt_id:{0}".format(attempt)],
                 )
             )
+        # Add script name as metadata
+        script_name = self._environment.get_environment_info()["script"]
+        metadata.append(
+            MetaDatum(
+                field="script-name",
+                value=script_name,
+                type="script-name",
+                tags=["attempt_id:{0}".format(attempt)],
+            )
+        )
+        # And add git metadata
+        git_info = self._get_git_info_as_dict()
+        if git_info:
+            metadata.append(
+                MetaDatum(
+                    field="git-info",
+                    value=json.dumps(git_info),
+                    type="git-info",
+                    tags=["attempt_id:{0}".format(attempt)],
+                )
+            )
         if metadata:
             self.register_metadata(run_id, step_name, task_id, metadata)
 

metaflow/metaflow_config.py

@@ -109,6 +109,12 @@ S3_WORKER_COUNT = from_conf("S3_WORKER_COUNT", 64)
 # top-level retries)
 S3_TRANSIENT_RETRY_COUNT = from_conf("S3_TRANSIENT_RETRY_COUNT", 20)
 
+# S3 retry configuration used in the aws client
+# Use the adaptive retry strategy by default
+S3_CLIENT_RETRY_CONFIG = from_conf(
+    "S3_CLIENT_RETRY_CONFIG", {"max_attempts": 10, "mode": "adaptive"}
+)
+
 # Threshold to start printing warnings for an AWS retry
 RETRY_WARNING_THRESHOLD = 3
 

metaflow/metaflow_environment.py

@@ -4,6 +4,7 @@ import sys
 
 from .util import get_username
 from . import metaflow_version
+from . import metaflow_git
 from metaflow.exception import MetaflowException
 from metaflow.extension_support import dump_module_info
 from metaflow.mflog import BASH_MFLOG, BASH_FLUSH_LOGS
@@ -197,6 +198,10 @@ class MetaflowEnvironment(object):
             "python_version_code": "%d.%d.%d" % sys.version_info[:3],
             "metaflow_version": metaflow_version.get_version(),
             "script": os.path.basename(os.path.abspath(sys.argv[0])),
+            # Add git info
+            **metaflow_git.get_repository_info(
+                path=os.path.dirname(os.path.abspath(sys.argv[0]))
+            ),
         }
         if R.use_r():
             env["metaflow_r_version"] = R.metaflow_r_version()
@@ -206,7 +211,7 @@ class MetaflowEnvironment(object):
             # Information about extension modules (to load them in the proper order)
             ext_key, ext_val = dump_module_info()
             env[ext_key] = ext_val
-        return env
+        return {k: v for k, v in env.items() if v is not None and v != ""}
 
     def executable(self, step_name, default=None):
         if default is not None:

metaflow/metaflow_git.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+"""Get git repository information for the package
+
+Functions to retrieve git repository details like URL, branch name, 
+and commit SHA for Metaflow code provenance tracking.
+"""
+
+import os
+import subprocess
+from typing import Dict, List, Optional, Tuple, Union
+
+# Cache for git information to avoid repeated subprocess calls
+_git_info_cache = None
+
+__all__ = ("get_repository_info",)
+
+
+def _call_git(
+    args: List[str], path=Union[str, os.PathLike]
+) -> Tuple[Optional[str], Optional[int], bool]:
+    """
+    Call git with provided args.
+
+    Returns
+    -------
+        tuple : Tuple containing
+            (stdout, exitcode, failure) of the call
+    """
+    try:
+        result = subprocess.run(
+            ["git", *args],
+            cwd=path,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        return result.stdout.strip(), result.returncode, False
+    except (OSError, subprocess.SubprocessError):
+        # Covers subprocess timeouts and other errors which would not lead to an exit code
+        return None, None, True
+
+
+def _get_repo_url(path: Union[str, os.PathLike]) -> Optional[str]:
+    """Get the repository URL from git config"""
+    stdout, returncode, _failed = _call_git(
+        ["config", "--get", "remote.origin.url"], path
+    )
+    if returncode == 0:
+        url = stdout
+        # Convert SSH URLs to HTTPS for clickable links
+        if url.startswith("git@"):
+            parts = url.split(":", 1)
+            if len(parts) == 2:
+                domain = parts[0].replace("git@", "")
+                repo_path = parts[1]
+                url = f"https://{domain}/{repo_path}"
+        return url
+    return None
+
+
+def _get_branch_name(path: Union[str, os.PathLike]) -> Optional[str]:
+    """Get the current git branch name"""
+    stdout, returncode, _failed = _call_git(["rev-parse", "--abbrev-ref", "HEAD"], path)
+    return stdout if returncode == 0 else None
+
+
+def _get_commit_sha(path: Union[str, os.PathLike]) -> Optional[str]:
+    """Get the current git commit SHA"""
+    stdout, returncode, _failed = _call_git(["rev-parse", "HEAD"], path)
+    return stdout if returncode == 0 else None
+
+
+def _is_in_git_repo(path: Union[str, os.PathLike]) -> bool:
+    """Check if we're currently in a git repository"""
+    stdout, returncode, _failed = _call_git(
+        ["rev-parse", "--is-inside-work-tree"], path
+    )
+    return returncode == 0 and stdout == "true"
+
+
+def _has_uncommitted_changes(path: Union[str, os.PathLike]) -> Optional[bool]:
+    """Check if the git repository has uncommitted changes"""
+    _stdout, returncode, failed = _call_git(
+        ["diff-index", "--quiet", "HEAD", "--"], path
+    )
+    if failed:
+        return None
+    return returncode != 0
+
+
+def get_repository_info(path: Union[str, os.PathLike]) -> Dict[str, Union[str, bool]]:
+    """Get git repository information for a path
+
+    Returns:
+        dict: Dictionary containing:
+            repo_url: Repository URL (converted to HTTPS if from SSH)
+            branch_name: Current branch name
+            commit_sha: Current commit SHA
+            has_uncommitted_changes: Boolean indicating if there are uncommitted changes
+    """
+    global _git_info_cache
+
+    if _git_info_cache is not None:
+        return _git_info_cache
+
+    _git_info_cache = {}
+    if _is_in_git_repo(path):
+        _git_info_cache = {
+            "repo_url": _get_repo_url(path),
+            "branch_name": _get_branch_name(path),
+            "commit_sha": _get_commit_sha(path),
+            "has_uncommitted_changes": _has_uncommitted_changes(path),
+        }
+
+    return _git_info_cache

metaflow/metaflow_version.py

@@ -27,11 +27,11 @@ if name == "nt":
         """find the path to the git executable on Windows"""
         # first see if git is in the path
         try:
-            check_output(["where", "/Q", "git"])
+            subprocess.check_output(["where", "/Q", "git"])
             # if this command succeeded, git is in the path
             return "git"
         # catch the exception thrown if git was not found
-        except CalledProcessError:
+        except subprocess.CalledProcessError:
             pass
         # There are several locations where git.exe may be hiding
         possible_locations = []

metaflow/plugins/__init__.py

@@ -75,6 +75,7 @@ FLOW_DECORATORS_DESC = [
 ENVIRONMENTS_DESC = [
     ("conda", ".pypi.conda_environment.CondaEnvironment"),
     ("pypi", ".pypi.pypi_environment.PyPIEnvironment"),
+    ("uv", ".uv.uv_environment.UVEnvironment"),
 ]
 
 # Add metadata providers here

metaflow/plugins/argo/argo_workflows.py

@@ -7,6 +7,7 @@ import sys
 from collections import defaultdict
 from hashlib import sha1
 from math import inf
+from typing import List
 
 from metaflow import JSONType, current
 from metaflow.decorators import flow_decorators
@@ -110,6 +111,7 @@ class ArgoWorkflows(object):
         notify_pager_duty_integration_key=None,
         notify_incident_io_api_key=None,
         incident_io_alert_source_config_id=None,
+        incident_io_metadata: List[str] = None,
         enable_heartbeat_daemon=True,
         enable_error_msg_capture=False,
     ):
@@ -161,6 +163,9 @@ class ArgoWorkflows(object):
         self.notify_pager_duty_integration_key = notify_pager_duty_integration_key
         self.notify_incident_io_api_key = notify_incident_io_api_key
         self.incident_io_alert_source_config_id = incident_io_alert_source_config_id
+        self.incident_io_metadata = self.parse_incident_io_metadata(
+            incident_io_metadata
+        )
         self.enable_heartbeat_daemon = enable_heartbeat_daemon
         self.enable_error_msg_capture = enable_error_msg_capture
         self.parameters = self._process_parameters()
@@ -287,6 +292,21 @@ class ArgoWorkflows(object):
 
         return True
 
+    @staticmethod
+    def parse_incident_io_metadata(metadata: List[str] = None):
+        "parse key value pairs into a dict for incident.io metadata if given"
+        parsed_metadata = None
+        if metadata is not None:
+            parsed_metadata = {}
+            for kv in metadata:
+                key, value = kv.split("=", 1)
+                if key in parsed_metadata:
+                    raise MetaflowException(
+                        "Incident.io Metadata *%s* provided multiple times" % key
+                    )
+                parsed_metadata[key] = value
+        return parsed_metadata
+
     @classmethod
     def trigger(cls, name, parameters=None):
         if parameters is None:
@@ -524,6 +544,7 @@ class ArgoWorkflows(object):
                 default_value = json.dumps(default_value)
 
             parameters[param.name] = dict(
+                python_var_name=var,
                 name=param.name,
                 value=default_value,
                 type=param_type,
@@ -838,7 +859,11 @@ class ArgoWorkflows(object):
                     Arguments().parameters(
                         [
                             Parameter(parameter["name"])
-                            .value(parameter["value"])
+                            .value(
+                                "'%s'" % parameter["value"]
+                                if parameter["type"] == "JSON"
+                                else parameter["value"]
+                            )
                             .description(parameter.get("description"))
                             # TODO: Better handle IncludeFile in Argo Workflows UI.
                             for parameter in self.parameters.values()
@@ -1594,11 +1619,7 @@ class ArgoWorkflows(object):
                         # {{foo.bar['param_name']}}.
                         # https://argoproj.github.io/argo-events/tutorials/02-parameterization/
                         # http://masterminds.github.io/sprig/strings.html
-                        (
-                            "--%s='{{workflow.parameters.%s}}'"
-                            if parameter["type"] == "JSON"
-                            else "--%s={{workflow.parameters.%s}}"
-                        )
+                        "--%s={{workflow.parameters.%s}}"
                         % (parameter["name"], parameter["name"])
                         for parameter in self.parameters.values()
                     ]
@@ -2551,9 +2572,12 @@ class ArgoWorkflows(object):
                             else None
                         ),
                         "metadata": {
-                            "run_status": "failed",
-                            "flow_name": self.flow.name,
-                            "run_id": "argo-{{workflow.name}}",
+                            **(self.incident_io_metadata or {}),
+                            **{
+                                "run_status": "failed",
+                                "flow_name": self.flow.name,
+                                "run_id": "argo-{{workflow.name}}",
+                            },
                         },
                     }
                 )
@@ -2602,9 +2626,12 @@ class ArgoWorkflows(object):
                             else None
                         ),
                         "metadata": {
-                            "run_status": "succeeded",
-                            "flow_name": self.flow.name,
-                            "run_id": "argo-{{workflow.name}}",
+                            **(self.incident_io_metadata or {}),
+                            **{
+                                "run_status": "succeeded",
+                                "flow_name": self.flow.name,
+                                "run_id": "argo-{{workflow.name}}",
+                            },
                         },
                     }
                 )
@@ -3156,15 +3183,37 @@ class ArgoWorkflows(object):
                                                 # NOTE: We need the conditional logic in order to successfully fall back to the default value
                                                 # when the event payload does not contain a key for a parameter.
                                                 # NOTE: Keys might contain dashes, so use the safer 'get' for fetching the value
-                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" | toRawJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
-                                                % (v, v),
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" %s) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                % (
+                                                    v,
+                                                    v,
+                                                    (
+                                                        "| toRawJson | squote"
+                                                        if self.parameters[
+                                                            parameter_name
+                                                        ]["type"]
+                                                        == "JSON"
+                                                        else "| toRawJson"
+                                                    ),
+                                                ),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for
                                                 # the parameters - there doesn't seem
                                                 # to be any way for us to skip
-                                                value=self.parameters[parameter_name][
-                                                    "value"
-                                                ],
+                                                value=(
+                                                    json.dumps(
+                                                        self.parameters[parameter_name][
+                                                            "value"
+                                                        ]
+                                                    )
+                                                    if self.parameters[parameter_name][
+                                                        "type"
+                                                    ]
+                                                    == "JSON"
+                                                    else self.parameters[
+                                                        parameter_name
+                                                    ]["value"]
+                                                ),
                                             )
                                             .dest(
                                                 # this undocumented (mis?)feature in

metaflow/plugins/argo/argo_workflows_cli.py

@@ -187,6 +187,13 @@ def argo_workflows(obj, name=None):
     default=None,
     help="Incident.io Alert source config ID. Example '01GW2G3V0S59R238FAHPDS1R66'",
 )
+@click.option(
+    "--incident-io-metadata",
+    default=None,
+    type=str,
+    multiple=True,
+    help="Incident.io Alert Custom Metadata field in the form of Key=Value",
+)
 @click.option(
     "--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
     default=False,
@@ -226,6 +233,7 @@ def create(
     notify_pager_duty_integration_key=None,
     notify_incident_io_api_key=None,
     incident_io_alert_source_config_id=None,
+    incident_io_metadata=None,
     enable_heartbeat_daemon=True,
     deployer_attribute_file=None,
     enable_error_msg_capture=False,
@@ -283,6 +291,7 @@ def create(
         notify_pager_duty_integration_key,
         notify_incident_io_api_key,
         incident_io_alert_source_config_id,
+        incident_io_metadata,
         enable_heartbeat_daemon,
         enable_error_msg_capture,
     )
@@ -459,6 +468,7 @@ def make_flow(
     notify_pager_duty_integration_key,
     notify_incident_io_api_key,
     incident_io_alert_source_config_id,
+    incident_io_metadata,
     enable_heartbeat_daemon,
     enable_error_msg_capture,
 ):
@@ -538,6 +548,7 @@ def make_flow(
         notify_pager_duty_integration_key=notify_pager_duty_integration_key,
         notify_incident_io_api_key=notify_incident_io_api_key,
         incident_io_alert_source_config_id=incident_io_alert_source_config_id,
+        incident_io_metadata=incident_io_metadata,
         enable_heartbeat_daemon=enable_heartbeat_daemon,
         enable_error_msg_capture=enable_error_msg_capture,
     )

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -19,6 +19,7 @@ def generate_fake_flow_file_contents(
 ):
     params_code = ""
     for _, param_details in param_info.items():
+        param_python_var_name = param_details["python_var_name"]
         param_name = param_details["name"]
         param_type = param_details["type"]
         param_help = param_details["description"]
@@ -26,21 +27,21 @@ def generate_fake_flow_file_contents(
 
         if param_type == "JSON":
             params_code += (
-                f"    {param_name} = Parameter('{param_name}', "
-                f"type=JSONType, help='{param_help}', required={param_required})\n"
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type=JSONType, help='''{param_help}''', required={param_required})\n"
             )
         elif param_type == "FilePath":
             is_text = param_details.get("is_text", True)
             encoding = param_details.get("encoding", "utf-8")
             params_code += (
-                f"    {param_name} = IncludeFile('{param_name}', "
-                f"is_text={is_text}, encoding='{encoding}', help='{param_help}', "
+                f"    {param_python_var_name} = IncludeFile('{param_name}', "
+                f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
                 f"required={param_required})\n"
             )
         else:
             params_code += (
-                f"    {param_name} = Parameter('{param_name}', "
-                f"type={param_type}, help='{param_help}', required={param_required})\n"
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type={param_type}, help='''{param_help}''', required={param_required})\n"
             )
 
     project_decorator = f"@project(name='{project_name}')\n" if project_name else ""

metaflow/plugins/aws/aws_client.py

@@ -14,6 +14,7 @@ class Boto3ClientProvider(object):
             AWS_SANDBOX_ENABLED,
             AWS_SANDBOX_STS_ENDPOINT_URL,
             AWS_SANDBOX_API_KEY,
+            S3_CLIENT_RETRY_CONFIG,
         )
 
         if session_vars is None:
@@ -37,10 +38,10 @@ class Boto3ClientProvider(object):
         if module == "s3" and (
             "config" not in client_params or client_params["config"].retries is None
         ):
-            # Use the adaptive retry strategy by default -- do not set anything if
-            # the user has already set something
+            # do not set anything if the user has already set something
             config = client_params.get("config", Config())
-            config.retries = {"max_attempts": 10, "mode": "adaptive"}
+            config.retries = S3_CLIENT_RETRY_CONFIG
+            client_params["config"] = config
 
         if AWS_SANDBOX_ENABLED:
             # role is ignored in the sandbox