metaflow 2.15.4__py2.py3-none-any.whl → 2.15.6__py2.py3-none-any.whl

metaflow/_vendor/v3_7/zipp.py

@@ -0,0 +1,329 @@
+import io
+import posixpath
+import zipfile
+import itertools
+import contextlib
+import sys
+import pathlib
+
+if sys.version_info < (3, 7):
+    from collections import OrderedDict
+else:
+    OrderedDict = dict
+
+
+__all__ = ['Path']
+
+
+def _parents(path):
+    """
+    Given a path with elements separated by
+    posixpath.sep, generate all parents of that path.
+
+    >>> list(_parents('b/d'))
+    ['b']
+    >>> list(_parents('/b/d/'))
+    ['/b']
+    >>> list(_parents('b/d/f/'))
+    ['b/d', 'b']
+    >>> list(_parents('b'))
+    []
+    >>> list(_parents(''))
+    []
+    """
+    return itertools.islice(_ancestry(path), 1, None)
+
+
+def _ancestry(path):
+    """
+    Given a path with elements separated by
+    posixpath.sep, generate all elements of that path
+
+    >>> list(_ancestry('b/d'))
+    ['b/d', 'b']
+    >>> list(_ancestry('/b/d/'))
+    ['/b/d', '/b']
+    >>> list(_ancestry('b/d/f/'))
+    ['b/d/f', 'b/d', 'b']
+    >>> list(_ancestry('b'))
+    ['b']
+    >>> list(_ancestry(''))
+    []
+    """
+    path = path.rstrip(posixpath.sep)
+    while path and path != posixpath.sep:
+        yield path
+        path, tail = posixpath.split(path)
+
+
+_dedupe = OrderedDict.fromkeys
+"""Deduplicate an iterable in original order"""
+
+
+def _difference(minuend, subtrahend):
+    """
+    Return items in minuend not in subtrahend, retaining order
+    with O(1) lookup.
+    """
+    return itertools.filterfalse(set(subtrahend).__contains__, minuend)
+
+
+class CompleteDirs(zipfile.ZipFile):
+    """
+    A ZipFile subclass that ensures that implied directories
+    are always included in the namelist.
+    """
+
+    @staticmethod
+    def _implied_dirs(names):
+        parents = itertools.chain.from_iterable(map(_parents, names))
+        as_dirs = (p + posixpath.sep for p in parents)
+        return _dedupe(_difference(as_dirs, names))
+
+    def namelist(self):
+        names = super(CompleteDirs, self).namelist()
+        return names + list(self._implied_dirs(names))
+
+    def _name_set(self):
+        return set(self.namelist())
+
+    def resolve_dir(self, name):
+        """
+        If the name represents a directory, return that name
+        as a directory (with the trailing slash).
+        """
+        names = self._name_set()
+        dirname = name + '/'
+        dir_match = name not in names and dirname in names
+        return dirname if dir_match else name
+
+    @classmethod
+    def make(cls, source):
+        """
+        Given a source (filename or zipfile), return an
+        appropriate CompleteDirs subclass.
+        """
+        if isinstance(source, CompleteDirs):
+            return source
+
+        if not isinstance(source, zipfile.ZipFile):
+            return cls(_pathlib_compat(source))
+
+        # Only allow for FastLookup when supplied zipfile is read-only
+        if 'r' not in source.mode:
+            cls = CompleteDirs
+
+        source.__class__ = cls
+        return source
+
+
+class FastLookup(CompleteDirs):
+    """
+    ZipFile subclass to ensure implicit
+    dirs exist and are resolved rapidly.
+    """
+
+    def namelist(self):
+        with contextlib.suppress(AttributeError):
+            return self.__names
+        self.__names = super(FastLookup, self).namelist()
+        return self.__names
+
+    def _name_set(self):
+        with contextlib.suppress(AttributeError):
+            return self.__lookup
+        self.__lookup = super(FastLookup, self)._name_set()
+        return self.__lookup
+
+
+def _pathlib_compat(path):
+    """
+    For path-like objects, convert to a filename for compatibility
+    on Python 3.6.1 and earlier.
+    """
+    try:
+        return path.__fspath__()
+    except AttributeError:
+        return str(path)
+
+
+class Path:
+    """
+    A pathlib-compatible interface for zip files.
+
+    Consider a zip file with this structure::
+
+        .
+        ├── a.txt
+        └── b
+            ├── c.txt
+            └── d
+                └── e.txt
+
+    >>> data = io.BytesIO()
+    >>> zf = zipfile.ZipFile(data, 'w')
+    >>> zf.writestr('a.txt', 'content of a')
+    >>> zf.writestr('b/c.txt', 'content of c')
+    >>> zf.writestr('b/d/e.txt', 'content of e')
+    >>> zf.filename = 'mem/abcde.zip'
+
+    Path accepts the zipfile object itself or a filename
+
+    >>> root = Path(zf)
+
+    From there, several path operations are available.
+
+    Directory iteration (including the zip file itself):
+
+    >>> a, b = root.iterdir()
+    >>> a
+    Path('mem/abcde.zip', 'a.txt')
+    >>> b
+    Path('mem/abcde.zip', 'b/')
+
+    name property:
+
+    >>> b.name
+    'b'
+
+    join with divide operator:
+
+    >>> c = b / 'c.txt'
+    >>> c
+    Path('mem/abcde.zip', 'b/c.txt')
+    >>> c.name
+    'c.txt'
+
+    Read text:
+
+    >>> c.read_text()
+    'content of c'
+
+    existence:
+
+    >>> c.exists()
+    True
+    >>> (b / 'missing.txt').exists()
+    False
+
+    Coercion to string:
+
+    >>> import os
+    >>> str(c).replace(os.sep, posixpath.sep)
+    'mem/abcde.zip/b/c.txt'
+
+    At the root, ``name``, ``filename``, and ``parent``
+    resolve to the zipfile. Note these attributes are not
+    valid and will raise a ``ValueError`` if the zipfile
+    has no filename.
+
+    >>> root.name
+    'abcde.zip'
+    >>> str(root.filename).replace(os.sep, posixpath.sep)
+    'mem/abcde.zip'
+    >>> str(root.parent)
+    'mem'
+    """
+
+    __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})"
+
+    def __init__(self, root, at=""):
+        """
+        Construct a Path from a ZipFile or filename.
+
+        Note: When the source is an existing ZipFile object,
+        its type (__class__) will be mutated to a
+        specialized type. If the caller wishes to retain the
+        original type, the caller should either create a
+        separate ZipFile object or pass a filename.
+        """
+        self.root = FastLookup.make(root)
+        self.at = at
+
+    def open(self, mode='r', *args, pwd=None, **kwargs):
+        """
+        Open this entry as text or binary following the semantics
+        of ``pathlib.Path.open()`` by passing arguments through
+        to io.TextIOWrapper().
+        """
+        if self.is_dir():
+            raise IsADirectoryError(self)
+        zip_mode = mode[0]
+        if not self.exists() and zip_mode == 'r':
+            raise FileNotFoundError(self)
+        stream = self.root.open(self.at, zip_mode, pwd=pwd)
+        if 'b' in mode:
+            if args or kwargs:
+                raise ValueError("encoding args invalid for binary operation")
+            return stream
+        return io.TextIOWrapper(stream, *args, **kwargs)
+
+    @property
+    def name(self):
+        return pathlib.Path(self.at).name or self.filename.name
+
+    @property
+    def suffix(self):
+        return pathlib.Path(self.at).suffix or self.filename.suffix
+
+    @property
+    def suffixes(self):
+        return pathlib.Path(self.at).suffixes or self.filename.suffixes
+
+    @property
+    def stem(self):
+        return pathlib.Path(self.at).stem or self.filename.stem
+
+    @property
+    def filename(self):
+        return pathlib.Path(self.root.filename).joinpath(self.at)
+
+    def read_text(self, *args, **kwargs):
+        with self.open('r', *args, **kwargs) as strm:
+            return strm.read()
+
+    def read_bytes(self):
+        with self.open('rb') as strm:
+            return strm.read()
+
+    def _is_child(self, path):
+        return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/")
+
+    def _next(self, at):
+        return self.__class__(self.root, at)
+
+    def is_dir(self):
+        return not self.at or self.at.endswith("/")
+
+    def is_file(self):
+        return self.exists() and not self.is_dir()
+
+    def exists(self):
+        return self.at in self.root._name_set()
+
+    def iterdir(self):
+        if not self.is_dir():
+            raise ValueError("Can't listdir a file")
+        subs = map(self._next, self.root.namelist())
+        return filter(self._is_child, subs)
+
+    def __str__(self):
+        return posixpath.join(self.root.filename, self.at)
+
+    def __repr__(self):
+        return self.__repr.format(self=self)
+
+    def joinpath(self, *other):
+        next = posixpath.join(self.at, *map(_pathlib_compat, other))
+        return self._next(self.root.resolve_dir(next))
+
+    __truediv__ = joinpath
+
+    @property
+    def parent(self):
+        if not self.at:
+            return self.filename.parent
+        parent_at = posixpath.dirname(self.at.rstrip('/'))
+        if parent_at:
+            parent_at += '/'
+        return self._next(parent_at)

metaflow/cmd/develop/stubs.py

@@ -24,7 +24,7 @@ def _check_stubs_supported():
             if _py_ver >= (3, 8):
                 from importlib import metadata
             elif _py_ver >= (3, 7):
-                from metaflow._vendor import importlib_metadata as metadata
+                from metaflow._vendor.v3_7 import importlib_metadata as metadata
             elif _py_ver >= (3, 6):
                 from metaflow._vendor.v3_6 import importlib_metadata as metadata
             else:

metaflow/extension_support/__init__.py

@@ -321,7 +321,7 @@ if _py_ver >= (3, 4):
     if _py_ver >= (3, 8):
         from importlib import metadata
     elif _py_ver >= (3, 7):
-        from metaflow._vendor import importlib_metadata as metadata
+        from metaflow._vendor.v3_7 import importlib_metadata as metadata
     elif _py_ver >= (3, 6):
         from metaflow._vendor.v3_6 import importlib_metadata as metadata
     else:

metaflow/plugins/argo/argo_client.py

@@ -256,12 +256,19 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def trigger_workflow_template(self, name, parameters={}):
+    def trigger_workflow_template(self, name, usertype, username, parameters={}):
         client = self._client.get()
         body = {
             "apiVersion": "argoproj.io/v1alpha1",
             "kind": "Workflow",
-            "metadata": {"generateName": name + "-"},
+            "metadata": {
+                "generateName": name + "-",
+                "annotations": {
+                    "metaflow/triggered_by_user": json.dumps(
+                        {"type": usertype, "name": username}
+                    )
+                },
+            },
             "spec": {
                 "workflowTemplateRef": {"name": name},
                 "arguments": {

metaflow/plugins/argo/argo_workflows.py

@@ -64,6 +64,7 @@ from metaflow.util import (
 )
 
 from .argo_client import ArgoClient
+from metaflow.util import resolve_identity
 
 
 class ArgoWorkflowsException(MetaflowException):
@@ -108,8 +109,7 @@ class ArgoWorkflows(object):
         notify_slack_webhook_url=None,
         notify_pager_duty_integration_key=None,
         notify_incident_io_api_key=None,
-        incident_io_success_severity_id=None,
-        incident_io_error_severity_id=None,
+        incident_io_alert_source_config_id=None,
         enable_heartbeat_daemon=True,
         enable_error_msg_capture=False,
     ):
@@ -160,8 +160,7 @@ class ArgoWorkflows(object):
         self.notify_slack_webhook_url = notify_slack_webhook_url
         self.notify_pager_duty_integration_key = notify_pager_duty_integration_key
         self.notify_incident_io_api_key = notify_incident_io_api_key
-        self.incident_io_success_severity_id = incident_io_success_severity_id
-        self.incident_io_error_severity_id = incident_io_error_severity_id
+        self.incident_io_alert_source_config_id = incident_io_alert_source_config_id
         self.enable_heartbeat_daemon = enable_heartbeat_daemon
         self.enable_error_msg_capture = enable_error_msg_capture
         self.parameters = self._process_parameters()
@@ -315,8 +314,16 @@ class ArgoWorkflows(object):
                     "Workflows before proceeding." % name
                 )
         try:
+            id_parts = resolve_identity().split(":")
+            parts_size = len(id_parts)
+            usertype = id_parts[0] if parts_size > 0 else "unknown"
+            username = id_parts[1] if parts_size > 1 else "unknown"
+
             return ArgoClient(namespace=KUBERNETES_NAMESPACE).trigger_workflow_template(
-                name, parameters
+                name,
+                usertype,
+                username,
+                parameters,
             )
         except Exception as e:
             raise ArgoWorkflowsException(str(e))
@@ -2505,25 +2512,49 @@ class ArgoWorkflows(object):
     def _incident_io_alert_template(self):
         if self.notify_incident_io_api_key is None:
             return None
-        if self.incident_io_error_severity_id is None:
+        if self.incident_io_alert_source_config_id is None:
             raise MetaflowException(
-                "Creating incidents for errors requires a severity id."
+                "Creating alerts for errors requires a alert source config ID."
             )
+        ui_links = self._incident_io_ui_urls_for_run()
         return Template("notify-incident-io-on-error").http(
             Http("POST")
-            .url("https://api.incident.io/v2/incidents")
+            .url(
+                "https://api.incident.io/v2/alert_events/http/%s"
+                % self.incident_io_alert_source_config_id
+            )
             .header("Content-Type", "application/json")
             .header("Authorization", "Bearer %s" % self.notify_incident_io_api_key)
             .body(
                 json.dumps(
                     {
                         "idempotency_key": "argo-{{workflow.name}}",  # use run id to deduplicate alerts.
-                        "visibility": "public",
-                        "severity_id": self.incident_io_error_severity_id,
-                        "name": "Flow %s has failed." % self.flow.name,
-                        "summary": "Metaflow run %s/argo-{{workflow.name}} failed! %s"
-                        % (self.flow.name, self._incident_io_ui_urls_for_run()),
-                        # TODO: Add support for custom field entries.
+                        "status": "firing",
+                        "title": "Flow %s has failed." % self.flow.name,
+                        "description": "Metaflow run {run_pathspec} failed!{urls}".format(
+                            run_pathspec="%s/argo-{{workflow.name}}" % self.flow.name,
+                            urls=(
+                                "\n\nSee details for the run at:\n\n"
+                                + "\n\n".join(ui_links)
+                                if ui_links
+                                else ""
+                            ),
+                        ),
+                        "source_url": (
+                            "%s/%s/%s"
+                            % (
+                                UI_URL.rstrip("/"),
+                                self.flow.name,
+                                "argo-{{workflow.name}}",
+                            )
+                            if UI_URL
+                            else None
+                        ),
+                        "metadata": {
+                            "run_status": "failed",
+                            "flow_name": self.flow.name,
+                            "run_id": "argo-{{workflow.name}}",
+                        },
                     }
                 )
             )
@@ -2532,27 +2563,49 @@ class ArgoWorkflows(object):
     def _incident_io_change_template(self):
         if self.notify_incident_io_api_key is None:
             return None
-        if self.incident_io_success_severity_id is None:
+        if self.incident_io_alert_source_config_id is None:
             raise MetaflowException(
-                "Creating incidents for successes requires a severity id."
+                "Creating alerts for successes requires an alert source config ID."
             )
+        ui_links = self._incident_io_ui_urls_for_run()
         return Template("notify-incident-io-on-success").http(
             Http("POST")
-            .url("https://api.incident.io/v2/incidents")
+            .url(
+                "https://api.incident.io/v2/alert_events/http/%s"
+                % self.incident_io_alert_source_config_id
+            )
             .header("Content-Type", "application/json")
             .header("Authorization", "Bearer %s" % self.notify_incident_io_api_key)
             .body(
                 json.dumps(
                     {
                         "idempotency_key": "argo-{{workflow.name}}",  # use run id to deduplicate alerts.
-                        "visibility": "public",
-                        "severity_id": self.incident_io_success_severity_id,
-                        # TODO: Do we need to make incident type configurable for successes? otherwise they are created as 'investigating'
-                        # "incident_type_id": ""
-                        "name": "Flow %s has succeeded." % self.flow.name,
-                        "summary": "Metaflow run %s/argo-{{workflow.name}} succeeded!%s"
-                        % (self.flow.name, self._incident_io_ui_urls_for_run()),
-                        # TODO: Add support for custom field entries.
+                        "status": "firing",
+                        "title": "Flow %s has succeeded." % self.flow.name,
+                        "description": "Metaflow run {run_pathspec} succeeded!{urls}".format(
+                            run_pathspec="%s/argo-{{workflow.name}}" % self.flow.name,
+                            urls=(
+                                "\n\nSee details for the run at:\n\n"
+                                + "\n\n".join(ui_links)
+                                if ui_links
+                                else ""
+                            ),
+                        ),
+                        "source_url": (
+                            "%s/%s/%s"
+                            % (
+                                UI_URL.rstrip("/"),
+                                self.flow.name,
+                                "argo-{{workflow.name}}",
+                            )
+                            if UI_URL
+                            else None
+                        ),
+                        "metadata": {
+                            "run_status": "succeeded",
+                            "flow_name": self.flow.name,
+                            "run_id": "argo-{{workflow.name}}",
+                        },
                     }
                 )
             )
@@ -2574,9 +2627,7 @@ class ArgoWorkflows(object):
                 "{{workflow.name}}",
             )
             links.append(url)
-        if links:
-            links = ["See details for the run at: ", *links]
-        return "\n\n".join(links)
+        return links
 
     def _pager_duty_change_template(self):
         # https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgy-send-a-change-event

metaflow/plugins/argo/argo_workflows_cli.py

@@ -130,6 +130,7 @@ def argo_workflows(obj, name=None):
     is_flag=True,
     default=False,
     help="Only print out JSON sent to Argo Workflows. Do not deploy anything.",
+    hidden=True,
 )
 @click.option(
     "--max-workers",
@@ -182,14 +183,9 @@ def argo_workflows(obj, name=None):
     help="Incident.io API V2 key for workflow success/failure notifications.",
 )
 @click.option(
-    "--incident-io-success-severity-id",
-    default=None,
-    help="Incident.io severity id for success alerts.",
-)
-@click.option(
-    "--incident-io-error-severity-id",
+    "--incident-io-alert-source-config-id",
     default=None,
-    help="Incident.io severity id for error alerts.",
+    help="Incident.io Alert source config ID. Example '01GW2G3V0S59R238FAHPDS1R66'",
 )
 @click.option(
     "--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
@@ -229,8 +225,7 @@ def create(
     notify_slack_webhook_url=None,
     notify_pager_duty_integration_key=None,
     notify_incident_io_api_key=None,
-    incident_io_success_severity_id=None,
-    incident_io_error_severity_id=None,
+    incident_io_alert_source_config_id=None,
     enable_heartbeat_daemon=True,
     deployer_attribute_file=None,
     enable_error_msg_capture=False,
@@ -287,8 +282,7 @@ def create(
         notify_slack_webhook_url,
         notify_pager_duty_integration_key,
         notify_incident_io_api_key,
-        incident_io_success_severity_id,
-        incident_io_error_severity_id,
+        incident_io_alert_source_config_id,
         enable_heartbeat_daemon,
         enable_error_msg_capture,
     )
@@ -464,8 +458,7 @@ def make_flow(
     notify_slack_webhook_url,
     notify_pager_duty_integration_key,
     notify_incident_io_api_key,
-    incident_io_success_severity_id,
-    incident_io_error_severity_id,
+    incident_io_alert_source_config_id,
     enable_heartbeat_daemon,
     enable_error_msg_capture,
 ):
@@ -488,19 +481,18 @@ def make_flow(
             "https://api.slack.com/messaging/webhooks to generate a webhook url.\n"
             " For notifications through PagerDuty, generate an integration key by following the instructions at "
             "https://support.pagerduty.com/docs/services-and-integrations#create-a-generic-events-api-integration\n"
-            " For notifications through Incident.io, generate an API key with a permission to create incidents."
+            " For notifications through Incident.io, generate an alert source config."
         )
 
-    if notify_incident_io_api_key:
-        if notify_on_error and incident_io_error_severity_id is None:
-            raise MetaflowException(
-                "Incident.io error notifications require a severity id. Please set one with --incident-io-error-severity-id"
-            )
+    if (
+        (notify_on_error or notify_on_success)
+        and notify_incident_io_api_key
+        and incident_io_alert_source_config_id is None
+    ):
+        raise MetaflowException(
+            "Incident.io alerts require an alert source configuration ID. Please set one with --incident-io-alert-source-config-id"
+        )
 
-        if notify_on_success and incident_io_success_severity_id is None:
-            raise MetaflowException(
-                "Incident.io success notifications require a severity id. Please set one with --incident-io-success-severity-id"
-            )
     # Attach @kubernetes and @environment decorator to the flow to
     # ensure that the related decorator hooks are invoked.
     decorators._attach_decorators(
@@ -545,8 +537,7 @@ def make_flow(
         notify_slack_webhook_url=notify_slack_webhook_url,
         notify_pager_duty_integration_key=notify_pager_duty_integration_key,
         notify_incident_io_api_key=notify_incident_io_api_key,
-        incident_io_success_severity_id=incident_io_success_severity_id,
-        incident_io_error_severity_id=incident_io_error_severity_id,
+        incident_io_alert_source_config_id=incident_io_alert_source_config_id,
         enable_heartbeat_daemon=enable_heartbeat_daemon,
         enable_error_msg_capture=enable_error_msg_capture,
     )

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -171,12 +171,16 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
         command_obj.sync_wait()
         return command_obj.process.returncode == 0
 
-    def wait_for_completion(self, timeout: Optional[int] = None):
+    def wait_for_completion(
+        self, check_interval: int = 5, timeout: Optional[int] = None
+    ):
         """
         Wait for the workflow to complete or timeout.
 
         Parameters
         ----------
+        check_interval: int, default: 5
+            Frequency of checking for workflow completion, in seconds.
         timeout : int, optional, default None
             Maximum time in seconds to wait for workflow completion.
             If None, waits indefinitely.
@@ -187,7 +191,6 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
             If the workflow does not complete within the specified timeout period.
         """
         start_time = time.time()
-        check_interval = 5
         while self.is_running:
             if timeout is not None and (time.time() - start_time) > timeout:
                 raise TimeoutError(