ob-metaflow 2.9.10.1__py2.py3-none-any.whl → 2.10.2.6__py2.py3-none-any.whl

metaflow/_vendor/v3_7/zipp.py

@@ -0,0 +1,329 @@
+import io
+import posixpath
+import zipfile
+import itertools
+import contextlib
+import sys
+import pathlib
+
+if sys.version_info < (3, 7):
+    from collections import OrderedDict
+else:
+    OrderedDict = dict
+
+
+__all__ = ['Path']
+
+
+def _parents(path):
+    """
+    Given a path with elements separated by
+    posixpath.sep, generate all parents of that path.
+
+    >>> list(_parents('b/d'))
+    ['b']
+    >>> list(_parents('/b/d/'))
+    ['/b']
+    >>> list(_parents('b/d/f/'))
+    ['b/d', 'b']
+    >>> list(_parents('b'))
+    []
+    >>> list(_parents(''))
+    []
+    """
+    return itertools.islice(_ancestry(path), 1, None)
+
+
+def _ancestry(path):
+    """
+    Given a path with elements separated by
+    posixpath.sep, generate all elements of that path
+
+    >>> list(_ancestry('b/d'))
+    ['b/d', 'b']
+    >>> list(_ancestry('/b/d/'))
+    ['/b/d', '/b']
+    >>> list(_ancestry('b/d/f/'))
+    ['b/d/f', 'b/d', 'b']
+    >>> list(_ancestry('b'))
+    ['b']
+    >>> list(_ancestry(''))
+    []
+    """
+    path = path.rstrip(posixpath.sep)
+    while path and path != posixpath.sep:
+        yield path
+        path, tail = posixpath.split(path)
+
+
+_dedupe = OrderedDict.fromkeys
+"""Deduplicate an iterable in original order"""
+
+
+def _difference(minuend, subtrahend):
+    """
+    Return items in minuend not in subtrahend, retaining order
+    with O(1) lookup.
+    """
+    return itertools.filterfalse(set(subtrahend).__contains__, minuend)
+
+
+class CompleteDirs(zipfile.ZipFile):
+    """
+    A ZipFile subclass that ensures that implied directories
+    are always included in the namelist.
+    """
+
+    @staticmethod
+    def _implied_dirs(names):
+        parents = itertools.chain.from_iterable(map(_parents, names))
+        as_dirs = (p + posixpath.sep for p in parents)
+        return _dedupe(_difference(as_dirs, names))
+
+    def namelist(self):
+        names = super(CompleteDirs, self).namelist()
+        return names + list(self._implied_dirs(names))
+
+    def _name_set(self):
+        return set(self.namelist())
+
+    def resolve_dir(self, name):
+        """
+        If the name represents a directory, return that name
+        as a directory (with the trailing slash).
+        """
+        names = self._name_set()
+        dirname = name + '/'
+        dir_match = name not in names and dirname in names
+        return dirname if dir_match else name
+
+    @classmethod
+    def make(cls, source):
+        """
+        Given a source (filename or zipfile), return an
+        appropriate CompleteDirs subclass.
+        """
+        if isinstance(source, CompleteDirs):
+            return source
+
+        if not isinstance(source, zipfile.ZipFile):
+            return cls(_pathlib_compat(source))
+
+        # Only allow for FastLookup when supplied zipfile is read-only
+        if 'r' not in source.mode:
+            cls = CompleteDirs
+
+        source.__class__ = cls
+        return source
+
+
+class FastLookup(CompleteDirs):
+    """
+    ZipFile subclass to ensure implicit
+    dirs exist and are resolved rapidly.
+    """
+
+    def namelist(self):
+        with contextlib.suppress(AttributeError):
+            return self.__names
+        self.__names = super(FastLookup, self).namelist()
+        return self.__names
+
+    def _name_set(self):
+        with contextlib.suppress(AttributeError):
+            return self.__lookup
+        self.__lookup = super(FastLookup, self)._name_set()
+        return self.__lookup
+
+
+def _pathlib_compat(path):
+    """
+    For path-like objects, convert to a filename for compatibility
+    on Python 3.6.1 and earlier.
+    """
+    try:
+        return path.__fspath__()
+    except AttributeError:
+        return str(path)
+
+
+class Path:
+    """
+    A pathlib-compatible interface for zip files.
+
+    Consider a zip file with this structure::
+
+        .
+        ├── a.txt
+        └── b
+            ├── c.txt
+            └── d
+                └── e.txt
+
+    >>> data = io.BytesIO()
+    >>> zf = zipfile.ZipFile(data, 'w')
+    >>> zf.writestr('a.txt', 'content of a')
+    >>> zf.writestr('b/c.txt', 'content of c')
+    >>> zf.writestr('b/d/e.txt', 'content of e')
+    >>> zf.filename = 'mem/abcde.zip'
+
+    Path accepts the zipfile object itself or a filename
+
+    >>> root = Path(zf)
+
+    From there, several path operations are available.
+
+    Directory iteration (including the zip file itself):
+
+    >>> a, b = root.iterdir()
+    >>> a
+    Path('mem/abcde.zip', 'a.txt')
+    >>> b
+    Path('mem/abcde.zip', 'b/')
+
+    name property:
+
+    >>> b.name
+    'b'
+
+    join with divide operator:
+
+    >>> c = b / 'c.txt'
+    >>> c
+    Path('mem/abcde.zip', 'b/c.txt')
+    >>> c.name
+    'c.txt'
+
+    Read text:
+
+    >>> c.read_text()
+    'content of c'
+
+    existence:
+
+    >>> c.exists()
+    True
+    >>> (b / 'missing.txt').exists()
+    False
+
+    Coercion to string:
+
+    >>> import os
+    >>> str(c).replace(os.sep, posixpath.sep)
+    'mem/abcde.zip/b/c.txt'
+
+    At the root, ``name``, ``filename``, and ``parent``
+    resolve to the zipfile. Note these attributes are not
+    valid and will raise a ``ValueError`` if the zipfile
+    has no filename.
+
+    >>> root.name
+    'abcde.zip'
+    >>> str(root.filename).replace(os.sep, posixpath.sep)
+    'mem/abcde.zip'
+    >>> str(root.parent)
+    'mem'
+    """
+
+    __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})"
+
+    def __init__(self, root, at=""):
+        """
+        Construct a Path from a ZipFile or filename.
+
+        Note: When the source is an existing ZipFile object,
+        its type (__class__) will be mutated to a
+        specialized type. If the caller wishes to retain the
+        original type, the caller should either create a
+        separate ZipFile object or pass a filename.
+        """
+        self.root = FastLookup.make(root)
+        self.at = at
+
+    def open(self, mode='r', *args, pwd=None, **kwargs):
+        """
+        Open this entry as text or binary following the semantics
+        of ``pathlib.Path.open()`` by passing arguments through
+        to io.TextIOWrapper().
+        """
+        if self.is_dir():
+            raise IsADirectoryError(self)
+        zip_mode = mode[0]
+        if not self.exists() and zip_mode == 'r':
+            raise FileNotFoundError(self)
+        stream = self.root.open(self.at, zip_mode, pwd=pwd)
+        if 'b' in mode:
+            if args or kwargs:
+                raise ValueError("encoding args invalid for binary operation")
+            return stream
+        return io.TextIOWrapper(stream, *args, **kwargs)
+
+    @property
+    def name(self):
+        return pathlib.Path(self.at).name or self.filename.name
+
+    @property
+    def suffix(self):
+        return pathlib.Path(self.at).suffix or self.filename.suffix
+
+    @property
+    def suffixes(self):
+        return pathlib.Path(self.at).suffixes or self.filename.suffixes
+
+    @property
+    def stem(self):
+        return pathlib.Path(self.at).stem or self.filename.stem
+
+    @property
+    def filename(self):
+        return pathlib.Path(self.root.filename).joinpath(self.at)
+
+    def read_text(self, *args, **kwargs):
+        with self.open('r', *args, **kwargs) as strm:
+            return strm.read()
+
+    def read_bytes(self):
+        with self.open('rb') as strm:
+            return strm.read()
+
+    def _is_child(self, path):
+        return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/")
+
+    def _next(self, at):
+        return self.__class__(self.root, at)
+
+    def is_dir(self):
+        return not self.at or self.at.endswith("/")
+
+    def is_file(self):
+        return self.exists() and not self.is_dir()
+
+    def exists(self):
+        return self.at in self.root._name_set()
+
+    def iterdir(self):
+        if not self.is_dir():
+            raise ValueError("Can't listdir a file")
+        subs = map(self._next, self.root.namelist())
+        return filter(self._is_child, subs)
+
+    def __str__(self):
+        return posixpath.join(self.root.filename, self.at)
+
+    def __repr__(self):
+        return self.__repr.format(self=self)
+
+    def joinpath(self, *other):
+        next = posixpath.join(self.at, *map(_pathlib_compat, other))
+        return self._next(self.root.resolve_dir(next))
+
+    __truediv__ = joinpath
+
+    @property
+    def parent(self):
+        if not self.at:
+            return self.filename.parent
+        parent_at = posixpath.dirname(self.at.rstrip('/'))
+        if parent_at:
+            parent_at += '/'
+        return self._next(parent_at)

metaflow/metaflow_config.py

@@ -312,6 +312,7 @@ ARGO_EVENTS_WEBHOOK_URL = from_conf("ARGO_EVENTS_WEBHOOK_URL")
 ARGO_EVENTS_INTERNAL_WEBHOOK_URL = from_conf(
     "ARGO_EVENTS_INTERNAL_WEBHOOK_URL", ARGO_EVENTS_WEBHOOK_URL
 )
+ARGO_EVENTS_WEBHOOK_AUTH = from_conf("ARGO_EVENTS_WEBHOOK_AUTH", "none")
 
 ARGO_WORKFLOWS_UI_URL = from_conf("ARGO_WORKFLOWS_UI_URL")
 
@@ -470,7 +471,7 @@ try:
                 DEBUG_OPTIONS.extend(o)
                 for typ in o:
                     vars()["DEBUG_%s" % typ.upper()] = from_conf(
-                        "DEBUG_%s" % typ.upper()
+                        "DEBUG_%s" % typ.upper(), False
                     )
             elif n == "get_pinned_conda_libs":
 

metaflow/metaflow_environment.py

@@ -193,7 +193,9 @@ class MetaflowEnvironment(object):
             env[ext_key] = ext_val
         return env
 
-    def executable(self, step_name):
+    def executable(self, step_name, default=None):
+        if default is not None:
+            return default
         return self._python()
 
     def _python(self):

metaflow/mflog/mflog.py

@@ -135,7 +135,13 @@ def merge_logs(logs):
                 missing.append(line)
         for line in missing:
             res = MFLogline(
-                False, None, MISSING_TIMESTAMP_STR, None, None, line, MISSING_TIMESTAMP
+                False,
+                None,
+                MISSING_TIMESTAMP_STR.encode("utf-8"),
+                None,
+                None,
+                line,
+                MISSING_TIMESTAMP,
             )
             yield res.utc_tstamp_str, res
 

metaflow/multicore_utils.py

@@ -4,6 +4,7 @@ import traceback
 from itertools import islice
 from multiprocessing import cpu_count
 from tempfile import NamedTemporaryFile
+import time
 import metaflow.tracing as tracing
 
 try:
@@ -70,8 +71,17 @@ def parallel_imap_unordered(func, iterable, max_parallel=None, dir=None):
 
     while pids:
 
-        pid, output_file = pids.pop()
-        if os.waitpid(pid, 0)[1]:
+        for idx, pid_info in enumerate(pids):
+            pid, output_file = pid_info
+            pid, exit_code = os.waitpid(pid, os.WNOHANG)
+            if pid:
+                pids.pop(idx)
+                break
+        else:
+            time.sleep(0.1)  # Wait a bit before re-checking
+            continue
+
+        if exit_code:
             raise MulticoreException("Child failed")
 
         with open(output_file, "rb") as f:

metaflow/plugins/__init__.py

@@ -41,10 +41,11 @@ STEP_DECORATORS_DESC = [
         "unbounded_test_foreach_internal",
         ".test_unbounded_foreach_decorator.InternalTestUnboundedForeachDecorator",
     ),
-    ("conda", ".conda.conda_step_decorator.CondaStepDecorator"),
     ("card", ".cards.card_decorator.CardDecorator"),
     ("pytorch_parallel", ".frameworks.pytorch.PytorchParallelDecorator"),
     ("airflow_internal", ".airflow.airflow_decorator.AirflowInternalDecorator"),
+    ("pypi", ".pypi.pypi_decorator.PyPIStepDecorator"),
+    ("conda", ".pypi.conda_decorator.CondaStepDecorator"),
 ]
 
 # Add new flow decorators here
@@ -53,15 +54,19 @@ STEP_DECORATORS_DESC = [
 # careful with the choice of name though - they become top-level
 # imports from the metaflow package.
 FLOW_DECORATORS_DESC = [
-    ("conda_base", ".conda.conda_flow_decorator.CondaFlowDecorator"),
     ("schedule", ".aws.step_functions.schedule_decorator.ScheduleDecorator"),
     ("project", ".project_decorator.ProjectDecorator"),
     ("trigger", ".events_decorator.TriggerDecorator"),
     ("trigger_on_finish", ".events_decorator.TriggerOnFinishDecorator"),
+    ("pypi_base", ".pypi.pypi_decorator.PyPIFlowDecorator"),
+    ("conda_base", ".pypi.conda_decorator.CondaFlowDecorator"),
 ]
 
 # Add environments here
-ENVIRONMENTS_DESC = [("conda", ".conda.conda_environment.CondaEnvironment")]
+ENVIRONMENTS_DESC = [
+    ("conda", ".pypi.conda_environment.CondaEnvironment"),
+    ("pypi", ".pypi.pypi_environment.PyPIEnvironment"),
+]
 
 # Add metadata providers here
 METADATA_PROVIDERS_DESC = [

metaflow/plugins/airflow/airflow.py

@@ -32,6 +32,9 @@ from metaflow.metaflow_config import (
     SERVICE_HEADERS,
     SERVICE_INTERNAL_URL,
 )
+
+from metaflow.metaflow_config_funcs import config_values
+
 from metaflow.parameters import (
     DelayedEvaluationParameter,
     JSONTypeClass,
@@ -335,6 +338,16 @@ class Airflow(object):
         metaflow_version["production_token"] = self.production_token
         env["METAFLOW_VERSION"] = json.dumps(metaflow_version)
 
+        # Temporary passing of *some* environment variables. Do not rely on this
+        # mechanism as it will be removed in the near future
+        env.update(
+            {
+                k: v
+                for k, v in config_values()
+                if k.startswith("METAFLOW_CONDA_") or k.startswith("METAFLOW_DEBUG_")
+            }
+        )
+
         # Extract the k8s decorators for constructing the arguments of the K8s Pod Operator on Airflow.
         k8s_deco = [deco for deco in node.decorators if deco.name == "kubernetes"][0]
         user_code_retries, _ = self._get_retries(node)

metaflow/plugins/argo/argo_client.py

@@ -52,6 +52,22 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
+    def get_workflow_templates(self):
+        client = self._client.get()
+        try:
+            return client.CustomObjectsApi().list_namespaced_custom_object(
+                group=self._group,
+                version=self._version,
+                namespace=self._namespace,
+                plural="workflowtemplates",
+            )["items"]
+        except client.rest.ApiException as e:
+            if e.status == 404:
+                return None
+            raise ArgoClientException(
+                json.loads(e.body)["message"] if e.body is not None else e.reason
+            )
+
     def register_workflow_template(self, name, workflow_template):
         # Unfortunately, Kubernetes client does not handle optimistic
         # concurrency control by itself unlike kubectl

metaflow/plugins/argo/argo_events.py

@@ -7,7 +7,11 @@ import uuid
 from datetime import datetime
 
 from metaflow.exception import MetaflowException
-from metaflow.metaflow_config import ARGO_EVENTS_WEBHOOK_URL
+from metaflow.metaflow_config import (
+    ARGO_EVENTS_WEBHOOK_URL,
+    ARGO_EVENTS_WEBHOOK_AUTH,
+    SERVICE_HEADERS,
+)
 
 
 class ArgoEventException(MetaflowException):
@@ -99,6 +103,8 @@ class ArgoEvent(object):
                 if self._access_token:
                     # TODO: Test with bearer tokens
                     headers = {"Authorization": "Bearer {}".format(self._access_token)}
+                if ARGO_EVENTS_WEBHOOK_AUTH == "service":
+                    headers.update(SERVICE_HEADERS)
                 # TODO: do we need to worry about certs?
 
                 # Use urllib to avoid introducing any dependency in Metaflow

metaflow/plugins/argo/argo_workflows.py

@@ -18,6 +18,7 @@ from metaflow.metaflow_config import (
     ARGO_EVENTS_INTERNAL_WEBHOOK_URL,
     ARGO_WORKFLOWS_ENV_VARS_TO_SKIP,
     ARGO_WORKFLOWS_KUBERNETES_SECRETS,
+    ARGO_EVENTS_WEBHOOK_AUTH,
     AWS_SECRETS_MANAGER_DEFAULT_REGION,
     AZURE_STORAGE_BLOB_SERVICE_ENDPOINT,
     CARD_AZUREROOT,
@@ -42,6 +43,9 @@ from metaflow.metaflow_config import (
     UI_URL,
     ARGO_WORKFLOWS_UI_URL,
 )
+
+from metaflow.metaflow_config_funcs import config_values
+
 from metaflow.mflog import BASH_SAVE_LOGS, bash_capture_logs, export_mflog_env_vars
 from metaflow.parameters import deploy_time_eval
 from metaflow.plugins.kubernetes.kubernetes import (
@@ -179,6 +183,25 @@ class ArgoWorkflows(object):
         # allowed by Metaflow - guaranteeing uniqueness.
         return name.replace("_", "-")
 
+    @staticmethod
+    def list_templates(flow_name, all=False):
+        client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
+
+        templates = client.get_workflow_templates()
+        if templates is None:
+            return []
+
+        template_names = [
+            template["metadata"]["name"]
+            for template in templates
+            if all
+            or flow_name
+            == template["metadata"]
+            .get("annotations", {})
+            .get("metaflow/flow_name", None)
+        ]
+        return template_names
+
     @staticmethod
     def delete(name):
         client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
@@ -583,6 +606,32 @@ class ArgoWorkflows(object):
                 }
             )
 
+        # Some more annotations to populate the Argo UI nicely
+        if self.tags:
+            annotations.update({"metaflow/tags": json.dumps(self.tags)})
+        if self.notify_on_error:
+            annotations.update(
+                {
+                    "metaflow/notify_on_error": json.dumps(
+                        {
+                            "slack": bool(self.notify_slack_webhook_url),
+                            "pager_duty": bool(self.notify_pager_duty_integration_key),
+                        }
+                    )
+                }
+            )
+        if self.notify_on_success:
+            annotations.update(
+                {
+                    "metaflow/notify_on_success": json.dumps(
+                        {
+                            "slack": bool(self.notify_slack_webhook_url),
+                            "pager_duty": bool(self.notify_pager_duty_integration_key),
+                        }
+                    )
+                }
+            )
+
         return (
             WorkflowTemplate()
             .metadata(
@@ -1138,6 +1187,18 @@ class ArgoWorkflows(object):
                     0
                 ].attributes["vars"]
             )
+
+            # Temporary passing of *some* environment variables. Do not rely on this
+            # mechanism as it will be removed in the near future
+            env.update(
+                {
+                    k: v
+                    for k, v in config_values()
+                    if k.startswith("METAFLOW_CONDA_")
+                    or k.startswith("METAFLOW_DEBUG_")
+                }
+            )
+
             env.update(
                 {
                     **{
@@ -1167,6 +1228,7 @@ class ArgoWorkflows(object):
                         "METAFLOW_ARGO_EVENTS_EVENT_SOURCE": ARGO_EVENTS_EVENT_SOURCE,
                         "METAFLOW_ARGO_EVENTS_SERVICE_ACCOUNT": ARGO_EVENTS_SERVICE_ACCOUNT,
                         "METAFLOW_ARGO_EVENTS_WEBHOOK_URL": ARGO_EVENTS_INTERNAL_WEBHOOK_URL,
+                        "METAFLOW_ARGO_EVENTS_WEBHOOK_AUTH": ARGO_EVENTS_WEBHOOK_AUTH,
                     },
                     **{
                         # Some optional values for bookkeeping

metaflow/plugins/argo/argo_workflows_cli.py

@@ -833,6 +833,21 @@ def terminate(obj, run_id, authorize=None):
         obj.echo("\nRun terminated.")
 
 
+@argo_workflows.command(help="List Argo Workflow templates for the flow.")
+@click.option(
+    "--all",
+    default=False,
+    is_flag=True,
+    type=bool,
+    help="list all Argo Workflow Templates (not just limited to this flow)",
+)
+@click.pass_obj
+def list_workflow_templates(obj, all=None):
+    templates = ArgoWorkflows.list_templates(obj.flow.name, all)
+    for template_name in templates:
+        obj.echo_always(template_name)
+
+
 def validate_run_id(
     workflow_name, token_prefix, authorize, run_id, instructions_fn=None
 ):

metaflow/plugins/aws/batch/batch.py

@@ -23,6 +23,9 @@ from metaflow.metaflow_config import (
     AWS_SECRETS_MANAGER_DEFAULT_REGION,
     S3_SERVER_SIDE_ENCRYPTION,
 )
+
+from metaflow.metaflow_config_funcs import config_values
+
 from metaflow.mflog import (
     export_mflog_env_vars,
     bash_capture_logs,
@@ -249,6 +252,13 @@ class Batch(object):
             .environment_variable("METAFLOW_CARD_S3ROOT", CARD_S3ROOT)
             .environment_variable("METAFLOW_RUNTIME_ENVIRONMENT", "aws-batch")
         )
+
+        # Temporary passing of *some* environment variables. Do not rely on this
+        # mechanism as it will be removed in the near future
+        for k, v in config_values():
+            if k.startswith("METAFLOW_CONDA_") or k.startswith("METAFLOW_DEBUG_"):
+                job.environment_variable(k, v)
+
         if DEFAULT_SECRETS_BACKEND_TYPE is not None:
             job.environment_variable(
                 "METAFLOW_DEFAULT_SECRETS_BACKEND_TYPE", DEFAULT_SECRETS_BACKEND_TYPE

metaflow/plugins/aws/batch/batch_cli.py

@@ -190,8 +190,7 @@ def step(
     if R.use_r():
         entrypoint = R.entrypoint()
     else:
-        if executable is None:
-            executable = ctx.obj.environment.executable(step_name)
+        executable = ctx.obj.environment.executable(step_name, executable)
         entrypoint = "%s -u %s" % (executable, os.path.basename(sys.argv[0]))
 
     top_args = " ".join(util.dict_to_cli_options(ctx.parent.parent.params))