ob-metaflow 2.12.5.1__py2.py3-none-any.whl → 2.12.6.1__py2.py3-none-any.whl

metaflow/metaflow_config.py

@@ -237,6 +237,7 @@ DEFAULT_RUNTIME_LIMIT = from_conf("DEFAULT_RUNTIME_LIMIT", 5 * 24 * 60 * 60)
 # Organization customizations
 ###
 UI_URL = from_conf("UI_URL")
+PAGERDUTY_TEMPLATE_URL = from_conf("PAGERDUTY_TEMPLATE_URL")
 
 # Contact information displayed when running the `metaflow` command.
 # Value should be a dictionary where:

metaflow/plugins/argo/argo_workflows.py

@@ -6,6 +6,7 @@ import shlex
 import sys
 from collections import defaultdict
 from hashlib import sha1
+from math import inf
 
 from metaflow import JSONType, current
 from metaflow.decorators import flow_decorators
@@ -45,6 +46,7 @@ from metaflow.metaflow_config import (
     SERVICE_HEADERS,
     SERVICE_INTERNAL_URL,
     UI_URL,
+    PAGERDUTY_TEMPLATE_URL,
 )
 from metaflow.metaflow_config_funcs import config_values
 from metaflow.mflog import BASH_SAVE_LOGS, bash_capture_logs, export_mflog_env_vars
@@ -60,7 +62,6 @@ from metaflow.util import (
     to_camelcase,
     to_unicode,
 )
-
 from .argo_client import ArgoClient
 
 
@@ -901,7 +902,9 @@ class ArgoWorkflows(object):
                                 "argo-{{workflow.name}}/%s/{{tasks.%s.outputs.parameters.task-id}}"
                                 % (n, self._sanitize(n))
                                 for n in node.in_funcs
-                            ]
+                            ],
+                            # NOTE: We set zlibmin to infinite because zlib compression for the Argo input-paths breaks template value substitution.
+                            zlibmin=inf,
                         )
                     )
                 ]
@@ -1796,10 +1799,18 @@ class ArgoWorkflows(object):
     def _pager_duty_notification_links(self):
         links = []
         if UI_URL:
+            if PAGERDUTY_TEMPLATE_URL:
+                pdproject = ''
+                pdbranch = ''
+                if getattr(current, "project_name", None):
+                    pdproject = current.project_name
+                    pdbranch = current.branch_name
+                href_val = PAGERDUTY_TEMPLATE_URL.format(pd_flow=self.flow.name, pd_namespace=KUBERNETES_NAMESPACE,pd_template=self.name,pd_project=pdproject, pd_branch=pdbranch)
+            else:
+                href_val = "%s/%s/%s"% (UI_URL.rstrip("/"), self.flow.name, "argo-{{workflow.name}}")
             links.append(
                 {
-                    "href": "%s/%s/%s"
-                    % (UI_URL.rstrip("/"), self.flow.name, "argo-{{workflow.name}}"),
+                    "href": href_val,
                     "text": "Metaflow UI",
                 }
             )
@@ -2155,7 +2166,8 @@ class ArgoWorkflows(object):
                                                 # everything within the body.
                                                 # NOTE: We need the conditional logic in order to successfully fall back to the default value
                                                 # when the event payload does not contain a key for a parameter.
-                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (.Input.body.payload.%s | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                # NOTE: Keys might contain dashes, so use the safer 'get' for fetching the value
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
                                                 % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for

metaflow/plugins/pypi/conda_decorator.py

@@ -100,6 +100,10 @@ class CondaStepDecorator(StepDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.
@@ -340,6 +344,10 @@ class CondaFlowDecorator(FlowDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.

metaflow/plugins/pypi/conda_environment.py

@@ -65,7 +65,7 @@ class CondaEnvironment(MetaflowEnvironment):
         micromamba = Micromamba()
         self.solvers = {"conda": micromamba, "pypi": Pip(micromamba)}
 
-    def init_environment(self, echo):
+    def init_environment(self, echo, only_steps=None):
         # The implementation optimizes for latency to ensure as many operations can
         # be turned into cheap no-ops as feasible. Otherwise, we focus on maintaining
         # a balance between latency and maintainability of code without re-implementing
@@ -77,6 +77,8 @@ class CondaEnvironment(MetaflowEnvironment):
         def environments(type_):
             seen = set()
             for step in self.flow:
+                if only_steps and step.name not in only_steps:
+                    continue
                 environment = self.get_environment(step)
                 if type_ in environment and environment["id_"] not in seen:
                     seen.add(environment["id_"])
@@ -286,10 +288,13 @@ class CondaEnvironment(MetaflowEnvironment):
             # PyPI registries, the usage of environment variable `GOOGLE_APPLICATION_CREDENTIALS`
             # demands that `keyrings.google-artifactregistry-auth` has to be installed
             # and available in the underlying python environment.
-            if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"):
-                environment["conda"]["packages"][
-                    "keyrings.google-artifactregistry-auth"
-                ] = ">=1.1.1"
+            
+            # commenting this out per https://outerboundsco.slack.com/archives/C040K733FND/p1719262399355449
+            # this should be a temporary workaround. Need to find a better fix
+            # if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"):
+            #     environment["conda"]["packages"][
+            #         "keyrings.google-artifactregistry-auth"
+            #     ] = ">=1.1.1"
 
         # Z combinator for a recursive lambda
         deep_sort = (lambda f: f(f))(

metaflow/plugins/pypi/pip.py

@@ -9,7 +9,6 @@ from itertools import chain, product
 from urllib.parse import unquote
 
 from metaflow.exception import MetaflowException
-from metaflow.util import which
 
 from .micromamba import Micromamba
 from .utils import pip_tags, wheel_tags
@@ -25,6 +24,23 @@ class PipException(MetaflowException):
         super(PipException, self).__init__(msg)
 
 
+class PipPackageNotFound(Exception):
+    "Wrapper for pip package resolve errors."
+
+    def __init__(self, error):
+        self.error = error
+        try:
+            # Parse the package spec from error message:
+            # ERROR: ERROR: Could not find a version that satisfies the requirement pkg==0.0.1 (from versions: none)
+            # ERROR: No matching distribution found for pkg==0.0.1
+            self.package_spec = re.search(
+                "ERROR: No matching distribution found for (.*)", self.error
+            )[1]
+            self.package_name = re.match("\w*", self.package_spec)[0]
+        except Exception:
+            pass
+
+
 METADATA_FILE = "{prefix}/.pip/metadata"
 INSTALLATION_MARKER = "{prefix}/.pip/id"
 
@@ -81,7 +97,16 @@ class Pip(object):
                     cmd.append(f"{package}{version}")
                 else:
                     cmd.append(f"{package}=={version}")
-            self._call(prefix, cmd)
+            try:
+                self._call(prefix, cmd)
+            except PipPackageNotFound as ex:
+                # pretty print package errors
+                raise PipException(
+                    "Could not find a binary distribution for %s \n"
+                    "for the platform %s\n\n"
+                    "Note that ***@pypi*** does not currently support source distributions"
+                    % (ex.package_spec, platform)
+                )
 
             def _format(dl_info):
                 res = {k: v for k, v in dl_info.items() if k in ["url"]}
@@ -302,11 +327,14 @@ class Pip(object):
                 .strip()
             )
         except subprocess.CalledProcessError as e:
+            errors = e.stderr.decode()
+            if "No matching distribution" in errors:
+                raise PipPackageNotFound(errors)
             raise PipException(
                 "command '{cmd}' returned error ({code}) {output}\n{stderr}".format(
                     cmd=" ".join(e.cmd),
                     code=e.returncode,
                     output=e.output.decode(),
-                    stderr=e.stderr.decode(),
+                    stderr=errors,
                 )
             )

metaflow/plugins/pypi/pypi_decorator.py

@@ -70,6 +70,10 @@ class PyPIStepDecorator(StepDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @pypi
         # implementation.
@@ -119,6 +123,10 @@ class PyPIFlowDecorator(FlowDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.5.1"
+metaflow_version = "2.12.6.1"

{ob_metaflow-2.12.5.1.dist-info → ob_metaflow-2.12.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.5.1
+Version: 2.12.6.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs ==2.12.5.1 ; extra == 'stubs'
+Requires-Dist: metaflow-stubs ==2.12.6.1 ; extra == 'stubs'
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.12.5.1.dist-info → ob_metaflow-2.12.6.1.dist-info}/RECORD

@@ -15,7 +15,7 @@ metaflow/graph.py,sha256=ZPxyG8uwVMk5YYgX4pQEQaPZtZM5Wy-G4NtJK73IEuA,11818
 metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=_kYAbAtsP7IG1Rd0FqNbo8I8Zs66_0WXbaZJFARO3dE,10394
-metaflow/metaflow_config.py,sha256=mEVBlA0bzUXk0O6744zsp73VPr2JfoVO0SLlYB_faHM,22507
+metaflow/metaflow_config.py,sha256=2lCcN96hN8Vq6oalxBu9ekUWqUXmZXQOswP69Rljyj4,22568
 metaflow/metaflow_config_funcs.py,sha256=pCaiQ2ez9wXixJI3ehmf3QiW9lUqFrZnBZx1my_0wIg,4874
 metaflow/metaflow_current.py,sha256=sCENPBiji3LcPbwgOG0ukGd_yEc5tST8EowES8DzRtA,7430
 metaflow/metaflow_environment.py,sha256=HJfhI3GrU-YbY7Etu9M-1q7EbwtEGFzCBhrUs45OLXY,7403
@@ -35,7 +35,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=m5womQ7y-jXehuMyHPfByDbZ4HwTJxzs869cPOlMR8s,13057
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=xOfWg75ZaYjB1bg-5CbGqOI6V4fvNH8DqNO1Zy0euCw,30
+metaflow/version.py,sha256=CfSlVQL5YZOGLEALNy7UrVy__xZYN6dZfwqsRrDx-WM,30
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -174,7 +174,7 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=k6guw0uaMO6BrdoAJhNJjJXEyijX3_u5egkdFG1QZlw,130416
+metaflow/plugins/argo/argo_workflows.py,sha256=p7GEKwdqJtZWyaioLAbWPnkBG925ii0Kv8G2Xf4i5sE,131266
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=sZTpgfmc50eT3e0qIxpVqUgWhTcYlO1HM4gU6Oaya8g,33259
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=K5t4uIk2IXPdK7v7DEjj3buSB8ikLjLycKjbZUYeiaw,6781
 metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
@@ -285,11 +285,11 @@ metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7IT
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=Hik3PZ_RQC8T6hEf-NE2Xr_jq2ZIUkpgUtJlx-rqJgU,5107
-metaflow/plugins/pypi/conda_decorator.py,sha256=-bPxNtZKjxqOo4sj89uIp8ZVrCIontWhAp7wwRFjYpg,14189
-metaflow/plugins/pypi/conda_environment.py,sha256=7k5GJ6S-EiSkDxnFtNpdKkCOiMKSnjOJ-2N_0fWjMHU,19230
+metaflow/plugins/pypi/conda_decorator.py,sha256=phrUvVC5QrfNwPqIByrXsnpRDg1SNVsfpl1wbAVrykI,14679
+metaflow/plugins/pypi/conda_environment.py,sha256=wvAaOOfBeR1LPvbKJmHzXHs7fblMoaVC-0uxCd2B-cM,19548
 metaflow/plugins/pypi/micromamba.py,sha256=wlVN2fm4WXFh3jVNtpDfu4XEz6VJKbmFNp0QvqlMIuI,12179
-metaflow/plugins/pypi/pip.py,sha256=MAgdyP7wK7Cp6iusG6S-jeKKDCxlA9k-jMqIGvyi0Ng,12472
-metaflow/plugins/pypi/pypi_decorator.py,sha256=syWk_oSQhIK9Y7OeOINMG2XVyxh9sj5uJhapwAXRBDw,5583
+metaflow/plugins/pypi/pip.py,sha256=uYPEHYV1_PtY4QA3NqUcVSPBAlRucGeY9tuyz7sB7aY,13641
+metaflow/plugins/pypi/pypi_decorator.py,sha256=Plmm4fhLECW-sj1QSFI84Gva7qqqwlJsqJ8laCRKIzw,6073
 metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
 metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
 metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
@@ -332,9 +332,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-ob_metaflow-2.12.5.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.12.5.1.dist-info/METADATA,sha256=99ECAT8w9bykd1KmAK2mMBu_eq3Be0lx0nGhYyUW8t0,5143
-ob_metaflow-2.12.5.1.dist-info/WHEEL,sha256=pOwdCRdxkCaq8tWTvnGIC-q_meWYHwmqvvQPkeaM2I4,109
-ob_metaflow-2.12.5.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-ob_metaflow-2.12.5.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.12.5.1.dist-info/RECORD,,
+ob_metaflow-2.12.6.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.12.6.1.dist-info/METADATA,sha256=Ce_SKp9yQS2l0zIEPFWuqAemKuz2cuRzK4BIxeqwl-w,5143
+ob_metaflow-2.12.6.1.dist-info/WHEEL,sha256=0XQbNV6JE5ziJsWjIU8TRRv0N6SohNonLWgP86g5fiI,109
+ob_metaflow-2.12.6.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+ob_metaflow-2.12.6.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.12.6.1.dist-info/RECORD,,

{ob_metaflow-2.12.5.1.dist-info → ob_metaflow-2.12.6.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any