ob-metaflow 2.18.11.1__tar.gz → 2.18.13.1__tar.gz

{ob_metaflow-2.18.11.1/ob_metaflow.egg-info → ob_metaflow-2.18.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.18.11.1
+Version: 2.18.13.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.11.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.13.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/graph.py

@@ -94,7 +94,7 @@ class DAGNode(object):
                 case_key = None
 
                 # handle string literals
-                if isinstance(key, ast.Str):
+                if hasattr(ast, "Str") and isinstance(key, ast.Str):
                     case_key = key.s
                 elif isinstance(key, ast.Constant):
                     case_key = key.value
@@ -171,7 +171,7 @@ class DAGNode(object):
                 # Get condition parameter
                 for keyword in tail.value.keywords:
                     if keyword.arg == "condition":
-                        if isinstance(keyword.value, ast.Str):
+                        if hasattr(ast, "Str") and isinstance(keyword.value, ast.Str):
                             condition_name = keyword.value.s
                         elif isinstance(keyword.value, ast.Constant) and isinstance(
                             keyword.value.value, str

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/plugins/argo/argo_workflows.py

@@ -122,6 +122,8 @@ class ArgoWorkflows(object):
         incident_io_metadata: List[str] = None,
         enable_heartbeat_daemon=True,
         enable_error_msg_capture=False,
+        workflow_title=None,
+        workflow_description=None,
     ):
         # Some high-level notes -
         #
@@ -186,6 +188,8 @@ class ArgoWorkflows(object):
         )
         self.enable_heartbeat_daemon = enable_heartbeat_daemon
         self.enable_error_msg_capture = enable_error_msg_capture
+        self.workflow_title = workflow_title
+        self.workflow_description = workflow_description
         self.parameters = self._process_parameters()
         self.config_parameters = self._process_config_parameters()
         self.triggers, self.trigger_options = self._process_triggers()
@@ -439,6 +443,25 @@ class ArgoWorkflows(object):
                     "metaflow/project_flow_name": current.project_flow_name,
                 }
             )
+
+        # Add Argo Workflows title and description annotations
+        # https://argo-workflows.readthedocs.io/en/latest/title-and-description/
+        # Use CLI-provided values or auto-populate from metadata
+        title = (
+            (self.workflow_title.strip() if self.workflow_title else None)
+            or current.get("project_flow_name")
+            or self.flow.name
+        )
+
+        description = (
+            self.workflow_description.strip() if self.workflow_description else None
+        ) or (self.flow.__doc__.strip() if self.flow.__doc__ else None)
+
+        if title:
+            annotations["workflows.argoproj.io/title"] = title
+        if description:
+            annotations["workflows.argoproj.io/description"] = description
+
         return annotations
 
     def _get_schedule(self):
@@ -595,7 +618,16 @@ class ArgoWorkflows(object):
             # the JSON equivalent of None to please argo-workflows. Unfortunately it
             # has the side effect of casting the parameter value to string null during
             # execution - which needs to be fixed imminently.
-            if not is_required or default_value is not None:
+            if default_value is None:
+                default_value = json.dumps(None)
+            elif param_type == "JSON":
+                if not isinstance(default_value, str):
+                    # once to serialize the default value if needed.
+                    default_value = json.dumps(default_value)
+                # adds outer quotes to param
+                default_value = json.dumps(default_value)
+            else:
+                # Make argo sensors happy
                 default_value = json.dumps(default_value)
 
             parameters[param.name] = dict(
@@ -903,7 +935,16 @@ class ArgoWorkflows(object):
                     .annotations(
                         {
                             **annotations,
-                            **self._base_annotations,
+                            **{
+                                k: v
+                                for k, v in self._base_annotations.items()
+                                if k
+                                # Skip custom title/description for workflows as this makes it harder to find specific runs.
+                                not in [
+                                    "workflows.argoproj.io/title",
+                                    "workflows.argoproj.io/description",
+                                ]
+                            },
                             **{"metaflow/run_id": "argo-{{workflow.name}}"},
                         }
                     )
@@ -918,11 +959,7 @@ class ArgoWorkflows(object):
                     Arguments().parameters(
                         [
                             Parameter(parameter["name"])
-                            .value(
-                                "'%s'" % parameter["value"]
-                                if parameter["type"] == "JSON"
-                                else parameter["value"]
-                            )
+                            .value(parameter["value"])
                             .description(parameter.get("description"))
                             # TODO: Better handle IncludeFile in Argo Workflows UI.
                             for parameter in self.parameters.values()
@@ -2031,7 +2068,7 @@ class ArgoWorkflows(object):
                         # {{foo.bar['param_name']}}.
                         # https://argoproj.github.io/argo-events/tutorials/02-parameterization/
                         # http://masterminds.github.io/sprig/strings.html
-                        "--%s={{workflow.parameters.%s}}"
+                        "--%s=\\\"$(python -m metaflow.plugins.argo.param_val {{=toBase64(workflow.parameters['%s'])}})\\\""
                         % (parameter["name"], parameter["name"])
                         for parameter in self.parameters.values()
                     ]
@@ -3861,37 +3898,27 @@ class ArgoWorkflows(object):
                                                 # NOTE: We need the conditional logic in order to successfully fall back to the default value
                                                 # when the event payload does not contain a key for a parameter.
                                                 # NOTE: Keys might contain dashes, so use the safer 'get' for fetching the value
-                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" %s) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}%s{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
                                                 % (
-                                                    v,
                                                     v,
                                                     (
-                                                        "| toRawJson | squote"
+                                                        '{{- $pv:=(get $.Input.body.payload "%s") -}}{{ if kindIs "string" $pv }}{{- $pv | toRawJson -}}{{- else -}}{{ $pv | toRawJson | toRawJson }}{{- end -}}'
+                                                        % v
                                                         if self.parameters[
                                                             parameter_name
                                                         ]["type"]
                                                         == "JSON"
-                                                        else "| toRawJson"
+                                                        else '{{- (get $.Input.body.payload "%s" | toRawJson) -}}'
+                                                        % v
                                                     ),
                                                 ),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for
                                                 # the parameters - there doesn't seem
                                                 # to be any way for us to skip
-                                                value=(
-                                                    json.dumps(
-                                                        self.parameters[parameter_name][
-                                                            "value"
-                                                        ]
-                                                    )
-                                                    if self.parameters[parameter_name][
-                                                        "type"
-                                                    ]
-                                                    == "JSON"
-                                                    else self.parameters[
-                                                        parameter_name
-                                                    ]["value"]
-                                                ),
+                                                value=self.parameters[parameter_name][
+                                                    "value"
+                                                ],
                                             )
                                             .dest(
                                                 # this undocumented (mis?)feature in

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -227,6 +227,18 @@ def argo_workflows(obj, name=None):
     show_default=True,
     help="Capture stack trace of first failed task in exit hook.",
 )
+@click.option(
+    "--workflow-title",
+    default=None,
+    type=str,
+    help="Custom title for the workflow displayed in Argo Workflows UI. Defaults to `project_flow_name`. Supports markdown formatting.",
+)
+@click.option(
+    "--workflow-description",
+    default=None,
+    type=str,
+    help="Custom description for the workflow displayed in Argo Workflows UI. Defaults to the flow's docstring if available. Supports markdown formatting and multi-line text.",
+)
 @click.pass_obj
 def create(
     obj,
@@ -248,6 +260,8 @@ def create(
     incident_io_alert_source_config_id=None,
     incident_io_metadata=None,
     enable_heartbeat_daemon=True,
+    workflow_title=None,
+    workflow_description=None,
     deployer_attribute_file=None,
     enable_error_msg_capture=False,
 ):
@@ -312,6 +326,8 @@ def create(
         incident_io_metadata,
         enable_heartbeat_daemon,
         enable_error_msg_capture,
+        workflow_title,
+        workflow_description,
     )
 
     if only_json:
@@ -658,6 +674,8 @@ def make_flow(
     incident_io_metadata,
     enable_heartbeat_daemon,
     enable_error_msg_capture,
+    workflow_title,
+    workflow_description,
 ):
     # TODO: Make this check less specific to Amazon S3 as we introduce
     #       support for more cloud object stores.
@@ -750,6 +768,8 @@ def make_flow(
         incident_io_metadata=incident_io_metadata,
         enable_heartbeat_daemon=enable_heartbeat_daemon,
         enable_error_msg_capture=enable_error_msg_capture,
+        workflow_title=workflow_title,
+        workflow_description=workflow_description,
     )
 
 

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/plugins/argo/capture_error.py

@@ -1,6 +1,6 @@
 import json
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 
 ###
 # Algorithm to determine 1st error:
@@ -26,6 +26,9 @@ def parse_workflow_failures():
 def group_failures_by_template(failures):
     groups = {}
     for failure in failures:
+        if failure.get("finishedAt", None) is None:
+            timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+            failure["finishedAt"] = timestamp
         groups.setdefault(failure["templateName"], []).append(failure)
     return groups
 
@@ -53,7 +56,7 @@ def determine_first_error():
     grouped_failures = group_failures_by_template(failures)
     for group in grouped_failures.values():
         group.sort(
-            key=lambda x: datetime.strptime(x["finishedAt"], "%Y-%m-%dT%H:%M:%SZ")
+            key=lambda g: datetime.strptime(g["finishedAt"], "%Y-%m-%dT%H:%M:%SZ")
         )
 
     earliest_group = grouped_failures[

ob_metaflow-2.18.13.1/metaflow/plugins/argo/param_val.py

@@ -0,0 +1,19 @@
+import sys
+import base64
+import json
+
+
+def parse_parameter_value(base64_value):
+    val = base64.b64decode(base64_value).decode("utf-8")
+
+    try:
+        return json.loads(val)
+    except json.decoder.JSONDecodeError:
+        # fallback to using the original value.
+        return val
+
+
+if __name__ == "__main__":
+    base64_val = sys.argv[1]
+
+    print(parse_parameter_value(base64_val))

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/plugins/cards/card_datastore.py

@@ -1,7 +1,3 @@
-""" 
-
-"""
-
 from collections import namedtuple
 from io import BytesIO
 import os
@@ -67,20 +63,18 @@ class CardDatastore(object):
             result = CARD_LOCALROOT
             if result is None:
                 current_path = os.getcwd()
-                check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR, CARD_SUFFIX)
+                check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
                 check_dir = os.path.realpath(check_dir)
                 orig_path = check_dir
                 while not os.path.isdir(check_dir):
                     new_path = os.path.dirname(current_path)
                     if new_path == current_path:
-                        break  # We are no longer making upward progress
+                        # No longer making upward progress so we
+                        # return the top level path
+                        return os.path.join(orig_path, CARD_SUFFIX)
                     current_path = new_path
-                    check_dir = os.path.join(
-                        current_path, DATASTORE_LOCAL_DIR, CARD_SUFFIX
-                    )
-                result = orig_path
-
-            return result
+                    check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
+                return os.path.join(check_dir, CARD_SUFFIX)
         else:
             # Let's make it obvious we need to update this block for each new datastore backend...
             raise NotImplementedError(

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/plugins/cards/card_modules/components.py

@@ -15,6 +15,7 @@ from .renderer_tools import render_safely
 from .json_viewer import JSONViewer as _JSONViewer, YAMLViewer as _YAMLViewer
 import uuid
 import inspect
+import textwrap
 
 
 def _warning_with_component(component, msg):
@@ -656,19 +657,38 @@ class Markdown(UserComponent):
     )
     ```
 
+    Multi-line strings with indentation are automatically dedented:
+    ```
+    current.card.append(
+        Markdown(f'''
+            # Header
+            - Item 1
+            - Item 2
+        ''')
+    )
+    ```
+
     Parameters
     ----------
     text : str
-        Text formatted in Markdown.
+        Text formatted in Markdown. Leading whitespace common to all lines
+        is automatically removed to support indented multi-line strings.
     """
 
     REALTIME_UPDATABLE = True
 
+    @staticmethod
+    def _dedent_text(text):
+        """Remove common leading whitespace from all lines."""
+        if text is None:
+            return None
+        return textwrap.dedent(text)
+
     def update(self, text=None):
-        self._text = text
+        self._text = self._dedent_text(text)
 
     def __init__(self, text=None):
-        self._text = text
+        self._text = self._dedent_text(text)
 
     @with_default_component_id
     @render_safely

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/runner/click_api.py

@@ -351,8 +351,10 @@ class MetaflowAPI(object):
         class_dict = {
             "__module__": "metaflow",
             "_API_NAME": flow_file,
-            "_internal_getattr": functools.partial(
-                _lazy_load_command, cli_collection, "_compute_flow_parameters"
+            "_internal_getattr": staticmethod(
+                functools.partial(
+                    _lazy_load_command, cli_collection, "_compute_flow_parameters"
+                )
             ),
             "__getattr__": getattr_wrapper,
         }

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/metaflow/runner/utils.py

@@ -109,7 +109,6 @@ def read_from_fifo_when_ready(
     content = bytearray()
     poll = select.poll()
     poll.register(fifo_fd, select.POLLIN)
-    max_timeout = 3  # Wait for 10 * 3 = 30 ms after last write
     while True:
         if check_process_exited(command_obj) and command_obj.process.returncode != 0:
             raise CalledProcessError(
@@ -137,15 +136,16 @@ def read_from_fifo_when_ready(
                 else:
                     # We had no events (just a timeout) and the read didn't return
                     # an exception so the file is still open; we continue waiting for data
-                    # Unfortunately, on MacOS, it seems that even *after* the file is
-                    # closed on the other end, we still don't get a BlockingIOError so
-                    # we hack our way and timeout if there is no write in 30ms which is
-                    # a relative eternity for file writes.
-                    if content:
-                        if max_timeout <= 0:
-                            break
-                        max_timeout -= 1
-                        continue
+                    # On some systems (notably MacOS), even after the file is closed on the
+                    # other end, we may not get a BlockingIOError or proper EOF signal.
+                    # Instead of using an arbitrary timeout, check if the writer process
+                    # has actually exited. If it has and we have content, we can safely
+                    # assume EOF. If the process is still running, continue waiting.
+                    if content and check_process_exited(command_obj):
+                        # Process has exited and we got an empty read with no poll events.
+                        # This is EOF - break out to return the content we've collected.
+                        break
+                    # else: process is still running, continue waiting for more data
         except BlockingIOError:
             has_blocking_error = True
             if content:

ob_metaflow-2.18.13.1/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.18.13.1"

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.18.11.1
+Version: 2.18.13.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.11.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.13.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/ob_metaflow.egg-info/SOURCES.txt

@@ -249,6 +249,7 @@ metaflow/plugins/argo/conditional_input_paths.py
 metaflow/plugins/argo/exit_hooks.py
 metaflow/plugins/argo/generate_input_paths.py
 metaflow/plugins/argo/jobset_input_paths.py
+metaflow/plugins/argo/param_val.py
 metaflow/plugins/aws/__init__.py
 metaflow/plugins/aws/aws_client.py
 metaflow/plugins/aws/aws_utils.py

{ob_metaflow-2.18.11.1 → ob_metaflow-2.18.13.1}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-metaflow-stubs==2.18.11.1
+metaflow-stubs==2.18.13.1

ob_metaflow-2.18.11.1/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.18.11.1"