metaflow 2.18.10__tar.gz → 2.18.12__tar.gz

{metaflow-2.18.10/metaflow.egg-info → metaflow-2.18.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.18.10
+Version: 2.18.12
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.10; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.12; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow/plugins/argo/argo_workflows.py

@@ -121,6 +121,8 @@ class ArgoWorkflows(object):
         incident_io_metadata: List[str] = None,
         enable_heartbeat_daemon=True,
         enable_error_msg_capture=False,
+        workflow_title=None,
+        workflow_description=None,
     ):
         # Some high-level notes -
         #
@@ -177,6 +179,8 @@ class ArgoWorkflows(object):
         )
         self.enable_heartbeat_daemon = enable_heartbeat_daemon
         self.enable_error_msg_capture = enable_error_msg_capture
+        self.workflow_title = workflow_title
+        self.workflow_description = workflow_description
         self.parameters = self._process_parameters()
         self.config_parameters = self._process_config_parameters()
         self.triggers, self.trigger_options = self._process_triggers()
@@ -430,6 +434,25 @@ class ArgoWorkflows(object):
                     "metaflow/project_flow_name": current.project_flow_name,
                 }
             )
+
+        # Add Argo Workflows title and description annotations
+        # https://argo-workflows.readthedocs.io/en/latest/title-and-description/
+        # Use CLI-provided values or auto-populate from metadata
+        title = (
+            (self.workflow_title.strip() if self.workflow_title else None)
+            or current.get("project_flow_name")
+            or self.flow.name
+        )
+
+        description = (
+            self.workflow_description.strip() if self.workflow_description else None
+        ) or (self.flow.__doc__.strip() if self.flow.__doc__ else None)
+
+        if title:
+            annotations["workflows.argoproj.io/title"] = title
+        if description:
+            annotations["workflows.argoproj.io/description"] = description
+
         return annotations
 
     def _get_schedule(self):
@@ -894,7 +917,16 @@ class ArgoWorkflows(object):
                     .annotations(
                         {
                             **annotations,
-                            **self._base_annotations,
+                            **{
+                                k: v
+                                for k, v in self._base_annotations.items()
+                                if k
+                                # Skip custom title/description for workflows as this makes it harder to find specific runs.
+                                not in [
+                                    "workflows.argoproj.io/title",
+                                    "workflows.argoproj.io/description",
+                                ]
+                            },
                             **{"metaflow/run_id": "argo-{{workflow.name}}"},
                         }
                     )

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -227,6 +227,18 @@ def argo_workflows(obj, name=None):
     show_default=True,
     help="Capture stack trace of first failed task in exit hook.",
 )
+@click.option(
+    "--workflow-title",
+    default=None,
+    type=str,
+    help="Custom title for the workflow displayed in Argo Workflows UI. Defaults to `project_flow_name`. Supports markdown formatting.",
+)
+@click.option(
+    "--workflow-description",
+    default=None,
+    type=str,
+    help="Custom description for the workflow displayed in Argo Workflows UI. Defaults to the flow's docstring if available. Supports markdown formatting and multi-line text.",
+)
 @click.pass_obj
 def create(
     obj,
@@ -248,6 +260,8 @@ def create(
     incident_io_alert_source_config_id=None,
     incident_io_metadata=None,
     enable_heartbeat_daemon=True,
+    workflow_title=None,
+    workflow_description=None,
     deployer_attribute_file=None,
     enable_error_msg_capture=False,
 ):
@@ -312,6 +326,8 @@ def create(
         incident_io_metadata,
         enable_heartbeat_daemon,
         enable_error_msg_capture,
+        workflow_title,
+        workflow_description,
     )
 
     if only_json:
@@ -658,6 +674,8 @@ def make_flow(
     incident_io_metadata,
     enable_heartbeat_daemon,
     enable_error_msg_capture,
+    workflow_title,
+    workflow_description,
 ):
     # TODO: Make this check less specific to Amazon S3 as we introduce
     #       support for more cloud object stores.
@@ -750,6 +768,8 @@ def make_flow(
         incident_io_metadata=incident_io_metadata,
         enable_heartbeat_daemon=enable_heartbeat_daemon,
         enable_error_msg_capture=enable_error_msg_capture,
+        workflow_title=workflow_title,
+        workflow_description=workflow_description,
     )
 
 

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow/plugins/argo/capture_error.py

@@ -1,6 +1,6 @@
 import json
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 
 ###
 # Algorithm to determine 1st error:
@@ -26,6 +26,9 @@ def parse_workflow_failures():
 def group_failures_by_template(failures):
     groups = {}
     for failure in failures:
+        if failure.get("finishedAt", None) is None:
+            timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+            failure["finishedAt"] = timestamp
         groups.setdefault(failure["templateName"], []).append(failure)
     return groups
 
@@ -53,7 +56,7 @@ def determine_first_error():
     grouped_failures = group_failures_by_template(failures)
     for group in grouped_failures.values():
         group.sort(
-            key=lambda x: datetime.strptime(x["finishedAt"], "%Y-%m-%dT%H:%M:%SZ")
+            key=lambda g: datetime.strptime(g["finishedAt"], "%Y-%m-%dT%H:%M:%SZ")
         )
 
     earliest_group = grouped_failures[

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow/plugins/cards/card_modules/basic.py

@@ -2,7 +2,7 @@ import base64
 import json
 import os
 from .card import MetaflowCard, MetaflowCardComponent, with_default_component_id
-from .convert_to_native_type import TaskToDict
+from .convert_to_native_type import TaskToDict, MAX_ARTIFACT_SIZE
 import uuid
 import inspect
 
@@ -376,9 +376,14 @@ class TaskInfoComponent(MetaflowCardComponent):
         components=[],
         runtime=False,
         flow=None,
+        max_artifact_size=None,
     ):
         self._task = task
         self._only_repr = only_repr
+        # Use the global MAX_ARTIFACT_SIZE constant if not specified
+        self._max_artifact_size = (
+            max_artifact_size if max_artifact_size is not None else MAX_ARTIFACT_SIZE
+        )
         self._graph = graph
         self._components = components
         self._page_title = page_title
@@ -394,9 +399,9 @@ class TaskInfoComponent(MetaflowCardComponent):
             a dictionary of form:
                 dict(metadata = {},components= [])
         """
-        task_data_dict = TaskToDict(only_repr=self._only_repr)(
-            self._task, graph=self._graph
-        )
+        task_data_dict = TaskToDict(
+            only_repr=self._only_repr, max_artifact_size=self._max_artifact_size
+        )(self._task, graph=self._graph)
         # ignore the name as an artifact
         if "name" in task_data_dict["data"]:
             del task_data_dict["data"]["name"]
@@ -693,10 +698,14 @@ class DefaultCard(MetaflowCard):
         **kwargs
     ):
         self._only_repr = True
+        # Default max artifact size uses the global MAX_ARTIFACT_SIZE constant (200MB)
+        self._max_artifact_size = MAX_ARTIFACT_SIZE
         self._graph = None if graph is None else transform_flow_graph(graph)
         self._flow = flow
         if "only_repr" in options:
             self._only_repr = options["only_repr"]
+        if "max_artifact_size" in options:
+            self._max_artifact_size = options["max_artifact_size"]
         self._components = components
 
     def render(self, task, runtime=False):
@@ -710,6 +719,7 @@ class DefaultCard(MetaflowCard):
             components=self._components,
             runtime=runtime,
             flow=self._flow,
+            max_artifact_size=self._max_artifact_size,
         ).render()
         pt = self._get_mustache()
         data_dict = dict(

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow/plugins/cards/card_modules/components.py

@@ -15,6 +15,7 @@ from .renderer_tools import render_safely
 from .json_viewer import JSONViewer as _JSONViewer, YAMLViewer as _YAMLViewer
 import uuid
 import inspect
+import textwrap
 
 
 def _warning_with_component(component, msg):
@@ -656,19 +657,38 @@ class Markdown(UserComponent):
     )
     ```
 
+    Multi-line strings with indentation are automatically dedented:
+    ```
+    current.card.append(
+        Markdown(f'''
+            # Header
+            - Item 1
+            - Item 2
+        ''')
+    )
+    ```
+
     Parameters
     ----------
     text : str
-        Text formatted in Markdown.
+        Text formatted in Markdown. Leading whitespace common to all lines
+        is automatically removed to support indented multi-line strings.
     """
 
     REALTIME_UPDATABLE = True
 
+    @staticmethod
+    def _dedent_text(text):
+        """Remove common leading whitespace from all lines."""
+        if text is None:
+            return None
+        return textwrap.dedent(text)
+
     def update(self, text=None):
-        self._text = text
+        self._text = self._dedent_text(text)
 
     def __init__(self, text=None):
-        self._text = text
+        self._text = self._dedent_text(text)
 
     @with_default_component_id
     @render_safely

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow/plugins/cards/card_modules/convert_to_native_type.py

@@ -8,7 +8,9 @@ TypeResolvedObject = namedtuple("TypeResolvedObject", ["data", "is_image", "is_t
 
 
 TIME_FORMAT = "%Y-%m-%d %I:%M:%S %p"
-MAX_ARTIFACT_SIZE = 1  # in 1 MB
+# Maximum artifact size to render in cards: 200MB (in bytes)
+# Artifacts larger than this will be skipped during card rendering to avoid memory issues
+MAX_ARTIFACT_SIZE = 256 * 1024 * 1024  # 256 MB = 268435456 bytes
 
 
 def _get_object_size(obj, seen=None):
@@ -44,7 +46,7 @@ def _full_classname(obj):
 
 
 class TaskToDict:
-    def __init__(self, only_repr=False, runtime=False):
+    def __init__(self, only_repr=False, runtime=False, max_artifact_size=None):
         # this dictionary holds all the supported functions
         import reprlib
         import pprint
@@ -61,6 +63,10 @@ class TaskToDict:
         self._repr = r
         self._runtime = runtime
         self._only_repr = only_repr
+        # Use the global MAX_ARTIFACT_SIZE constant if not specified
+        self._max_artifact_size = (
+            max_artifact_size if max_artifact_size is not None else MAX_ARTIFACT_SIZE
+        )
         self._supported_types = {
             "tuple": self._parse_tuple,
             "NoneType": self._parse_nonetype,
@@ -110,6 +116,19 @@ class TaskToDict:
         task_data_dict = {}
         type_inferred_objects = {"images": {}, "tables": {}}
         for data in task:
+            # Check if artifact size exceeds the maximum allowed size
+            if data.size > self._max_artifact_size:
+                # Skip artifacts that are too large
+                task_data_dict[data.id] = dict(
+                    type="skipped",
+                    data=f"<artifact too large: {data.size} bytes, max: {self._max_artifact_size} bytes>",
+                    large_object=True,
+                    supported_type=False,
+                    only_repr=self._only_repr,
+                    name=data.id,
+                )
+                continue
+
             try:
                 data_object = data.data
                 task_data_dict[data.id] = self._convert_to_native_type(data_object)
@@ -241,7 +260,8 @@ class TaskToDict:
             supported_type = True
             type_parsing_func = self._supported_types[obj_type_name]
             data_obj = type_parsing_func(data_object)
-            if _get_object_size(data_obj) * 1e-6 > MAX_ARTIFACT_SIZE:
+            # Secondary check: if the in-memory object size exceeds our limit, use repr instead
+            if _get_object_size(data_obj) > self._max_artifact_size:
                 data_obj = rep.repr(data_obj)
                 large_object = True
         else:

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow/runner/utils.py

@@ -109,7 +109,6 @@ def read_from_fifo_when_ready(
     content = bytearray()
     poll = select.poll()
     poll.register(fifo_fd, select.POLLIN)
-    max_timeout = 3  # Wait for 10 * 3 = 30 ms after last write
     while True:
         if check_process_exited(command_obj) and command_obj.process.returncode != 0:
             raise CalledProcessError(
@@ -137,15 +136,16 @@ def read_from_fifo_when_ready(
                 else:
                     # We had no events (just a timeout) and the read didn't return
                     # an exception so the file is still open; we continue waiting for data
-                    # Unfortunately, on MacOS, it seems that even *after* the file is
-                    # closed on the other end, we still don't get a BlockingIOError so
-                    # we hack our way and timeout if there is no write in 30ms which is
-                    # a relative eternity for file writes.
-                    if content:
-                        if max_timeout <= 0:
-                            break
-                        max_timeout -= 1
-                        continue
+                    # On some systems (notably MacOS), even after the file is closed on the
+                    # other end, we may not get a BlockingIOError or proper EOF signal.
+                    # Instead of using an arbitrary timeout, check if the writer process
+                    # has actually exited. If it has and we have content, we can safely
+                    # assume EOF. If the process is still running, continue waiting.
+                    if content and check_process_exited(command_obj):
+                        # Process has exited and we got an empty read with no poll events.
+                        # This is EOF - break out to return the content we've collected.
+                        break
+                    # else: process is still running, continue waiting for more data
         except BlockingIOError:
             has_blocking_error = True
             if content:

metaflow-2.18.12/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.18.12"

{metaflow-2.18.10 → metaflow-2.18.12/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.18.10
+Version: 2.18.12
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.10; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.12; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.18.10 → metaflow-2.18.12}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.18.10
+metaflow-stubs==2.18.12

metaflow-2.18.10/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.18.10"