ob-metaflow 2.18.9.2__tar.gz → 2.18.11.1__tar.gz

{ob_metaflow-2.18.9.2/ob_metaflow.egg-info → ob_metaflow-2.18.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.18.9.2
+Version: 2.18.11.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.9.2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.11.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.18.9.2 → ob_metaflow-2.18.11.1}/devtools/Tiltfile

@@ -184,6 +184,7 @@ if "postgresql" in enabled_components:
             'auth.username=metaflow',
             'auth.password=metaflow123',
             'auth.database=metaflow',
+            'image.repository=bitnamilegacy/postgresql',
             'primary.persistence.enabled=false',
             'primary.resources.requests.memory=128Mi',
             'primary.resources.requests.cpu=50m',

{ob_metaflow-2.18.9.2 → ob_metaflow-2.18.11.1}/metaflow/plugins/argo/argo_workflows.py

@@ -1432,11 +1432,7 @@ class ArgoWorkflows(object):
             dag_tasks.append(dag_task)
             # End the workflow if we have reached the end of the flow
             if node.type == "end":
-                return [
-                    Template(self.flow.name).dag(
-                        DAGTemplate().fail_fast().tasks(dag_tasks)
-                    )
-                ] + templates, dag_tasks
+                return templates, dag_tasks
             # For split nodes traverse all the children
             if node.type == "split":
                 for n in node.out_funcs:
@@ -1559,7 +1555,6 @@ class ArgoWorkflows(object):
                         parent_foreach,
                         seen,
                     )
-
                 return _visit(
                     self.graph[self._matching_conditional_join(node)],
                     exit_node,
@@ -1815,7 +1810,11 @@ class ArgoWorkflows(object):
             for daemon_template in self._daemon_templates()
         ]
 
-        templates, _ = _visit(node=self.graph["start"], dag_tasks=daemon_tasks)
+        templates, dag_tasks = _visit(node=self.graph["start"], dag_tasks=daemon_tasks)
+        # Add the DAG template only after fully traversing the graph so we are guaranteed to have all the dag_tasks collected.
+        templates.append(
+            Template(self.flow.name).dag(DAGTemplate().fail_fast().tasks(dag_tasks))
+        )
         return templates
 
     # Visit every node and yield ContainerTemplates.

{ob_metaflow-2.18.9.2 → ob_metaflow-2.18.11.1}/metaflow/plugins/cards/card_modules/basic.py

@@ -2,7 +2,7 @@ import base64
 import json
 import os
 from .card import MetaflowCard, MetaflowCardComponent, with_default_component_id
-from .convert_to_native_type import TaskToDict
+from .convert_to_native_type import TaskToDict, MAX_ARTIFACT_SIZE
 import uuid
 import inspect
 
@@ -376,9 +376,14 @@ class TaskInfoComponent(MetaflowCardComponent):
         components=[],
         runtime=False,
         flow=None,
+        max_artifact_size=None,
     ):
         self._task = task
         self._only_repr = only_repr
+        # Use the global MAX_ARTIFACT_SIZE constant if not specified
+        self._max_artifact_size = (
+            max_artifact_size if max_artifact_size is not None else MAX_ARTIFACT_SIZE
+        )
         self._graph = graph
         self._components = components
         self._page_title = page_title
@@ -394,9 +399,9 @@ class TaskInfoComponent(MetaflowCardComponent):
             a dictionary of form:
                 dict(metadata = {},components= [])
         """
-        task_data_dict = TaskToDict(only_repr=self._only_repr)(
-            self._task, graph=self._graph
-        )
+        task_data_dict = TaskToDict(
+            only_repr=self._only_repr, max_artifact_size=self._max_artifact_size
+        )(self._task, graph=self._graph)
         # ignore the name as an artifact
         if "name" in task_data_dict["data"]:
             del task_data_dict["data"]["name"]
@@ -693,10 +698,14 @@ class DefaultCard(MetaflowCard):
         **kwargs
     ):
         self._only_repr = True
+        # Default max artifact size uses the global MAX_ARTIFACT_SIZE constant (200MB)
+        self._max_artifact_size = MAX_ARTIFACT_SIZE
         self._graph = None if graph is None else transform_flow_graph(graph)
         self._flow = flow
         if "only_repr" in options:
             self._only_repr = options["only_repr"]
+        if "max_artifact_size" in options:
+            self._max_artifact_size = options["max_artifact_size"]
         self._components = components
 
     def render(self, task, runtime=False):
@@ -710,6 +719,7 @@ class DefaultCard(MetaflowCard):
             components=self._components,
             runtime=runtime,
             flow=self._flow,
+            max_artifact_size=self._max_artifact_size,
         ).render()
         pt = self._get_mustache()
         data_dict = dict(

{ob_metaflow-2.18.9.2 → ob_metaflow-2.18.11.1}/metaflow/plugins/cards/card_modules/convert_to_native_type.py

@@ -8,7 +8,9 @@ TypeResolvedObject = namedtuple("TypeResolvedObject", ["data", "is_image", "is_t
 
 
 TIME_FORMAT = "%Y-%m-%d %I:%M:%S %p"
-MAX_ARTIFACT_SIZE = 1  # in 1 MB
+# Maximum artifact size to render in cards: 200MB (in bytes)
+# Artifacts larger than this will be skipped during card rendering to avoid memory issues
+MAX_ARTIFACT_SIZE = 256 * 1024 * 1024  # 256 MB = 268435456 bytes
 
 
 def _get_object_size(obj, seen=None):
@@ -44,7 +46,7 @@ def _full_classname(obj):
 
 
 class TaskToDict:
-    def __init__(self, only_repr=False, runtime=False):
+    def __init__(self, only_repr=False, runtime=False, max_artifact_size=None):
         # this dictionary holds all the supported functions
         import reprlib
         import pprint
@@ -61,6 +63,10 @@ class TaskToDict:
         self._repr = r
         self._runtime = runtime
         self._only_repr = only_repr
+        # Use the global MAX_ARTIFACT_SIZE constant if not specified
+        self._max_artifact_size = (
+            max_artifact_size if max_artifact_size is not None else MAX_ARTIFACT_SIZE
+        )
         self._supported_types = {
             "tuple": self._parse_tuple,
             "NoneType": self._parse_nonetype,
@@ -110,6 +116,19 @@ class TaskToDict:
         task_data_dict = {}
         type_inferred_objects = {"images": {}, "tables": {}}
         for data in task:
+            # Check if artifact size exceeds the maximum allowed size
+            if data.size > self._max_artifact_size:
+                # Skip artifacts that are too large
+                task_data_dict[data.id] = dict(
+                    type="skipped",
+                    data=f"<artifact too large: {data.size} bytes, max: {self._max_artifact_size} bytes>",
+                    large_object=True,
+                    supported_type=False,
+                    only_repr=self._only_repr,
+                    name=data.id,
+                )
+                continue
+
             try:
                 data_object = data.data
                 task_data_dict[data.id] = self._convert_to_native_type(data_object)
@@ -241,7 +260,8 @@ class TaskToDict:
             supported_type = True
             type_parsing_func = self._supported_types[obj_type_name]
             data_obj = type_parsing_func(data_object)
-            if _get_object_size(data_obj) * 1e-6 > MAX_ARTIFACT_SIZE:
+            # Secondary check: if the in-memory object size exceeds our limit, use repr instead
+            if _get_object_size(data_obj) > self._max_artifact_size:
                 data_obj = rep.repr(data_obj)
                 large_object = True
         else:

{ob_metaflow-2.18.9.2 → ob_metaflow-2.18.11.1}/metaflow/plugins/datatools/s3/s3op.py

@@ -595,11 +595,23 @@ class S3Ops(object):
                 # - the trailing slash is significant in S3
                 if "Contents" in page:
                     for key in page.get("Contents", []):
-                        url = url_base + key["Key"]
+                        key_path = key["Key"]
+                        # filter out sibling directories that share the same prefix
+                        if delimiter == "":  # recursive mode
+                            normalized_prefix = prefix_url.path
+                            if not normalized_prefix.endswith("/"):
+                                normalized_prefix += "/"
+                            # Only include keys that are actually under our directory
+                            if not (
+                                key_path.startswith(normalized_prefix)
+                                and len(key_path) > len(normalized_prefix)
+                            ):
+                                continue
+                        url = url_base + key_path
                         urlobj = S3Url(
                             url=url,
                             bucket=prefix_url.bucket,
-                            path=key["Key"],
+                            path=key_path,
                             local=generate_local_path(url),
                             prefix=prefix_url.url,
                         )

ob_metaflow-2.18.11.1/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.18.11.1"

{ob_metaflow-2.18.9.2 → ob_metaflow-2.18.11.1/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.18.9.2
+Version: 2.18.11.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.9.2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.11.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.18.9.2 → ob_metaflow-2.18.11.1}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-metaflow-stubs==2.18.9.2
+metaflow-stubs==2.18.11.1

ob_metaflow-2.18.9.2/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.18.9.2"