metaflow 2.18.10__py2.py3-none-any.whl → 2.18.11__py2.py3-none-any.whl

metaflow/plugins/cards/card_modules/basic.py

@@ -2,7 +2,7 @@ import base64
 import json
 import os
 from .card import MetaflowCard, MetaflowCardComponent, with_default_component_id
-from .convert_to_native_type import TaskToDict
+from .convert_to_native_type import TaskToDict, MAX_ARTIFACT_SIZE
 import uuid
 import inspect
 
@@ -376,9 +376,14 @@ class TaskInfoComponent(MetaflowCardComponent):
         components=[],
         runtime=False,
         flow=None,
+        max_artifact_size=None,
     ):
         self._task = task
         self._only_repr = only_repr
+        # Use the global MAX_ARTIFACT_SIZE constant if not specified
+        self._max_artifact_size = (
+            max_artifact_size if max_artifact_size is not None else MAX_ARTIFACT_SIZE
+        )
         self._graph = graph
         self._components = components
         self._page_title = page_title
@@ -394,9 +399,9 @@ class TaskInfoComponent(MetaflowCardComponent):
             a dictionary of form:
                 dict(metadata = {},components= [])
         """
-        task_data_dict = TaskToDict(only_repr=self._only_repr)(
-            self._task, graph=self._graph
-        )
+        task_data_dict = TaskToDict(
+            only_repr=self._only_repr, max_artifact_size=self._max_artifact_size
+        )(self._task, graph=self._graph)
         # ignore the name as an artifact
         if "name" in task_data_dict["data"]:
             del task_data_dict["data"]["name"]
@@ -693,10 +698,14 @@ class DefaultCard(MetaflowCard):
         **kwargs
     ):
         self._only_repr = True
+        # Default max artifact size uses the global MAX_ARTIFACT_SIZE constant (200MB)
+        self._max_artifact_size = MAX_ARTIFACT_SIZE
         self._graph = None if graph is None else transform_flow_graph(graph)
         self._flow = flow
         if "only_repr" in options:
             self._only_repr = options["only_repr"]
+        if "max_artifact_size" in options:
+            self._max_artifact_size = options["max_artifact_size"]
         self._components = components
 
     def render(self, task, runtime=False):
@@ -710,6 +719,7 @@ class DefaultCard(MetaflowCard):
             components=self._components,
             runtime=runtime,
             flow=self._flow,
+            max_artifact_size=self._max_artifact_size,
         ).render()
         pt = self._get_mustache()
         data_dict = dict(

metaflow/plugins/cards/card_modules/convert_to_native_type.py

@@ -8,7 +8,9 @@ TypeResolvedObject = namedtuple("TypeResolvedObject", ["data", "is_image", "is_t
 
 
 TIME_FORMAT = "%Y-%m-%d %I:%M:%S %p"
-MAX_ARTIFACT_SIZE = 1  # in 1 MB
+# Maximum artifact size to render in cards: 200MB (in bytes)
+# Artifacts larger than this will be skipped during card rendering to avoid memory issues
+MAX_ARTIFACT_SIZE = 256 * 1024 * 1024  # 256 MB = 268435456 bytes
 
 
 def _get_object_size(obj, seen=None):
@@ -44,7 +46,7 @@ def _full_classname(obj):
 
 
 class TaskToDict:
-    def __init__(self, only_repr=False, runtime=False):
+    def __init__(self, only_repr=False, runtime=False, max_artifact_size=None):
         # this dictionary holds all the supported functions
         import reprlib
         import pprint
@@ -61,6 +63,10 @@ class TaskToDict:
         self._repr = r
         self._runtime = runtime
         self._only_repr = only_repr
+        # Use the global MAX_ARTIFACT_SIZE constant if not specified
+        self._max_artifact_size = (
+            max_artifact_size if max_artifact_size is not None else MAX_ARTIFACT_SIZE
+        )
         self._supported_types = {
             "tuple": self._parse_tuple,
             "NoneType": self._parse_nonetype,
@@ -110,6 +116,19 @@ class TaskToDict:
         task_data_dict = {}
         type_inferred_objects = {"images": {}, "tables": {}}
         for data in task:
+            # Check if artifact size exceeds the maximum allowed size
+            if data.size > self._max_artifact_size:
+                # Skip artifacts that are too large
+                task_data_dict[data.id] = dict(
+                    type="skipped",
+                    data=f"<artifact too large: {data.size} bytes, max: {self._max_artifact_size} bytes>",
+                    large_object=True,
+                    supported_type=False,
+                    only_repr=self._only_repr,
+                    name=data.id,
+                )
+                continue
+
             try:
                 data_object = data.data
                 task_data_dict[data.id] = self._convert_to_native_type(data_object)
@@ -241,7 +260,8 @@ class TaskToDict:
             supported_type = True
             type_parsing_func = self._supported_types[obj_type_name]
             data_obj = type_parsing_func(data_object)
-            if _get_object_size(data_obj) * 1e-6 > MAX_ARTIFACT_SIZE:
+            # Secondary check: if the in-memory object size exceeds our limit, use repr instead
+            if _get_object_size(data_obj) > self._max_artifact_size:
                 data_obj = rep.repr(data_obj)
                 large_object = True
         else:

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.18.10"
+metaflow_version = "2.18.11"

{metaflow-2.18.10.dist-info → metaflow-2.18.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.18.10
+Version: 2.18.11
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.18.10; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.18.11; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.18.10.dist-info → metaflow-2.18.11.dist-info}/RECORD

@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=g2SOU_CRzJLgDM_UGF9QDMANMAIHAsDRXE6S76_YzsY,14594
 metaflow/vendor.py,sha256=A82CGHfStZGDP5pQ5XzRjFkbN1ZC-vFmghXIrzMDDNg,5868
-metaflow/version.py,sha256=fm3PVktu1KSNr7dMY4-BletucOknAbjiLAaLr9y1JkU,29
+metaflow/version.py,sha256=q3OkFrD1pVtgUDvcF5KfpBd3SSEX2AEsbdqVS9ReFy4,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -284,11 +284,11 @@ metaflow/plugins/cards/exception.py,sha256=2UqlNb-Kxpg6cuLu2sBEIPTIElwlVBsSpeCgD
 metaflow/plugins/cards/metadata.py,sha256=tACaw7_XNAICZ4A25celIbgxUF0CxHh7BBpFzKrMLTo,487
 metaflow/plugins/cards/card_modules/__init__.py,sha256=WI2IAsFiKGyqPrHtO9S9-MbyVtUTgWJNL4xjJaBErRo,3437
 metaflow/plugins/cards/card_modules/base.html,sha256=Y208ZKIZqEWWUcoBFTLTdWKAG0C8xH5lmyCRSjaN2FY,21004
-metaflow/plugins/cards/card_modules/basic.py,sha256=b6aBg7800CjjkQb0J_TOd3JujVR10X2QoXYdwiXFbkE,25831
+metaflow/plugins/cards/card_modules/basic.py,sha256=bKU_b_Wfs2OC6me6kPp1Jm_SLvaJ4dkUCMAt2VLampk,26430
 metaflow/plugins/cards/card_modules/bundle.css,sha256=zlYjv5rt7lMqiQzd_OAe4QdQeM3J3YbwljnEghlbTaU,28052
 metaflow/plugins/cards/card_modules/card.py,sha256=6sbqP5mwf7QWvQvX2N_bC78H9ixuI5sQ8612Q5islys,4627
 metaflow/plugins/cards/card_modules/components.py,sha256=hF204MUyJ2DZXSPhEt7d23isjdEf4TGK8pzmVkmXyQ0,45574
-metaflow/plugins/cards/card_modules/convert_to_native_type.py,sha256=Vcjqn5rfC0kVMdhqDwsYEjknXTbkG_ppraQrQGaQY_E,16245
+metaflow/plugins/cards/card_modules/convert_to_native_type.py,sha256=opjGOvWg7trOqzhTRZ7h1cFlYrNj68Wn3_EyDTIKEvw,17320
 metaflow/plugins/cards/card_modules/json_viewer.py,sha256=DWOcQPk6DXbeGuYEqv1gGrQW1eMVYlCXv3kMsVLnhNc,7113
 metaflow/plugins/cards/card_modules/main.css,sha256=avu7BTB9qj0M8LvqNLUhikUFQhmAJhQQ7REcUgh9zMw,11725
 metaflow/plugins/cards/card_modules/main.js,sha256=wT4PhcwoFwVNOGVZYBkjqfYKROuPSPWGjS-zCQZErz8,1075727
@@ -430,12 +430,12 @@ metaflow/user_decorators/mutable_flow.py,sha256=EywKTN3cnXPQF_s62wQaC4a4aH14j8oe
 metaflow/user_decorators/mutable_step.py,sha256=-BY0UDXf_RCAEnC5JlLzEXGdiw1KD9oSrSxS_SWaB9Y,16791
 metaflow/user_decorators/user_flow_decorator.py,sha256=2yDwZq9QGv9W-7kEuKwa8o4ZkTvuHJ5ESz7VVrGViAI,9890
 metaflow/user_decorators/user_step_decorator.py,sha256=4558NR8RJtN22OyTwCXO80bAMhMTaRGMoX12b1GMcPc,27232
-metaflow-2.18.10.data/data/share/metaflow/devtools/Makefile,sha256=TT4TCq8ALSfqYyGqDPocN5oPcZe2FqoCZxmGO1LmyCc,13760
-metaflow-2.18.10.data/data/share/metaflow/devtools/Tiltfile,sha256=b6l_fjDO0wtxOkO85lFvuf3HDa6wnzHhhBG8yv1kQqk,23949
-metaflow-2.18.10.data/data/share/metaflow/devtools/pick_services.sh,sha256=PGjQeDIigFHeoQ0asmYNdYDPIOdeYy1UYvkw2wdN4zg,2209
-metaflow-2.18.10.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.18.10.dist-info/METADATA,sha256=EpxLRWGGk1VpW6VbbCa0CGQnVsoaTy4kXEPib4uCsNU,6743
-metaflow-2.18.10.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-metaflow-2.18.10.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-metaflow-2.18.10.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.18.10.dist-info/RECORD,,
+metaflow-2.18.11.data/data/share/metaflow/devtools/Makefile,sha256=TT4TCq8ALSfqYyGqDPocN5oPcZe2FqoCZxmGO1LmyCc,13760
+metaflow-2.18.11.data/data/share/metaflow/devtools/Tiltfile,sha256=b6l_fjDO0wtxOkO85lFvuf3HDa6wnzHhhBG8yv1kQqk,23949
+metaflow-2.18.11.data/data/share/metaflow/devtools/pick_services.sh,sha256=PGjQeDIigFHeoQ0asmYNdYDPIOdeYy1UYvkw2wdN4zg,2209
+metaflow-2.18.11.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.18.11.dist-info/METADATA,sha256=7lnNSVzLGJoGR8_RscIBX2R55lbQPIU1QqUOlUuWiUo,6743
+metaflow-2.18.11.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+metaflow-2.18.11.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+metaflow-2.18.11.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.18.11.dist-info/RECORD,,