ob-metaflow-extensions 1.1.136__py2.py3-none-any.whl → 1.1.138__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py

@@ -267,7 +267,7 @@ class DockerEnvironment(MetaflowEnvironment):
             packages = get_pinned_conda_libs(python, self.datastore_type)
             packages.update(dependencies.attributes["packages"] if dependencies else {})
 
-            return {
+            requested = {
                 "python": python,
                 "pypi_packages": (
                     packages if isinstance(dependencies, PyPIStepDecorator) else None
@@ -277,15 +277,35 @@ class DockerEnvironment(MetaflowEnvironment):
                 ),
                 "base_image": base_image,
             }
+            dedup_key = hashlib.sha256(
+                json.dumps(requested).encode("utf-8")
+            ).hexdigest()
+
+            return step.name, dedup_key, requested
 
         with ThreadPoolExecutor() as executor:
             prepared_args = list(executor.map(prepare_step, steps))
-            for i, args in enumerate(prepared_args, 1):
-                args["ref"] = f"#{i:02d}"
-            futures = [executor.submit(_cached_bake, **args) for args in prepared_args]
+            # Deduplicate the requests for baking images of steps.
+            # We do not want to bake the same image twice.
+            dedup_requests = {}
+            for step_name, key, args in prepared_args:
+                if key not in dedup_requests:
+                    dedup_requests[key] = {"step_names": set(), "args": args}
+                dedup_requests[key]["step_names"].add(step_name)
+
+            # unique futures
+            futures = []
+            for i, kv in enumerate(dedup_requests.items(), 1):
+                key, value = kv
+                future = executor.submit(
+                    _cached_bake, **{**value["args"], "ref": f"#{i:02d}"}
+                )
+                futures.append({"step_names": value["step_names"], "future": future})
+
             results = {}
-            for step, future in zip(steps, futures):
-                results[step.name] = future.result()
+            for item in futures:
+                for step_name in item["step_names"]:
+                    results[step_name] = item["future"].result()
 
             return results
 

{ob_metaflow_extensions-1.1.136.dist-info → ob_metaflow_extensions-1.1.138.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.136
+Version: 1.1.138
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial
 Description-Content-Type: text/markdown
 Requires-Dist: boto3
 Requires-Dist: kubernetes
-Requires-Dist: ob-metaflow (==2.15.3.1)
+Requires-Dist: ob-metaflow (==2.15.6.1)
 
 # Outerbounds platform package
 

{ob_metaflow_extensions-1.1.136.dist-info → ob_metaflow_extensions-1.1.138.dist-info}/RECORD

@@ -9,7 +9,7 @@ metaflow_extensions/outerbounds/plugins/card_utilities/async_cards.py,sha256=6rQ
 metaflow_extensions/outerbounds/plugins/card_utilities/extra_components.py,sha256=D8rgCaSc7PuLD0MHJjqsjN0g0PQMN1H-ySOJqi5uIOE,19111
 metaflow_extensions/outerbounds/plugins/card_utilities/injector.py,sha256=UQtHrviFs7Z5LBh0nbaIv8aEnHnyXFjs9TiGCjtcIWc,2452
 metaflow_extensions/outerbounds/plugins/fast_bakery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py,sha256=i6F3FXwvEhkmUCTHDJ4VmSoL6vKyQhC_YRCtY6F4EkA,14209
+metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py,sha256=Tl520HdBteg-aDOM7mnnJJpdDCZc49BmFFmLUc_vTi8,15018
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py,sha256=cj63FrdioggipQFP8GwgxU3FYe6IyzjGSUGYxLQZ4nQ,5189
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py,sha256=kqFyu2bJSnc9_9aYfBpz5xK6L6luWFZK_NMuh8f1eVk,1494
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_decorator.py,sha256=MXSIp05-jvt8Q2uGaLKjtuM_ToLeRLxhtMbfHc9Kcko,1515
@@ -53,7 +53,7 @@ metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3u
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
 metaflow_extensions/outerbounds/toplevel/plugins/ollama/__init__.py,sha256=GRSz2zwqkvlmFS6bcfYD_CX6CMko9DHQokMaH1iBshA,47
 metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
-ob_metaflow_extensions-1.1.136.dist-info/METADATA,sha256=4VKMvHaHd76Iw0NhwQOr8tmfIZRupUssZaVM_ZxcZ_M,520
-ob_metaflow_extensions-1.1.136.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.136.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.136.dist-info/RECORD,,
+ob_metaflow_extensions-1.1.138.dist-info/METADATA,sha256=3r5V79L8IdAPZmfzevOm2d-g9nGvsQjrCB_PvfG6IhY,520
+ob_metaflow_extensions-1.1.138.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.138.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.138.dist-info/RECORD,,