ob-metaflow-extensions 1.1.136__tar.gz → 1.1.138__tar.gz

{ob-metaflow-extensions-1.1.136 → ob-metaflow-extensions-1.1.138}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.136
+Version: 1.1.138
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.136 → ob-metaflow-extensions-1.1.138}/metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py

@@ -267,7 +267,7 @@ class DockerEnvironment(MetaflowEnvironment):
             packages = get_pinned_conda_libs(python, self.datastore_type)
             packages.update(dependencies.attributes["packages"] if dependencies else {})
 
-            return {
+            requested = {
                 "python": python,
                 "pypi_packages": (
                     packages if isinstance(dependencies, PyPIStepDecorator) else None
@@ -277,15 +277,35 @@ class DockerEnvironment(MetaflowEnvironment):
                 ),
                 "base_image": base_image,
             }
+            dedup_key = hashlib.sha256(
+                json.dumps(requested).encode("utf-8")
+            ).hexdigest()
+
+            return step.name, dedup_key, requested
 
         with ThreadPoolExecutor() as executor:
             prepared_args = list(executor.map(prepare_step, steps))
-            for i, args in enumerate(prepared_args, 1):
-                args["ref"] = f"#{i:02d}"
-            futures = [executor.submit(_cached_bake, **args) for args in prepared_args]
+            # Deduplicate the requests for baking images of steps.
+            # We do not want to bake the same image twice.
+            dedup_requests = {}
+            for step_name, key, args in prepared_args:
+                if key not in dedup_requests:
+                    dedup_requests[key] = {"step_names": set(), "args": args}
+                dedup_requests[key]["step_names"].add(step_name)
+
+            # unique futures
+            futures = []
+            for i, kv in enumerate(dedup_requests.items(), 1):
+                key, value = kv
+                future = executor.submit(
+                    _cached_bake, **{**value["args"], "ref": f"#{i:02d}"}
+                )
+                futures.append({"step_names": value["step_names"], "future": future})
+
             results = {}
-            for step, future in zip(steps, futures):
-                results[step.name] = future.result()
+            for item in futures:
+                for step_name in item["step_names"]:
+                    results[step_name] = item["future"].result()
 
             return results
 

{ob-metaflow-extensions-1.1.136 → ob-metaflow-extensions-1.1.138}/ob_metaflow_extensions.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.136
+Version: 1.1.138
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

ob-metaflow-extensions-1.1.138/ob_metaflow_extensions.egg-info/requires.txt

@@ -0,0 +1,3 @@
+boto3
+kubernetes
+ob-metaflow==2.15.6.1

{ob-metaflow-extensions-1.1.136 → ob-metaflow-extensions-1.1.138}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 from pathlib import Path
 
 
-version = "1.1.136"
+version = "1.1.138"
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()
 
@@ -18,6 +18,6 @@ setup(
     install_requires=[
         "boto3",
         "kubernetes",
-        "ob-metaflow == 2.15.3.1",
+        "ob-metaflow == 2.15.6.1",
     ],
 )

ob-metaflow-extensions-1.1.136/ob_metaflow_extensions.egg-info/requires.txt

@@ -1,3 +0,0 @@
-boto3
-kubernetes
-ob-metaflow==2.15.3.1