ob-metaflow 2.15.11.1__py2.py3-none-any.whl → 2.15.11.2__py2.py3-none-any.whl

metaflow/metaflow_config.py

@@ -453,7 +453,15 @@ ESCAPE_HATCH_WARNING = from_conf("ESCAPE_HATCH_WARNING", True)
 ###
 # Debug configuration
 ###
-DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client", "tracing", "stubgen", "userconf"]
+DEBUG_OPTIONS = [
+    "subcommand",
+    "sidecar",
+    "s3client",
+    "tracing",
+    "stubgen",
+    "userconf",
+    "conda",
+]
 
 for typ in DEBUG_OPTIONS:
     vars()["DEBUG_%s" % typ.upper()] = from_conf("DEBUG_%s" % typ.upper(), False)

metaflow/plugins/pypi/conda_environment.py

@@ -13,6 +13,7 @@ from hashlib import sha256
 from io import BufferedIOBase, BytesIO
 from urllib.parse import unquote, urlparse
 
+from metaflow.debug import debug
 from metaflow.exception import MetaflowException
 from metaflow.metaflow_config import get_pinned_conda_libs
 from metaflow.metaflow_environment import MetaflowEnvironment
@@ -117,6 +118,11 @@ class CondaEnvironment(MetaflowEnvironment):
             )
 
         def cache(storage, results, type_):
+            debug.conda_exec(
+                "Caching packages for %s environments %s"
+                % (type_, [result[0] for result in results])
+            )
+
             def _path(url, local_path):
                 # Special handling for VCS packages
                 if url.startswith("git+"):
@@ -169,14 +175,25 @@ class CondaEnvironment(MetaflowEnvironment):
                 )
                 for url, package in local_packages.items()
             ]
+            debug.conda_exec(
+                "Caching %s new packages to the datastore for %s environment %s"
+                % (
+                    len(list_of_path_and_filehandle),
+                    type_,
+                    [result[0] for result in results],
+                )
+            )
             storage.save_bytes(
                 list_of_path_and_filehandle,
                 len_hint=len(list_of_path_and_filehandle),
+                # overwrite=True,
             )
             for id_, packages, _, platform in results:
                 if id_ in dirty:
                     self.write_to_environment_manifest([id_, platform, type_], packages)
 
+            debug.conda_exec("Finished caching packages.")
+
         storage = None
         if self.datastore_type not in ["local"]:
             # Initialize storage for caching if using a remote datastore
@@ -192,44 +209,85 @@ class CondaEnvironment(MetaflowEnvironment):
         #  6. Create and cache PyPI environments in parallel
         with ThreadPoolExecutor() as executor:
             # Start all conda solves in parallel
-            conda_futures = [
+            debug.conda_exec("Solving packages for Conda environments..")
+            conda_solve_futures = [
                 executor.submit(lambda x: solve(*x, "conda"), env)
                 for env in environments("conda")
             ]
+            conda_create_futures = []
 
             pypi_envs = {env[0]: env for env in environments("pypi")}
-            pypi_futures = []
+            pypi_solve_futures = []
+            pypi_create_futures = []
 
+            cache_futures = []
             # Process conda results sequentially for downloads
-            for future in as_completed(conda_futures):
+            for future in as_completed(conda_solve_futures):
                 result = future.result()
                 # Sequential conda download
+                debug.conda_exec(
+                    "Downloading packages for Conda environment %s" % result[0]
+                )
                 self.solvers["conda"].download(*result)
                 # Parallel conda create and cache
-                create_future = executor.submit(self.solvers["conda"].create, *result)
+                conda_create_future = executor.submit(
+                    self.solvers["conda"].create, *result
+                )
                 if storage:
-                    executor.submit(cache, storage, [result], "conda")
+                    cache_futures.append(
+                        executor.submit(cache, storage, [result], "conda")
+                    )
 
                 # Queue PyPI solve to start after conda create
                 if result[0] in pypi_envs:
+                    debug.conda_exec(
+                        "Solving packages for PyPI environment %s" % result[0]
+                    )
                     # solve pypi envs uniquely
                     pypi_env = pypi_envs.pop(result[0])
 
                     def pypi_solve(env):
-                        create_future.result()  # Wait for conda create
+                        conda_create_future.result()  # Wait for conda create
                         return solve(*env, "pypi")
 
-                    pypi_futures.append(executor.submit(pypi_solve, pypi_env))
+                    pypi_solve_futures.append(executor.submit(pypi_solve, pypi_env))
+                else:
+                    # add conda create future to the generic list
+                    conda_create_futures.append(conda_create_future)
 
             # Process PyPI results sequentially for downloads
-            for solve_future in pypi_futures:
+            for solve_future in as_completed(pypi_solve_futures):
                 result = solve_future.result()
                 # Sequential PyPI download
+                debug.conda_exec(
+                    "Downloading packages for PyPI environment %s" % result[0]
+                )
                 self.solvers["pypi"].download(*result)
                 # Parallel PyPI create and cache
-                executor.submit(self.solvers["pypi"].create, *result)
+                pypi_create_futures.append(
+                    executor.submit(self.solvers["pypi"].create, *result)
+                )
                 if storage:
-                    executor.submit(cache, storage, [result], "pypi")
+                    cache_futures.append(
+                        executor.submit(cache, storage, [result], "pypi")
+                    )
+
+            # Raise exceptions for conda create
+            debug.conda_exec("Checking results for Conda create..")
+            for future in as_completed(conda_create_futures):
+                future.result()
+
+            # Raise exceptions for pypi create
+            debug.conda_exec("Checking results for PyPI create..")
+            for future in as_completed(pypi_create_futures):
+                future.result()
+
+            # Raise exceptions for caching
+            debug.conda_exec("Checking results for caching..")
+            for future in as_completed(cache_futures):
+                # check for result in order to raise any exceptions.
+                future.result()
+
         self.logger("Virtual environment(s) bootstrapped!")
 
     def executable(self, step_name, default=None):

metaflow/plugins/pypi/micromamba.py

@@ -6,6 +6,7 @@ import subprocess
 import tempfile
 import time
 
+from metaflow.debug import debug
 from metaflow.exception import MetaflowException
 from metaflow.util import which
 
@@ -71,6 +72,7 @@ class Micromamba(object):
 
             # Install Micromamba on the fly.
             # TODO: Make this optional at some point.
+            debug.conda_exec("No Micromamba binary found. Installing micromamba")
             _install_micromamba(self._path_to_hidden_micromamba)
             self._bin = which(
                 os.path.join(self._path_to_hidden_micromamba, "bin/micromamba")
@@ -98,6 +100,7 @@ class Micromamba(object):
         #    environment
         # 4. Multiple solves can progress at the same time while relying on the same
         #    index
+        debug.conda_exec("Solving packages for conda environment %s" % id_)
         with tempfile.TemporaryDirectory() as tmp_dir:
             env = {
                 "MAMBA_ADD_PIP_AS_PYTHON_DEPENDENCY": "true",
@@ -158,6 +161,7 @@ class Micromamba(object):
         if self.path_to_environment(id_, platform):
             return
 
+        debug.conda_exec("Downloading packages for conda environment %s" % id_)
         with tempfile.TemporaryDirectory() as tmp_dir:
             env = {
                 "CONDA_SUBDIR": platform,
@@ -190,6 +194,7 @@ class Micromamba(object):
         if platform != self.platform() or self.path_to_environment(id_, platform):
             return
 
+        debug.conda_exec("Creating local Conda environment %s" % id_)
         prefix = "{env_dirs}/{keyword}/{platform}/{id}".format(
             env_dirs=self.info()["envs_dirs"][0],
             platform=platform,

metaflow/plugins/pypi/pip.py

@@ -8,6 +8,7 @@ from concurrent.futures import ThreadPoolExecutor
 from itertools import chain, product
 from urllib.parse import unquote
 
+from metaflow.debug import debug
 from metaflow.exception import MetaflowException
 
 from .micromamba import Micromamba
@@ -66,6 +67,7 @@ class Pip(object):
             msg += "for id {id}".format(id=id_)
             raise PipException(msg)
 
+        debug.conda_exec("Solving packages for PyPI environment %s" % id_)
         with tempfile.TemporaryDirectory() as tmp_dir:
             report = "{tmp_dir}/report.json".format(tmp_dir=tmp_dir)
             implementations, platforms, abis = zip(
@@ -152,6 +154,7 @@ class Pip(object):
         custom_index_url, extra_index_urls = self.indices(prefix)
 
         # build wheels if needed
+        debug.conda_exec("Building wheels for PyPI environment %s if necessary" % id_)
         with ThreadPoolExecutor() as executor:
 
             def _build(key, package):
@@ -212,6 +215,7 @@ class Pip(object):
             ]
         )
 
+        debug.conda_exec("Downloading packages for PyPI environment %s" % id_)
         cmd = [
             "download",
             "--no-deps",
@@ -251,6 +255,7 @@ class Pip(object):
         # Pip can't install packages if the underlying virtual environment doesn't
         # share the same platform
         if self.micromamba.platform() == platform:
+            debug.conda_exec("Installing packages for local PyPI environment %s" % id_)
             cmd = [
                 "install",
                 "--no-compile",

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.15.11.1"
+metaflow_version = "2.15.11.2"

{ob_metaflow-2.15.11.1.dist-info → ob_metaflow-2.15.11.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.15.11.1
+Version: 2.15.11.2
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.11.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.11.2; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.15.11.1.dist-info → ob_metaflow-2.15.11.2.dist-info}/RECORD

@@ -16,7 +16,7 @@ metaflow/includefile.py,sha256=RtISGl1V48qjkJBakUZ9yPpHV102h7pOIFiKP8PLHpc,20927
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
-metaflow/metaflow_config.py,sha256=HU22KmyqqAais02_cycnnKGuVJccGlKURxTFszMse-o,23935
+metaflow/metaflow_config.py,sha256=C2XAUDNsq8DzzDEHznuK3q10uq8WVdWbAusJCKz9nGM,23975
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
 metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
 metaflow/metaflow_environment.py,sha256=CWG90qpfz9iJ6hHhFlAmMVNALn2v_5eTVk3mFbQR4Pw,8379
@@ -37,7 +37,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=mJBkV5tShIyCsLDeM1zygQGeciQVMrVPm_qI8Oi33G0,14656
 metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
-metaflow/version.py,sha256=7dCHBe9m2nXGQzS67rMdLChDGb6TDIgh0LX7ACbMuDU,31
+metaflow/version.py,sha256=y8wc2OrMrJ04jszycuxbojNRVeCxsev_fb2-TjVp-LE,31
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -331,10 +331,10 @@ metaflow/plugins/metadata_providers/service.py,sha256=9j0db_EOGFdb49YTgr9q4EWqVA
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=SNONquX6QnTbu7htmhaQeVeZ2ofaFaUCDScRIrTTERc,14718
 metaflow/plugins/pypi/conda_decorator.py,sha256=N0HGiaS1mRsa6qT4eYzu2C3DHtas22QIXowW4vEl44M,15961
-metaflow/plugins/pypi/conda_environment.py,sha256=c8mJv6LF9nbube5yvMwTPwVHjCr2khscz0F0e3m0h00,22483
-metaflow/plugins/pypi/micromamba.py,sha256=LLJ2dGGOEyld07W8iI6dtE01h2Y1PQnBhU-dMBssZ3c,16502
+metaflow/plugins/pypi/conda_environment.py,sha256=XZH3Glfn0_ZYFhbS7W9u2-V8NmUJ0MNltD9smeftB-Y,24791
+metaflow/plugins/pypi/micromamba.py,sha256=XAuZulI7O2Q_Zgtx28yjAqHco6tHiPQxoy_vsf1kWm8,16843
 metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
-metaflow/plugins/pypi/pip.py,sha256=H0cIy8odpZ-JTn4SwF0b74tuC3uRU7X8TdAQJ2kODG8,13971
+metaflow/plugins/pypi/pip.py,sha256=WhPyA18RoVT40sqbZRJdCEij4euL9PZwLfm1SrAKqmU,14333
 metaflow/plugins/pypi/pypi_decorator.py,sha256=ybNgo-T5Z_0W2KNuED0pdjyI0qygZ4a1MXAzKqdHt_E,7250
 metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
 metaflow/plugins/pypi/utils.py,sha256=W8OhDrhz7YIE-2MSSN5Rj7AmESwsN5YDmdnro152J4w,3551
@@ -393,12 +393,12 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 metaflow/user_configs/config_decorators.py,sha256=qCKVAvd0NKgaCxQ2OThes5-DYHXq6A1HqURubYNeFdw,20481
 metaflow/user_configs/config_options.py,sha256=m6jccSpzI4qUJ7vyYkYBIf8G3V0Caunxg_k7zg4Zlqg,21067
 metaflow/user_configs/config_parameters.py,sha256=oeJGVKu1ao_YQX6Lg6P2FEv5k5-_F4sARLlVpTW9ezM,15502
-ob_metaflow-2.15.11.1.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
-ob_metaflow-2.15.11.1.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
-ob_metaflow-2.15.11.1.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
-ob_metaflow-2.15.11.1.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.15.11.1.dist-info/METADATA,sha256=vRkRWqRQkqF4tUU16497HAeOlu5szQlflfl6Ru4YwGE,5937
-ob_metaflow-2.15.11.1.dist-info/WHEEL,sha256=oSJJyWjO7Z2XSScFQUpXG1HL-N0sFMqqeKVVbZTPkWc,109
-ob_metaflow-2.15.11.1.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-ob_metaflow-2.15.11.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.15.11.1.dist-info/RECORD,,
+ob_metaflow-2.15.11.2.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
+ob_metaflow-2.15.11.2.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
+ob_metaflow-2.15.11.2.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
+ob_metaflow-2.15.11.2.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.15.11.2.dist-info/METADATA,sha256=SUOXuM04ColPZum-XJ6n-5BNr_bUJ-vNCiWS7uoWG9k,5937
+ob_metaflow-2.15.11.2.dist-info/WHEEL,sha256=oSJJyWjO7Z2XSScFQUpXG1HL-N0sFMqqeKVVbZTPkWc,109
+ob_metaflow-2.15.11.2.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+ob_metaflow-2.15.11.2.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.15.11.2.dist-info/RECORD,,