metaflow 2.19.2__py2.py3-none-any.whl → 2.19.4__py2.py3-none-any.whl

metaflow/datastore/flow_datastore.py

@@ -167,7 +167,8 @@ class FlowDataStore(object):
         if attempt is not None and attempt <= metaflow_config.MAX_ATTEMPTS - 1:
             attempt_range = range(attempt + 1) if include_prior else [attempt]
         for task_url in task_urls:
-            task_splits = task_url.split("/")
+            # task_url can have a trailing slash, so strip this to avoid empty strings in the split
+            task_splits = task_url.rstrip("/").split("/")
             # Usually it is flow, run, step, task (so 4 components) -- if we have a
             # fifth one, there is a specific attempt number listed as well.
             task_attempt_range = attempt_range

metaflow/metaflow_config.py

@@ -605,7 +605,7 @@ def get_pinned_conda_libs(python_version, datastore_type):
         pins["google-auth"] = ">=2.11.0"
         pins["google-cloud-secret-manager"] = ">=2.10.0"
         pins["simple-gcp-object-downloader"] = ">=0.1.0"
-        pins["packaging"] = ">=24.2"
+        pins["packaging"] = ">=24.0"
     elif datastore_type == "local":
         pass
     else:

metaflow/plugins/env_escape/client_modules.py

@@ -1,5 +1,6 @@
 import atexit
 import importlib
+import importlib.util
 import itertools
 import pickle
 import re
@@ -41,6 +42,8 @@ class _WrappedModule(object):
     def __getattr__(self, name):
         if name == "__loader__":
             return self._loader
+        if name == "__spec__":
+            return importlib.util.spec_from_loader(self._prefix, self._loader)
         if name in ("__name__", "__package__"):
             return self._prefix
         if name in ("__file__", "__path__"):
@@ -71,7 +74,8 @@ class _WrappedModule(object):
             # Try to see if this is a submodule that we can load
             m = None
             try:
-                m = self._loader.load_module(".".join([self._prefix, name]))
+                submodule_name = ".".join([self._prefix, name])
+                m = importlib.import_module(submodule_name)
             except ImportError:
                 pass
             if m is None:
@@ -117,7 +121,28 @@ class _WrappedModule(object):
 
 
 class ModuleImporter(object):
-    # This ModuleImporter implements the Importer Protocol defined in PEP 302
+    """
+    A custom import hook that proxies module imports to a different Python environment.
+
+    This class implements the MetaPathFinder and Loader protocols (PEP 451) to enable
+    "environment escape" - allowing the current Python process to import and use modules
+    from a different Python interpreter with potentially different versions or packages.
+
+    When a module is imported through this importer:
+    1. A client spawns a server process in the target Python environment
+    2. The module is loaded in the remote environment
+    3. A _WrappedModule proxy is returned that forwards all operations (function calls,
+       attribute access, etc.) to the remote environment via RPC
+    4. Data is serialized/deserialized using pickle for cross-environment communication
+
+    Args:
+        python_executable: Path to the Python interpreter for the remote environment
+        pythonpath: Python path to use in the remote environment
+        max_pickle_version: Maximum pickle protocol version supported by remote interpreter
+        config_dir: Directory containing configuration for the environment escape
+        module_prefixes: List of module name prefixes to handle
+    """
+
     def __init__(
         self,
         python_executable,
@@ -135,84 +160,89 @@ class ModuleImporter(object):
         self._handled_modules = None
         self._aliases = {}
 
-    def find_module(self, fullname, path=None):
+    def find_spec(self, fullname, path=None, target=None):
         if self._handled_modules is not None:
             if get_canonical_name(fullname, self._aliases) in self._handled_modules:
-                return self
+                return importlib.util.spec_from_loader(fullname, self)
             return None
         if any([fullname.startswith(prefix) for prefix in self._module_prefixes]):
             # We potentially handle this
-            return self
+            return importlib.util.spec_from_loader(fullname, self)
         return None
 
-    def load_module(self, fullname):
-        if fullname in sys.modules:
-            return sys.modules[fullname]
-        if self._client is None:
-            if sys.version_info[0] < 3:
-                raise NotImplementedError(
-                    "Environment escape imports are not supported in Python 2"
-                )
-            # We initialize a client and query the modules we handle
-            # The max_pickle_version is the pickle version that the server (so
-            # the underlying interpreter we call into) supports; we determine
-            # what version the current environment support and take the minimum
-            # of those two
-            max_pickle_version = min(self._max_pickle_version, pickle.HIGHEST_PROTOCOL)
-
-            self._client = Client(
-                self._module_prefixes,
-                self._python_executable,
-                self._pythonpath,
-                max_pickle_version,
-                self._config_dir,
-            )
-            atexit.register(_clean_client, self._client)
-
-            # Get information about overrides and what the server knows about
-            exports = self._client.get_exports()
-
-            prefixes = set()
-            export_classes = exports.get("classes", [])
-            export_functions = exports.get("functions", [])
-            export_values = exports.get("values", [])
-            export_exceptions = exports.get("exceptions", [])
-            self._aliases = exports.get("aliases", {})
-            for name in itertools.chain(
-                export_classes,
-                export_functions,
-                export_values,
-                (e[0] for e in export_exceptions),
-            ):
-                splits = name.rsplit(".", 1)
-                prefixes.add(splits[0])
-            # We will make sure that we create modules even for "empty" prefixes
-            # because packages are always loaded hierarchically so if we have
-            # something in `a.b.c` but nothing directly in `a`, we still need to
-            # create a module named `a`. There is probably a better way of doing this
-            all_prefixes = list(prefixes)
-            for prefix in all_prefixes:
-                parts = prefix.split(".")
-                cur = parts[0]
-                for i in range(1, len(parts)):
-                    prefixes.add(cur)
-                    cur = ".".join([cur, parts[i]])
-
-            # We now know all the modules that we can handle. We update
-            # handled_module and return the module if we have it or raise ImportError
-            self._handled_modules = {}
-            for prefix in prefixes:
-                self._handled_modules[prefix] = _WrappedModule(
-                    self, prefix, exports, self._client
-                )
+    def create_module(self, spec):
+        # Return the pre-created wrapped module for this spec
+        self._initialize_client()
+
+        fullname = spec.name
         canonical_fullname = get_canonical_name(fullname, self._aliases)
-        # Modules are created canonically but we need to return something for any
-        # of the aliases.
-        module = self._handled_modules.get(canonical_fullname)
-        if module is None:
-            raise ImportError
-        sys.modules[fullname] = module
-        return module
+        # Modules are created canonically but we need to handle any of the aliases.
+        wrapped_module = self._handled_modules.get(canonical_fullname)
+        if wrapped_module is None:
+            raise ImportError(f"No module named '{fullname}'")
+        return wrapped_module
+
+    def exec_module(self, module):
+        # No initialization needed since the wrapped module returned by
+        # create_module() is fully initialized
+        pass
+
+    def _initialize_client(self):
+        if self._client is not None:
+            return
+
+        # We initialize a client and query the modules we handle
+        # The max_pickle_version is the pickle version that the server (so
+        # the underlying interpreter we call into) supports; we determine
+        # what version the current environment support and take the minimum
+        # of those two
+        max_pickle_version = min(self._max_pickle_version, pickle.HIGHEST_PROTOCOL)
+
+        self._client = Client(
+            self._module_prefixes,
+            self._python_executable,
+            self._pythonpath,
+            max_pickle_version,
+            self._config_dir,
+        )
+        atexit.register(_clean_client, self._client)
+
+        # Get information about overrides and what the server knows about
+        exports = self._client.get_exports()
+
+        prefixes = set()
+        export_classes = exports.get("classes", [])
+        export_functions = exports.get("functions", [])
+        export_values = exports.get("values", [])
+        export_exceptions = exports.get("exceptions", [])
+        self._aliases = exports.get("aliases", {})
+        for name in itertools.chain(
+            export_classes,
+            export_functions,
+            export_values,
+            (e[0] for e in export_exceptions),
+        ):
+            splits = name.rsplit(".", 1)
+            prefixes.add(splits[0])
+        # We will make sure that we create modules even for "empty" prefixes
+        # because packages are always loaded hierarchically so if we have
+        # something in `a.b.c` but nothing directly in `a`, we still need to
+        # create a module named `a`. There is probably a better way of doing this
+        all_prefixes = list(prefixes)
+        for prefix in all_prefixes:
+            parts = prefix.split(".")
+            cur = parts[0]
+            for i in range(1, len(parts)):
+                prefixes.add(cur)
+                cur = ".".join([cur, parts[i]])
+
+        # We now know all the modules that we can handle. We update
+        # handled_module and return the module if we have it or raise ImportError
+        self._handled_modules = {}
+        for prefix in prefixes:
+            self._handled_modules[prefix] = _WrappedModule(
+                self, prefix, exports, self._client
+            )
 
 
 def create_modules(python_executable, pythonpath, max_pickle_version, path, prefixes):

metaflow/user_decorators/mutable_flow.py

@@ -239,7 +239,7 @@ class MutableFlow:
                     "Mutable flow removing parameter %s from flow" % var
                 )
                 # Reset so that we don't list it again
-                del self._flow_cls._flow_state[_FlowState.CACHED_PARAMETERS]
+                self._flow_cls._flow_state.pop(_FlowState.CACHED_PARAMETERS, None)
                 return True
         debug.userconf_exec(
             "Mutable flow failed to remove parameter %s from flow" % parameter_name

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.19.2"
+metaflow_version = "2.19.4"

{metaflow-2.19.2.data → metaflow-2.19.4.data}/data/share/metaflow/devtools/Tiltfile

@@ -630,7 +630,7 @@ if "metadata-service" in enabled_components:
             'metadatadb.database=metaflow',
             'metadatadb.host=postgresql',
             'image.repository=public.ecr.aws/outerbounds/metaflow_metadata_service',
-            'image.tag=2.4.13-2-g70af4ed',
+            'image.tag=2.5.0',
             'resources.requests.cpu=25m',
             'resources.requests.memory=64Mi',
             'resources.limits.cpu=50m',

{metaflow-2.19.2.dist-info → metaflow-2.19.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.19.2
+Version: 2.19.4
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.19.2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.19.4; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.19.2.dist-info → metaflow-2.19.4.dist-info}/RECORD

@@ -16,7 +16,7 @@ metaflow/includefile.py,sha256=NXERo_halboBCjvnS5iCjnBMyCMlQMnT2mRe6kxl2xU,21978
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=A2NdUq_MnQal_RUCMC8ZOSR0VYZGyi2mSgwPQB0UzQo,15343
 metaflow/meta_files.py,sha256=vlgJHI8GJUKzXoxdrVoH8yyCF5bhFgwYemUgnyd1wgM,342
-metaflow/metaflow_config.py,sha256=fyIr5xzL3j6NLmetxK9hbOHgWc77v2HwuWxLOi6GDXo,25670
+metaflow/metaflow_config.py,sha256=gI_y2SfJYCBAb1n_dE0oBn9u-vkOUsZvJ2Dm8nEPx30,25670
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
 metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
 metaflow/metaflow_environment.py,sha256=DWe9yJttDURtUNWmNg6TKr36CQpo_iY-fXG5yxGTQjE,11506
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=nyX0NWJAIRk1gImohMYvuS2ZoQup9VO1aiNEIN4Y1Ik,21035
 metaflow/vendor.py,sha256=A82CGHfStZGDP5pQ5XzRjFkbN1ZC-vFmghXIrzMDDNg,5868
-metaflow/version.py,sha256=us3V2hf9-rJI2KZbxaGFpE_ogCdurd3WSix2rp_fwzU,28
+metaflow/version.py,sha256=SxC8AHEAgIIfEqI3iYfosaum_4H5Fptkn4K5f_8UzJc,28
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -174,7 +174,7 @@ metaflow/datastore/content_addressed_store.py,sha256=2ZDn9wpurtHYNwBbwca0ipzHciV
 metaflow/datastore/datastore_set.py,sha256=WDgGeySHtt5hr6u3L1XMmd3b_yAJz6_2Ip_bj_Ygsms,2614
 metaflow/datastore/datastore_storage.py,sha256=7V43QuiWDQ_Q4oHw9y7Z7X9lYj3GI-LV1-xB3d2Tt5k,9038
 metaflow/datastore/exceptions.py,sha256=r7Ab5FvHIzyFh6kwiptA1lO5nLqWg0xRBoeYGefvapA,373
-metaflow/datastore/flow_datastore.py,sha256=UeZ-dagR2ru8dmMWLThTat7tQnh3v4YcGskYUanhrYg,14727
+metaflow/datastore/flow_datastore.py,sha256=-9S5pWPfyL4yVjVSkNljQ_y1Wq6pfaFH6GglmG5BoTc,14839
 metaflow/datastore/inputs.py,sha256=i43dXr2xvgtsgKMO9allgCR18bk80GeayeQFyUTH36w,449
 metaflow/datastore/spin_datastore.py,sha256=5bhtVNWl4131pMKtBN72DLHxc72QX46JO0sV7r6IpVY,3035
 metaflow/datastore/task_datastore.py,sha256=i9hBON-J0ECV_1uaqAtUGzPcd949mnYQBsqaJfVt15Q,38332
@@ -317,7 +317,7 @@ metaflow/plugins/datatools/s3/s3tail.py,sha256=boQjQGQMI-bvTqcMP2y7uSlSYLcvWOy7J
 metaflow/plugins/datatools/s3/s3util.py,sha256=FgRgaVmEq7-i2dV7q8XK5w5PfFt-xJjZa8WrK8IJfdI,3769
 metaflow/plugins/env_escape/__init__.py,sha256=tGNUZnmPvk52eNs__VK443b3CZ7ogEFTT-s9_n_HF8Q,8837
 metaflow/plugins/env_escape/client.py,sha256=GsFZqjhGttd4eMU_CTw14sfNBV4vBKUUShp4SR0D8k8,25066
-metaflow/plugins/env_escape/client_modules.py,sha256=hQAcwz41wd6NnHezdI2Tv37ngO3CYuJ2WG8Wak6z59s,9295
+metaflow/plugins/env_escape/client_modules.py,sha256=-4eeNz_F09CfKCoyJc6Zxv4OFY7alpbWHHYoFFm05Ik,10580
 metaflow/plugins/env_escape/consts.py,sha256=jafRUdqZnkeKSgpdmRcTmnsVhEhVAjfQ6TKErbRH7wo,1000
 metaflow/plugins/env_escape/data_transferer.py,sha256=wm1Aqf0rTWaq8JgqpiRN0g3N3hX7YAMuhCLRrbOP_9E,12696
 metaflow/plugins/env_escape/exception_transferer.py,sha256=rpcpwDYRf5XqsZvShyQe5eRQ9uBGNdqmyAeB1h-Lofw,5746
@@ -430,16 +430,16 @@ metaflow/user_configs/config_options.py,sha256=d3hKA6WRPe21PdTl7sBnxIp5sE6zBpRtg
 metaflow/user_configs/config_parameters.py,sha256=2LJnDPJ2NuZrM7ZrdOHNSbPw-X3BhM_tJgy0bxBAWAA,20975
 metaflow/user_decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/user_decorators/common.py,sha256=0u9NRLQ95TfJCjWVEOGT4MJ9WoQgAMBM9kJ2gJGlVjk,5362
-metaflow/user_decorators/mutable_flow.py,sha256=NPPcvSyeqYCTYNi9PaxmDISxLFKYqzwMMppSfqR7lvE,19498
+metaflow/user_decorators/mutable_flow.py,sha256=c7cYqj7100K1UYGQCF1B_AttyHwdcwCZWslnZfZnZPo,19504
 metaflow/user_decorators/mutable_step.py,sha256=-BY0UDXf_RCAEnC5JlLzEXGdiw1KD9oSrSxS_SWaB9Y,16791
 metaflow/user_decorators/user_flow_decorator.py,sha256=2yDwZq9QGv9W-7kEuKwa8o4ZkTvuHJ5ESz7VVrGViAI,9890
 metaflow/user_decorators/user_step_decorator.py,sha256=4I1TQlQDMaMSuAJAAm3wo3bFETODC77RtXW11EuUI4A,27837
-metaflow-2.19.2.data/data/share/metaflow/devtools/Makefile,sha256=_otS4tft0V5IzmMJPeVPxoN1_1aUiwC9-2Nu54k40ao,14499
-metaflow-2.19.2.data/data/share/metaflow/devtools/Tiltfile,sha256=b6l_fjDO0wtxOkO85lFvuf3HDa6wnzHhhBG8yv1kQqk,23949
-metaflow-2.19.2.data/data/share/metaflow/devtools/pick_services.sh,sha256=PGjQeDIigFHeoQ0asmYNdYDPIOdeYy1UYvkw2wdN4zg,2209
-metaflow-2.19.2.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.19.2.dist-info/METADATA,sha256=GRzj__ayQkG3VkzUx80YIugmW9P21yCA6nW3Oum78AQ,6739
-metaflow-2.19.2.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-metaflow-2.19.2.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-metaflow-2.19.2.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.19.2.dist-info/RECORD,,
+metaflow-2.19.4.data/data/share/metaflow/devtools/Makefile,sha256=_otS4tft0V5IzmMJPeVPxoN1_1aUiwC9-2Nu54k40ao,14499
+metaflow-2.19.4.data/data/share/metaflow/devtools/Tiltfile,sha256=ziXgaSg19d4eRDDw3mQkpwsPqeb9lorkfeVQNLqcL5E,23937
+metaflow-2.19.4.data/data/share/metaflow/devtools/pick_services.sh,sha256=PGjQeDIigFHeoQ0asmYNdYDPIOdeYy1UYvkw2wdN4zg,2209
+metaflow-2.19.4.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.19.4.dist-info/METADATA,sha256=W2QpM9_r1MlG6JLR-evG00brDWYzQnbOwHtBZvew3X0,6739
+metaflow-2.19.4.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+metaflow-2.19.4.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+metaflow-2.19.4.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.19.4.dist-info/RECORD,,