metaflow-netflixext 1.3.2.dev0__tar.gz → 1.3.4.dev0__tar.gz

{metaflow_netflixext-1.3.2.dev0/metaflow_netflixext.egg-info → metaflow_netflixext-1.3.4.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow-netflixext
-Version: 1.3.2.dev0
+Version: 1.3.4.dev0
 Summary: Metaflow extensions from Netflix
 Author: Netflix Metaflow Developers
 Author-email: metaflow-dev@netflix.com

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/conda.py

@@ -685,6 +685,7 @@ class Conda(object):
         env_id: EnvID,
         local_only: bool = False,
         use_latest: str = CONDA_USE_REMOTE_LATEST,
+        full_id_unique_keys: Optional[Dict[str, str]] = None,
     ) -> Optional[ResolvedEnvironment]:
         """
         Returns the resolved environment for a given environment ID.
@@ -712,7 +713,9 @@ class Conda(object):
             The ResolvedEnvironment corresponding to the input EnvID
         """
         # First look if we have from_env_id locally
-        env = self._cached_environment.env_for(*env_id)
+        env = self._cached_environment.env_for(
+            *env_id, full_id_unique_keys=full_id_unique_keys
+        )
 
         debug.conda_exec("%s%sfound locally" % (str(env_id), " " if env else " not "))
         if env:
@@ -760,6 +763,25 @@ class Conda(object):
                         "%s found as latest remotely: %s"
                         % (str(env_id), str(env.env_id))
                     )
+
+        # In case the resolved files changed but the user dependency stays the same,
+        # which happens when pyproject.toml stays the same but the user does
+        # `uv lock --upgrade` -- the full_id_unique_keys which includes information
+        # from the resolved files hash will be different, and the cache environment
+        # will be invalidated.
+        if full_id_unique_keys and env:
+            if env._full_id_unique_keys != full_id_unique_keys:
+                debug.conda_exec(
+                    "%s found remotely but unique keys do not match, unique key in cache is %s"
+                    " unique key in request is %s"
+                    % (
+                        str(env.env_id),
+                        str(env._full_id_unique_keys),
+                        str(full_id_unique_keys),
+                    )
+                )
+                return None
+
         return env
 
     def environments(

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/conda_common_decorator.py

@@ -45,6 +45,10 @@ class StepRequirementIface:
     def file_paths(self) -> Dict[str, List[str]]:
         return {}
 
+    @property
+    def full_id_unique_keys(self) -> Dict[str, str]:
+        return {}
+
     @property
     def extras(self) -> Dict[str, List[str]]:
         return {}
@@ -102,6 +106,7 @@ class StepRequirement(StepRequirementIface):
         self._sources = {}  # type: Dict[str, List[str]]
         self._file_paths = {}  # type: Dict[str, List[str]]
         self._extras = {}  # type: Dict[str, List[str]]
+        self._full_id_unique_keys = {}  # type: Dict[str, str]
         self._default_disabled = {
             UBF_CONTROL: None,
             UBF_TASK: None,
@@ -117,6 +122,7 @@ class StepRequirement(StepRequirementIface):
         n._packages = copy.deepcopy(self._packages)
         n._sources = copy.deepcopy(self._sources)
         n._file_paths = copy.deepcopy(self._file_paths)
+        n._full_id_unique_keys = copy.deepcopy(self._full_id_unique_keys)
         n._extras = copy.deepcopy(self._extras)
         n._default_disabled = copy.deepcopy(self._default_disabled)
         return n
@@ -195,6 +201,14 @@ class StepRequirement(StepRequirementIface):
     def file_paths(self, value: Dict[str, List[str]]):
         self._file_paths = value
 
+    @property
+    def full_id_unique_keys(self) -> Dict[str, str]:
+        return copy.deepcopy(self._full_id_unique_keys)
+
+    @full_id_unique_keys.setter
+    def full_id_unique_keys(self, value: Dict[str, str]):
+        self._full_id_unique_keys = value
+
     @property
     def extras(self) -> Dict[str, List[str]]:
         return copy.deepcopy(self._extras)
@@ -305,6 +319,13 @@ class StepRequirement(StepRequirementIface):
         for category, extras in other_extras.items():
             self._extras.setdefault(category, []).extend(extras or [])
 
+        other_full_id_unique_keys = other.full_id_unique_keys
+        for key, value in other_full_id_unique_keys.items():
+            # We use override rather than list extension here for same keys,
+            # as we assume it rare that different decorators will have same
+            # full_id_unique_keys.
+            self._full_id_unique_keys[key] = value
+
         # Special handling for pathspec/name
         if other.from_name is not None and other.from_pathspec is not None:
             raise InvalidEnvironmentException(

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/conda_environment.py

@@ -135,6 +135,7 @@ class CondaEnvironment(MetaflowEnvironment):
                 step.name,
                 base_env,
                 base_from_full_id=base_from_full_id,
+                full_id_unique_keys=req.full_id_unique_keys,
             )
 
         resolver.resolve_environments(echo)
@@ -372,7 +373,11 @@ class CondaEnvironment(MetaflowEnvironment):
         elif step_info[2]:
             # In this case, we should know about the environment -- it will have
             # _default flag at this time
-            resolved_env = conda.environment(step_info[2][0], local_only=True)
+            resolved_env = conda.environment(
+                step_info[2][0],
+                local_only=True,
+                full_id_unique_keys=step_info[1].full_id_unique_keys,
+            )
             if resolved_env:
                 return resolved_env.env_id
             return None

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/conda_flow_mutator.py

@@ -9,7 +9,11 @@ from metaflow import FlowMutator
 from .conda_common_decorator import StepRequirementMixin
 from .conda_flow_decorator import PackageRequirementFlowDecorator
 from .parsers import req_parser, toml_parser
-from .utils import call_binary, determine_uv_env_python_version
+from .utils import (
+    call_binary,
+    determine_uv_env_python_version,
+    compute_file_hash,
+)
 
 
 if TYPE_CHECKING:
@@ -171,6 +175,9 @@ class ResolvedEnvironmentBaseFlowMutator(FlowMutator):
         # See _parse_requirements_txt() for the format of this dict.
         user_deps: Dict[str, str],
         user_sources: List[str],
+        # takes a full_id_unique_keys dict rather than uv_content_hash
+        # to allow more flexibility in the future.
+        full_id_unique_keys: Dict[str, str],
     ):
         ResolvedEnvironmentBaseFlowMutator._verify_no_conflict_step_decorators(
             step_name, step
@@ -181,6 +188,7 @@ class ResolvedEnvironmentBaseFlowMutator(FlowMutator):
                 "path": pylock_toml_path,
                 "user_deps_for_hash": user_deps,
                 "user_sources_for_hash": user_sources,
+                "full_id_unique_keys": full_id_unique_keys,
             },
         )
 
@@ -290,14 +298,23 @@ class ResolvedUVEnvFlowDecorator(ResolvedEnvironmentBaseFlowMutator):
                 f"must exist in the specified path {proj_config_dir}"
             )
 
-        temp_toml_file_path, user_deps, user_sources = self._handle_uv_lock_scenario(
-            os.path.join(proj_config_dir, "uv.lock"),
-            os.path.join(proj_config_dir, "pyproject.toml"),
+        temp_toml_file_path, user_deps, user_sources, uv_lock_file_content_hash = (
+            self._handle_uv_lock_scenario(
+                os.path.join(proj_config_dir, "uv.lock"),
+                os.path.join(proj_config_dir, "pyproject.toml"),
+            )
         )
 
         for step_name, step in mutable_flow.steps:
             ResolvedEnvironmentBaseFlowMutator._mutate_step(
-                step_name, step, temp_toml_file_path, user_deps, user_sources
+                step_name,
+                step,
+                temp_toml_file_path,
+                user_deps,
+                user_sources,
+                full_id_unique_keys={
+                    "uv_lock_content_hash": uv_lock_file_content_hash,
+                },
             )
 
     # @staticmethod
@@ -306,10 +323,11 @@ class ResolvedUVEnvFlowDecorator(ResolvedEnvironmentBaseFlowMutator):
     # ) -> tuple[Optional[str], Dict[str, str]]:
     #     raise NotImplementedError("This will be supported in a stacked PR soon.")
 
+    # Returns: (temp_pylock_toml_path, user_deps_dict, user_sources_list, uv_lock_content_hash)
     @staticmethod
     def _handle_uv_lock_scenario(
         uv_lock_path: str, pyproject_toml_path: str
-    ) -> Tuple[Optional[str], Dict[str, str], List[str]]:
+    ) -> Tuple[str, Dict[str, str], List[str], str]:
         # 1. read pyproject.toml
         user_deps, user_srcs = (
             ResolvedUVEnvFlowDecorator._parse_pyproject_toml_to_user_deps_dict(
@@ -337,7 +355,12 @@ class ResolvedUVEnvFlowDecorator(ResolvedEnvironmentBaseFlowMutator):
             cwd=cwd,
         )
 
-        return temp_path, user_deps, user_srcs
+        # Calculate uv lock file content hash, to be passed to PylockToml Resolver,
+        # and eventually ResolvedEnvironment, to be a component of full_id hash calculation,
+        # and will be used to invalidate the cache if pyproject.toml is the same but uv.lock changes.
+        uv_lock_content_hash = compute_file_hash(uv_lock_path)
+
+        return temp_path, user_deps, user_srcs, uv_lock_content_hash
 
     @staticmethod
     def _parse_pyproject_toml_to_user_deps_dict(

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/conda_step_decorator.py

@@ -392,6 +392,16 @@ class PylockTomlInternalDecorator(StepRequirementMixin, StepDecorator):
         "user_deps_for_hash": {},
         # The purpose is similar to user_deps_for_hash.
         "user_sources_for_hash": [],
+        # When a pylock_internal decorator is generated, the originating
+        # ResolvedUvEnvFlowDecorator calculates the hash of uv.lock, assigns it
+        # to the decorator's attributes["full_id_unique_keys"]["uv_content_hash"],
+        # and eventually this uv content hash value will be passed to
+        # PylockTomlInternalDecorator.resolve()'s parameters.
+        #
+        # This is used to detect a uv.lock change with unchanged user
+        # dependencies in pyproject.toml, caused by design by uv's non-determinism
+        # in resolution when ">=version" specifiers are involved.
+        "full_id_unique_keys": {},
     }
 
     name = "pylock_toml_internal"
@@ -445,6 +455,10 @@ class PylockTomlInternalDecorator(StepRequirementMixin, StepDecorator):
     def python(self) -> Optional[str]:
         return self.attributes["user_deps_for_hash"].get("python")
 
+    @property
+    def full_id_unique_keys(self) -> Dict[str, str]:
+        return cast(Dict[str, str], self.attributes["full_id_unique_keys"])
+
 
 class CondaEnvInternalDecorator(StepDecorator):
     name = "conda_env_internal"

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/env_descr.py

@@ -793,6 +793,8 @@ class ResolvedEnvironment:
         co_resolved: Optional[List[str]] = None,
         env_type: EnvType = EnvType.MIXED,
         accurate_source: bool = True,
+        full_id_unique_keys: Optional[Dict[str, str]] = None,
+        # Sample format: {'uv_lock_content': '33746974cb80adfea6047b5c5713c59c0dbc721da3179501c1bad261241cd3d3'}
     ):
         self._env_type = env_type
         self._user_dependencies = dict_to_tstr(user_dependencies)
@@ -803,6 +805,7 @@ class ResolvedEnvironment:
         self._user_extra_args = dict_to_tstr(user_extra_args) if user_extra_args else []
 
         self._accurate_source = accurate_source
+        self._full_id_unique_keys = full_id_unique_keys or {}
 
         if not env_id:
             env_req_id = ResolvedEnvironment.get_req_id(
@@ -813,13 +816,10 @@ class ResolvedEnvironment:
 
             env_full_id = "_unresolved"
             if all_packages is not None:
-                env_full_id = self._compute_hash(
-                    [
-                        "%s#%s" % (p.filename, p.pkg_hash(p.url_format))
-                        for p in sorted(all_packages, key=lambda p: p.filename)
-                    ]
-                    + [arch or arch_id()]
+                env_full_id = self._compute_full_id(
+                    all_packages, arch or arch_id(), self._full_id_unique_keys
                 )
+
             self._env_id = EnvID(
                 req_id=env_req_id, full_id=env_full_id, arch=arch or arch_id()
             )
@@ -937,10 +937,12 @@ class ResolvedEnvironment:
             Unique identifier for this environment.
         """
         if self._env_id.full_id in ("_default", "_unresolved") and self._all_packages:
-            all_packages = sorted(self._all_packages, key=lambda p: p.filename)
-            env_full_id = self._compute_hash(
-                [p.filename for p in all_packages] + [self._env_id.arch or arch_id()]
+            env_full_id = self._compute_full_id(
+                self._all_packages,
+                self._env_id.arch or arch_id(),
+                self._full_id_unique_keys,
             )
+
             self._env_id = self._env_id._replace(full_id=env_full_id)
         return self._env_id
 
@@ -1225,6 +1227,7 @@ class ResolvedEnvironment:
             "resolved_archs": self._co_resolved,
             "env_type": self._env_type.value,
             "accurate_source": self._accurate_source,
+            "full_id_unique_keys": self._full_id_unique_keys,
         }
 
     @classmethod
@@ -1267,12 +1270,54 @@ class ResolvedEnvironment:
             co_resolved=d["resolved_archs"],
             env_type=env_type,
             accurate_source=d.get("accurate_source", True),
+            full_id_unique_keys=d.get("full_id_unique_keys", None),
         )
 
     @staticmethod
-    def _compute_hash(inputs: Iterable[str]):
+    def _compute_hash(inputs: Iterable[str]) -> str:
         return sha1(b" ".join([s.encode("ascii") for s in inputs])).hexdigest()
 
+    @staticmethod
+    def _unique_keys_to_list(
+        unique_keys: Dict[str, str],
+    ) -> List[str]:
+        # Using "{k}:{v}" not "{k}#{v}" to avoid mixing with package filename#hash,
+        # in case this helps debuggability.
+        return [f"{k}:{v}" for k, v in sorted(unique_keys.items())]
+
+    # There are two levels of hashing for the environment:
+    # * the req_id which is based on the user-requested dependencies and sources.
+    # * the full_id which is based on the resolved user dependencies.
+    #
+    # Originally, full_id is composed of only the resolved dependencies and the
+    # architecture, and a req_id can be mapped to a recently resolved full_id.
+    # This justifies the caching mechanism such as the "_default" key under the
+    # req_id entry in the environment manifest.
+    #
+    # However, as we introduced uv,
+    # we found operations such as `uv lock --upgrade` can introduce different
+    # resolved dependencies while keeping the same user-requested dependencies.
+    # As a result, full_id needs to take into consideration full_id_unique_keys
+    # as well.
+    @staticmethod
+    def _compute_full_id(
+        all_packages: Sequence[PackageSpecification],
+        arch: str,
+        full_id_unique_keys: Optional[Dict[str, str]] = None,
+    ) -> str:
+        to_hash_list = [
+            "%s#%s" % (p.filename, p.pkg_hash(p.url_format))
+            for p in sorted(all_packages, key=lambda p: p.filename)
+        ] + [arch]
+        to_hash_list.extend(
+            ResolvedEnvironment._unique_keys_to_list(full_id_unique_keys)
+            if full_id_unique_keys
+            else []
+        )
+
+        env_full_id = ResolvedEnvironment._compute_hash(to_hash_list)
+        return env_full_id
+
 
 class MetaflowResolvedEnvironment:
     """
@@ -1511,7 +1556,11 @@ class CachedEnvironmentInfo:
         # Missing AliasType.REQ_FULL_ID but we don't record aliases for that.
 
     def env_for(
-        self, req_id: str, full_id: str = "_default", arch: Optional[str] = None
+        self,
+        req_id: str,
+        full_id: str = "_default",
+        arch: Optional[str] = None,
+        full_id_unique_keys: Optional[Dict[str, str]] = None,
     ) -> Optional[ResolvedEnvironment]:
         arch = arch or arch_id()
         per_arch_envs = self._resolved_environments.get(arch)
@@ -1520,7 +1569,16 @@ class CachedEnvironmentInfo:
             if per_req_id_envs:
                 if full_id == "_default":
                     full_id = per_req_id_envs.get("_default", "_invalid")  # type: ignore
-                return per_req_id_envs.get(full_id)  # type: ignore
+                env = per_req_id_envs.get(full_id)  # type: ignore
+
+                if (
+                    env
+                    and isinstance(env, ResolvedEnvironment)
+                    and env._full_id_unique_keys != full_id_unique_keys
+                ):
+                    env = None
+                return env  # type: ignore
+
         return None
 
     def envs_for(

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/envsresolver.py

@@ -97,6 +97,7 @@ class EnvsResolver(object):
         force: bool = False,
         local_only: bool = False,
         use_latest: str = CONDA_USE_REMOTE_LATEST,
+        full_id_unique_keys: Optional[Dict[str, str]] = None,
     ) -> Tuple[
         EnvType,
         EnvID,
@@ -184,11 +185,14 @@ class EnvsResolver(object):
                 co_resolved=base_env.co_resolved_archs,
                 env_type=base_env.env_type,
                 accurate_source=base_env.is_info_accurate,
+                full_id_unique_keys=full_id_unique_keys,
             )
             resolved_env.dirty = True
         else:
             resolved_env = (
-                conda.environment(env_id, local_only, use_latest) if not force else None
+                conda.environment(env_id, local_only, use_latest, full_id_unique_keys)
+                if not force
+                else None
             )
 
         return (
@@ -216,6 +220,7 @@ class EnvsResolver(object):
         use_latest: str = CONDA_USE_REMOTE_LATEST,
         force: bool = False,
         force_co_resolve: bool = False,
+        full_id_unique_keys: Optional[Dict[str, str]] = None,
     ):
         """
         Add an environment to resolve to this EnvsResolver. The EnvsResolver will resolve
@@ -306,6 +311,7 @@ class EnvsResolver(object):
             force=force,
             local_only=local_only,
             use_latest=use_latest,
+            full_id_unique_keys=full_id_unique_keys,
         )
 
         # Check if we have already requested this environment
@@ -331,6 +337,7 @@ class EnvsResolver(object):
                 "sources": user_sources,
                 "extras": extras,
                 "file_paths": file_paths,
+                "full_id_unique_keys": full_id_unique_keys,
                 "conda_format": (
                     [CONDA_PREFERRED_FORMAT]
                     if CONDA_PREFERRED_FORMAT and CONDA_PREFERRED_FORMAT != "none"
@@ -764,6 +771,7 @@ class EnvsResolver(object):
                         "resolved": builder_env,
                         "need_caching": builder_env is None,
                         "env_type": EnvType.CONDA_ONLY,
+                        "full_id_unique_keys": {},
                     }
                 self._builder_envs[builder_env_id] = builder_env_info
                 builders_by_req_id.setdefault(env_id.req_id, []).append(builder_env_id)
@@ -926,6 +934,7 @@ class EnvsResolver(object):
                 builder_envs,
                 env_desc["base"],
                 env_desc["file_paths"],
+                env_desc["full_id_unique_keys"],
             )
 
             if env_desc["base"]:

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/resolvers/conda_resolver.py

@@ -47,6 +47,8 @@ class CondaResolver(Resolver):
         builder_envs: Optional[List[ResolvedEnvironment]] = None,
         base_env: Optional[ResolvedEnvironment] = None,
         file_paths: Dict[str, List[str]] = {},
+        # full_id_unique_keys is not used in CondaResolver.
+        full_id_unique_keys: Optional[Dict[str, str]] = None,
     ) -> Tuple[ResolvedEnvironment, Optional[List[ResolvedEnvironment]]]:
         if base_env:
             local_packages = [
@@ -174,6 +176,10 @@ class CondaResolver(Resolver):
                 architecture,
                 all_packages=packages,
                 env_type=env_type,
+                # full_id_unique_keys is a uv specific cache invalidation mechanism,
+                # and PipResolver shouldn't need it. We still pass this along as {}
+                # to make code consistent.
+                full_id_unique_keys=full_id_unique_keys,
             ),
             builder_envs,
         )

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/resolvers/pip_resolver.py

@@ -62,6 +62,7 @@ class PipResolver(Resolver):
         builder_envs: Optional[List[ResolvedEnvironment]] = None,
         base_env: Optional[ResolvedEnvironment] = None,
         file_paths: Dict[str, List[str]] = {},
+        full_id_unique_keys: Dict[str, str] = {},
     ) -> Tuple[ResolvedEnvironment, Optional[List[ResolvedEnvironment]]]:
         if base_env:
             # For base environments, we may have built packages already so for those
@@ -594,6 +595,10 @@ class PipResolver(Resolver):
                 architecture,
                 all_packages=packages,
                 env_type=env_type,
+                # full_id_unique_keys is a uv specific cache invalidation mechanism,
+                # and PipResolver shouldn't need it. We still pass this along as {}
+                # to make code consistent.
+                full_id_unique_keys=full_id_unique_keys,
             ),
             builder_envs,
         )

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/resolvers/pylock_toml_resolver.py

@@ -18,7 +18,7 @@ from metaflow._vendor.packaging.tags import (
     Tag,
 )
 
-from metaflow_extensions.netflix_ext.plugins.conda.conda import CondaException
+from ..utils import CondaException
 
 from . import Resolver
 from typing import Callable, Dict, Iterable, List, Any, Optional, Tuple

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0}/metaflow_extensions/netflix_ext/plugins/conda/utils.py

@@ -1415,6 +1415,18 @@ def filter_user_reqs_by_markers(
     return new_deps
 
 
+def compute_file_hash(file_path: str) -> str:
+    if not file_path:
+        return ""
+
+    try:
+        with open(file_path, "rb") as f:
+            file_content_hash = hashlib.sha256(f.read()).hexdigest()
+            return file_content_hash
+    except (IOError, OSError):
+        raise IOError(f"Could not read file '{file_path}' for hashing") from None
+
+
 class WithDir:
     # WARNING: os.chdir is not compatible with thread processing so do not use in
     # a context where multiple threads can exist.

{metaflow_netflixext-1.3.2.dev0 → metaflow_netflixext-1.3.4.dev0/metaflow_netflixext.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow-netflixext
-Version: 1.3.2.dev0
+Version: 1.3.4.dev0
 Summary: Metaflow extensions from Netflix
 Author: Netflix Metaflow Developers
 Author-email: metaflow-dev@netflix.com