ob-metaflow 2.16.6.4rc0__tar.gz → 2.16.6.5rc1__tar.gz

{ob_metaflow-2.16.6.4rc0/ob_metaflow.egg-info → ob_metaflow-2.16.6.5rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.6.4rc0
+Version: 2.16.6.5rc1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.6.4rc0; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.6.5rc1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/devtools/Tiltfile

@@ -14,6 +14,17 @@
 version_settings(constraint='>=0.22.2')
 allow_k8s_contexts('minikube')
 
+# Version configuration for components
+JOBSET_VERSION = os.getenv("JOBSET_VERSION", "v0.6.0")
+
+# Argo Workflows versions
+ARGO_WORKFLOWS_HELM_CHART_VERSION = os.getenv("ARGO_WORKFLOWS_HELM_CHART_VERSION", "0.45.2")  # Helm chart version
+ARGO_WORKFLOWS_IMAGE_TAG = os.getenv("ARGO_WORKFLOWS_IMAGE_TAG", "v3.6.0")  # Argo Workflows application version
+
+# Argo Events versions  
+ARGO_EVENTS_HELM_CHART_VERSION = os.getenv("ARGO_EVENTS_HELM_CHART_VERSION", "2.4.8")  # Helm chart version
+ARGO_EVENTS_IMAGE_TAG = os.getenv("ARGO_EVENTS_IMAGE_TAG", "v1.9.2")  # Argo Events application version
+
 components = {
     "metadata-service": ["postgresql"],
     "ui": ["postgresql", "minio"],
@@ -21,9 +32,10 @@ components = {
     "postgresql": [],
     "argo-workflows": [],
     "argo-events": ["argo-workflows"],
+    "jobset": [],
 }
 
-services_env = os.getenv("SERVICES", "").strip().lower()
+services_env = os.getenv("SERVICES", "all").strip().lower()
 
 if services_env:
     if services_env == "all":
@@ -205,6 +217,7 @@ if "postgresql" in enabled_components:
 if "argo-workflows" in enabled_components:
     helm_remote(
         'argo-workflows',
+        version=ARGO_WORKFLOWS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -220,7 +233,9 @@ if "argo-workflows" in enabled_components:
             'controller.resources.requests.memory=128Mi',
             'controller.resources.requests.cpu=50m',
             'controller.resources.limits.memory=256Mi',
-            'controller.resources.limits.cpu=100m'
+            'controller.resources.limits.cpu=100m',
+            # Image version overrides
+            'images.tag=%s' % ARGO_WORKFLOWS_IMAGE_TAG,
         ]
     )
 
@@ -307,6 +322,7 @@ if "argo-workflows" in enabled_components:
 if "argo-events" in enabled_components:
     helm_remote(
         'argo-events',
+        version=ARGO_EVENTS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -334,6 +350,8 @@ if "argo-events" in enabled_components:
             'configs.jetstream.versions[1].natsImage=nats:2.9.15',
             'configs.jetstream.versions[1].startCommand=/nats-server',
             'configs.jetstream.versions[1].version=2.9.15',
+            # Image version overrides
+            'global.image.tag=%s' % ARGO_EVENTS_IMAGE_TAG,
         ]
     )
 
@@ -541,6 +559,29 @@ if "argo-events" in enabled_components:
     config_resources.append('argo-events-controller-manager')
     config_resources.append('argo-events-webhook-eventsource-svc')
 
+#################################################
+# JOBSET
+#################################################
+if "jobset" in enabled_components:
+    # Apply JobSet manifests directly from GitHub releases
+    jobset_manifest_url = "https://github.com/kubernetes-sigs/jobset/releases/download/%s/manifests.yaml" % JOBSET_VERSION
+
+    cmd = "curl -sSL %s" % (jobset_manifest_url)
+    k8s_yaml(
+        local(
+            cmd,
+        )
+    )
+
+    k8s_resource(
+        'jobset-controller-manager',
+        labels=['jobset'],
+    )
+
+    metaflow_config["METAFLOW_KUBERNETES_JOBSET_ENABLED"] = "true"
+    
+    config_resources.append('jobset-controller-manager')
+
 #################################################
 # METADATA SERVICE
 #################################################

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/devtools/pick_services.sh

@@ -18,6 +18,7 @@ SERVICE_OPTIONS=(
     "ui"
     "argo-workflows"
     "argo-events"
+    "jobset"
 )
 
 gum style "$LOGO" \

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/_vendor/imghdr/__init__.py

@@ -1,14 +1,9 @@
 """Recognize image file formats based on their first few bytes."""
 
 from os import PathLike
-import warnings
 
 __all__ = ["what"]
 
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
 #-------------------------#
 # Recognize image headers #
 #-------------------------#

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/client/core.py

@@ -831,12 +831,10 @@ class MetaflowCode(object):
         )
         self._code_obj = BytesIO(blobdata)
         self._info = MetaflowPackage.cls_get_info(self._code_metadata, self._code_obj)
-        self._code_obj.seek(0)
         if self._info:
             self._flowspec = MetaflowPackage.cls_get_content(
                 self._code_metadata, self._code_obj, self._info["script"]
             )
-            self._code_obj.seek(0)
         else:
             raise MetaflowInternalError("Code package metadata is invalid.")
 
@@ -887,9 +885,7 @@ class MetaflowCode(object):
             TarFile for everything in this code package
         """
         if self._backend.type == "tgz":
-            to_return = self._backend.cls_open(self._code_obj)
-            self._code_obj.seek(0)
-            return to_return
+            return self._backend.cls_open(self._code_obj)
         raise RuntimeError("Archive is not a tarball")
 
     def extract(self) -> TemporaryDirectory:
@@ -925,7 +921,6 @@ class MetaflowCode(object):
         MetaflowPackage.cls_extract_into(
             self._code_metadata, self._code_obj, tmp.name, ContentType.USER_CONTENT
         )
-        self._code_obj.seek(0)
         return tmp
 
     @property

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/extension_support/__init__.py

@@ -205,10 +205,9 @@ def package_mfext_all():
     # the packaged metaflow_extensions directory "self-contained" so that
     # python doesn't go and search other parts of the system for more
     # metaflow_extensions.
-    if _all_packages:
-        yield os.path.join(
-            os.path.dirname(os.path.abspath(__file__)), "_empty_file.py"
-        ), os.path.join(EXT_PKG, "__init__.py")
+    yield os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), "_empty_file.py"
+    ), os.path.join(EXT_PKG, "__init__.py")
 
     for p in _all_packages:
         for path_tuple in package_mfext_package(p):

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/metaflow_environment.py

@@ -203,19 +203,6 @@ class MetaflowEnvironment(object):
                     "mfcontent_version": 1,
                 }
             )
-
-        extra_exports = []
-        for k, v in MetaflowPackage.get_post_extract_env_vars(
-            code_package_metadata, dest_dir="$(pwd)"
-        ).items():
-            if k.endswith(":"):
-                # If the value ends with a colon, we override the existing value
-                extra_exports.append("export %s=%s" % (k[:-1], v))
-            else:
-                extra_exports.append(
-                    "export %s=%s:$(printenv %s)" % (k, v.replace('"', '\\"'), k)
-                )
-
         cmds = (
             [
                 BASH_MFLOG,
@@ -239,7 +226,12 @@ class MetaflowEnvironment(object):
             + MetaflowPackage.get_extract_commands(
                 code_package_metadata, "job.tar", dest_dir="."
             )
-            + extra_exports
+            + [
+                "export %s=%s:$(printenv %s)" % (k, v.replace('"', '\\"'), k)
+                for k, v in MetaflowPackage.get_post_extract_env_vars(
+                    code_package_metadata, dest_dir="."
+                ).items()
+            ]
             + [
                 "mflog 'Task is starting.'",
                 "flush_mflogs",

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/package/__init__.py

@@ -17,6 +17,7 @@ from ..packaging_sys.utils import suffix_filter, walk
 from ..metaflow_config import DEFAULT_PACKAGE_SUFFIXES
 from ..exception import MetaflowException
 from ..user_configs.config_parameters import dump_config_values
+from ..util import get_metaflow_root
 from .. import R
 
 DEFAULT_SUFFIXES_LIST = DEFAULT_PACKAGE_SUFFIXES.split(",")
@@ -75,22 +76,12 @@ class MetaflowPackage(object):
             from ..user_decorators.user_flow_decorator import FlowMutatorMeta
             from ..user_decorators.user_step_decorator import UserStepDecoratorMeta
 
-            # Be very defensive here to filter modules in case there are
-            # some badly behaved modules that have weird values for
-            # METAFLOW_PACKAGE_POLICY for example.
-            try:
-                if (
-                    m.__name__ in FlowMutatorMeta._import_modules
-                    or m.__name__ in UserStepDecoratorMeta._import_modules
-                    or (
-                        hasattr(m, "METAFLOW_PACKAGE_POLICY")
-                        and m.METAFLOW_PACKAGE_POLICY == "include"
-                    )
-                ):
-                    return True
-                return False
-            except:
-                return False
+            if (
+                m.__name__ in FlowMutatorMeta._import_modules
+                or m.__name__ in UserStepDecoratorMeta._import_modules
+                or hasattr(m, "METAFLOW_PACKAGE")
+            ):
+                return True
 
         if mfcontent is None:
             self._mfcontent = MetaflowCodeContentV1(criteria=_module_selector)
@@ -359,10 +350,10 @@ class MetaflowPackage(object):
         """
         backend = cls.get_backend(pkg_metadata)
         with backend.cls_open(archive) as opened_archive:
-            include_members = MetaflowCodeContent.get_archive_content_members(
+            include_names = MetaflowCodeContent.get_archive_content_names(
                 opened_archive, content_types, backend
             )
-            backend.cls_extract_members(opened_archive, include_members, dest_dir)
+            backend.extract_members(include_names, dest_dir)
 
     def user_tuples(self, timeout: Optional[float] = None):
         # Wait for at least the blob to be formed

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/packaging_sys/__init__.py

@@ -118,7 +118,9 @@ class MetaflowCodeContent:
         return handling_cls.get_filename_impl(mfcontent_info, filename, content_type)
 
     @classmethod
-    def get_env_vars_for_packaged_metaflow(cls, dest_dir: str) -> Dict[str, str]:
+    def get_env_vars_for_packaged_metaflow(
+        cls, dest_dir: str
+    ) -> Optional[Dict[str, str]]:
         """
         Get the environment variables that are needed to run Metaflow when it is
         packaged. This is typically used to set the PYTHONPATH to include the
@@ -126,19 +128,17 @@ class MetaflowCodeContent:
 
         Returns
         -------
-        Dict[str, str]
+        Optional[Dict[str, str]]
             The environment variables that are needed to run Metaflow when it is
-            packaged it present.
+            packaged -- None if there are no such variables (not packaged for example)
         """
-        mfcontent_info = cls._extract_mfcontent_info(dest_dir)
+        mfcontent_info = cls._extract_mfcontent_info()
         if mfcontent_info is None:
             # No MFCONTENT_MARKER file found -- this is not a packaged Metaflow code
             # package so no environment variables to set.
-            return {}
+            return None
         handling_cls = cls._get_mfcontent_class(mfcontent_info)
-        v = handling_cls.get_post_extract_env_vars_impl(dest_dir)
-        v["METAFLOW_EXTRACTED_ROOT:"] = dest_dir
-        return v
+        return handling_cls.get_post_extract_env_vars_impl(dest_dir)
 
     @classmethod
     def get_archive_info(
@@ -216,15 +216,15 @@ class MetaflowCodeContent:
         )
 
     @classmethod
-    def get_archive_content_members(
+    def get_archive_content_names(
         cls,
         archive: Any,
         content_types: Optional[int] = None,
         packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
-    ) -> List[Any]:
+    ) -> List[str]:
         mfcontent_info = cls._extract_archive_mfcontent_info(archive, packaging_backend)
         handling_cls = cls._get_mfcontent_class(mfcontent_info)
-        return handling_cls.get_archive_content_members_impl(
+        return handling_cls.get_archive_content_names_impl(
             mfcontent_info, archive, content_types, packaging_backend
         )
 
@@ -276,9 +276,7 @@ class MetaflowCodeContent:
                 "Invalid package -- unknown version %s in info: %s"
                 % (version_id, cls._mappings)
             )
-        v = cls._mappings[version_id].get_post_extract_env_vars_impl(dest_dir)
-        v["METAFLOW_EXTRACTED_ROOT:"] = dest_dir
-        return v
+        return cls._mappings[version_id].get_post_extract_env_vars_impl(dest_dir)
 
     # Implement the _impl methods in the base subclass (in this file). These need to
     # happen with as few imports as possible to prevent circular dependencies.
@@ -339,14 +337,14 @@ class MetaflowCodeContent:
         raise NotImplementedError("get_archive_filename_impl not implemented")
 
     @classmethod
-    def get_archive_content_members_impl(
+    def get_archive_content_names_impl(
         cls,
         mfcontent_info: Optional[Dict[str, Any]],
         archive: Any,
         content_types: Optional[int] = None,
         packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
-    ) -> List[Any]:
-        raise NotImplementedError("get_archive_content_members_impl not implemented")
+    ) -> List[str]:
+        raise NotImplementedError("get_archive_content_names_impl not implemented")
 
     @classmethod
     def get_post_extract_env_vars_impl(cls, dest_dir: str) -> Dict[str, str]:
@@ -525,22 +523,19 @@ class MetaflowCodeContent:
         return mfcontent_info
 
     @classmethod
-    def _extract_mfcontent_info(
-        cls, target_dir: Optional[str] = None
-    ) -> Optional[Dict[str, Any]]:
-        target_dir = target_dir or "_local"
-        if target_dir in cls._cached_mfcontent_info:
-            return cls._cached_mfcontent_info[target_dir]
+    def _extract_mfcontent_info(cls) -> Optional[Dict[str, Any]]:
+        if "_local" in cls._cached_mfcontent_info:
+            return cls._cached_mfcontent_info["_local"]
 
         mfcontent_info = None  # type: Optional[Dict[str, Any]]
-        if target_dir == "_local":
-            root = os.environ.get("METAFLOW_EXTRACTED_ROOT", get_metaflow_root())
-        else:
-            root = target_dir
-        if os.path.exists(os.path.join(root, MFCONTENT_MARKER)):
-            with open(os.path.join(root, MFCONTENT_MARKER), "r", encoding="utf-8") as f:
+        if os.path.exists(os.path.join(get_metaflow_root(), MFCONTENT_MARKER)):
+            with open(
+                os.path.join(get_metaflow_root(), MFCONTENT_MARKER),
+                "r",
+                encoding="utf-8",
+            ) as f:
                 mfcontent_info = json.load(f)
-        cls._cached_mfcontent_info[target_dir] = mfcontent_info
+        cls._cached_mfcontent_info["_local"] = mfcontent_info
         return mfcontent_info
 
     def get_package_version(self) -> int:
@@ -632,13 +627,13 @@ class MetaflowCodeContentV0(MetaflowCodeContent, version_id=0):
         return None
 
     @classmethod
-    def get_archive_content_members_impl(
+    def get_archive_content_names_impl(
         cls,
         mfcontent_info: Optional[Dict[str, Any]],
         archive: Any,
         content_types: Optional[int] = None,
         packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
-    ) -> List[Any]:
+    ) -> List[str]:
         """
         For V0, we use a static list of known files to classify the content
         """
@@ -654,20 +649,16 @@ class MetaflowCodeContentV0(MetaflowCodeContent, version_id=0):
             "condav2-1.cnd": ContentType.OTHER_CONTENT.value,
         }
         to_return = []
-        for member in packaging_backend.cls_list_members(archive):
-            filename = packaging_backend.cls_member_name(member)
-            added = False
+        for filename in packaging_backend.cls_list_members(archive):
             for prefix, classification in known_prefixes.items():
                 if (
                     prefix[-1] == "/" and filename.startswith(prefix)
                 ) or prefix == filename:
                     if content_types & classification:
-                        to_return.append(member)
-                        added = True
-                        break
-            if not added and content_types & ContentType.USER_CONTENT.value:
-                # Everything else is user content
-                to_return.append(member)
+                        to_return.append(filename)
+                elif content_types & ContentType.USER_CONTENT.value:
+                    # Everything else is user content
+                    to_return.append(filename)
         return to_return
 
     @classmethod
@@ -714,7 +705,7 @@ class MetaflowCodeContentV1Base(MetaflowCodeContent, version_id=1):
         cls, mfcontent_info: Optional[Dict[str, Any]], filename: str, in_archive: bool
     ) -> str:
         if in_archive:
-            return os.path.join(cls._other_dir, filename)
+            return filename
         return os.path.join(get_metaflow_root(), "..", cls._other_dir, filename)
 
     @classmethod
@@ -722,7 +713,7 @@ class MetaflowCodeContentV1Base(MetaflowCodeContent, version_id=1):
         cls, mfcontent_info: Optional[Dict[str, Any]], filename: str, in_archive: bool
     ) -> str:
         if in_archive:
-            return os.path.join(cls._code_dir, filename)
+            return filename
         return os.path.join(get_metaflow_root(), filename)
 
     @classmethod
@@ -841,38 +832,37 @@ class MetaflowCodeContentV1Base(MetaflowCodeContent, version_id=1):
         return None
 
     @classmethod
-    def get_archive_content_members_impl(
+    def get_archive_content_names_impl(
         cls,
         mfcontent_info: Optional[Dict[str, Any]],
         archive: Any,
         content_types: Optional[int] = None,
         packaging_backend: Type[PackagingBackend] = TarPackagingBackend,
-    ) -> List[Any]:
+    ) -> List[str]:
         to_return = []
         module_content = set(mfcontent_info.get("module_files", []))
-        for member in packaging_backend.cls_list_members(archive):
-            filename = packaging_backend.cls_member_name(member)
+        for filename in packaging_backend.cls_list_members(archive):
             if filename.startswith(cls._other_dir) and (
                 content_types & ContentType.OTHER_CONTENT.value
             ):
-                to_return.append(member)
+                to_return.append(filename)
             elif filename.startswith(cls._code_dir):
                 # Special case for marker which is a other content even if in code.
-                if filename == MFCONTENT_MARKER:
+                if filename == f"{cls._code_dir}/{MFCONTENT_MARKER}":
                     if content_types & ContentType.OTHER_CONTENT.value:
-                        to_return.append(member)
+                        to_return.append(filename)
                     else:
                         continue
                 # Here it is either module or code
                 if os.path.join(cls._code_dir, filename) in module_content:
                     if content_types & ContentType.MODULE_CONTENT.value:
-                        to_return.append(member)
+                        to_return.append(filename)
                 elif content_types & ContentType.CODE_CONTENT.value:
-                    to_return.append(member)
+                    to_return.append(filename)
             else:
                 if content_types & ContentType.USER_CONTENT.value:
                     # Everything else is user content
-                    to_return.append(member)
+                    to_return.append(filename)
         return to_return
 
     @classmethod

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/packaging_sys/backend.py

@@ -57,15 +57,6 @@ class PackagingBackend(ABC):
         """Open the archive from the given content."""
         pass
 
-    @classmethod
-    @abstractmethod
-    def cls_member_name(cls, member: Union[Any, str]) -> str:
-        """
-        Returns the name of the member as a string.
-        This is used to ensure consistent naming across different archive formats.
-        """
-        pass
-
     @classmethod
     @abstractmethod
     def cls_has_member(cls, archive: Any, name: str) -> bool:
@@ -81,20 +72,14 @@ class PackagingBackend(ABC):
     def cls_extract_members(
         cls,
         archive: Any,
-        members: Optional[List[Any]] = None,
+        members: Optional[List[str]] = None,
         dest_dir: str = ".",
     ) -> None:
         pass
 
     @classmethod
     @abstractmethod
-    def cls_list_names(cls, archive: Any) -> Optional[List[str]]:
-        pass
-
-    @classmethod
-    @abstractmethod
-    def cls_list_members(cls, archive: Any) -> Optional[List[Any]]:
-        """List all members in the archive."""
+    def cls_list_members(cls, archive: Any) -> Optional[List[str]]:
         pass
 
     def has_member(self, name: str) -> bool:
@@ -108,17 +93,17 @@ class PackagingBackend(ABC):
         raise ValueError("Cannot get member from an uncreated archive")
 
     def extract_members(
-        self, members: Optional[List[Any]] = None, dest_dir: str = "."
+        self, members: Optional[List[str]] = None, dest_dir: str = "."
     ) -> None:
         if self._archive:
             self.cls_extract_members(self._archive, members, dest_dir)
         else:
             raise ValueError("Cannot extract from an uncreated archive")
 
-    def list_names(self) -> Optional[List[str]]:
+    def list_members(self) -> Optional[List[str]]:
         if self._archive:
-            return self.cls_list_names(self._archive)
-        raise ValueError("Cannot list names from an uncreated archive")
+            return self.cls_list_members(self._archive)
+        raise ValueError("Cannot list members from an uncreated archive")
 
     def __enter__(self):
         self.create()

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/packaging_sys/tar_backend.py

@@ -1,7 +1,7 @@
 import tarfile
 
 from io import BytesIO
-from typing import Any, IO, List, Optional, Union
+from typing import IO, List, Optional, Union
 
 from .backend import PackagingBackend
 
@@ -56,13 +56,6 @@ class TarPackagingBackend(PackagingBackend):
     def cls_open(cls, content: IO[bytes]) -> tarfile.TarFile:
         return tarfile.open(fileobj=content, mode="r:gz")
 
-    @classmethod
-    def cls_member_name(cls, member: Union[tarfile.TarInfo, str]) -> str:
-        """
-        Returns the name of the member as a string.
-        """
-        return member.name if isinstance(member, tarfile.TarInfo) else member
-
     @classmethod
     def cls_has_member(cls, archive: tarfile.TarFile, name: str) -> bool:
         try:
@@ -83,17 +76,11 @@ class TarPackagingBackend(PackagingBackend):
     def cls_extract_members(
         cls,
         archive: tarfile.TarFile,
-        members: Optional[List[Any]] = None,
+        members: Optional[List[str]] = None,
         dest_dir: str = ".",
     ) -> None:
         archive.extractall(path=dest_dir, members=members)
 
     @classmethod
-    def cls_list_members(
-        cls, archive: tarfile.TarFile
-    ) -> Optional[List[tarfile.TarInfo]]:
-        return archive.getmembers() or None
-
-    @classmethod
-    def cls_list_names(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
+    def cls_list_members(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
         return archive.getnames() or None

{ob_metaflow-2.16.6.4rc0 → ob_metaflow-2.16.6.5rc1}/metaflow/packaging_sys/v1.py

@@ -61,25 +61,23 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
         else:
             new_modules = []
 
-        self._modules = {}  # type: Dict[str, _ModuleInfo]
-        # We do this explicitly module by module to harden it against misbehaving
-        # modules like the one in:
-        # https://github.com/Netflix/metaflow/issues/2512
-        # We will silently ignore modules that are not well built.
-        for name, mod in new_modules:
-            try:
-                minfo = _ModuleInfo(
-                    name,
-                    set(
-                        Path(p).resolve().as_posix()
-                        for p in getattr(mod, "__path__", [mod.__file__])
-                    ),
-                    mod,
-                    True,  # This is a Metaflow module (see filter below)
-                )
-            except:
-                continue
-            self._modules[name] = minfo
+        self._modules = {
+            name: _ModuleInfo(
+                name,
+                set(
+                    Path(p).resolve().as_posix()
+                    for p in getattr(mod, "__path__", [mod.__file__])
+                ),
+                mod,
+                True,  # This is a Metaflow module (see filter below)
+            )
+            for (name, mod) in new_modules
+        }
+
+        # Filter the modules
+        self._modules = {
+            name: info for name, info in self._modules.items() if criteria(info.module)
+        }
 
         # Contain metadata information regarding the distributions packaged.
         # This allows Metaflow to "fake" distribution information when packaged
@@ -357,14 +355,16 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
                 )
                 yield json.dumps(self.create_mfcontent_info()).encode(
                     "utf-8"
-                ), MFCONTENT_MARKER
+                ), os.path.join(self._code_dir, MFCONTENT_MARKER)
             else:
                 for k in self._other_content.keys():
                     yield "<generated %s content>" % (os.path.basename(k)), k
                 yield "<generated %s content>" % (
                     os.path.basename(self._dist_info_file)
                 ), os.path.join(self._other_dir, self._dist_info_file)
-                yield "<generated %s content>" % MFCONTENT_MARKER, MFCONTENT_MARKER
+                yield "<generated %s content>" % MFCONTENT_MARKER, os.path.join(
+                    self._code_dir, MFCONTENT_MARKER
+                )
 
     def _metaflow_distribution_files(self) -> Generator[Tuple[str, str], None, None]:
         debug.package_exec("Including Metaflow from '%s'" % self._metaflow_root)