ob-metaflow 2.16.6.5rc3__py2.py3-none-any.whl → 2.16.8.2__py2.py3-none-any.whl

metaflow/packaging_sys/tar_backend.py

@@ -1,7 +1,7 @@
 import tarfile
 
 from io import BytesIO
-from typing import IO, List, Optional, Union
+from typing import Any, IO, List, Optional, Union
 
 from .backend import PackagingBackend
 
@@ -56,6 +56,13 @@ class TarPackagingBackend(PackagingBackend):
     def cls_open(cls, content: IO[bytes]) -> tarfile.TarFile:
         return tarfile.open(fileobj=content, mode="r:gz")
 
+    @classmethod
+    def cls_member_name(cls, member: Union[tarfile.TarInfo, str]) -> str:
+        """
+        Returns the name of the member as a string.
+        """
+        return member.name if isinstance(member, tarfile.TarInfo) else member
+
     @classmethod
     def cls_has_member(cls, archive: tarfile.TarFile, name: str) -> bool:
         try:
@@ -76,11 +83,17 @@ class TarPackagingBackend(PackagingBackend):
     def cls_extract_members(
         cls,
         archive: tarfile.TarFile,
-        members: Optional[List[str]] = None,
+        members: Optional[List[Any]] = None,
         dest_dir: str = ".",
     ) -> None:
         archive.extractall(path=dest_dir, members=members)
 
     @classmethod
-    def cls_list_members(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
+    def cls_list_members(
+        cls, archive: tarfile.TarFile
+    ) -> Optional[List[tarfile.TarInfo]]:
+        return archive.getmembers() or None
+
+    @classmethod
+    def cls_list_names(cls, archive: tarfile.TarFile) -> Optional[List[str]]:
         return archive.getnames() or None

metaflow/packaging_sys/v1.py

@@ -61,23 +61,25 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
         else:
             new_modules = []
 
-        self._modules = {
-            name: _ModuleInfo(
-                name,
-                set(
-                    Path(p).resolve().as_posix()
-                    for p in getattr(mod, "__path__", [mod.__file__])
-                ),
-                mod,
-                True,  # This is a Metaflow module (see filter below)
-            )
-            for (name, mod) in new_modules
-        }
-
-        # Filter the modules
-        self._modules = {
-            name: info for name, info in self._modules.items() if criteria(info.module)
-        }
+        self._modules = {}  # type: Dict[str, _ModuleInfo]
+        # We do this explicitly module by module to harden it against misbehaving
+        # modules like the one in:
+        # https://github.com/Netflix/metaflow/issues/2512
+        # We will silently ignore modules that are not well built.
+        for name, mod in new_modules:
+            try:
+                minfo = _ModuleInfo(
+                    name,
+                    set(
+                        Path(p).resolve().as_posix()
+                        for p in getattr(mod, "__path__", [mod.__file__])
+                    ),
+                    mod,
+                    True,  # This is a Metaflow module (see filter below)
+                )
+            except:
+                continue
+            self._modules[name] = minfo
 
         # Contain metadata information regarding the distributions packaged.
         # This allows Metaflow to "fake" distribution information when packaged
@@ -355,16 +357,14 @@ class MetaflowCodeContentV1(MetaflowCodeContentV1Base):
                 )
                 yield json.dumps(self.create_mfcontent_info()).encode(
                     "utf-8"
-                ), os.path.join(self._code_dir, MFCONTENT_MARKER)
+                ), MFCONTENT_MARKER
             else:
                 for k in self._other_content.keys():
                     yield "<generated %s content>" % (os.path.basename(k)), k
                 yield "<generated %s content>" % (
                     os.path.basename(self._dist_info_file)
                 ), os.path.join(self._other_dir, self._dist_info_file)
-                yield "<generated %s content>" % MFCONTENT_MARKER, os.path.join(
-                    self._code_dir, MFCONTENT_MARKER
-                )
+                yield "<generated %s content>" % MFCONTENT_MARKER, MFCONTENT_MARKER
 
     def _metaflow_distribution_files(self) -> Generator[Tuple[str, str], None, None]:
         debug.package_exec("Including Metaflow from '%s'" % self._metaflow_root)

metaflow/plugins/cards/card_modules/convert_to_native_type.py

@@ -146,7 +146,13 @@ class TaskToDict:
             # Python 3.13 removes the standard ``imghdr`` module. Metaflow
             # vendors a copy so we can keep using ``what`` to detect image
             # formats irrespective of the Python version.
-            from metaflow._vendor import imghdr
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.filterwarnings(
+                    "ignore", category=DeprecationWarning, module="imghdr"
+                )
+                from metaflow._vendor import imghdr
 
             resp = imghdr.what(None, h=data_object)
             # Only accept types supported on the web

metaflow/plugins/kubernetes/kubernetes_job.py

@@ -522,12 +522,10 @@ class RunningJob(object):
         # 3. If the pod object hasn't shown up yet, we set the parallelism to 0
         #    to preempt it.
         client = self._client.get()
-
         if not self.is_done:
             if self.is_running:
                 # Case 1.
                 from kubernetes.stream import stream
-
                 api_instance = client.CoreV1Api
                 try:
                     # TODO: stream opens a web-socket connection. It may
@@ -593,6 +591,10 @@ class RunningJob(object):
             return self.id
         return "job %s" % self._name
 
+    @property
+    def is_unschedulable(self):
+        return self._job["metadata"]["annotations"].get("metaflow/job_status", "") == "Unsatisfiable_Resource_Request"
+
     @property
     def is_done(self):
         # Check if the container is done. As a side effect, also refreshes self._job and
@@ -606,6 +608,7 @@ class RunningJob(object):
                 or bool(self._job["status"].get("failed"))
                 or self._are_pod_containers_done
                 or (self._job["spec"]["parallelism"] == 0)
+                or self.is_unschedulable
             )
 
         if not done():
@@ -663,6 +666,7 @@ class RunningJob(object):
             bool(self._job["status"].get("failed"))
             or self._has_any_container_failed
             or (self._job["spec"]["parallelism"] == 0)
+            or self.is_unschedulable
         )
         return retval
 
@@ -760,6 +764,8 @@ class RunningJob(object):
                 return 0, None
             # Best effort since Pod object can disappear on us at anytime
             else:
+                if self.is_unschedulable:
+                    return 1, self._job["metadata"]["annotations"].get("metaflow/job_status_reason", "")
                 if self._pod.get("status", {}).get("phase") not in (
                     "Succeeded",
                     "Failed",

metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -6,6 +6,7 @@ from collections import namedtuple
 from metaflow.exception import MetaflowException
 from metaflow.metaflow_config import KUBERNETES_JOBSET_GROUP, KUBERNETES_JOBSET_VERSION
 from metaflow.tracing import inject_tracing_vars
+from metaflow._vendor import yaml
 
 from .kube_utils import qos_requests_and_limits
 
@@ -1025,14 +1026,6 @@ class KubernetesArgoJobSet(object):
 
     def dump(self):
         client = self._kubernetes_sdk
-
-        def _best_effort_yaml_dump(_data):
-            try:
-                import yaml
-            except ImportError:
-                return False, json.dumps(_data)
-            return True, yaml.dump(_data, default_flow_style=False, indent=2)
-
         js_dict = client.ApiClient().sanitize_for_serialization(
             dict(
                 apiVersion=self._group + "/" + self._version,
@@ -1058,7 +1051,7 @@ class KubernetesArgoJobSet(object):
                 status=None,
             )
         )
-        yaml_coverted, data = _best_effort_yaml_dump(js_dict)
+        data = yaml.dump(js_dict, default_flow_style=False, indent=2)
         # The values we populate in the Jobset manifest (for Argo Workflows) piggybacks on the Argo Workflow's templating engine.
         # Even though Argo Workflows's templating helps us constructing all the necessary IDs and populating the fields
         # required by Metaflow, we run into one glitch. When we construct JSON/YAML serializable objects,
@@ -1073,9 +1066,6 @@ class KubernetesArgoJobSet(object):
         # Since the value of `num_parallel` can be dynamic and can change from run to run, we need to ensure that the
         # value can be passed-down dynamically and is **explicitly set as a integer** in the Jobset Manifest submitted as a
         # part of the Argo Workflow
-        quoted_substring = '"{{=asInt(inputs.parameters.workerCount)}}"'
-        if yaml_coverted:
-            quoted_substring = "'{{=asInt(inputs.parameters.workerCount)}}'"
-
+        quoted_substring = "'{{=asInt(inputs.parameters.workerCount)}}'"
         unquoted_substring = "{{=asInt(inputs.parameters.workerCount)}}"
         return data.replace(quoted_substring, unquoted_substring)

metaflow/plugins/pypi/conda_decorator.py

@@ -243,9 +243,11 @@ class CondaStepDecorator(StepDecorator):
         # Ensure local installation of Metaflow is visible to user code
         python_path = self.__class__._metaflow_home.name
         addl_env_vars = {}
-        if self.__class__._addl_env_vars is not None:
+        if self.__class__._addl_env_vars:
             for key, value in self.__class__._addl_env_vars.items():
-                if key == "PYTHONPATH":
+                if key.endswith(":"):
+                    addl_env_vars[key[:-1]] = value
+                elif key == "PYTHONPATH":
                     addl_env_vars[key] = os.pathsep.join([value, python_path])
                 else:
                     addl_env_vars[key] = value

metaflow/runner/click_api.py

@@ -43,6 +43,7 @@ from metaflow._vendor.click.types import (
 )
 from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
+from metaflow.flowspec import _FlowState
 from metaflow.includefile import FilePathClass
 from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
@@ -171,7 +172,6 @@ def _lazy_load_command(
     _self,
     name: str,
 ):
-
     # Context is not used in get_command so we can pass None. Since we pin click,
     # this won't change from under us.
 
@@ -516,6 +516,11 @@ class MetaflowAPI(object):
         # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
         # it will init all parameters (config_options will be None)
         # We ignore any errors if we don't check the configs in the click API.
+
+        # Init all values in the flow mutators and then process them
+        for decorator in self._flow_cls._flow_state.get(_FlowState.FLOW_MUTATORS, []):
+            decorator.external_init()
+
         new_cls = self._flow_cls._process_config_decorators(
             config_options, process_configs=CLICK_API_PROCESS_CONFIG
         )
@@ -541,14 +546,16 @@ def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
 
     for each_param in cmd_obj.params:
         if isinstance(each_param, click.Argument):
-            arg_params_sigs[each_param.name], annotations[each_param.name] = (
-                get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
-            )
+            (
+                arg_params_sigs[each_param.name],
+                annotations[each_param.name],
+            ) = get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
             arg_parameters[each_param.name] = each_param
         elif isinstance(each_param, click.Option):
-            opt_params_sigs[each_param.name], annotations[each_param.name] = (
-                get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
-            )
+            (
+                opt_params_sigs[each_param.name],
+                annotations[each_param.name],
+            ) = get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
             opt_parameters[each_param.name] = each_param
 
         defaults[each_param.name] = each_param.default

metaflow/runner/subprocess_manager.py

@@ -152,12 +152,20 @@ class SubprocessManager(object):
         int
             The process ID of the subprocess.
         """
-        updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
-            get_metaflow_root()
-        )
-        if updated_env:
-            env = env or {}
-            env.update(updated_env)
+        env = env or {}
+        installed_root = os.environ.get("METAFLOW_EXTRACTED_ROOT", get_metaflow_root())
+
+        for k, v in MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
+            installed_root
+        ).items():
+            if k.endswith(":"):
+                # Override
+                env[k[:-1]] = v
+            elif k in env:
+                env[k] = "%s:%s" % (v, env[k])
+            else:
+                env[k] = v
+
         command_obj = CommandManager(command, env, cwd)
         pid = command_obj.run(show_output=show_output)
         self.commands[pid] = command_obj
@@ -188,12 +196,12 @@ class SubprocessManager(object):
         int
             The process ID of the subprocess.
         """
-        updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
-            get_metaflow_root()
-        )
-        if updated_env:
-            env = env or {}
-            env.update(updated_env)
+        env = env or {}
+        if "PYTHONPATH" in env:
+            env["PYTHONPATH"] = "%s:%s" % (get_metaflow_root(), env["PYTHONPATH"])
+        else:
+            env["PYTHONPATH"] = get_metaflow_root()
+
         command_obj = CommandManager(command, env, cwd)
         pid = await command_obj.async_run()
         self.commands[pid] = command_obj

metaflow/vendor.py

@@ -63,14 +63,29 @@ def find_vendored_libs(vendor_dir, whitelist, whitelist_dirs):
     return vendored_libs, paths
 
 
-def fetch_licenses(*info_dir, vendor_dir):
-    for file in chain.from_iterable(map(iter_subtree, info_dir)):
-        if "LICENSE" in file.name:
-            library = file.parent.name.split("-")[0]
-            shutil.copy(file, vendor_dir / ("%s.LICENSE" % library))
-        else:
+def fetch_licenses(*info_dirs, vendor_dir):
+    for dist_info in info_dirs:
+        metadata_file = dist_info / "METADATA"
+        if not metadata_file.exists():
+            continue
+
+        project_name = None
+        for line in metadata_file.read_text("utf-8").splitlines():
+            if line.startswith("Name: "):
+                project_name = line.split("Name: ", 1)[1].strip()
+                break
+        if not project_name:
             continue
 
+        for item in dist_info.iterdir():
+            if item.is_file() and re.search(r"(LICENSE|COPYING)", item.name, re.I):
+                shutil.copy(item, vendor_dir / f"{project_name}.LICENSE")
+            elif item.is_dir() and item.name.lower() == "licenses":
+                for license_file in item.iterdir():
+                    if license_file.is_file():
+                        dest_name = f"{project_name}.{license_file.name}"
+                        shutil.copy(license_file, vendor_dir / dest_name)
+
 
 def vendor(vendor_dir):
     # remove everything
@@ -108,6 +123,8 @@ def vendor(vendor_dir):
                 "-r",
                 "_vendor/vendor_%s.txt" % subdir,
                 "--no-compile",
+                "--no-binary",
+                ":all:",
             ]
         )
 

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.16.6.5rc3"
+metaflow_version = "2.16.8.2"

{ob_metaflow-2.16.6.5rc3.data → ob_metaflow-2.16.8.2.data}/data/share/metaflow/devtools/Tiltfile

@@ -14,17 +14,6 @@
 version_settings(constraint='>=0.22.2')
 allow_k8s_contexts('minikube')
 
-# Version configuration for components
-JOBSET_VERSION = os.getenv("JOBSET_VERSION", "v0.6.0")
-
-# Argo Workflows versions
-ARGO_WORKFLOWS_HELM_CHART_VERSION = os.getenv("ARGO_WORKFLOWS_HELM_CHART_VERSION", "0.45.2")  # Helm chart version
-ARGO_WORKFLOWS_IMAGE_TAG = os.getenv("ARGO_WORKFLOWS_IMAGE_TAG", "v3.6.0")  # Argo Workflows application version
-
-# Argo Events versions  
-ARGO_EVENTS_HELM_CHART_VERSION = os.getenv("ARGO_EVENTS_HELM_CHART_VERSION", "2.4.8")  # Helm chart version
-ARGO_EVENTS_IMAGE_TAG = os.getenv("ARGO_EVENTS_IMAGE_TAG", "v1.9.2")  # Argo Events application version
-
 components = {
     "metadata-service": ["postgresql"],
     "ui": ["postgresql", "minio"],
@@ -32,10 +21,9 @@ components = {
     "postgresql": [],
     "argo-workflows": [],
     "argo-events": ["argo-workflows"],
-    "jobset": [],
 }
 
-services_env = os.getenv("SERVICES", "all").strip().lower()
+services_env = os.getenv("SERVICES", "").strip().lower()
 
 if services_env:
     if services_env == "all":
@@ -217,7 +205,6 @@ if "postgresql" in enabled_components:
 if "argo-workflows" in enabled_components:
     helm_remote(
         'argo-workflows',
-        version=ARGO_WORKFLOWS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -233,9 +220,7 @@ if "argo-workflows" in enabled_components:
             'controller.resources.requests.memory=128Mi',
             'controller.resources.requests.cpu=50m',
             'controller.resources.limits.memory=256Mi',
-            'controller.resources.limits.cpu=100m',
-            # Image version overrides
-            'images.tag=%s' % ARGO_WORKFLOWS_IMAGE_TAG,
+            'controller.resources.limits.cpu=100m'
         ]
     )
 
@@ -322,7 +307,6 @@ if "argo-workflows" in enabled_components:
 if "argo-events" in enabled_components:
     helm_remote(
         'argo-events',
-        version=ARGO_EVENTS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -350,8 +334,6 @@ if "argo-events" in enabled_components:
             'configs.jetstream.versions[1].natsImage=nats:2.9.15',
             'configs.jetstream.versions[1].startCommand=/nats-server',
             'configs.jetstream.versions[1].version=2.9.15',
-            # Image version overrides
-            'global.image.tag=%s' % ARGO_EVENTS_IMAGE_TAG,
         ]
     )
 
@@ -559,62 +541,6 @@ if "argo-events" in enabled_components:
     config_resources.append('argo-events-controller-manager')
     config_resources.append('argo-events-webhook-eventsource-svc')
 
-#################################################
-# JOBSET
-#################################################
-if "jobset" in enabled_components:
-    # Apply JobSet manifests directly from GitHub releases
-    jobset_manifest_url = "https://github.com/kubernetes-sigs/jobset/releases/download/%s/manifests.yaml" % JOBSET_VERSION
-
-    cmd = "curl -sSL %s" % (jobset_manifest_url)
-    k8s_yaml(
-        local(
-            cmd,
-        )
-    )
-
-    k8s_resource(
-        'jobset-controller-manager',
-        labels=['jobset'],
-    )
-
-    metaflow_config["METAFLOW_KUBERNETES_JOBSET_ENABLED"] = "true"
-    
-    config_resources.append('jobset-controller-manager')
-
-    # ClusterRole for jobset operations
-    k8s_yaml(encode_yaml({
-        'apiVersion': 'rbac.authorization.k8s.io/v1',
-        'kind': 'ClusterRole',
-        'metadata': {
-            'name': 'jobset-full-access'
-        },
-        'rules': [{
-            'apiGroups': ['jobset.x-k8s.io'],
-            'resources': ['jobsets'],
-            'verbs': ['*']
-        }]
-    }))
-
-    # ClusterRoleBinding for default service account to access jobsets
-    k8s_yaml(encode_yaml({
-        'apiVersion': 'rbac.authorization.k8s.io/v1',
-        'kind': 'ClusterRoleBinding',
-        'metadata': {
-            'name': 'default-jobset-binding'
-        },
-        'subjects': [{
-            'kind': 'ServiceAccount',
-            'name': 'default',
-            'namespace': 'default'
-        }],
-        'roleRef': {
-            'kind': 'ClusterRole',
-            'name': 'jobset-full-access',
-            'apiGroup': 'rbac.authorization.k8s.io'
-        }
-    }))
-
 #################################################
 # METADATA SERVICE
 #################################################

{ob_metaflow-2.16.6.5rc3.data → ob_metaflow-2.16.8.2.data}/data/share/metaflow/devtools/pick_services.sh

@@ -18,7 +18,6 @@ SERVICE_OPTIONS=(
     "ui"
     "argo-workflows"
     "argo-events"
-    "jobset"
 )
 
 gum style "$LOGO" \

{ob_metaflow-2.16.6.5rc3.dist-info → ob_metaflow-2.16.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.6.5rc3
+Version: 2.16.8.2
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.6.5rc3; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.8.2; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description