ob-metaflow-extensions 1.1.168rc5__py2.py3-none-any.whl → 1.1.170rc0__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/__init__.py

@@ -52,10 +52,6 @@ def get_boto3_session(role_arn=None, session_vars=None):
 
     token_info = get_token("/generate/aws")
 
-    # Check if the assume_role decorator has set a role ARN via environment variable
-    # This takes precedence over CSPR role
-    decorator_role_arn = os.environ.get(OBP_ASSUME_ROLE_ARN_ENV_VAR)
-
     # Write token to a file. The file name is derived from the user name
     # so it works with multiple users on the same machine.
     #
@@ -76,13 +72,18 @@ def get_boto3_session(role_arn=None, session_vars=None):
     if token_info.get("cspr_role_arn"):
         cspr_role = token_info["cspr_role_arn"]
 
-    # If assume_role decorator is used, prioritize it over CSPR role
-    effective_role = decorator_role_arn or cspr_role
-    if effective_role:
-        # If we have either a decorator role or CSPR role, set up AWS config
+    # Check if the assume_role decorator has set a CSPR ARN via environment variable
+    # This takes precedence over CSPR role that comes from the token_info response
+    decorator_role_arn = os.environ.get(OBP_ASSUME_ROLE_ARN_ENV_VAR)
+    if decorator_role_arn:
+        cspr_role = decorator_role_arn
+
+    if cspr_role:
+        # If CSPR role is set, we set it as the default role to assume
+        # for the AWS SDK. We do this by writing an AWS config file
         # with two profiles. One to get credentials for the task role
         # in exchange for the OIDC token, and second to assume the
-        # effective role using the task role credentials.
+        # CSPR role using the task role credentials.
         import configparser
         from io import StringIO
 
@@ -94,9 +95,9 @@ def get_boto3_session(role_arn=None, session_vars=None):
             "web_identity_token_file": token_file,
         }
 
-        # Effective role profile (decorator role or CSPR role)
-        aws_config["profile effective"] = {
-            "role_arn": effective_role,
+        # CSPR role profile (default)
+        aws_config["profile cspr"] = {
+            "role_arn": cspr_role,
             "source_profile": "task",
         }
 
@@ -112,7 +113,7 @@ def get_boto3_session(role_arn=None, session_vars=None):
             tmp_aws_config_file = f.name
         os.rename(tmp_aws_config_file, aws_config_file)
         os.environ["AWS_CONFIG_FILE"] = aws_config_file
-        os.environ["AWS_PROFILE"] = "effective"
+        os.environ["AWS_PROFILE"] = "cspr"
     else:
         os.environ["AWS_WEB_IDENTITY_TOKEN_FILE"] = token_file
         os.environ["AWS_ROLE_ARN"] = token_info["role_arn"]
@@ -124,29 +125,21 @@ def get_boto3_session(role_arn=None, session_vars=None):
     if token_info.get("region"):
         os.environ["AWS_DEFAULT_REGION"] = token_info["region"]
 
-    if effective_role:
-        if role_arn == decorator_role_arn or role_arn == USE_CSPR_ROLE_ARN_IF_SET:
-            # We have either a decorator role or CSPR role, use the effective profile
-            # The generated AWS config will be used here since we set the
-            # AWS_CONFIG_FILE environment variable above.
-            session = boto3.session.Session(profile_name="effective")
+    if cspr_role:
+        #  The generated AWS config will be used here since we set the
+        #  AWS_CONFIG_FILE environment variable above.
+        if role_arn == USE_CSPR_ROLE_ARN_IF_SET:
+            # Otherwise start from the default profile, assuming CSPR role
+            session = boto3.session.Session(profile_name="cspr")
         else:
             session = boto3.session.Session(profile_name="task")
     else:
-        # No decorator role or CSPR role, use default session
         # Not using AWS config, just AWS_WEB_IDENTITY_TOKEN_FILE + AWS_ROLE_ARN
         session = boto3.session.Session()
 
-    if (
-        role_arn
-        and role_arn != USE_CSPR_ROLE_ARN_IF_SET
-        and role_arn != decorator_role_arn
-    ):
-        # If the user provided a role_arn that's different from the decorator role,
-        # we assume that role using the current session credentials.
-        # This works for both cases:
-        # 1. No decorator role: Task role -> Secrets role
-        # 2. With decorator role: Decorator role -> Secrets role
+    if role_arn and role_arn != USE_CSPR_ROLE_ARN_IF_SET:
+        # If the user provided a role_arn, we assume that role
+        # using the task role credentials. CSPR role is not used.
         fetcher = botocore.credentials.AssumeRoleCredentialFetcher(
             client_creator=session._session.create_client,
             source_credentials=session._session.get_credentials(),
@@ -162,8 +155,8 @@ def get_boto3_session(role_arn=None, session_vars=None):
     else:
         # If the user didn't provide a role_arn, or if the role_arn
         # is set to USE_CSPR_ROLE_ARN_IF_SET, we return the default
-        # session which would use the effective role (decorator or CSPR) if set,
-        # or the task role otherwise.
+        # session which would use the CSPR role if it is set on the
+        # server, and the task role otherwise.
         return session
 
 

metaflow_extensions/outerbounds/plugins/apps/app_cli.py

@@ -1,6 +1,9 @@
 from metaflow._vendor import click
+import os
 
+os.environ["APPS_CLI_LOADING_IN_METAFLOW"] = "true"
 OUTERBOUNDS_APP_CLI_AVAILABLE = True
+
 try:
     import outerbounds.apps.app_cli as ob_apps_cli
 except ImportError:

metaflow_extensions/outerbounds/plugins/kubernetes/pod_killer.py

@@ -39,7 +39,7 @@ def derive_job_outcome(job_status: "V1JobStatus"):
 
     # This means that the job has neither finished or succedded.
     if job_status.active:
-        return JobOutcomes.KILL
+        return JobOutcomes.DELETE
 
     # This means that the job is not active. Had started. There is not succedded/fail.
     # This is a weird state. Better to just kill the job
@@ -47,7 +47,7 @@ def derive_job_outcome(job_status: "V1JobStatus"):
 
 
 class PodKiller:
-    def __init__(self, kubernetes_client, echo_func, namespace):
+    def __init__(self, kubernetes_client, echo_func, namespace, progress_bar=None):
         self.client = kubernetes_client
         self.echo = echo_func
         self.api_instance = self.client.CoreV1Api()
@@ -55,6 +55,7 @@ class PodKiller:
         self._namespace = namespace
         self.jobset_api = None
         self.jobset_api = self.client.CustomObjectsApi()
+        self.progress_bar = progress_bar
 
     def _delete_jobset(self, owner_ref, namespace):
         """Delete a JobSet if it's the owner of a job."""
@@ -147,20 +148,31 @@ class PodKiller:
 
     def _find_matching_jobs(self, flow_name, run_id=None, user=None):
         """Find jobs that match the flow_name, run_id, and user criteria using similar logic to _find_active_pods"""
+
+        def paginated_job_finder(namespace):
+            continue_token = None
+            while True:
+                response = self.job_api.list_namespaced_job(
+                    namespace=namespace, limit=100, _continue=continue_token
+                )
+                yield response.items
+                continue_token = response.metadata._continue
+                if not continue_token:
+                    break
+
         try:
-            jobs = self.job_api.list_namespaced_job(namespace=self._namespace)
             matching_jobs = []
-            for _job in jobs.items:
-                job = _job.to_dict()
-                _match = self._metaflow_matching_spec(
-                    run_id=run_id,
-                    user=user,
-                    flow_name=flow_name,
-                    annotations=job.get("metadata", {}).get("annotations", {}),
-                    labels=job.get("metadata", {}).get("labels", {}),
-                )
-                if _match:
-                    matching_jobs.append(_job)
+            for _jobs in paginated_job_finder(self._namespace):
+                for job in _jobs:
+                    _match = self._metaflow_matching_spec(
+                        run_id=run_id,
+                        user=user,
+                        flow_name=flow_name,
+                        annotations=job.metadata.annotations,
+                        labels=job.metadata.labels,
+                    )
+                    if _match:
+                        matching_jobs.append(job)
             return matching_jobs
         except Exception as e:
             self.echo(f"Error finding jobs: {str(e)}\n")
@@ -171,25 +183,38 @@ class PodKiller:
         if not self.jobset_api:
             return []
 
+        def paginated_jobset_finder(namespace):
+            continue_token = None
+            responses = []
+            while True:
+                response = self.jobset_api.list_namespaced_custom_object(
+                    group="jobset.x-k8s.io",
+                    version="v1alpha2",
+                    namespace=namespace,
+                    plural="jobsets",
+                    limit=100,
+                    **({"_continue": continue_token} if continue_token else {}),
+                )
+                continue_token = response.get("metadata", {}).get("continue", None)
+                responses.append(response)
+                if not continue_token:
+                    break
+            return responses
+
         try:
-            jobsets = self.jobset_api.list_namespaced_custom_object(
-                group="jobset.x-k8s.io",
-                version="v1alpha2",
-                namespace=self._namespace,
-                plural="jobsets",
-            )
             matching_jobsets = []
 
-            for jobset in jobsets.get("items", []):
-                _match = self._metaflow_matching_spec(
-                    run_id=run_id,
-                    user=user,
-                    flow_name=flow_name,
-                    annotations=jobset.get("metadata", {}).get("annotations", {}),
-                    labels=jobset.get("metadata", {}).get("labels", {}),
-                )
-                if _match:
-                    matching_jobsets.append(jobset)
+            for jobset_response in paginated_jobset_finder(self._namespace):
+                for jobset in jobset_response.get("items", []):
+                    _match = self._metaflow_matching_spec(
+                        run_id=run_id,
+                        user=user,
+                        flow_name=flow_name,
+                        annotations=jobset.get("metadata", {}).get("annotations", {}),
+                        labels=jobset.get("metadata", {}).get("labels", {}),
+                    )
+                    if _match:
+                        matching_jobsets.append(jobset)
 
             return matching_jobsets
         except Exception as e:
@@ -270,9 +295,20 @@ class PodKiller:
             self.echo(f"Unknown outcome {outcome} for JobSet {jobset_name}")
             return False
 
+    def extract_matching_jobs_and_jobsets(self, flow_name, run_id, user):
+        """Extract matching jobs and jobsets based on the flow_name, run_id, and user criteria"""
+        jobs = self._find_matching_jobs(flow_name, run_id, user)
+        jobsets = self._find_matching_jobsets(flow_name, run_id, user)
+        return [(j, derive_job_outcome(j.status)) for j in jobs], [
+            (j, derive_jobset_outcome(j.get("status", {}))) for j in jobsets
+        ]
+
     def process_matching_jobs_and_jobsets(self, flow_name, run_id, user):
         """Process all matching jobs and jobsets based on their derived outcomes"""
         results = []
+        progress_update = lambda x: x
+        if self.progress_bar:
+            progress_update = lambda x: self.progress_bar.update(1, x)
 
         # Process matching jobs
         _jobs, _jobsets = [], []
@@ -282,6 +318,7 @@ class PodKiller:
             result = self._handle_job_outcome(job, outcome)
             # results.append(result)
             if result is not None:
+                progress_update("💀 Killing Job %s" % job.metadata.name)
                 results.append(result)
                 _jobs.append(result)
 
@@ -292,7 +329,46 @@ class PodKiller:
             outcome = derive_jobset_outcome(jobset_status)
             result = self._handle_jobset_outcome(jobset, outcome)
             if result is not None:
+                progress_update(
+                    "💀 Deleting JobSet %s"
+                    % jobset.get("metadata", {}).get("name", "unknown")
+                )
                 results.append(result)
                 _jobsets.append(result)
 
         return results, len(_jobs), len(_jobsets)
+
+    def process_matching_jobs_and_jobsets_force_all(self, flow_name, run_id, user):
+        """Force process ALL matching jobs and jobsets regardless of their status/outcome"""
+        results = []
+        progress_update = lambda x: x
+        if self.progress_bar:
+            progress_update = lambda x: self.progress_bar.update(1, x)
+
+        # Process matching jobs - FORCE DELETE ALL
+        _jobs, _jobsets = [], []
+        jobs = self._find_matching_jobs(flow_name, run_id, user)
+        for job in jobs:
+            # Force DELETE outcome regardless of actual status
+            result = self._handle_job_outcome(job, JobOutcomes.DELETE)
+            progress_update("🔥 FORCE Deleting Job %s" % job.metadata.name)
+            results.append(
+                result if result is not None else True
+            )  # Treat None as success for force mode
+            _jobs.append(result if result is not None else True)
+
+        # Process matching jobsets - FORCE DELETE ALL
+        jobsets = self._find_matching_jobsets(flow_name, run_id, user)
+        for jobset in jobsets:
+            # Force DELETE outcome regardless of actual status
+            result = self._handle_jobset_outcome(jobset, JobOutcomes.DELETE)
+            progress_update(
+                "🔥 FORCE Deleting JobSet %s"
+                % jobset.get("metadata", {}).get("name", "unknown")
+            )
+            results.append(
+                result if result is not None else True
+            )  # Treat None as success for force mode
+            _jobsets.append(result if result is not None else True)
+
+        return results, len(_jobs), len(_jobsets)

metaflow_extensions/outerbounds/toplevel/ob_internal.py

@@ -1 +1,2 @@
+from ..plugins.kubernetes.pod_killer import PodKiller
 from ..plugins.fast_bakery.baker import bake_image

{ob_metaflow_extensions-1.1.168rc5.dist-info → ob_metaflow_extensions-1.1.170rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.168rc5
+Version: 1.1.170rc0
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob_metaflow_extensions-1.1.168rc5.dist-info → ob_metaflow_extensions-1.1.170rc0.dist-info}/RECORD

@@ -1,11 +1,11 @@
 metaflow_extensions/outerbounds/__init__.py,sha256=Gb8u06s9ClQsA_vzxmkCzuMnigPy7kKcDnLfb7eB-64,514
 metaflow_extensions/outerbounds/remote_config.py,sha256=pEFJuKDYs98eoB_-ryPjVi9b_c4gpHMdBHE14ltoxIU,4672
 metaflow_extensions/outerbounds/config/__init__.py,sha256=JsQGRuGFz28fQWjUvxUgR8EKBLGRdLUIk_buPLJplJY,1225
-metaflow_extensions/outerbounds/plugins/__init__.py,sha256=8NoHkOVUbd7YWK1S7GjAe7FzB_dTj-uPIZfla7-t-jE,14129
+metaflow_extensions/outerbounds/plugins/__init__.py,sha256=rWDE4k-KbsFsCoqct1Rw7w3bSV0jLunx6LMBKD2SYEA,13747
 metaflow_extensions/outerbounds/plugins/auth_server.py,sha256=_Q9_2EL0Xy77bCRphkwT1aSu8gQXRDOH-Z-RxTUO8N4,2202
 metaflow_extensions/outerbounds/plugins/perimeters.py,sha256=QXh3SFP7GQbS-RAIxUOPbhPzQ7KDFVxZkTdKqFKgXjI,2697
 metaflow_extensions/outerbounds/plugins/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow_extensions/outerbounds/plugins/apps/app_cli.py,sha256=erfKCC6zKuwax0ye9j-tqIkAVdCg7MJfbRGhMhViSzU,575
+metaflow_extensions/outerbounds/plugins/apps/app_cli.py,sha256=Uv9viEmQ0_6ogVHO8O_FGQyXR6rV5HR5agsP3gGnKm0,638
 metaflow_extensions/outerbounds/plugins/apps/app_utils.py,sha256=sw9whU17lAzlD2K2kEDNjlk1Ib-2xE2UNhJkmzD8Qv8,8543
 metaflow_extensions/outerbounds/plugins/apps/consts.py,sha256=iHsyqbUg9k-rgswCs1Jxf5QZIxR1V-peCDRjgr9kdBM,177
 metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py,sha256=VkmiMdNYHhNdt-Qm9AVv7aE2LWFsIFEc16YcOYjwF6Q,8568
@@ -28,7 +28,7 @@ metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py,sha256=kq
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_decorator.py,sha256=MXSIp05-jvt8Q2uGaLKjtuM_ToLeRLxhtMbfHc9Kcko,1515
 metaflow_extensions/outerbounds/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
 metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py,sha256=sjBhQ4aa-i1UkKsJyTswdDLYOBAFIvHRco4r7wfs9Tc,5003
-metaflow_extensions/outerbounds/plugins/kubernetes/pod_killer.py,sha256=R483_Qg8eRifx5Z-3iHSQsFL6hUDmNTVLBlPWudCsQA,11404
+metaflow_extensions/outerbounds/plugins/kubernetes/pod_killer.py,sha256=3b9y6gYEqWkN-_-GOqcAV2pOJOeCjpVmfW-nOMCJ4Z0,14924
 metaflow_extensions/outerbounds/plugins/nim/card.py,sha256=dXOJvsZed5NyYyxYLPDvtwg9z_X4azL9HTJGYaiNriY,4690
 metaflow_extensions/outerbounds/plugins/nim/nim_decorator.py,sha256=50YVvC7mcZYlPluM0Wq1UtufhzlQb-RxzZkTOJJ3LkM,3439
 metaflow_extensions/outerbounds/plugins/nim/nim_manager.py,sha256=y8U71106KJtrC6nlhsNnzX9Xkv3RnyZ1KEpRFwqZZFk,13686
@@ -78,7 +78,7 @@ metaflow_extensions/outerbounds/profilers/__init__.py,sha256=wa_jhnCBr82TBxoS0e8
 metaflow_extensions/outerbounds/profilers/gpu.py,sha256=3Er8uKQzfm_082uadg4yn_D4Y-iSCgzUfFmguYxZsz4,27485
 metaflow_extensions/outerbounds/toplevel/__init__.py,sha256=qWUJSv_r5hXJ7jV_On4nEasKIfUCm6_UjkjXWA_A1Ts,90
 metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=_fKWv_-O1k5Nk5A1q05Ioh-PSsFXGL-jiAt7zfl8pIE,2999
-metaflow_extensions/outerbounds/toplevel/ob_internal.py,sha256=53xM6d_UYT3uGFFA59UzxN23H5QMO5_F39pALpmGy04,51
+metaflow_extensions/outerbounds/toplevel/ob_internal.py,sha256=VdoPHis20NUkasUiM8d6JEq-X8QVO4eMDaMBAJgKKLg,105
 metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py,sha256=WUuhz2YQfI4fz7nIcipwwWq781eaoHEk7n4GAn1npDg,63
 metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3uILlEZ6ntBLKeNyqn3If8nIXZFq_Apd7Dhco,70
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
@@ -86,7 +86,7 @@ metaflow_extensions/outerbounds/toplevel/plugins/ollama/__init__.py,sha256=GRSz2
 metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
 metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py,sha256=uTVkdSk3xZ7hEKYfdlyVteWj5KeDwaM1hU9WT-_YKfI,50
 metaflow_extensions/outerbounds/toplevel/plugins/vllm/__init__.py,sha256=ekcgD3KVydf-a0xMI60P4uy6ePkSEoFHiGnDq1JM940,45
-ob_metaflow_extensions-1.1.168rc5.dist-info/METADATA,sha256=1a5w7uL3QYY5Oif4eylHkXQQO9cfaKqWZ59YGtXBWgo,524
-ob_metaflow_extensions-1.1.168rc5.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.168rc5.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.168rc5.dist-info/RECORD,,
+ob_metaflow_extensions-1.1.170rc0.dist-info/METADATA,sha256=B5xv_U5U4uooT7aqY74T1AT5sVkxF7oZ36j56HLkllI,524
+ob_metaflow_extensions-1.1.170rc0.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.170rc0.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.170rc0.dist-info/RECORD,,