ob-metaflow 2.16.6.5rc2__tar.gz → 2.16.6.5rc3__tar.gz

{ob_metaflow-2.16.6.5rc2/ob_metaflow.egg-info → ob_metaflow-2.16.6.5rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.6.5rc2
+Version: 2.16.6.5rc3
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.6.5rc2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.6.5rc3; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.16.6.5rc2 → ob_metaflow-2.16.6.5rc3}/devtools/Tiltfile

@@ -582,6 +582,39 @@ if "jobset" in enabled_components:
     
     config_resources.append('jobset-controller-manager')
 
+    # ClusterRole for jobset operations
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRole',
+        'metadata': {
+            'name': 'jobset-full-access'
+        },
+        'rules': [{
+            'apiGroups': ['jobset.x-k8s.io'],
+            'resources': ['jobsets'],
+            'verbs': ['*']
+        }]
+    }))
+
+    # ClusterRoleBinding for default service account to access jobsets
+    k8s_yaml(encode_yaml({
+        'apiVersion': 'rbac.authorization.k8s.io/v1',
+        'kind': 'ClusterRoleBinding',
+        'metadata': {
+            'name': 'default-jobset-binding'
+        },
+        'subjects': [{
+            'kind': 'ServiceAccount',
+            'name': 'default',
+            'namespace': 'default'
+        }],
+        'roleRef': {
+            'kind': 'ClusterRole',
+            'name': 'jobset-full-access',
+            'apiGroup': 'rbac.authorization.k8s.io'
+        }
+    }))
+
 #################################################
 # METADATA SERVICE
 #################################################

{ob_metaflow-2.16.6.5rc2 → ob_metaflow-2.16.6.5rc3}/metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -310,6 +310,43 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
         return cls(deployer=d)
 
+    @classmethod
+    def get_triggered_run(
+        cls, identifier: str, run_id: str, metadata: Optional[str] = None
+    ):
+        """
+        Retrieves a `ArgoWorkflowsTriggeredRun` object from an identifier, a run id and
+        optional metadata.
+
+        Parameters
+        ----------
+        identifier : str
+            Deployer specific identifier for the workflow to retrieve
+        run_id : str
+            Run ID for the which to fetch the triggered run object
+        metadata : str, optional, default None
+            Optional deployer specific metadata.
+
+        Returns
+        -------
+        ArgoWorkflowsTriggeredRun
+            A `ArgoWorkflowsTriggeredRun` object representing the
+            triggered run on argo workflows.
+        """
+        deployed_flow_obj = cls.from_deployment(identifier, metadata)
+        return ArgoWorkflowsTriggeredRun(
+            deployer=deployed_flow_obj.deployer,
+            content=json.dumps(
+                {
+                    "metadata": deployed_flow_obj.deployer.metadata,
+                    "pathspec": "/".join(
+                        (deployed_flow_obj.deployer.flow_name, run_id)
+                    ),
+                    "name": run_id,
+                }
+            ),
+        )
+
     @property
     def production_token(self) -> Optional[str]:
         """

{ob_metaflow-2.16.6.5rc2 → ob_metaflow-2.16.6.5rc3}/metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -84,6 +84,22 @@ class StepFunctionsDeployedFlow(DeployedFlow):
             "from_deployment is not implemented for StepFunctions"
         )
 
+    @classmethod
+    def get_triggered_run(
+        cls, identifier: str, run_id: str, metadata: Optional[str] = None
+    ):
+        """
+        This method is not currently implemented for Step Functions.
+
+        Raises
+        ------
+        NotImplementedError
+            This method is not implemented for Step Functions.
+        """
+        raise NotImplementedError(
+            "get_triggered_run is not implemented for StepFunctions"
+        )
+
     @property
     def production_token(self: DeployedFlow) -> Optional[str]:
         """

{ob_metaflow-2.16.6.5rc2 → ob_metaflow-2.16.6.5rc3}/metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -1026,33 +1026,39 @@ class KubernetesArgoJobSet(object):
     def dump(self):
         client = self._kubernetes_sdk
 
-        data = json.dumps(
-            client.ApiClient().sanitize_for_serialization(
-                dict(
-                    apiVersion=self._group + "/" + self._version,
-                    kind="JobSet",
-                    metadata=client.api_client.ApiClient().sanitize_for_serialization(
-                        client.V1ObjectMeta(
-                            name=self.name,
-                            labels=self._labels,
-                            annotations=self._annotations,
-                        )
-                    ),
-                    spec=dict(
-                        replicatedJobs=[self.control.dump(), self.worker.dump()],
-                        suspend=False,
-                        startupPolicy=None,
-                        successPolicy=None,
-                        # The Failure Policy helps setting the number of retries for the jobset.
-                        # but we don't rely on it and instead rely on either the local scheduler
-                        # or the Argo Workflows to handle retries.
-                        failurePolicy=None,
-                        network=None,
-                    ),
-                    status=None,
-                )
+        def _best_effort_yaml_dump(_data):
+            try:
+                import yaml
+            except ImportError:
+                return False, json.dumps(_data)
+            return True, yaml.dump(_data, default_flow_style=False, indent=2)
+
+        js_dict = client.ApiClient().sanitize_for_serialization(
+            dict(
+                apiVersion=self._group + "/" + self._version,
+                kind="JobSet",
+                metadata=client.api_client.ApiClient().sanitize_for_serialization(
+                    client.V1ObjectMeta(
+                        name=self.name,
+                        labels=self._labels,
+                        annotations=self._annotations,
+                    )
+                ),
+                spec=dict(
+                    replicatedJobs=[self.control.dump(), self.worker.dump()],
+                    suspend=False,
+                    startupPolicy=None,
+                    successPolicy=None,
+                    # The Failure Policy helps setting the number of retries for the jobset.
+                    # but we don't rely on it and instead rely on either the local scheduler
+                    # or the Argo Workflows to handle retries.
+                    failurePolicy=None,
+                    network=None,
+                ),
+                status=None,
             )
         )
+        yaml_coverted, data = _best_effort_yaml_dump(js_dict)
         # The values we populate in the Jobset manifest (for Argo Workflows) piggybacks on the Argo Workflow's templating engine.
         # Even though Argo Workflows's templating helps us constructing all the necessary IDs and populating the fields
         # required by Metaflow, we run into one glitch. When we construct JSON/YAML serializable objects,
@@ -1067,7 +1073,9 @@ class KubernetesArgoJobSet(object):
         # Since the value of `num_parallel` can be dynamic and can change from run to run, we need to ensure that the
         # value can be passed-down dynamically and is **explicitly set as a integer** in the Jobset Manifest submitted as a
         # part of the Argo Workflow
-
         quoted_substring = '"{{=asInt(inputs.parameters.workerCount)}}"'
+        if yaml_coverted:
+            quoted_substring = "'{{=asInt(inputs.parameters.workerCount)}}'"
+
         unquoted_substring = "{{=asInt(inputs.parameters.workerCount)}}"
         return data.replace(quoted_substring, unquoted_substring)

{ob_metaflow-2.16.6.5rc2 → ob_metaflow-2.16.6.5rc3}/metaflow/runner/deployer.py

@@ -231,6 +231,67 @@ class DeployedFlowMeta(type):
                 }
             )
 
+            def _get_triggered_run_injected_method():
+                def f(
+                    cls,
+                    identifier: str,
+                    run_id: str,
+                    metadata: Optional[str] = None,
+                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
+                ) -> "TriggeredRun":
+                    """
+                    Retrieves a `TriggeredRun` object from an identifier, a run id and optional
+                    metadata. The `impl` parameter specifies the deployer implementation
+                    to use (like `argo-workflows`).
+
+                    Parameters
+                    ----------
+                    identifier : str
+                        Deployer specific identifier for the workflow to retrieve
+                    run_id : str
+                        Run ID for the which to fetch the triggered run object
+                    metadata : str, optional, default None
+                        Optional deployer specific metadata.
+                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
+                        The default implementation to use if not specified
+
+                    Returns
+                    -------
+                    TriggeredRun
+                        A `TriggeredRun` object representing the triggered run corresponding
+                        to the identifier and the run id.
+                    """
+                    if impl in allowed_providers:
+                        return (
+                            allowed_providers[impl]
+                            .deployed_flow_type()
+                            .get_triggered_run(identifier, run_id, metadata)
+                        )
+                    else:
+                        raise ValueError(
+                            f"No deployer '{impl}' exists; valid deployers are: "
+                            f"{list(allowed_providers.keys())}"
+                        )
+
+                f.__name__ = "get_triggered_run"
+                return f
+
+            def _per_type_get_triggered_run_injected_method(method_name, impl):
+                def f(
+                    cls,
+                    identifier: str,
+                    run_id: str,
+                    metadata: Optional[str] = None,
+                ):
+                    return (
+                        allowed_providers[impl]
+                        .deployed_flow_type()
+                        .get_triggered_run(identifier, run_id, metadata)
+                    )
+
+                f.__name__ = method_name
+                return f
+
             def _from_deployment_injected_method():
                 def f(
                     cls,
@@ -347,10 +408,16 @@ class DeployedFlowMeta(type):
                 "list_deployed_flows",
                 classmethod(_list_deployed_flows_injected_method()),
             )
+            setattr(
+                cls,
+                "get_triggered_run",
+                classmethod(_get_triggered_run_injected_method()),
+            )
 
             for impl in allowed_providers:
                 from_deployment_method_name = f"from_{impl}"
                 list_deployed_flows_method_name = f"list_{impl}"
+                get_triggered_run_method_name = f"get_triggered_{impl}_run"
 
                 setattr(
                     cls,
@@ -372,6 +439,16 @@ class DeployedFlowMeta(type):
                     ),
                 )
 
+                setattr(
+                    cls,
+                    get_triggered_run_method_name,
+                    classmethod(
+                        _per_type_get_triggered_run_injected_method(
+                            get_triggered_run_method_name, impl
+                        )
+                    ),
+                )
+
         return cls
 
 

ob_metaflow-2.16.6.5rc3/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.16.6.5rc3"

{ob_metaflow-2.16.6.5rc2 → ob_metaflow-2.16.6.5rc3/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.6.5rc2
+Version: 2.16.6.5rc3
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.6.5rc2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.6.5rc3; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.16.6.5rc2 → ob_metaflow-2.16.6.5rc3}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-metaflow-stubs==2.16.6.5rc2
+metaflow-stubs==2.16.6.5rc3

ob_metaflow-2.16.6.5rc2/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.16.6.5rc2"