ob-metaflow 2.12.25.2__tar.gz → 2.12.26.1__tar.gz

{ob_metaflow-2.12.25.2/ob_metaflow.egg-info → ob_metaflow-2.12.26.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.25.2
+Version: 2.12.26.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.25.2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.26.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/client/core.py

@@ -670,7 +670,7 @@ class MetaflowObject(object):
         origin_pathspec = None
         if self._NAME == "run":
             latest_step = next(self.steps())
-            if latest_step:
+            if latest_step and latest_step.task:
                 # If we had a step
                 task = latest_step.task
                 origin_run_id = [
@@ -1888,9 +1888,10 @@ class Run(MetaflowObject):
         # TODO: A more optimized way of figuring out if a run has remote steps (and thus a codepackage) available.
         # This might require changes to the metadata-service as well.
         for step in self:
-            code = step.task.code
-            if code:
-                return code
+            if step.task:
+                code = step.task.code
+                if code:
+                    return code
 
     @property
     def data(self) -> Optional[MetaflowData]:
@@ -2152,7 +2153,7 @@ class Run(MetaflowObject):
         Trigger, optional
             Container of triggering events
         """
-        if "start" in self:
+        if "start" in self and self["start"].task:
             meta = self["start"].task.metadata_dict.get("execution-triggers")
             if meta:
                 return Trigger(json.loads(meta))

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/argo/argo_client.py

@@ -10,6 +10,14 @@ class ArgoClientException(MetaflowException):
     headline = "Argo Client error"
 
 
+class ArgoResourceNotFound(MetaflowException):
+    headline = "Resource not found"
+
+
+class ArgoNotPermitted(MetaflowException):
+    headline = "Operation not permitted"
+
+
 class ArgoClient(object):
     def __init__(self, namespace=None):
         self._client = KubernetesClient()
@@ -140,9 +148,7 @@ class ArgoClient(object):
             if e.status == 404:
                 return None
             else:
-                raise ArgoClientException(
-                    json.loads(e.body)["message"] if e.body is not None else e.reason
-                )
+                raise wrap_api_error(e)
 
     def delete_workflow_template(self, name):
         """
@@ -164,9 +170,7 @@ class ArgoClient(object):
             if e.status == 404:
                 return None
             else:
-                raise ArgoClientException(
-                    json.loads(e.body)["message"] if e.body is not None else e.reason
-                )
+                raise wrap_api_error(e)
 
     def terminate_workflow(self, name):
         client = self._client.get()
@@ -428,6 +432,18 @@ class ArgoClient(object):
         except client.rest.ApiException as e:
             if e.status == 404:
                 return None
-            raise ArgoClientException(
-                json.loads(e.body)["message"] if e.body is not None else e.reason
-            )
+            raise wrap_api_error(e)
+
+
+def wrap_api_error(error):
+    message = (
+        json.loads(error.body)["message"] if error.body is not None else error.reason
+    )
+    # catch all
+    ex = ArgoClientException(message)
+    if error.status == 404:
+        # usually handled outside this function as most cases want to return None instead.
+        ex = ArgoResourceNotFound(message)
+    if error.status == 403:
+        ex = ArgoNotPermitted(message)
+    return ex

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/argo/argo_workflows.py

@@ -2311,7 +2311,9 @@ class ArgoWorkflows(object):
             and k not in set(ARGO_WORKFLOWS_ENV_VARS_TO_SKIP.split(","))
         }
         return [
-            Template("error-msg-capture-hook").container(
+            Template("error-msg-capture-hook")
+            .service_account_name(resources["service_account"])
+            .container(
                 to_camelcase(
                     kubernetes_sdk.V1Container(
                         name="main",
@@ -2462,7 +2464,7 @@ class ArgoWorkflows(object):
         Use Slack's Block Kit to add general information about the environment and
         execution metadata, including a link to the UI and an optional message.
         """
-        ui_link = "%s%s/argo-{{workflow.name}}" % (UI_URL, self.flow.name)
+        ui_link = "%s/%s/argo-{{workflow.name}}" % (UI_URL.rstrip("/"), self.flow.name)
         # fmt: off
         if getattr(current, "project_name", None):
             # Add @project metadata when available.

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -1016,13 +1016,7 @@ def validate_run_id(
 
     if project_name is not None:
         # Verify we are operating on the correct project.
-        # Perform match with separators to avoid substrings matching
-        # e.g. 'test_proj' and 'test_project' should count as a mismatch.
-        project_part = "%s." % sanitize_for_argo(project_name)
-        if (
-            current.get("project_name") != project_name
-            and project_part not in workflow_name
-        ):
+        if current.get("project_name") != project_name:
             raise RunIdMismatch(
                 "The workflow belongs to the project *%s*. "
                 "Please use the project decorator or --name to target the correct project"
@@ -1030,13 +1024,7 @@ def validate_run_id(
             )
 
         # Verify we are operating on the correct branch.
-        # Perform match with separators to avoid substrings matching.
-        # e.g. 'user.tes' and 'user.test' should count as a mismatch.
-        branch_part = ".%s." % sanitize_for_argo(branch_name)
-        if (
-            current.get("branch_name") != branch_name
-            and branch_part not in workflow_name
-        ):
+        if current.get("branch_name") != branch_name:
             raise RunIdMismatch(
                 "The workflow belongs to the branch *%s*. "
                 "Please use --branch, --production or --name to target the correct branch"

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/aws/batch/batch_decorator.py

@@ -134,6 +134,10 @@ class BatchDecorator(StepDecorator):
     package_sha = None
     run_time_limit = None
 
+    # Conda environment support
+    supports_conda_environment = True
+    target_platform = "linux-64"
+
     def __init__(self, attributes=None, statically_defined=False):
         super(BatchDecorator, self).__init__(attributes, statically_defined)
 

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/azure/azure_tail.py

@@ -70,7 +70,7 @@ class AzureTail(object):
         if data is None:
             return None
         if data:
-            buf = BytesIO(data)
+            buf = BytesIO(self._tail + data)
             self._pos += len(data)
             self._tail = b""
             return buf

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/cards/card_decorator.py

@@ -117,6 +117,7 @@ class CardDecorator(StepDecorator):
         # `'%s-%s'%(evt_name,step_name)` ensures that we capture this once per @card per @step.
         # Since there can be many steps checking if event is registered for `evt_name` will only make it check it once for all steps.
         # Hence, we have `_is_event_registered('%s-%s'%(evt_name,step_name))`
+        self._is_runtime_card = False
         evt = "%s-%s" % (evt_name, step_name)
         if not self._is_event_registered(evt):
             # We set the total count of decorators so that we can use it for

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/gcp/gs_tail.py

@@ -11,15 +11,15 @@ from metaflow.plugins.gcp.gs_utils import parse_gs_full_path
 class GSTail(object):
     def __init__(self, blob_full_uri):
         """Location should be something like gs://<bucket_name>/blob"""
-        bucket_name, blob_name = parse_gs_full_path(blob_full_uri)
-        if not blob_name:
+        self.bucket_name, self.blob_name = parse_gs_full_path(blob_full_uri)
+        if not self.blob_name:
             raise MetaflowException(
                 msg="Failed to parse blob_full_uri into gs://<bucket_name>/<blob_name> (got %s)"
                 % blob_full_uri
             )
         client = get_gs_storage_client()
-        bucket = client.bucket(bucket_name)
-        self._blob_client = bucket.blob(blob_name)
+        self.bucket = client.bucket(self.bucket_name)
+        self._blob_client = self.bucket.blob(self.blob_name)
         self._pos = 0
         self._tail = b""
 
@@ -46,7 +46,11 @@ class GSTail(object):
     def _make_range_request(self):
         try:
             # Yes we read to the end... memory blow up is possible. We can improve by specifying length param
-            return self._blob_client.download_as_bytes(start=self._pos)
+            # NOTE: We must re-instantiate the whole client here due to a behavior with the GS library,
+            # otherwise download_as_bytes will simply return the same content for consecutive requests with the same attributes,
+            # even if the blob has grown in size.
+            blob_client = self.bucket.blob(self.blob_name)
+            return blob_client.download_as_bytes(start=self._pos)
         except NotFound:
             return None
         except ClientError as e:
@@ -63,7 +67,7 @@ class GSTail(object):
         if data is None:
             return None
         if data:
-            buf = BytesIO(data)
+            buf = BytesIO(self._tail + data)
             self._pos += len(data)
             self._tail = b""
             return buf

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/kubernetes/kubernetes.py

@@ -726,12 +726,23 @@ class Kubernetes(object):
         wait_for_launch(self._job)
 
         # 2) Tail logs until the job has finished
+        self._output_final_logs = False
+
+        def _has_updates():
+            if self._job.is_running:
+                return True
+            # Make sure to output final tail for a job that has finished.
+            if not self._output_final_logs:
+                self._output_final_logs = True
+                return True
+            return False
+
         tail_logs(
             prefix=prefix(),
             stdout_tail=stdout_tail,
             stderr_tail=stderr_tail,
             echo=echo,
-            has_log_updates=lambda: self._job.is_running,
+            has_log_updates=_has_updates,
         )
         # 3) Fetch remaining logs
         #

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -2,6 +2,7 @@ import json
 import os
 import platform
 import sys
+import time
 
 from metaflow import current
 from metaflow.decorators import StepDecorator
@@ -106,6 +107,9 @@ class KubernetesDecorator(StepDecorator):
     compute_pool : str, optional, default None
         Compute pool to be used for for this step.
         If not specified, any accessible compute pool within the perimeter is used.
+    hostname_resolution_timeout: int, default 10 * 60
+        Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
+        Only applicable when @parallel is used.
     """
 
     name = "kubernetes"
@@ -132,11 +136,16 @@ class KubernetesDecorator(StepDecorator):
         "port": None,
         "compute_pool": None,
         "executable": None,
+        "hostname_resolution_timeout": 10 * 60,
     }
     package_url = None
     package_sha = None
     run_time_limit = None
 
+    # Conda environment support
+    supports_conda_environment = True
+    target_platform = "linux-64"
+
     def __init__(self, attributes=None, statically_defined=False):
         super(KubernetesDecorator, self).__init__(attributes, statically_defined)
 
@@ -401,7 +410,7 @@ class KubernetesDecorator(StepDecorator):
             cli_args.command_args.append(self.package_url)
 
             # skip certain keys as CLI arguments
-            _skip_keys = ["compute_pool"]
+            _skip_keys = ["compute_pool", "hostname_resolution_timeout"]
             # --namespace is used to specify Metaflow namespace (a different
             # concept from k8s namespace).
             for k, v in self.attributes.items():
@@ -493,7 +502,9 @@ class KubernetesDecorator(StepDecorator):
             num_parallel = flow._parallel_ubf_iter.num_parallel
 
         if num_parallel and num_parallel > 1:
-            _setup_multinode_environment()
+            _setup_multinode_environment(
+                ubf_context, self.attributes["hostname_resolution_timeout"]
+            )
             # current.parallel.node_index will be correctly available over here.
             meta.update({"parallel-node-index": current.parallel.node_index})
             if ubf_context == UBF_CONTROL:
@@ -557,18 +568,44 @@ class KubernetesDecorator(StepDecorator):
 
 
 # TODO: Unify this method with the multi-node setup in @batch
-def _setup_multinode_environment():
-    # TODO [FIXME SOON]
-    # Even if Kubernetes may deploy control pods before worker pods, there is always a
-    # possibility that the worker pods may start before the control. In the case that this happens,
-    # the worker pods will not be able to resolve the control pod's IP address and this will cause
-    # the worker pods to fail. This function should account for this in the near future.
+def _setup_multinode_environment(ubf_context, hostname_resolution_timeout):
     import socket
 
+    def _wait_for_hostname_resolution(max_wait_timeout=10 * 60):
+        """
+        keep trying to resolve the hostname of the control task until the hostname is resolved
+        or the max_wait_timeout is reached. This is a workaround for the issue where the control
+        task is not scheduled before the worker task and the worker task fails because it cannot
+        resolve the hostname of the control task.
+        """
+        start_time = time.time()
+        while True:
+            try:
+                return socket.gethostbyname(os.environ["MF_MASTER_ADDR"])
+            except socket.gaierror:
+                if time.time() - start_time > max_wait_timeout:
+                    raise MetaflowException(
+                        "Failed to get host by name for MF_MASTER_ADDR after waiting for {} seconds.".format(
+                            max_wait_timeout
+                        )
+                    )
+                time.sleep(1)
+
     try:
-        os.environ["MF_PARALLEL_MAIN_IP"] = socket.gethostbyname(
-            os.environ["MF_MASTER_ADDR"]
-        )
+        # Even if Kubernetes may deploy control pods before worker pods, there is always a
+        # possibility that the worker pods may start before the control. In the case that this happens,
+        # the worker pods will not be able to resolve the control pod's IP address and this will cause
+        # the worker pods to fail. So if the worker pods are requesting a hostname resolution, we will
+        # make it wait for the name to be resolved within a reasonable timeout period.
+        if ubf_context != UBF_CONTROL:
+            os.environ["MF_PARALLEL_MAIN_IP"] = _wait_for_hostname_resolution(
+                hostname_resolution_timeout
+            )
+        else:
+            os.environ["MF_PARALLEL_MAIN_IP"] = socket.gethostbyname(
+                os.environ["MF_MASTER_ADDR"]
+            )
+
         os.environ["MF_PARALLEL_NUM_NODES"] = os.environ["MF_WORLD_SIZE"]
         os.environ["MF_PARALLEL_NODE_INDEX"] = (
             str(0)

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/parallel_decorator.py

@@ -13,18 +13,19 @@ class ParallelDecorator(StepDecorator):
     MF Add To Current
     -----------------
     parallel -> metaflow.metaflow_current.Parallel
+        Returns a namedtuple with relevant information about the parallel task.
 
         @@ Returns
         -------
         Parallel
             `namedtuple` with the following fields:
-                - main_ip : str
+                - main_ip (`str`)
                     The IP address of the control task.
-                - num_nodes : int
+                - num_nodes (`int`)
                     The total number of tasks created by @parallel
-                - node_index : int
+                - node_index (`int`)
                     The index of the current task in all the @parallel tasks.
-                - control_task_id : Optional[str]
+                - control_task_id (`Optional[str]`)
                     The task ID of the control task. Available to all tasks.
 
     is_parallel -> bool

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/plugins/pypi/conda_environment.py

@@ -269,11 +269,19 @@ class CondaEnvironment(MetaflowEnvironment):
         # Resolve `linux-64` Conda environments if @batch or @kubernetes are in play
         target_platform = conda_platform()
         for decorator in step.decorators:
-            # TODO: rather than relying on decorator names, rely on attributes
-            #       to make them extensible.
-            if decorator.name in ["batch", "kubernetes", "nvidia", "snowpark", "slurm"]:
+            # NOTE: Keep the list of supported decorator names for backward compatibility purposes.
+            # Older versions did not implement the 'support_conda_environment' attribute.
+            if getattr(
+                decorator, "supports_conda_environment", False
+            ) or decorator.name in [
+                "batch",
+                "kubernetes",
+                "nvidia",
+                "snowpark",
+                "slurm",
+            ]:
                 # TODO: Support arm architectures
-                target_platform = "linux-64"
+                target_platform = getattr(decorator, "target_platform", "linux-64")
                 break
 
         environment["conda"]["platforms"] = [target_platform]

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/runner/metaflow_runner.py

@@ -8,7 +8,7 @@ from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run, metadata
 
-from .utils import handle_timeout, clear_and_set_os_environ
+from .utils import handle_timeout
 from .subprocess_manager import CommandManager, SubprocessManager
 
 
@@ -249,8 +249,7 @@ class Runner(object):
         self.flow_file = flow_file
         self.show_output = show_output
 
-        self.old_env = os.environ.copy()
-        self.env_vars = self.old_env.copy()
+        self.env_vars = os.environ.copy()
         self.env_vars.update(env or {})
         if profile:
             self.env_vars["METAFLOW_PROFILE"] = profile
@@ -268,22 +267,12 @@ class Runner(object):
         return self
 
     def __get_executing_run(self, tfp_runner_attribute, command_obj):
-        # When two 'Runner' executions are done sequentially i.e. one after the other
-        # the 2nd run kinda uses the 1st run's previously set metadata and
-        # environment variables.
-
-        # It is thus necessary to set them to correct values before we return
-        # the Run object.
-
         content = handle_timeout(
             tfp_runner_attribute, command_obj, self.file_read_timeout
         )
         content = json.loads(content)
         pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
 
-        # Set the environment variables to what they were before the run executed.
-        clear_and_set_os_environ(self.old_env)
-
         # Set the correct metadata from the runner_attribute file corresponding to this run.
         metadata_for_flow = content.get("metadata")
         metadata(metadata_for_flow)

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/runner/nbdeploy.py

@@ -5,8 +5,6 @@ from typing import Dict, Optional
 from metaflow import Deployer
 from metaflow.runner.utils import get_current_cell, format_flowfile
 
-DEFAULT_DIR = tempfile.gettempdir()
-
 
 class NBDeployerInitializationError(Exception):
     """Custom exception for errors during NBDeployer initialization."""
@@ -46,8 +44,8 @@ class NBDeployer(object):
         Additional environment variables to set. This overrides the
         environment set for this process.
     base_dir : Optional[str], default None
-        The directory to run the subprocess in; if not specified, a temporary
-        directory is used.
+        The directory to run the subprocess in; if not specified, the current
+        working directory is used.
     **kwargs : Any
         Additional arguments that you would pass to `python myflow.py` i.e. options
         listed in `python myflow.py --help`
@@ -60,7 +58,7 @@ class NBDeployer(object):
         show_output: bool = True,
         profile: Optional[str] = None,
         env: Optional[Dict] = None,
-        base_dir: str = DEFAULT_DIR,
+        base_dir: Optional[str] = None,
         file_read_timeout: int = 3600,
         **kwargs,
     ):
@@ -78,7 +76,7 @@ class NBDeployer(object):
         self.show_output = show_output
         self.profile = profile
         self.env = env
-        self.cwd = base_dir
+        self.cwd = base_dir if base_dir is not None else os.getcwd()
         self.file_read_timeout = file_read_timeout
         self.top_level_kwargs = kwargs
 

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/runner/nbrun.py

@@ -5,8 +5,6 @@ from typing import Dict, Optional
 from metaflow import Runner
 from metaflow.runner.utils import get_current_cell, format_flowfile
 
-DEFAULT_DIR = tempfile.gettempdir()
-
 
 class NBRunnerInitializationError(Exception):
     """Custom exception for errors during NBRunner initialization."""
@@ -43,8 +41,8 @@ class NBRunner(object):
         Additional environment variables to set for the Run. This overrides the
         environment set for this process.
     base_dir : Optional[str], default None
-        The directory to run the subprocess in; if not specified, a temporary
-        directory is used.
+        The directory to run the subprocess in; if not specified, the current
+        working directory is used.
     file_read_timeout : int, default 3600
         The timeout until which we try to read the runner attribute file.
     **kwargs : Any
@@ -59,7 +57,7 @@ class NBRunner(object):
         show_output: bool = True,
         profile: Optional[str] = None,
         env: Optional[Dict] = None,
-        base_dir: str = DEFAULT_DIR,
+        base_dir: Optional[str] = None,
         file_read_timeout: int = 3600,
         **kwargs,
     ):
@@ -84,7 +82,7 @@ class NBRunner(object):
         if profile:
             self.env_vars["METAFLOW_PROFILE"] = profile
 
-        self.base_dir = base_dir
+        self.base_dir = base_dir if base_dir is not None else os.getcwd()
         self.file_read_timeout = file_read_timeout
 
         if not self.cell:

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/metaflow/runner/utils.py

@@ -36,11 +36,6 @@ def format_flowfile(cell):
     return "\n".join(lines)
 
 
-def clear_and_set_os_environ(env: Dict):
-    os.environ.clear()
-    os.environ.update(env)
-
-
 def check_process_status(command_obj: "CommandManager"):
     if isinstance(command_obj.process, asyncio.subprocess.Process):
         return command_obj.process.returncode is not None

ob_metaflow-2.12.26.1/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.12.26.1"

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.25.2
+Version: 2.12.26.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.25.2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.26.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.12.25.2 → ob_metaflow-2.12.26.1}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-metaflow-stubs==2.12.25.2
+metaflow-stubs==2.12.26.1

ob_metaflow-2.12.25.2/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.12.25.2"