ob-metaflow 2.12.19.1__py2.py3-none-any.whl → 2.12.22.1__py2.py3-none-any.whl

metaflow/plugins/parallel_decorator.py

@@ -24,6 +24,8 @@ class ParallelDecorator(StepDecorator):
                     The total number of tasks created by @parallel
                 - node_index : int
                     The index of the current task in all the @parallel tasks.
+                - control_task_id : Optional[str]
+                    The task ID of the control task. Available to all tasks.
 
     is_parallel -> bool
         True if the current step is a @parallel step.
@@ -67,6 +69,7 @@ class ParallelDecorator(StepDecorator):
                     main_ip=os.environ.get("MF_PARALLEL_MAIN_IP", "127.0.0.1"),
                     num_nodes=int(os.environ.get("MF_PARALLEL_NUM_NODES", "1")),
                     node_index=int(os.environ.get("MF_PARALLEL_NODE_INDEX", "0")),
+                    control_task_id=os.environ.get("MF_PARALLEL_CONTROL_TASK_ID", None),
                 )
             ),
         )
@@ -177,6 +180,7 @@ def _local_multinode_control_task_step_func(
     num_parallel = foreach_iter.num_parallel
     os.environ["MF_PARALLEL_NUM_NODES"] = str(num_parallel)
     os.environ["MF_PARALLEL_MAIN_IP"] = "127.0.0.1"
+    os.environ["MF_PARALLEL_CONTROL_TASK_ID"] = str(current.task_id)
 
     run_id = current.run_id
     step_name = current.step_name

metaflow/plugins/pypi/bootstrap.py

@@ -103,6 +103,7 @@ if __name__ == "__main__":
         echo "@EXPLICIT" > "$tmpfile";
         ls -d {conda_pkgs_dir}/*/* >> "$tmpfile";
         export PATH=$PATH:$(pwd)/micromamba;
+        export CONDA_PKGS_DIRS=$(pwd)/micromamba/pkgs;
         micromamba create --yes --offline --no-deps --safety-checks=disabled --no-extra-safety-checks --prefix {prefix} --file "$tmpfile";
         rm "$tmpfile"''',
     ]
@@ -123,6 +124,7 @@ if __name__ == "__main__":
             [
                 f"""set -e;
                 export PATH=$PATH:$(pwd)/micromamba;
+                export CONDA_PKGS_DIRS=$(pwd)/micromamba/pkgs;
                 micromamba run --prefix {prefix} python -m pip --disable-pip-version-check install --root-user-action=ignore --no-compile {pypi_pkgs_dir}/*.whl --no-user"""
             ]
         )

metaflow/plugins/pypi/conda_decorator.py

@@ -12,7 +12,7 @@ from metaflow.metadata import MetaDatum
 from metaflow.metaflow_environment import InvalidEnvironmentException
 from metaflow.util import get_metaflow_root
 
-from ... import INFO_FILE
+from ...info_file import INFO_FILE
 
 
 class CondaStepDecorator(StepDecorator):
@@ -353,6 +353,12 @@ class CondaFlowDecorator(FlowDecorator):
     def flow_init(
         self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
     ):
+        # NOTE: Important for extensions implementing custom virtual environments.
+        # Without this steps will not have an implicit conda step decorator on them unless the environment adds one in its decospecs.
+        from metaflow import decorators
+
+        decorators._attach_decorators(flow, ["conda"])
+
         # @conda uses a conda environment to create a virtual environment.
         # The conda environment can be created through micromamba.
         _supported_virtual_envs = ["conda"]

metaflow/runner/click_api.py

@@ -1,3 +1,4 @@
+import os
 import sys
 
 if sys.version_info < (3, 7):
@@ -12,6 +13,7 @@ import importlib
 import inspect
 import itertools
 import uuid
+import json
 from collections import OrderedDict
 from typing import Any, Callable, Dict, List, Optional
 from typing import OrderedDict as TOrderedDict
@@ -37,6 +39,9 @@ from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
 from metaflow.parameters import JSONTypeClass, flow_context
 
+# Define a recursive type alias for JSON
+JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
+
 click_to_python_types = {
     StringParamType: str,
     IntParamType: int,
@@ -48,7 +53,7 @@ click_to_python_types = {
     Tuple: tuple,
     Choice: str,
     File: str,
-    JSONTypeClass: str,
+    JSONTypeClass: JSON,
     FilePathClass: str,
 }
 
@@ -82,6 +87,11 @@ def _method_sanity_check(
                 % (supplied_k, annotations[supplied_k], defaults[supplied_k])
             )
 
+        # because Click expects stringified JSON..
+        supplied_v = (
+            json.dumps(supplied_v) if annotations[supplied_k] == JSON else supplied_v
+        )
+
         if supplied_k in possible_arg_params:
             cli_name = possible_arg_params[supplied_k].opts[0].strip("-")
             method_params["args"][cli_name] = supplied_v
@@ -142,6 +152,8 @@ loaded_modules = {}
 
 
 def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
+    if not os.path.exists(flow_file):
+        raise FileNotFoundError("Flow file not present at '%s'" % flow_file)
     # Check if the module has already been loaded
     if flow_file in loaded_modules:
         module = loaded_modules[flow_file]

metaflow/runner/deployer.py

@@ -5,6 +5,8 @@ import time
 import importlib
 import functools
 import tempfile
+
+from subprocess import CalledProcessError
 from typing import Optional, Dict, ClassVar
 
 from metaflow.exception import MetaflowNotFound
@@ -25,6 +27,8 @@ def handle_timeout(
         Temporary file that stores runner attribute data.
     command_obj : CommandManager
         Command manager object that encapsulates the running command details.
+    file_read_timeout : int
+        Timeout for reading the file.
 
     Returns
     -------
@@ -39,10 +43,10 @@ def handle_timeout(
     """
     try:
         content = read_from_file_when_ready(
-            tfp_runner_attribute.name, timeout=file_read_timeout
+            tfp_runner_attribute.name, command_obj, timeout=file_read_timeout
         )
         return content
-    except TimeoutError as e:
+    except (CalledProcessError, TimeoutError) as e:
         stdout_log = open(command_obj.log_files["stdout"]).read()
         stderr_log = open(command_obj.log_files["stderr"]).read()
         command = " ".join(command_obj.command)
@@ -397,6 +401,9 @@ class DeployerImpl(object):
             self.name = content.get("name")
             self.flow_name = content.get("flow_name")
             self.metadata = content.get("metadata")
+            # Additional info is used to pass additional deployer specific information.
+            # It is used in non-OSS deployers (extensions).
+            self.additional_info = content.get("additional_info", {})
 
             if command_obj.process.returncode == 0:
                 deployed_flow = DeployedFlow(deployer=self)

metaflow/runner/metaflow_runner.py

@@ -2,6 +2,8 @@ import importlib
 import os
 import sys
 import tempfile
+
+from subprocess import CalledProcessError
 from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run, metadata
@@ -275,13 +277,13 @@ class Runner(object):
 
             # Set the correct metadata from the runner_attribute file corresponding to this run.
             content = read_from_file_when_ready(
-                tfp_runner_attribute.name, timeout=self.file_read_timeout
+                tfp_runner_attribute.name, command_obj, timeout=self.file_read_timeout
             )
             metadata_for_flow, pathspec = content.rsplit(":", maxsplit=1)
             metadata(metadata_for_flow)
             run_object = Run(pathspec, _namespace_check=False)
             return ExecutingRun(self, command_obj, run_object)
-        except TimeoutError as e:
+        except (CalledProcessError, TimeoutError) as e:
             stdout_log = open(command_obj.log_files["stdout"]).read()
             stderr_log = open(command_obj.log_files["stderr"]).read()
             command = " ".join(command_obj.command)

metaflow/runner/subprocess_manager.py

@@ -1,18 +1,22 @@
 import asyncio
 import os
+import time
 import shutil
 import signal
 import subprocess
 import sys
 import tempfile
 import threading
-import time
 from typing import Callable, Dict, Iterator, List, Optional, Tuple
 
 
 def kill_process_and_descendants(pid, termination_timeout):
+    # TODO: there's a race condition that new descendants might
+    # spawn b/w the invocations of 'pkill' and 'kill'.
+    # Needs to be fixed in future.
     try:
         subprocess.check_call(["pkill", "-TERM", "-P", str(pid)])
+        subprocess.check_call(["kill", "-TERM", str(pid)])
     except subprocess.CalledProcessError:
         pass
 
@@ -20,6 +24,7 @@ def kill_process_and_descendants(pid, termination_timeout):
 
     try:
         subprocess.check_call(["pkill", "-KILL", "-P", str(pid)])
+        subprocess.check_call(["kill", "-KILL", str(pid)])
     except subprocess.CalledProcessError:
         pass
 
@@ -436,13 +441,13 @@ class CommandManager(object):
         if self.run_called:
             shutil.rmtree(self.temp_dir, ignore_errors=True)
 
-    async def kill(self, termination_timeout: float = 1):
+    async def kill(self, termination_timeout: float = 5):
         """
         Kill the subprocess and its descendants.
 
         Parameters
         ----------
-        termination_timeout : float, default 1
+        termination_timeout : float, default 5
             The time to wait after sending a SIGTERM to the process and its descendants
             before sending a SIGKILL.
         """

metaflow/runner/utils.py

@@ -1,7 +1,12 @@
 import os
 import ast
 import time
-from typing import Dict
+
+from subprocess import CalledProcessError
+from typing import Dict, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .subprocess_manager import CommandManager
 
 
 def get_current_cell(ipython):
@@ -35,11 +40,23 @@ def clear_and_set_os_environ(env: Dict):
     os.environ.update(env)
 
 
-def read_from_file_when_ready(file_path: str, timeout: float = 5):
+def read_from_file_when_ready(
+    file_path: str, command_obj: "CommandManager", timeout: float = 5
+):
     start_time = time.time()
     with open(file_path, "r", encoding="utf-8") as file_pointer:
         content = file_pointer.read()
         while not content:
+            if command_obj.process.poll() is not None:
+                # Check to make sure the file hasn't been read yet to avoid a race
+                # where the file is written between the end of this while loop and the
+                # poll call above.
+                content = file_pointer.read()
+                if content:
+                    break
+                raise CalledProcessError(
+                    command_obj.process.returncode, command_obj.command
+                )
             if time.time() - start_time > timeout:
                 raise TimeoutError(
                     "Timeout while waiting for file content from '%s'" % file_path

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.19.1"
+metaflow_version = "2.12.22.1"

{ob_metaflow-2.12.19.1.dist-info → ob_metaflow-2.12.22.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.19.1
+Version: 2.12.22.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.19.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.22.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.12.19.1.dist-info → ob_metaflow-2.12.22.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
-metaflow/__init__.py,sha256=PRCu_YigqL-wzR79NZqMzqtm66VgUkqFQ506CO-eUCg,6340
+metaflow/__init__.py,sha256=mrvSL4mgwJOGkNWmk0cdSJSFz1aIhF5K4mdLgOfQbME,6010
 metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
 metaflow/cli.py,sha256=fa6dx6F2PKtq0oeMxRM2efA9LyE108jGZQvYCtiUS94,34274
 metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
@@ -10,20 +10,21 @@ metaflow/decorators.py,sha256=hbJwRlZuLbl6t7fOsTiH7Sux4Lx6zgEEpldIXoM5TQc,21540
 metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
 metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=KC1LHJQzzYkWib0DeQ4l_A2r8VaudywsSqIQuq1RDZU,4954
-metaflow/flowspec.py,sha256=0-NnCKiX4oJ21Spb5PJkl2IvHoJxBcpi8CQrZEGB2Ag,27178
+metaflow/flowspec.py,sha256=4w7Hm1aFSmVBOq0x9-LtFAq4eDjFb5F6Rdyh_NW41BQ,27270
 metaflow/graph.py,sha256=HFJ7V_bPSht_NHIm8BejrSqOX2fyBQpVOczRCliRw08,11975
 metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
+metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=5rj1MlpluxyPTSINjtMoJ7viotyNzfjtBJSAihlAwMU,10870
-metaflow/metaflow_config.py,sha256=MxVBlIXpLmvudB09WwuWzcYZtNmnIKdxhHuLaNG9IZA,23018
+metaflow/metaflow_config.py,sha256=J11vmCjjffr2Koddo1OuymZFheW-0UaTD6rec2Hp0Gk,23017
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
-metaflow/metaflow_current.py,sha256=5Kri7fzj-rtIJVr5xh5kPKwZ0T73_4egZybzlDR-fgc,7136
-metaflow/metaflow_environment.py,sha256=D7zHYqe8aLZVwJ20nx19vmsNW29Kf3PVE8hbBjVQin8,8115
+metaflow/metaflow_current.py,sha256=pC-EMnAsnvBLvLd61W6MvfiCKcboryeui9f6r8z_sg8,7161
+metaflow/metaflow_environment.py,sha256=rojFyGdyY56sN1HaEb1-0XX53Q3XPNnl0SaH-8xXZ8w,7987
 metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
-metaflow/metaflow_version.py,sha256=mPQ6g_3XjNdi0NrxDzwlW8ZH0nMyYpwqmJ04P7TIdP0,4774
+metaflow/metaflow_version.py,sha256=lOsrhMfHdKXBPS-673sZb48dK2FJhr34rIwLqMFeG9I,7291
 metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
 metaflow/multicore_utils.py,sha256=vdTNgczVLODifscUbbveJbuSDOl3Y9pAxhr7sqYiNf4,4760
-metaflow/package.py,sha256=sOvRpnvqVaQa6eR8lwcfb5HYCGqmpYFPm-cLgOEdl04,7377
+metaflow/package.py,sha256=QutDP6WzjwGk1UCKXqBfXa9F10Q--FlRr0J7fwlple0,7399
 metaflow/parameters.py,sha256=l8qnhBG9C4wf_FkXWjq5sapUA6npLdR7pyB0PPQ-KF0,15712
 metaflow/procpoll.py,sha256=U2tE4iK_Mwj2WDyVTx_Uglh6xZ-jixQOo4wrM9OOhxg,2859
 metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=XY6g1qJ5R4LqOmhFOzlgSJpLjqBMlxLI69hn8Pur3ig,31
+metaflow/version.py,sha256=MeZM-8MzkvMaKzlO6Il3_BkQ6liKzbdGUDS7Zf9ygEI,31
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -110,11 +111,11 @@ metaflow/_vendor/v3_6/importlib_metadata/_meta.py,sha256=_F48Hu_jFxkfKWz5wcYS8vO
 metaflow/_vendor/v3_6/importlib_metadata/_text.py,sha256=HCsFksZpJLeTP3NEk_ngrAeXVRRtTrtyh9eOABoRP4A,2166
 metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
-metaflow/client/core.py,sha256=L4COrMyQgSAWkXIPoXFFQ3OqGnoHfePsBZdJQvShU1E,74162
+metaflow/client/core.py,sha256=jnFTP_XLTLJCHRqaxGKYDn1ZwB_v6a1AAkULlLmJGOk,74171
 metaflow/client/filecache.py,sha256=Wy0yhhCqC1JZgebqi7z52GCwXYnkAqMZHTtxThvwBgM,15229
 metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
-metaflow/cmd/main_cli.py,sha256=Yunxh0QFUPgGYMe0eeGrbmxtZGVdJPCgkd4P1x_j8B4,2950
+metaflow/cmd/main_cli.py,sha256=E546zT_jYQKysmjwfpEgzZd5QMsyirs28M2s0OPU93E,2966
 metaflow/cmd/tutorials_cmd.py,sha256=8FdlKkicTOhCIDKcBR5b0Oz6giDvS-EMY3o9skIrRqw,5156
 metaflow/cmd/util.py,sha256=jS_0rUjOnGGzPT65fzRLdGjrYAOOLA4jU2S0HJLV0oc,406
 metaflow/cmd/develop/__init__.py,sha256=p1Sy8yU1MEKSrH5ttOWOZvNcI1qYu6J6jghdTHwPgOw,689
@@ -128,7 +129,7 @@ metaflow/datastore/exceptions.py,sha256=r7Ab5FvHIzyFh6kwiptA1lO5nLqWg0xRBoeYGefv
 metaflow/datastore/flow_datastore.py,sha256=kbJcOLYnvPHgJfZ_WWkD9LJSX1PHI1K6f9oVUu08A9U,10235
 metaflow/datastore/inputs.py,sha256=i43dXr2xvgtsgKMO9allgCR18bk80GeayeQFyUTH36w,449
 metaflow/datastore/task_datastore.py,sha256=-aDvRUUEaWq0chiGzCCSYmsTP8T-gjs0eowvy2Xyaug,36167
-metaflow/extension_support/__init__.py,sha256=FF-GdIkxiHSqdwCN5xkW5A0DQn3B8GpnphIFDBpLEr0,49432
+metaflow/extension_support/__init__.py,sha256=2z0c4R8zsVmEFOMGT2Jujsl6xveDVa9KLll7moL58NE,52984
 metaflow/extension_support/_empty_file.py,sha256=HENjnM4uAfeNygxMB_feCCWORFoSat9n_QwzSx2oXPw,109
 metaflow/extension_support/cmd.py,sha256=hk8iBUUINqvKCDxInKgWpum8ThiRZtHSJP7qBASHzl8,5711
 metaflow/extension_support/integrations.py,sha256=AWAh-AZ-vo9IxuAVEjGw3s8p_NMm2DKHYx10oC51gPU,5506
@@ -150,7 +151,7 @@ metaflow/plugins/environment_decorator.py,sha256=6m9j2B77d-Ja_l_9CTJ__0O6aB2a8Qt
 metaflow/plugins/events_decorator.py,sha256=c2GcH6Mspbey3wBkjM5lqxaNByFOzYDQdllLpXzRNv8,18283
 metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
 metaflow/plugins/package_cli.py,sha256=-J6D4cupHfWSZ4GEFo2yy9Je9oL3owRWm5pEJwaiqd4,1649
-metaflow/plugins/parallel_decorator.py,sha256=wXBmEnnOmWI46-Lcry_CArkujj6YOrfzAwVVw1kgpPI,8591
+metaflow/plugins/parallel_decorator.py,sha256=hRYTdw3iJjxBEws2rtrZaCHojOaDoO0Pu08ckl-Z5bU,8876
 metaflow/plugins/project_decorator.py,sha256=eJOe0Ea7CbUCReEhR_XQvRkhV6jyRqDxM72oZI7EMCk,5336
 metaflow/plugins/resources_decorator.py,sha256=3MMZ7uptDf99795_RcSOq4h0N3OFlKpd3ahIEsozBBs,1333
 metaflow/plugins/retry_decorator.py,sha256=tz_2Tq6GLg3vjDBZp0KKVTk3ADlCvqaWTSf7blmFdUw,1548
@@ -174,12 +175,11 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=BzlJ4_3Lz4MZgxrnr2xpkEts0ZB4CBRzG76WK3ony1w,170676
-metaflow/plugins/argo/argo_workflows_cli.py,sha256=X2j_F0xF8-K30ebM4dSLOTteDKXbr-jMN18oMpl5S6Y,36313
+metaflow/plugins/argo/argo_workflows.py,sha256=HFCebPkWZA4zhWlRJ37cfsSvjOgVwmzsRFgw8UJ3Rg0,172892
+metaflow/plugins/argo/argo_workflows_cli.py,sha256=E_PyhOtxuS2F8DwhBANsRZCMxpeZ5rfID8eksfSOPm8,37231
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=yprszMdbE3rBTcEA9VR0IEnPjTprUauZBc4SBb-Q7sA,7878
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=wSSZtThn_VPvE_Wu6NB1L0Q86LmBJh9g009v_lpvBPM,8125
 metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
-metaflow/plugins/argo/daemon.py,sha256=dJOS_UUISXBYffi3oGVKPwq4Pa4P_nGBGL15piPaPto,1776
 metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
 metaflow/plugins/argo/jobset_input_paths.py,sha256=_JhZWngA6p9Q_O2fx3pdzKI0WE-HPRHz_zFvY2pHPTQ,525
 metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -188,7 +188,7 @@ metaflow/plugins/aws/aws_utils.py,sha256=Y9C1asbn6C3WuFaOCOka_hNAy__HQYx1tXjCTiW
 metaflow/plugins/aws/batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/batch/batch.py,sha256=e9ssahWM18GnipPK2sqYB-ztx9w7Eoo7YtWyEtufYxs,17787
 metaflow/plugins/aws/batch/batch_cli.py,sha256=6PTbyajRgdy0XmjyJLBTdKdiOB84dcovQQ8sFXlJqko,11749
-metaflow/plugins/aws/batch/batch_client.py,sha256=s9ZHhxQPPoBQijLUgn6_16QOaD4-22U_44uJbp-yLkI,28565
+metaflow/plugins/aws/batch/batch_client.py,sha256=ddlGG0Vk1mkO7tcvJjDvNAVsVLOlqddF7MA1kKfHSqM,28830
 metaflow/plugins/aws/batch/batch_decorator.py,sha256=dyC3xl_iB0BQT6mzWSa3oRBH3YE734XpIhxKue38KcU,17526
 metaflow/plugins/aws/secrets_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.py,sha256=JtFUVu00Cg0FzAizgrPLXmrMqsT7YeQMkQlgeivUxcE,7986
@@ -279,18 +279,19 @@ metaflow/plugins/gcp/gs_tail.py,sha256=Jl_wvnzU7dub07A-DOAuP5FeccNIrPM-CeL1xKFs1
 metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRmvow,2030
 metaflow/plugins/gcp/includefile_support.py,sha256=vIDeR-MiJuUh-2S2pV7Z7FBkhIWwtHXaRrj76MWGRiY,3869
 metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/plugins/kubernetes/kubernetes.py,sha256=sP5BDnjv-5cLm8i7E5YiJygWN4yNRQHt-hhOmysKm1k,31051
-metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=qBDdr1Lvtt-RO9pB-9_HTOPdzAmDvvJ0aiQ1OoCcrMU,10892
-metaflow/plugins/kubernetes/kubernetes_client.py,sha256=GKg-gT3qhXMRQV-sG1YyoOf3Z32NXr_wwEN2ytMVSEg,2471
-metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=MATnWGxfC62MJnRPR-LqybxoWolZb7CRh1Tkv5GbqLE,26162
+metaflow/plugins/kubernetes/kube_utils.py,sha256=fYDlvqi8jYPsWijDwT6Z2qhQswyFqv7tiwtic_I80Vg,749
+metaflow/plugins/kubernetes/kubernetes.py,sha256=bKBqgZXnIDkoa4xKtKoV6InPtYQy4CujfvcbQ3Pvsbc,31305
+metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=sFZ9Zrjef85vCO0MGpUF-em8Pw3dePFb3hbX3PtAH4I,13463
+metaflow/plugins/kubernetes/kubernetes_client.py,sha256=LuKX9rejRySIajnfmDoNkQ6HrOMjwIMbHAOT7wgF9i8,6362
+metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=_RnMafj_oV2uh3qzib0qC3U7YqqP0bo-u2CHi0FbroI,26511
 metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
 metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=sCVuy-PRN60s5HWpnRvFmnljS41k70sBoPO3nOtl2ZE,40800
 metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
-metaflow/plugins/pypi/bootstrap.py,sha256=Tvc4_QKIx-A8j5Aq8ccWZrrxNM8csN40rK8HmxDx-Z8,5106
-metaflow/plugins/pypi/conda_decorator.py,sha256=D5B6xnBc2KUNPhaD0J062CGM1Jzo6D2bxhSU87KrmCM,15372
+metaflow/plugins/pypi/bootstrap.py,sha256=LSRjpDhtpvYqCC0lDPIWvZsrHW7PXx626NDzdYjFeVM,5224
+metaflow/plugins/pypi/conda_decorator.py,sha256=fPeXxvmg51oSFTnlguNlcWUIdXHA9OuMnp9ElaxQPFo,15695
 metaflow/plugins/pypi/conda_environment.py,sha256=NSBcN4oD_RZONOUrnubLh-ur8EfS0h9PM4r7GNZczyc,19685
 metaflow/plugins/pypi/micromamba.py,sha256=67FiIZZz0Kig9EcN7bZLObsE6Z1MFyo4Dp93fd3Grcc,12178
 metaflow/plugins/pypi/pip.py,sha256=7B06mPOs5MvY33xbzPVYZlBr1iKMYaN-n8uulL9zSVg,13649
@@ -301,13 +302,13 @@ metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcw
 metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpLZDBybXLergiDlN63E0u3x8,294
 metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
 metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/runner/click_api.py,sha256=vrrIb5DtVYhCW_hB7wPgj6Q0o-h1bBSWiYnKTOyC454,13452
-metaflow/runner/deployer.py,sha256=nArjnErc0rOaZW612VRKDOT5594jwzeu86w5zW1LX6U,13558
-metaflow/runner/metaflow_runner.py,sha256=AO9nwr5qUbZWmsbFdjkUJrvFlaylz7WvxslvHsIqDYc,15371
+metaflow/runner/click_api.py,sha256=Qfg4BOz5K2LaXTYBsi1y4zTfNIsGGHBVF3UkorX_-o8,13878
+metaflow/runner/deployer.py,sha256=ddYv2UFE7lOFD3EPg4REzMdeLqg5WQhAXkdgMD5bUY8,13920
+metaflow/runner/metaflow_runner.py,sha256=zRlnM2j5QLBh9OboPUuaQ4_WpwFaqrCedKBOpjjtxDw,15449
 metaflow/runner/nbdeploy.py,sha256=fP1s_5MeiDyT_igP82pB5EUqX9rOy2s06Hyc-OUbOvQ,4115
 metaflow/runner/nbrun.py,sha256=lmvhzMCz7iC9LSPGRijifW1wMXxa4RW_jVmpdjQi22E,7261
-metaflow/runner/subprocess_manager.py,sha256=0knxWZYJx8srMv6wTPYKOC6tn4-airnyI7Vbqfb3iXY,19567
-metaflow/runner/utils.py,sha256=FibdEj8CDnx1a-Je5KUQTwHuNbtkFm1unXGarj0D8ok,1394
+metaflow/runner/subprocess_manager.py,sha256=FJh2C6wJlNnvwE00wyK6zxjf8R49PRRHeVcnPvlO4-0,19839
+metaflow/runner/utils.py,sha256=jFlQf9fVvNRd4c6VKv0KwQ8kez_re7kC2VCRgHVMSsM,2051
 metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
 metaflow/sidecar/sidecar.py,sha256=EspKXvPPNiyRToaUZ51PS5TT_PzrBNAurn_wbFnmGr0,1334
 metaflow/sidecar/sidecar_messages.py,sha256=zPsCoYgDIcDkkvdC9MEpJTJ3y6TSGm2JWkRc4vxjbFA,1071
@@ -344,9 +345,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-ob_metaflow-2.12.19.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.12.19.1.dist-info/METADATA,sha256=dlRjUFXgU42VFrt4kFUosjHtZgpgCwHnItdgQvXtx1w,5143
-ob_metaflow-2.12.19.1.dist-info/WHEEL,sha256=O8iI_RpvXnU2dkdPNf_wSffaC9hi-VsSQKnuYEptk_s,109
-ob_metaflow-2.12.19.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-ob_metaflow-2.12.19.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.12.19.1.dist-info/RECORD,,
+ob_metaflow-2.12.22.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.12.22.1.dist-info/METADATA,sha256=zA1Cn4lel8UBUz2u78BMzN2CI9h1NvEVc58S43IUAVM,5143
+ob_metaflow-2.12.22.1.dist-info/WHEEL,sha256=AHX6tWk3qWuce7vKLrj7lnulVHEdWoltgauo8bgCXgU,109
+ob_metaflow-2.12.22.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+ob_metaflow-2.12.22.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.12.22.1.dist-info/RECORD,,

{ob_metaflow-2.12.19.1.dist-info → ob_metaflow-2.12.22.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.1)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

metaflow/plugins/argo/daemon.py

@@ -1,59 +0,0 @@
-from collections import namedtuple
-from time import sleep
-from metaflow.metaflow_config import DEFAULT_METADATA
-from metaflow.metaflow_environment import MetaflowEnvironment
-from metaflow.plugins import METADATA_PROVIDERS
-from metaflow._vendor import click
-
-
-class CliState:
-    pass
-
-
-@click.group()
-@click.option("--flow_name", required=True)
-@click.option("--run_id", required=True)
-@click.option(
-    "--tag",
-    "tags",
-    multiple=True,
-    default=None,
-    help="Annotate all objects produced by Argo Workflows runs "
-    "with the given tag. You can specify this option multiple "
-    "times to attach multiple tags.",
-)
-@click.pass_context
-def cli(ctx, flow_name, run_id, tags=None):
-    ctx.obj = CliState()
-    ctx.obj.flow_name = flow_name
-    ctx.obj.run_id = run_id
-    ctx.obj.tags = tags
-    # Use a dummy flow to initialize the environment and metadata service,
-    # as we only need a name for the flow object.
-    flow = namedtuple("DummyFlow", "name")
-    dummyflow = flow(flow_name)
-
-    # Initialize a proper metadata service instance
-    environment = MetaflowEnvironment(dummyflow)
-
-    ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == DEFAULT_METADATA][0](
-        environment, dummyflow, None, None
-    )
-
-
-@cli.command(help="start heartbeat process for a run")
-@click.pass_obj
-def heartbeat(obj):
-    # Try to register a run in case the start task has not taken care of it yet.
-    obj.metadata.register_run_id(obj.run_id, obj.tags)
-    # Start run heartbeat
-    obj.metadata.start_run_heartbeat(obj.flow_name, obj.run_id)
-    # Keepalive loop
-    while True:
-        # Do not pollute daemon logs with anything unnecessary,
-        # as they might be extremely long running.
-        sleep(10)
-
-
-if __name__ == "__main__":
-    cli()