metaflow-stubs 2.12.22__py2.py3-none-any.whl → 2.12.24__py2.py3-none-any.whl

metaflow-stubs/runner/deployer.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.620731                                        #
+# MF version: 2.12.24                                                            #
+# Generated on 2024-10-04T11:37:46.985801                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -9,137 +9,17 @@ from __future__ import annotations
 import typing
 if typing.TYPE_CHECKING:
     import metaflow.runner.deployer
-    import metaflow.exception
     import metaflow.runner.subprocess_manager
+    import metaflow.exception
 
 class MetaflowNotFound(metaflow.exception.MetaflowException, metaclass=type):
     ...
 
-class CommandManager(object, metaclass=type):
-    def __init__(self, command: typing.List[str], env: typing.Optional[typing.Dict[str, str]] = None, cwd: typing.Optional[str] = None):
-        """
-        Create a new CommandManager object.
-        This does not run the process itself but sets it up.
-        
-        Parameters
-        ----------
-        command : List[str]
-            The command to run in List form.
-        env : Optional[Dict[str, str]], default None
-            Environment variables to set for the subprocess; if not specified,
-            the current enviornment variables are used.
-        cwd : Optional[str], default None
-            The directory to run the subprocess in; if not specified, the current
-            directory is used.
-        """
-        ...
-    def __aenter__(self) -> metaflow.runner.subprocess_manager.CommandManager:
-        ...
-    def __aexit__(self, exc_type, exc_value, traceback):
-        ...
-    def wait(self, timeout: typing.Optional[float] = None, stream: typing.Optional[str] = None):
-        """
-        Wait for the subprocess to finish, optionally with a timeout
-        and optionally streaming its output.
-        
-        You can only call `wait` if `async_run` has already been called.
-        
-        Parameters
-        ----------
-        timeout : Optional[float], default None
-            The maximum time to wait for the subprocess to finish.
-            If the timeout is reached, the subprocess is killed.
-        stream : Optional[str], default None
-            If specified, the specified stream is printed to stdout. `stream` can
-            be one of `stdout` or `stderr`.
-        """
-        ...
-    def run(self, show_output: bool = False):
-        """
-        Run the subprocess synchronously. This can only be called once.
-        
-        This also waits on the process implicitly.
-        
-        Parameters
-        ----------
-        show_output : bool, default False
-            Suppress the 'stdout' and 'stderr' to the console by default.
-            They can be accessed later by reading the files present in:
-                - self.log_files["stdout"]
-                - self.log_files["stderr"]
-        """
-        ...
-    def async_run(self):
-        """
-        Run the subprocess asynchronously. This can only be called once.
-        
-        Once this is called, you can then wait on the process (using `wait`), stream
-        logs (using `stream_logs`) or kill it (using `kill`).
-        """
-        ...
-    def stream_log(self, stream: str, position: typing.Optional[int] = None, timeout_per_line: typing.Optional[float] = None, log_write_delay: float = 0.01) -> typing.Iterator[typing.Tuple[int, str]]:
-        """
-        Stream logs from the subprocess line by line.
-        
-        Parameters
-        ----------
-        stream : str
-            The stream to stream logs from. Can be one of "stdout" or "stderr".
-        position : Optional[int], default None
-            The position in the log file to start streaming from. If None, it starts
-            from the beginning of the log file. This allows resuming streaming from
-            a previously known position
-        timeout_per_line : Optional[float], default None
-            The time to wait for a line to be read from the log file. If None, it
-            waits indefinitely. If the timeout is reached, a LogReadTimeoutError
-            is raised. Note that this timeout is *per line* and not cumulative so this
-            function may take significantly more time than `timeout_per_line`
-        log_write_delay : float, default 0.01
-            Improves the probability of getting whole lines. This setting is for
-            advanced use cases.
-        
-        Yields
-        ------
-        Tuple[int, str]
-            A tuple containing the position in the log file and the line read. The
-            position returned can be used to feed into another `stream_logs` call
-            for example.
-        """
-        ...
-    def emit_logs(self, stream: str = "stdout", custom_logger: typing.Callable[..., None] = print):
-        """
-        Helper function that can easily emit all the logs for a given stream.
-        
-        This function will only terminate when all the log has been printed.
-        
-        Parameters
-        ----------
-        stream : str, default "stdout"
-            The stream to emit logs for. Can be one of "stdout" or "stderr".
-        custom_logger : Callable[..., None], default print
-            A custom logger function that takes in a string and "emits" it. By default,
-            the log is printed to stdout.
-        """
-        ...
-    def cleanup(self):
-        """
-        Clean up log files for a running subprocesses.
-        """
-        ...
-    def kill(self, termination_timeout: float = 5):
-        """
-        Kill the subprocess and its descendants.
-        
-        Parameters
-        ----------
-        termination_timeout : float, default 5
-            The time to wait after sending a SIGTERM to the process and its descendants
-            before sending a SIGKILL.
-        """
-        ...
-    ...
-
 class SubprocessManager(object, metaclass=type):
+    """
+    A manager for subprocesses. The subprocess manager manages one or more
+    CommandManager objects, each of which manages an individual subprocess.
+    """
     def __init__(self):
         ...
     def __aenter__(self) -> metaflow.runner.subprocess_manager.SubprocessManager:
@@ -215,10 +95,7 @@ class SubprocessManager(object, metaclass=type):
         ...
     ...
 
-def read_from_file_when_ready(file_path: str, command_obj: "CommandManager", timeout: float = 5):
-    ...
-
-def handle_timeout(tfp_runner_attribute, command_obj: metaflow.runner.subprocess_manager.CommandManager, file_read_timeout: int):
+def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
     """
     Handle the timeout for a running subprocess command that reads a file
     and raises an error with appropriate logs if a TimeoutError occurs.
@@ -273,6 +150,30 @@ def get_lower_level_group(api, top_level_kwargs: typing.Dict, _type: typing.Opti
     ...
 
 class Deployer(object, metaclass=type):
+    """
+    Use the `Deployer` class to configure and access one of the production
+    orchestrators supported by Metaflow.
+    
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to deploy.
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default.
+    profile : Optional[str], default None
+        Metaflow profile to use for the deployment. If not specified, the default
+        profile is used.
+    env : Optional[Dict[str, str]], default None
+        Additional environment variables to set for the deployment.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the deployer attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the deployment command.
+    """
     def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
         ...
     def _Deployer__make_function(self, deployer_class):
@@ -293,6 +194,12 @@ class Deployer(object, metaclass=type):
     ...
 
 class TriggeredRun(object, metaclass=type):
+    """
+    TriggeredRun class represents a run that has been triggered on a production orchestrator.
+    
+    Only when the `start` task starts running, the `run` object corresponding to the run
+    becomes available.
+    """
     def __init__(self, deployer: DeployerImpl, content: str):
         ...
     def wait_for_run(self, timeout = None):
@@ -327,11 +234,43 @@ class TriggeredRun(object, metaclass=type):
     ...
 
 class DeployedFlow(object, metaclass=type):
+    """
+    DeployedFlow class represents a flow that has been deployed.
+    
+    Parameters
+    ----------
+    deployer : DeployerImpl
+        Instance of the deployer implementation.
+    """
     def __init__(self, deployer: DeployerImpl):
         ...
     ...
 
 class DeployerImpl(object, metaclass=type):
+    """
+    Base class for deployer implementations. Each implementation should define a TYPE
+    class variable that matches the name of the CLI group.
+    
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to deploy.
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default.
+    profile : Optional[str], default None
+        Metaflow profile to use for the deployment. If not specified, the default
+        profile is used.
+    env : Optional[Dict], default None
+        Additional environment variables to set for the deployment.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the deployer attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the deployment command.
+    """
     def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
         ...
     def __enter__(self) -> DeployerImpl:

metaflow-stubs/runner/metaflow_runner.pyi

@@ -1,20 +1,40 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.617579                                        #
+# MF version: 2.12.24                                                            #
+# Generated on 2024-10-04T11:37:46.983038                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.runner.metaflow_runner
-    import metaflow.events
-    import metaflow.runner.subprocess_manager
     import metaflow.client.core
+    import metaflow.runner.subprocess_manager
     import datetime
+    import metaflow.events
+    import metaflow.runner.metaflow_runner
 
 class Run(metaflow.client.core.MetaflowObject, metaclass=type):
+    """
+    A `Run` represents an execution of a `Flow`. It is a container of `Step`s.
+    
+    Attributes
+    ----------
+    data : MetaflowData
+        a shortcut to run['end'].task.data, i.e. data produced by this run.
+    successful : bool
+        True if the run completed successfully.
+    finished : bool
+        True if the run completed.
+    finished_at : datetime
+        Time this run finished.
+    code : MetaflowCode
+        Code package for this run (if present). See `MetaflowCode`.
+    trigger : MetaflowTrigger
+        Information about event(s) that triggered this run (if present). See `MetaflowTrigger`.
+    end_task : Task
+        `Task` for the end step (if it is present already).
+    """
     def steps(self, *tags: str) -> typing.Iterator[metaflow.client.core.Step]:
         """
         [Legacy function - do not use]
@@ -274,13 +294,40 @@ def metadata(ms: str) -> str:
     """
     ...
 
-def clear_and_set_os_environ(env: typing.Dict):
+def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
+    """
+    Handle the timeout for a running subprocess command that reads a file
+    and raises an error with appropriate logs if a TimeoutError occurs.
+    
+    Parameters
+    ----------
+    tfp_runner_attribute : NamedTemporaryFile
+        Temporary file that stores runner attribute data.
+    command_obj : CommandManager
+        Command manager object that encapsulates the running command details.
+    file_read_timeout : int
+        Timeout for reading the file.
+    
+    Returns
+    -------
+    str
+        Content read from the temporary file.
+    
+    Raises
+    ------
+    RuntimeError
+        If a TimeoutError occurs, it raises a RuntimeError with the command's
+        stdout and stderr logs.
+    """
     ...
 
-def read_from_file_when_ready(file_path: str, command_obj: "CommandManager", timeout: float = 5):
+def clear_and_set_os_environ(env: typing.Dict):
     ...
 
 class CommandManager(object, metaclass=type):
+    """
+    A manager for an individual subprocess.
+    """
     def __init__(self, command: typing.List[str], env: typing.Optional[typing.Dict[str, str]] = None, cwd: typing.Optional[str] = None):
         """
         Create a new CommandManager object.
@@ -319,6 +366,8 @@ class CommandManager(object, metaclass=type):
             be one of `stdout` or `stderr`.
         """
         ...
+    def sync_wait(self):
+        ...
     def run(self, show_output: bool = False):
         """
         Run the subprocess synchronously. This can only be called once.
@@ -391,13 +440,13 @@ class CommandManager(object, metaclass=type):
         Clean up log files for a running subprocesses.
         """
         ...
-    def kill(self, termination_timeout: float = 5):
+    def kill(self, termination_timeout: float = 2):
         """
         Kill the subprocess and its descendants.
         
         Parameters
         ----------
-        termination_timeout : float, default 5
+        termination_timeout : float, default 2
             The time to wait after sending a SIGTERM to the process and its descendants
             before sending a SIGKILL.
         """
@@ -405,6 +454,10 @@ class CommandManager(object, metaclass=type):
     ...
 
 class SubprocessManager(object, metaclass=type):
+    """
+    A manager for subprocesses. The subprocess manager manages one or more
+    CommandManager objects, each of which manages an individual subprocess.
+    """
     def __init__(self):
         ...
     def __aenter__(self) -> metaflow.runner.subprocess_manager.SubprocessManager:
@@ -481,6 +534,22 @@ class SubprocessManager(object, metaclass=type):
     ...
 
 class ExecutingRun(object, metaclass=type):
+    """
+    This class contains a reference to a `metaflow.Run` object representing
+    the currently executing or finished run, as well as metadata related
+    to the process.
+    
+    `ExecutingRun` is returned by methods in `Runner` and `NBRunner`. It is not
+    meant to be instantiated directly.
+    
+    This class works as a context manager, allowing you to use a pattern like
+    ```python
+    with Runner(...).run() as running:
+        ...
+    ```
+    Note that you should use either this object as the context manager or
+    `Runner`, not both in a nested manner.
+    """
     def __init__(self, runner: Runner, command_obj: metaflow.runner.subprocess_manager.CommandManager, run_obj: metaflow.client.core.Run):
         """
         Create a new ExecutingRun -- this should not be done by the user directly but
@@ -541,9 +610,10 @@ class ExecutingRun(object, metaclass=type):
         for executing the run.
         
         The return value is one of the following strings:
+        - `timeout` indicates that the run timed out.
         - `running` indicates a currently executing run.
         - `failed` indicates a failed run.
-        - `successful` a successful run.
+        - `successful` indicates a successful run.
         
         Returns
         -------
@@ -603,6 +673,44 @@ class ExecutingRun(object, metaclass=type):
     ...
 
 class Runner(object, metaclass=type):
+    """
+    Metaflow's Runner API that presents a programmatic interface
+    to run flows and perform other operations either synchronously or asynchronously.
+    The class expects a path to the flow file along with optional arguments
+    that match top-level options on the command-line.
+    
+    This class works as a context manager, calling `cleanup()` to remove
+    temporary files at exit.
+    
+    Example:
+    ```python
+    with Runner('slowflow.py', pylint=False) as runner:
+        result = runner.run(alpha=5, tags=["abc", "def"], max_workers=5)
+        print(result.run.finished)
+    ```
+    
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to run
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default,
+        Only applicable for synchronous 'run' and 'resume' functions.
+    profile : Optional[str], default None
+        Metaflow profile to use to run this run. If not specified, the default
+        profile is used (or the one already set using `METAFLOW_PROFILE`)
+    env : Optional[Dict], default None
+        Additional environment variables to set for the Run. This overrides the
+        environment set for this process.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the runner attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the `run` command.
+    """
     def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
         ...
     def __enter__(self) -> Runner:

metaflow-stubs/runner/nbdeploy.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.621303                                        #
+# MF version: 2.12.24                                                            #
+# Generated on 2024-10-04T11:37:46.986533                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -9,6 +9,30 @@ from __future__ import annotations
 import typing
 
 class Deployer(object, metaclass=type):
+    """
+    Use the `Deployer` class to configure and access one of the production
+    orchestrators supported by Metaflow.
+    
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to deploy.
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default.
+    profile : Optional[str], default None
+        Metaflow profile to use for the deployment. If not specified, the default
+        profile is used.
+    env : Optional[Dict[str, str]], default None
+        Additional environment variables to set for the deployment.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the deployer attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the deployment command.
+    """
     def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
         ...
     def _Deployer__make_function(self, deployer_class):
@@ -40,9 +64,49 @@ def format_flowfile(cell):
 DEFAULT_DIR: str
 
 class NBDeployerInitializationError(Exception, metaclass=type):
+    """
+    Custom exception for errors during NBDeployer initialization.
+    """
     ...
 
 class NBDeployer(object, metaclass=type):
+    """
+    A  wrapper over `Deployer` for deploying flows defined in a Jupyter
+    notebook cell.
+    
+    Instantiate this class on the last line of a notebook cell where
+    a `flow` is defined. In contrast to `Deployer`, this class is not
+    meant to be used in a context manager.
+    
+    ```python
+    deployer = NBDeployer(FlowName)
+    ar = deployer.argo_workflows(name="madhur")
+    ar_obj = ar.create()
+    result = ar_obj.trigger(alpha=300)
+    print(result.status)
+    print(result.run)
+    result.terminate()
+    ```
+    
+    Parameters
+    ----------
+    flow : FlowSpec
+        Flow defined in the same cell
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default,
+    profile : Optional[str], default None
+        Metaflow profile to use to deploy this run. If not specified, the default
+        profile is used (or the one already set using `METAFLOW_PROFILE`)
+    env : Optional[Dict[str, str]], default None
+        Additional environment variables to set. This overrides the
+        environment set for this process.
+    base_dir : Optional[str], default None
+        The directory to run the subprocess in; if not specified, a temporary
+        directory is used.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` i.e. options
+        listed in `python myflow.py --help`
+    """
     def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", file_read_timeout: int = 3600, **kwargs):
         ...
     def cleanup(self):

metaflow-stubs/runner/nbrun.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.618784                                        #
+# MF version: 2.12.24                                                            #
+# Generated on 2024-10-04T11:37:46.984347                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -11,6 +11,44 @@ if typing.TYPE_CHECKING:
     import metaflow.runner.metaflow_runner
 
 class Runner(object, metaclass=type):
+    """
+    Metaflow's Runner API that presents a programmatic interface
+    to run flows and perform other operations either synchronously or asynchronously.
+    The class expects a path to the flow file along with optional arguments
+    that match top-level options on the command-line.
+    
+    This class works as a context manager, calling `cleanup()` to remove
+    temporary files at exit.
+    
+    Example:
+    ```python
+    with Runner('slowflow.py', pylint=False) as runner:
+        result = runner.run(alpha=5, tags=["abc", "def"], max_workers=5)
+        print(result.run.finished)
+    ```
+    
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to run
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default,
+        Only applicable for synchronous 'run' and 'resume' functions.
+    profile : Optional[str], default None
+        Metaflow profile to use to run this run. If not specified, the default
+        profile is used (or the one already set using `METAFLOW_PROFILE`)
+    env : Optional[Dict], default None
+        Additional environment variables to set for the Run. This overrides the
+        environment set for this process.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the runner attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the `run` command.
+    """
     def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
         ...
     def __enter__(self) -> metaflow.runner.metaflow_runner.Runner:
@@ -114,9 +152,48 @@ def format_flowfile(cell):
 DEFAULT_DIR: str
 
 class NBRunnerInitializationError(Exception, metaclass=type):
+    """
+    Custom exception for errors during NBRunner initialization.
+    """
     ...
 
 class NBRunner(object, metaclass=type):
+    """
+    A  wrapper over `Runner` for executing flows defined in a Jupyter
+    notebook cell.
+    
+    Instantiate this class on the last line of a notebook cell where
+    a `flow` is defined. In contrast to `Runner`, this class is not
+    meant to be used in a context manager. Instead, use a blocking helper
+    function like `nbrun` (which calls `cleanup()` internally) or call
+    `cleanup()` explictly when using non-blocking APIs.
+    
+    ```python
+    run = NBRunner(FlowName).nbrun()
+    ```
+    
+    Parameters
+    ----------
+    flow : FlowSpec
+        Flow defined in the same cell
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default,
+        Only applicable for synchronous 'run' and 'resume' functions.
+    profile : Optional[str], default None
+        Metaflow profile to use to run this run. If not specified, the default
+        profile is used (or the one already set using `METAFLOW_PROFILE`)
+    env : Optional[Dict], default None
+        Additional environment variables to set for the Run. This overrides the
+        environment set for this process.
+    base_dir : Optional[str], default None
+        The directory to run the subprocess in; if not specified, a temporary
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the runner attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the `run` command.
+    """
     def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", file_read_timeout: int = 3600, **kwargs):
         ...
     def nbrun(self, **kwargs):