ob-metaflow-stubs 6.0.3.103rc3__py2.py3-none-any.whl → 6.0.3.105__py2.py3-none-any.whl

metaflow-stubs/plugins/resources_decorator.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.579835                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.668413                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -11,5 +11,36 @@ if typing.TYPE_CHECKING:
     import metaflow.decorators
 
 class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    """
+    Specifies the resources needed when executing this step.
+    
+    Use `@resources` to specify the resource requirements
+    independently of the specific compute layer (`@batch`, `@kubernetes`).
+    
+    You can choose the compute layer on the command line by executing e.g.
+    ```
+    python myflow.py run --with batch
+    ```
+    or
+    ```
+    python myflow.py run --with kubernetes
+    ```
+    which executes the flow on the desired system using the
+    requirements specified in `@resources`.
+    
+    Parameters
+    ----------
+    cpu : int, default 1
+        Number of CPUs required for this step.
+    gpu : int, default 0
+        Number of GPUs required for this step.
+    disk : int, optional, default None
+        Disk size (in MB) required for this step. Only applies on Kubernetes.
+    memory : int, default 4096
+        Memory size (in MB) required for this step.
+    shared_memory : int, optional, default None
+        The value for the size (in MiB) of the /dev/shm volume for this step.
+        This parameter maps to the `--shm-size` option in Docker.
+    """
     ...
 

metaflow-stubs/plugins/retry_decorator.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.579133                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.668792                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -20,6 +20,25 @@ class MetaflowException(Exception, metaclass=type):
 MAX_ATTEMPTS: int
 
 class RetryDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    """
+    Specifies the number of times the task corresponding
+    to a step needs to be retried.
+    
+    This decorator is useful for handling transient errors, such as networking issues.
+    If your task contains operations that can't be retried safely, e.g. database updates,
+    it is advisable to annotate it with `@retry(times=0)`.
+    
+    This can be used in conjunction with the `@catch` decorator. The `@catch`
+    decorator will execute a no-op task after all retries have been exhausted,
+    ensuring that the flow execution can continue.
+    
+    Parameters
+    ----------
+    times : int, default 3
+        Number of times to retry this task.
+    minutes_between_retries : int, default 2
+        Number of minutes between retries.
+    """
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
         ...
     def step_task_retry_count(self):

metaflow-stubs/plugins/secrets/__init__.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.580140                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.672213                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.611009                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.710233                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/secrets/secrets_decorator.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.610776                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.710006                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -69,6 +69,15 @@ def get_secrets_backend_provider(secrets_backend_type):
     ...
 
 class SecretsDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    """
+    Specifies secrets to be retrieved and injected as environment variables prior to
+    the execution of a step.
+    
+    Parameters
+    ----------
+    sources : List[Union[str, Dict[str, Any]]], default: []
+        List of secret specs, defining how the secrets are to be retrieved
+    """
     def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_user_code_retries, ubf_context, inputs):
         ...
     ...

metaflow-stubs/plugins/storage_executor.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.578677                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.671104                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -15,6 +15,10 @@ class MetaflowException(Exception, metaclass=type):
     ...
 
 class StorageExecutor(object, metaclass=type):
+    """
+    Thin wrapper around a ProcessPoolExecutor, or a ThreadPoolExecutor where
+    the former may be unsafe.
+    """
     def __init__(self, use_processes = False):
         ...
     def warm_up(self):

metaflow-stubs/plugins/tag_cli.pyi

@@ -1,18 +1,18 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.584078                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.676371                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.exception
-    import datetime
-    import metaflow.events
     import metaflow.metaflow_current
+    import metaflow.events
+    import datetime
     import metaflow.client.core
+    import metaflow.exception
 
 def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
     """
@@ -35,6 +35,17 @@ def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
     ...
 
 class Flow(metaflow.client.core.MetaflowObject, metaclass=type):
+    """
+    A Flow represents all existing flows with a certain name, in other words,
+    classes derived from `FlowSpec`. A container of `Run` objects.
+    
+    Attributes
+    ----------
+    latest_run : Run
+        Latest `Run` (in progress or completed, successfully or not) of this flow.
+    latest_successful_run : Run
+        Latest successfully completed `Run` of this flow.
+    """
     def __init__(self, *args, **kwargs):
         ...
     @property
@@ -121,6 +132,26 @@ class Flow(metaflow.client.core.MetaflowObject, metaclass=type):
     ...
 
 class Run(metaflow.client.core.MetaflowObject, metaclass=type):
+    """
+    A `Run` represents an execution of a `Flow`. It is a container of `Step`s.
+    
+    Attributes
+    ----------
+    data : MetaflowData
+        a shortcut to run['end'].task.data, i.e. data produced by this run.
+    successful : bool
+        True if the run completed successfully.
+    finished : bool
+        True if the run completed.
+    finished_at : datetime
+        Time this run finished.
+    code : MetaflowCode
+        Code package for this run (if present). See `MetaflowCode`.
+    trigger : MetaflowTrigger
+        Information about event(s) that triggered this run (if present). See `MetaflowTrigger`.
+    end_task : Task
+        `Task` for the end step (if it is present already).
+    """
     def steps(self, *tags: str) -> typing.Iterator[metaflow.client.core.Step]:
         """
         [Legacy function - do not use]

metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.575783                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.667387                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -25,6 +25,9 @@ UBF_TASK: str
 CONTROL_TASK_TAG: str
 
 class MetaDatum(tuple, metaclass=type):
+    """
+    MetaDatum(field, value, type, tags)
+    """
     @staticmethod
     def __new__(_cls, field, value, type, tags):
         """
@@ -44,6 +47,10 @@ class MetaDatum(tuple, metaclass=type):
     ...
 
 class InternalTestUnboundedForeachInput(metaflow.unbounded_foreach.UnboundedForeachInput, metaclass=type):
+    """
+    Test class that wraps around values (any iterator) and simulates an
+    unbounded-foreach instead of a bounded foreach.
+    """
     def __init__(self, iterable):
         ...
     def __iter__(self):

metaflow-stubs/plugins/timeout_decorator.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.577020                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.668079                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -26,6 +26,27 @@ class TimeoutException(metaflow.exception.MetaflowException, metaclass=type):
     ...
 
 class TimeoutDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    """
+    Specifies a timeout for your step.
+    
+    This decorator is useful if this step may hang indefinitely.
+    
+    This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
+    A timeout is considered to be an exception thrown by the step. It will cause the step to be
+    retried if needed and the exception will be caught by the `@catch` decorator, if present.
+    
+    Note that all the values specified in parameters are added together so if you specify
+    60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
+    
+    Parameters
+    ----------
+    seconds : int, default 0
+        Number of seconds to wait prior to timing out.
+    minutes : int, default 0
+        Number of minutes to wait prior to timing out.
+    hours : int, default 0
+        Number of hours to wait prior to timing out.
+    """
     def __init__(self, *args, **kwargs):
         ...
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):

metaflow-stubs/procpoll.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.561402                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.652549                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/profilers/__init__.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.533402                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.625299                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/pylint_wrapper.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.560980                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.652150                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/runner/__init__.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.546241                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.638301                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/runner/deployer.pyi

@@ -1,145 +1,25 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1)                              #
-# Generated on 2024-09-20T21:42:44.559559                                        #
+# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1)                              #
+# Generated on 2024-10-04T10:13:10.650677                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.runner.subprocess_manager
-    import metaflow.exception
     import metaflow.runner.deployer
+    import metaflow.exception
+    import metaflow.runner.subprocess_manager
 
 class MetaflowNotFound(metaflow.exception.MetaflowException, metaclass=type):
     ...
 
-class CommandManager(object, metaclass=type):
-    def __init__(self, command: typing.List[str], env: typing.Optional[typing.Dict[str, str]] = None, cwd: typing.Optional[str] = None):
-        """
-        Create a new CommandManager object.
-        This does not run the process itself but sets it up.
-        
-        Parameters
-        ----------
-        command : List[str]
-            The command to run in List form.
-        env : Optional[Dict[str, str]], default None
-            Environment variables to set for the subprocess; if not specified,
-            the current enviornment variables are used.
-        cwd : Optional[str], default None
-            The directory to run the subprocess in; if not specified, the current
-            directory is used.
-        """
-        ...
-    def __aenter__(self) -> metaflow.runner.subprocess_manager.CommandManager:
-        ...
-    def __aexit__(self, exc_type, exc_value, traceback):
-        ...
-    def wait(self, timeout: typing.Optional[float] = None, stream: typing.Optional[str] = None):
-        """
-        Wait for the subprocess to finish, optionally with a timeout
-        and optionally streaming its output.
-        
-        You can only call `wait` if `async_run` has already been called.
-        
-        Parameters
-        ----------
-        timeout : Optional[float], default None
-            The maximum time to wait for the subprocess to finish.
-            If the timeout is reached, the subprocess is killed.
-        stream : Optional[str], default None
-            If specified, the specified stream is printed to stdout. `stream` can
-            be one of `stdout` or `stderr`.
-        """
-        ...
-    def run(self, show_output: bool = False):
-        """
-        Run the subprocess synchronously. This can only be called once.
-        
-        This also waits on the process implicitly.
-        
-        Parameters
-        ----------
-        show_output : bool, default False
-            Suppress the 'stdout' and 'stderr' to the console by default.
-            They can be accessed later by reading the files present in:
-                - self.log_files["stdout"]
-                - self.log_files["stderr"]
-        """
-        ...
-    def async_run(self):
-        """
-        Run the subprocess asynchronously. This can only be called once.
-        
-        Once this is called, you can then wait on the process (using `wait`), stream
-        logs (using `stream_logs`) or kill it (using `kill`).
-        """
-        ...
-    def stream_log(self, stream: str, position: typing.Optional[int] = None, timeout_per_line: typing.Optional[float] = None, log_write_delay: float = 0.01) -> typing.Iterator[typing.Tuple[int, str]]:
-        """
-        Stream logs from the subprocess line by line.
-        
-        Parameters
-        ----------
-        stream : str
-            The stream to stream logs from. Can be one of "stdout" or "stderr".
-        position : Optional[int], default None
-            The position in the log file to start streaming from. If None, it starts
-            from the beginning of the log file. This allows resuming streaming from
-            a previously known position
-        timeout_per_line : Optional[float], default None
-            The time to wait for a line to be read from the log file. If None, it
-            waits indefinitely. If the timeout is reached, a LogReadTimeoutError
-            is raised. Note that this timeout is *per line* and not cumulative so this
-            function may take significantly more time than `timeout_per_line`
-        log_write_delay : float, default 0.01
-            Improves the probability of getting whole lines. This setting is for
-            advanced use cases.
-        
-        Yields
-        ------
-        Tuple[int, str]
-            A tuple containing the position in the log file and the line read. The
-            position returned can be used to feed into another `stream_logs` call
-            for example.
-        """
-        ...
-    def emit_logs(self, stream: str = "stdout", custom_logger: typing.Callable[..., None] = print):
-        """
-        Helper function that can easily emit all the logs for a given stream.
-        
-        This function will only terminate when all the log has been printed.
-        
-        Parameters
-        ----------
-        stream : str, default "stdout"
-            The stream to emit logs for. Can be one of "stdout" or "stderr".
-        custom_logger : Callable[..., None], default print
-            A custom logger function that takes in a string and "emits" it. By default,
-            the log is printed to stdout.
-        """
-        ...
-    def cleanup(self):
-        """
-        Clean up log files for a running subprocesses.
-        """
-        ...
-    def kill(self, termination_timeout: float = 5):
-        """
-        Kill the subprocess and its descendants.
-        
-        Parameters
-        ----------
-        termination_timeout : float, default 5
-            The time to wait after sending a SIGTERM to the process and its descendants
-            before sending a SIGKILL.
-        """
-        ...
-    ...
-
 class SubprocessManager(object, metaclass=type):
+    """
+    A manager for subprocesses. The subprocess manager manages one or more
+    CommandManager objects, each of which manages an individual subprocess.
+    """
     def __init__(self):
         ...
     def __aenter__(self) -> metaflow.runner.subprocess_manager.SubprocessManager:
@@ -215,10 +95,7 @@ class SubprocessManager(object, metaclass=type):
         ...
     ...
 
-def read_from_file_when_ready(file_path: str, command_obj: "CommandManager", timeout: float = 5):
-    ...
-
-def handle_timeout(tfp_runner_attribute, command_obj: metaflow.runner.subprocess_manager.CommandManager, file_read_timeout: int):
+def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
     """
     Handle the timeout for a running subprocess command that reads a file
     and raises an error with appropriate logs if a TimeoutError occurs.
@@ -273,6 +150,30 @@ def get_lower_level_group(api, top_level_kwargs: typing.Dict, _type: typing.Opti
     ...
 
 class Deployer(object, metaclass=type):
+    """
+    Use the `Deployer` class to configure and access one of the production
+    orchestrators supported by Metaflow.
+    
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to deploy.
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default.
+    profile : Optional[str], default None
+        Metaflow profile to use for the deployment. If not specified, the default
+        profile is used.
+    env : Optional[Dict[str, str]], default None
+        Additional environment variables to set for the deployment.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the deployer attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the deployment command.
+    """
     def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
         ...
     def _Deployer__make_function(self, deployer_class):
@@ -293,6 +194,12 @@ class Deployer(object, metaclass=type):
     ...
 
 class TriggeredRun(object, metaclass=type):
+    """
+    TriggeredRun class represents a run that has been triggered on a production orchestrator.
+    
+    Only when the `start` task starts running, the `run` object corresponding to the run
+    becomes available.
+    """
     def __init__(self, deployer: DeployerImpl, content: str):
         ...
     def wait_for_run(self, timeout = None):
@@ -327,11 +234,43 @@ class TriggeredRun(object, metaclass=type):
     ...
 
 class DeployedFlow(object, metaclass=type):
+    """
+    DeployedFlow class represents a flow that has been deployed.
+    
+    Parameters
+    ----------
+    deployer : DeployerImpl
+        Instance of the deployer implementation.
+    """
     def __init__(self, deployer: DeployerImpl):
         ...
     ...
 
 class DeployerImpl(object, metaclass=type):
+    """
+    Base class for deployer implementations. Each implementation should define a TYPE
+    class variable that matches the name of the CLI group.
+    
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to deploy.
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default.
+    profile : Optional[str], default None
+        Metaflow profile to use for the deployment. If not specified, the default
+        profile is used.
+    env : Optional[Dict], default None
+        Additional environment variables to set for the deployment.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the deployer attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the deployment command.
+    """
     def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
         ...
     def __enter__(self) -> DeployerImpl: