metaflow-stubs 2.12.22__py2.py3-none-any.whl → 2.12.23__py2.py3-none-any.whl

metaflow-stubs/plugins/datatools/__init__.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.633982                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.993990                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -22,6 +22,13 @@ class MetaflowLocalURLException(metaflow.exception.MetaflowException, metaclass=
     ...
 
 class Local(object, metaclass=type):
+    """
+    This class allows you to access the local filesystem in a way similar to the S3 datatools
+    client. It is a stripped down version for now and only implements the functionality needed
+    for this use case.
+    
+    In the future, we may want to allow it to be used in a way similar to the S3() client.
+    """
     @classmethod
     def get_root_from_config(cls, echo, create_on_absent = True):
         ...
@@ -47,6 +54,59 @@ class MetaflowS3Exception(metaflow.exception.MetaflowException, metaclass=type):
     ...
 
 class S3(object, metaclass=type):
+    """
+    The Metaflow S3 client.
+    
+    This object manages the connection to S3 and a temporary diretory that is used
+    to download objects. Note that in most cases when the data fits in memory, no local
+    disk IO is needed as operations are cached by the operating system, which makes
+    operations fast as long as there is enough memory available.
+    
+    The easiest way is to use this object as a context manager:
+    ```
+    with S3() as s3:
+        data = [obj.blob for obj in s3.get_many(urls)]
+    print(data)
+    ```
+    The context manager takes care of creating and deleting a temporary directory
+    automatically. Without a context manager, you must call `.close()` to delete
+    the directory explicitly:
+    ```
+    s3 = S3()
+    data = [obj.blob for obj in s3.get_many(urls)]
+    s3.close()
+    ```
+    You can customize the location of the temporary directory with `tmproot`. It
+    defaults to the current working directory.
+    
+    To make it easier to deal with object locations, the client can be initialized
+    with an S3 path prefix. There are three ways to handle locations:
+    
+    1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
+       initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
+       with the current run ID. This mode makes it easy to version data based
+       on the run ID consistently. You can use the `bucket` and `prefix` to
+       override parts of `DATATOOLS_S3ROOT`.
+    
+    2. Specify an S3 prefix explicitly with `s3root`,
+       e.g. `S3(s3root='s3://mybucket/some/path')`.
+    
+    3. Specify nothing, i.e. `S3()`, in which case all operations require
+       a full S3 url prefixed with `s3://`.
+    
+    Parameters
+    ----------
+    tmproot : str, default: '.'
+        Where to store the temporary directory.
+    bucket : str, optional
+        Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
+    prefix : str, optional
+        Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
+    run : FlowSpec or Run, optional
+        Derive path prefix from the current or a past run ID, e.g. S3(run=self).
+    s3root : str, optional
+        If `run` is not specified, use this as the S3 prefix.
+    """
     @classmethod
     def get_root_from_config(cls, echo, create_on_absent = True):
         ...

metaflow-stubs/plugins/datatools/local.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.626555                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.989233                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -28,6 +28,13 @@ class MetaflowLocalNotFound(metaflow.exception.MetaflowException, metaclass=type
     ...
 
 class LocalObject(object, metaclass=type):
+    """
+    This object represents a local object. It is a very thin wrapper
+    to allow it to be used in the same way as the S3Object (only as needed
+    in the IncludeFile use case)
+    
+    Get or list calls return one or more of LocalObjects.
+    """
     def __init__(self, url, path):
         ...
     @property
@@ -59,6 +66,13 @@ class LocalObject(object, metaclass=type):
     ...
 
 class Local(object, metaclass=type):
+    """
+    This class allows you to access the local filesystem in a way similar to the S3 datatools
+    client. It is a stripped down version for now and only implements the functionality needed
+    for this use case.
+    
+    In the future, we may want to allow it to be used in a way similar to the S3() client.
+    """
     @classmethod
     def get_root_from_config(cls, echo, create_on_absent = True):
         ...

metaflow-stubs/plugins/datatools/s3/__init__.pyi

@@ -1,18 +1,21 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.677599                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.017862                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.exception
     import metaflow.plugins.datatools.s3.s3
+    import metaflow.exception
     import io
 
 class RangeInfo(tuple, metaclass=type):
+    """
+    RangeInfo(total_size, request_offset, request_length)
+    """
     @staticmethod
     def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
         """
@@ -34,6 +37,59 @@ class RangeInfo(tuple, metaclass=type):
     ...
 
 class S3(object, metaclass=type):
+    """
+    The Metaflow S3 client.
+    
+    This object manages the connection to S3 and a temporary diretory that is used
+    to download objects. Note that in most cases when the data fits in memory, no local
+    disk IO is needed as operations are cached by the operating system, which makes
+    operations fast as long as there is enough memory available.
+    
+    The easiest way is to use this object as a context manager:
+    ```
+    with S3() as s3:
+        data = [obj.blob for obj in s3.get_many(urls)]
+    print(data)
+    ```
+    The context manager takes care of creating and deleting a temporary directory
+    automatically. Without a context manager, you must call `.close()` to delete
+    the directory explicitly:
+    ```
+    s3 = S3()
+    data = [obj.blob for obj in s3.get_many(urls)]
+    s3.close()
+    ```
+    You can customize the location of the temporary directory with `tmproot`. It
+    defaults to the current working directory.
+    
+    To make it easier to deal with object locations, the client can be initialized
+    with an S3 path prefix. There are three ways to handle locations:
+    
+    1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
+       initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
+       with the current run ID. This mode makes it easy to version data based
+       on the run ID consistently. You can use the `bucket` and `prefix` to
+       override parts of `DATATOOLS_S3ROOT`.
+    
+    2. Specify an S3 prefix explicitly with `s3root`,
+       e.g. `S3(s3root='s3://mybucket/some/path')`.
+    
+    3. Specify nothing, i.e. `S3()`, in which case all operations require
+       a full S3 url prefixed with `s3://`.
+    
+    Parameters
+    ----------
+    tmproot : str, default: '.'
+        Where to store the temporary directory.
+    bucket : str, optional
+        Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
+    prefix : str, optional
+        Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
+    run : FlowSpec or Run, optional
+        Derive path prefix from the current or a past run ID, e.g. S3(run=self).
+    s3root : str, optional
+        If `run` is not specified, use this as the S3 prefix.
+    """
     @classmethod
     def get_root_from_config(cls, echo, create_on_absent = True):
         ...
@@ -325,6 +381,9 @@ class S3(object, metaclass=type):
     ...
 
 class S3GetObject(tuple, metaclass=type):
+    """
+    S3GetObject(key, offset, length)
+    """
     @staticmethod
     def __new__(_cls, key: str, offset: int, length: int):
         """
@@ -346,6 +405,13 @@ class S3GetObject(tuple, metaclass=type):
     ...
 
 class S3Object(object, metaclass=type):
+    """
+    This object represents a path or an object in S3,
+    with an optional local copy.
+    
+    `S3Object`s are not instantiated directly, but they are returned
+    by many methods of the `S3` client.
+    """
     def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[metaflow.plugins.datatools.s3.s3.RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
         ...
     @property
@@ -549,6 +615,9 @@ class S3Object(object, metaclass=type):
     ...
 
 class S3PutObject(tuple, metaclass=type):
+    """
+    S3PutObject(key, value, path, content_type, encryption, metadata)
+    """
     @staticmethod
     def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
         """

metaflow-stubs/plugins/datatools/s3/s3.pyi

@@ -1,24 +1,32 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.609180                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.969693                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.flowspec
     import metaflow.exception
-    import metaflow.metaflow_current
     import metaflow.plugins.datatools.s3.s3
+    import io
     import metaflow.datastore.inputs
+    import metaflow.metaflow_current
+    import metaflow.flowspec
     import typing
-    import io
 
 TYPE_CHECKING: bool
 
 class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
+    """
+    Main class from which all Flows should inherit.
+    
+    Attributes
+    ----------
+    index
+    input
+    """
     def __init__(self, use_cli = True):
         """
         Construct a FlowSpec
@@ -290,6 +298,9 @@ def ensure_unicode(x):
     ...
 
 class S3GetObject(tuple, metaclass=type):
+    """
+    S3GetObject(key, offset, length)
+    """
     @staticmethod
     def __new__(_cls, key: str, offset: int, length: int):
         """
@@ -311,6 +322,9 @@ class S3GetObject(tuple, metaclass=type):
     ...
 
 class S3PutObject(tuple, metaclass=type):
+    """
+    S3PutObject(key, value, path, content_type, encryption, metadata)
+    """
     @staticmethod
     def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
         """
@@ -332,6 +346,9 @@ class S3PutObject(tuple, metaclass=type):
     ...
 
 class RangeInfo(tuple, metaclass=type):
+    """
+    RangeInfo(total_size, request_offset, request_length)
+    """
     @staticmethod
     def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
         """
@@ -371,6 +388,13 @@ class MetaflowS3InvalidRange(metaflow.exception.MetaflowException, metaclass=typ
     ...
 
 class S3Object(object, metaclass=type):
+    """
+    This object represents a path or an object in S3,
+    with an optional local copy.
+    
+    `S3Object`s are not instantiated directly, but they are returned
+    by many methods of the `S3` client.
+    """
     def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
         ...
     @property
@@ -587,6 +611,59 @@ class S3Client(object, metaclass=type):
     ...
 
 class S3(object, metaclass=type):
+    """
+    The Metaflow S3 client.
+    
+    This object manages the connection to S3 and a temporary diretory that is used
+    to download objects. Note that in most cases when the data fits in memory, no local
+    disk IO is needed as operations are cached by the operating system, which makes
+    operations fast as long as there is enough memory available.
+    
+    The easiest way is to use this object as a context manager:
+    ```
+    with S3() as s3:
+        data = [obj.blob for obj in s3.get_many(urls)]
+    print(data)
+    ```
+    The context manager takes care of creating and deleting a temporary directory
+    automatically. Without a context manager, you must call `.close()` to delete
+    the directory explicitly:
+    ```
+    s3 = S3()
+    data = [obj.blob for obj in s3.get_many(urls)]
+    s3.close()
+    ```
+    You can customize the location of the temporary directory with `tmproot`. It
+    defaults to the current working directory.
+    
+    To make it easier to deal with object locations, the client can be initialized
+    with an S3 path prefix. There are three ways to handle locations:
+    
+    1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
+       initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
+       with the current run ID. This mode makes it easy to version data based
+       on the run ID consistently. You can use the `bucket` and `prefix` to
+       override parts of `DATATOOLS_S3ROOT`.
+    
+    2. Specify an S3 prefix explicitly with `s3root`,
+       e.g. `S3(s3root='s3://mybucket/some/path')`.
+    
+    3. Specify nothing, i.e. `S3()`, in which case all operations require
+       a full S3 url prefixed with `s3://`.
+    
+    Parameters
+    ----------
+    tmproot : str, default: '.'
+        Where to store the temporary directory.
+    bucket : str, optional
+        Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
+    prefix : str, optional
+        Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
+    run : FlowSpec or Run, optional
+        Derive path prefix from the current or a past run ID, e.g. S3(run=self).
+    s3root : str, optional
+        If `run` is not specified, use this as the S3 prefix.
+    """
     @classmethod
     def get_root_from_config(cls, echo, create_on_absent = True):
         ...

metaflow-stubs/plugins/datatools/s3/s3tail.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.713188                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.080294                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/datatools/s3/s3util.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.644075                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.006164                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/debug_logger.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.636974                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.998981                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/debug_monitor.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.637245                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.999272                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/environment_decorator.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.634180                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.994485                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -11,6 +11,14 @@ if typing.TYPE_CHECKING:
     import metaflow.decorators
 
 class EnvironmentDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    """
+    Specifies environment variables to be set prior to the execution of a step.
+    
+    Parameters
+    ----------
+    vars : Dict[str, str], default {}
+        Dictionary of environment variables to set.
+    """
     def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
         ...
     ...

metaflow-stubs/plugins/events_decorator.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.635638                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.997576                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -21,11 +21,115 @@ class MetaflowException(Exception, metaclass=type):
     ...
 
 class TriggerDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
+    """
+    Specifies the event(s) that this flow depends on.
+    
+    ```
+    @trigger(event='foo')
+    ```
+    or
+    ```
+    @trigger(events=['foo', 'bar'])
+    ```
+    
+    Additionally, you can specify the parameter mappings
+    to map event payload to Metaflow parameters for the flow.
+    ```
+    @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
+    ```
+    or
+    ```
+    @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
+                     {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
+    ```
+    
+    'parameters' can also be a list of strings and tuples like so:
+    ```
+    @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
+    ```
+    This is equivalent to:
+    ```
+    @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
+    ```
+    
+    Parameters
+    ----------
+    event : Union[str, Dict[str, Any]], optional, default None
+        Event dependency for this flow.
+    events : List[Union[str, Dict[str, Any]]], default []
+        Events dependency for this flow.
+    options : Dict[str, Any], default {}
+        Backend-specific configuration for tuning eventing behavior.
+    
+    MF Add To Current
+    -----------------
+    trigger -> metaflow.events.Trigger
+        Returns `Trigger` if the current run is triggered by an event
+    
+        @@ Returns
+        -------
+        Trigger
+            `Trigger` if triggered by an event
+    """
     def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
         ...
     ...
 
 class TriggerOnFinishDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
+    """
+    Specifies the flow(s) that this flow depends on.
+    
+    ```
+    @trigger_on_finish(flow='FooFlow')
+    ```
+    or
+    ```
+    @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
+    ```
+    This decorator respects the @project decorator and triggers the flow
+    when upstream runs within the same namespace complete successfully
+    
+    Additionally, you can specify project aware upstream flow dependencies
+    by specifying the fully qualified project_flow_name.
+    ```
+    @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
+    ```
+    or
+    ```
+    @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
+    ```
+    
+    You can also specify just the project or project branch (other values will be
+    inferred from the current project or project branch):
+    ```
+    @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
+    ```
+    
+    Note that `branch` is typically one of:
+      - `prod`
+      - `user.bob`
+      - `test.my_experiment`
+      - `prod.staging`
+    
+    Parameters
+    ----------
+    flow : Union[str, Dict[str, str]], optional, default None
+        Upstream flow dependency for this flow.
+    flows : List[Union[str, Dict[str, str]]], default []
+        Upstream flow dependencies for this flow.
+    options : Dict[str, Any], default {}
+        Backend-specific configuration for tuning eventing behavior.
+    
+    MF Add To Current
+    -----------------
+    trigger -> metaflow.events.Trigger
+        Returns `Trigger` if the current run is triggered by an event
+    
+        @@ Returns
+        -------
+        Trigger
+            `Trigger` if triggered by an event
+    """
     def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
         ...
     def get_top_level_options(self):

metaflow-stubs/plugins/frameworks/__init__.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.634538                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.994842                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/frameworks/pytorch.pyi

@@ -1,20 +1,41 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.694478                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.043936                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.plugins.parallel_decorator
     import metaflow.metaflow_current
+    import metaflow.plugins.parallel_decorator
     import metaflow.decorators
 
 current: metaflow.metaflow_current.Current
 
 class ParallelDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    """
+    MF Add To Current
+    -----------------
+    parallel -> metaflow.metaflow_current.Parallel
+    
+        @@ Returns
+        -------
+        Parallel
+            `namedtuple` with the following fields:
+                - main_ip : str
+                    The IP address of the control task.
+                - num_nodes : int
+                    The total number of tasks created by @parallel
+                - node_index : int
+                    The index of the current task in all the @parallel tasks.
+                - control_task_id : Optional[str]
+                    The task ID of the control task. Available to all tasks.
+    
+    is_parallel -> bool
+        True if the current step is a @parallel step.
+    """
     def __init__(self, attributes = None, statically_defined = False):
         ...
     def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):

metaflow-stubs/plugins/gcp/__init__.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.636193                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.998196                                        #
 ##################################################################################
 
 from __future__ import annotations