metaflow-stubs 2.12.22__py2.py3-none-any.whl → 2.12.23__py2.py3-none-any.whl

metaflow-stubs/includefile.pyi

@@ -1,18 +1,18 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.603042                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.963435                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.plugins.datatools.s3.s3
-    import typing
+    import metaflow._vendor.click.types
     import io
     import metaflow.parameters
-    import metaflow._vendor.click.types
+    import metaflow.plugins.datatools.s3.s3
+    import typing
 
 class MetaflowException(Exception, metaclass=type):
     def __init__(self, msg = "", lineno = None):
@@ -22,6 +22,18 @@ class MetaflowException(Exception, metaclass=type):
     ...
 
 class DelayedEvaluationParameter(object, metaclass=type):
+    """
+    This is a very simple wrapper to allow parameter "conversion" to be delayed until
+    the `_set_constants` function in FlowSpec. Typically, parameters are converted
+    by click when the command line option is processed. For some parameters, like
+    IncludeFile, this is too early as it would mean we would trigger the upload
+    of the file too early. If a parameter converts to a DelayedEvaluationParameter
+    object through the usual click mechanisms, `_set_constants` knows to invoke the
+    __call__ method on that DelayedEvaluationParameter; in that case, the __call__
+    method is invoked without any parameter. The return_str parameter will be used
+    by schedulers when they need to convert DelayedEvaluationParameters to a
+    string to store them
+    """
     def __init__(self, name, field, fun):
         ...
     def __call__(self, return_str = False):
@@ -29,6 +41,14 @@ class DelayedEvaluationParameter(object, metaclass=type):
     ...
 
 class DeployTimeField(object, metaclass=type):
+    """
+    This a wrapper object for a user-defined function that is called
+    at deploy time to populate fields in a Parameter. The wrapper
+    is needed to make Click show the actual value returned by the
+    function instead of a function pointer in its help text. Also, this
+    object curries the context argument for the function, and pretty
+    prints any exceptions that occur during evaluation.
+    """
     def __init__(self, parameter_name, parameter_type, field, fun, return_str = True, print_representation = None):
         ...
     def __call__(self, deploy_time = False):
@@ -43,6 +63,49 @@ class DeployTimeField(object, metaclass=type):
     ...
 
 class Parameter(object, metaclass=type):
+    """
+    Defines a parameter for a flow.
+    
+    Parameters must be instantiated as class variables in flow classes, e.g.
+    ```
+    class MyFlow(FlowSpec):
+        param = Parameter('myparam')
+    ```
+    in this case, the parameter is specified on the command line as
+    ```
+    python myflow.py run --myparam=5
+    ```
+    and its value is accessible through a read-only artifact like this:
+    ```
+    print(self.param == 5)
+    ```
+    Note that the user-visible parameter name, `myparam` above, can be
+    different from the artifact name, `param` above.
+    
+    The parameter value is converted to a Python type based on the `type`
+    argument or to match the type of `default`, if it is set.
+    
+    Parameters
+    ----------
+    name : str
+        User-visible parameter name.
+    default : str or float or int or bool or `JSONType` or a function.
+        Default value for the parameter. Use a special `JSONType` class to
+        indicate that the value must be a valid JSON object. A function
+        implies that the parameter corresponds to a *deploy-time parameter*.
+        The type of the default value is used as the parameter `type`.
+    type : Type, default None
+        If `default` is not specified, define the parameter type. Specify
+        one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
+        to the type of `default` or `str` if none specified.
+    help : str, optional
+        Help text to show in `run --help`.
+    required : bool, default False
+        Require that the user specified a value for the parameter.
+        `required=True` implies that the `default` is not used.
+    show_default : bool, default True
+        If True, show the default value in the help text.
+    """
     def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], metaflow.parameters.JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
         ...
     def __repr__(self):
@@ -61,6 +124,9 @@ class Parameter(object, metaclass=type):
     ...
 
 class ParameterContext(tuple, metaclass=type):
+    """
+    ParameterContext(flow_name, user_name, parameter_name, logger, ds_type)
+    """
     @staticmethod
     def __new__(_cls, flow_name: str, user_name: str, parameter_name: str, logger: typing.Callable[..., None], ds_type: str):
         """
@@ -82,6 +148,13 @@ class ParameterContext(tuple, metaclass=type):
     ...
 
 class Local(object, metaclass=type):
+    """
+    This class allows you to access the local filesystem in a way similar to the S3 datatools
+    client. It is a stripped down version for now and only implements the functionality needed
+    for this use case.
+    
+    In the future, we may want to allow it to be used in a way similar to the S3() client.
+    """
     @classmethod
     def get_root_from_config(cls, echo, create_on_absent = True):
         ...
@@ -104,6 +177,59 @@ class Local(object, metaclass=type):
     ...
 
 class S3(object, metaclass=type):
+    """
+    The Metaflow S3 client.
+    
+    This object manages the connection to S3 and a temporary diretory that is used
+    to download objects. Note that in most cases when the data fits in memory, no local
+    disk IO is needed as operations are cached by the operating system, which makes
+    operations fast as long as there is enough memory available.
+    
+    The easiest way is to use this object as a context manager:
+    ```
+    with S3() as s3:
+        data = [obj.blob for obj in s3.get_many(urls)]
+    print(data)
+    ```
+    The context manager takes care of creating and deleting a temporary directory
+    automatically. Without a context manager, you must call `.close()` to delete
+    the directory explicitly:
+    ```
+    s3 = S3()
+    data = [obj.blob for obj in s3.get_many(urls)]
+    s3.close()
+    ```
+    You can customize the location of the temporary directory with `tmproot`. It
+    defaults to the current working directory.
+    
+    To make it easier to deal with object locations, the client can be initialized
+    with an S3 path prefix. There are three ways to handle locations:
+    
+    1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
+       initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
+       with the current run ID. This mode makes it easy to version data based
+       on the run ID consistently. You can use the `bucket` and `prefix` to
+       override parts of `DATATOOLS_S3ROOT`.
+    
+    2. Specify an S3 prefix explicitly with `s3root`,
+       e.g. `S3(s3root='s3://mybucket/some/path')`.
+    
+    3. Specify nothing, i.e. `S3()`, in which case all operations require
+       a full S3 url prefixed with `s3://`.
+    
+    Parameters
+    ----------
+    tmproot : str, default: '.'
+        Where to store the temporary directory.
+    bucket : str, optional
+        Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
+    prefix : str, optional
+        Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
+    run : FlowSpec or Run, optional
+        Derive path prefix from the current or a past run ID, e.g. S3(run=self).
+    s3root : str, optional
+        If `run` is not specified, use this as the S3 prefix.
+    """
     @classmethod
     def get_root_from_config(cls, echo, create_on_absent = True):
         ...
@@ -469,6 +595,33 @@ class FilePathClass(metaflow._vendor.click.types.ParamType, metaclass=type):
     ...
 
 class IncludeFile(metaflow.parameters.Parameter, metaclass=type):
+    """
+    Includes a local file as a parameter for the flow.
+    
+    `IncludeFile` behaves like `Parameter` except that it reads its value from a file instead of
+    the command line. The user provides a path to a file on the command line. The file contents
+    are saved as a read-only artifact which is available in all steps of the flow.
+    
+    Parameters
+    ----------
+    name : str
+        User-visible parameter name.
+    default : Union[str, Callable[ParameterContext, str]]
+        Default path to a local file. A function
+        implies that the parameter corresponds to a *deploy-time parameter*.
+    is_text : bool, default True
+        Convert the file contents to a string using the provided `encoding`.
+        If False, the artifact is stored in `bytes`.
+    encoding : str, optional, default 'utf-8'
+        Use this encoding to decode the file contexts if `is_text=True`.
+    required : bool, default False
+        Require that the user specified a value for the parameter.
+        `required=True` implies that the `default` is not used.
+    help : str, optional
+        Help text to show in `run --help`.
+    show_default : bool, default True
+        If True, show the default value in the help text.
+    """
     def __init__(self, name: str, required: bool = False, is_text: bool = True, encoding: str = "utf-8", help: typing.Optional[str] = None, **kwargs: typing.Dict[str, str]):
         ...
     def load_parameter(self, v):

metaflow-stubs/info_file.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.586556                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.946138                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/metadata/metadata.pyi

@@ -1,15 +1,15 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.650748                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.013692                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.exception
     import metaflow.metadata.metadata
+    import metaflow.exception
 
 class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
     ...
@@ -28,6 +28,9 @@ def validate_tag(tag):
     ...
 
 class DataArtifact(tuple, metaclass=type):
+    """
+    DataArtifact(name, ds_type, ds_root, url, type, sha)
+    """
     @staticmethod
     def __new__(_cls, name, ds_type, ds_root, url, type, sha):
         """
@@ -47,6 +50,9 @@ class DataArtifact(tuple, metaclass=type):
     ...
 
 class MetaDatum(tuple, metaclass=type):
+    """
+    MetaDatum(field, value, type, tags)
+    """
     @staticmethod
     def __new__(_cls, field, value, type, tags):
         """

metaflow-stubs/metadata/util.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.713487                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.080603                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/metaflow_config.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.588831                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.948523                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/metaflow_current.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.716345                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.085581                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -9,16 +9,19 @@ from __future__ import annotations
 import typing
 if typing.TYPE_CHECKING:
     import metaflow.events
-    import metaflow.metaflow_current
+    import metaflow.plugins.cards.component_serializer
     import metaflow
+    import metaflow.metaflow_current
     import typing
-    import metaflow.plugins.cards.component_serializer
 
 TYPE_CHECKING: bool
 
 TEMPDIR: str
 
 class Parallel(tuple, metaclass=type):
+    """
+    Parallel(main_ip, num_nodes, node_index, control_task_id)
+    """
     @staticmethod
     def __new__(_cls, main_ip, num_nodes, node_index, control_task_id):
         """
@@ -221,24 +224,6 @@ class Current(object, metaclass=type):
     def graph(_):
         ...
     @property
-    def card(self) -> "metaflow.plugins.cards.component_serializer.CardComponentCollector":
-        """
-        (only in the presence of the @card decorator)
-        
-        The `@card` decorator makes the cards available through the `current.card`
-        object. If multiple `@card` decorators are present, you can add an `ID` to
-        distinguish between them using `@card(id=ID)` as the decorator. You will then
-        be able to access that specific card using `current.card[ID].
-        
-        Methods available are `append` and `extend`
-        
-        Returns
-        -------
-        CardComponentCollector
-            The or one of the cards attached to this step.
-        """
-        ...
-    @property
     def parallel(self) -> "metaflow.metaflow_current.Parallel":
         """
         (only in the presence of the @parallel decorator)
@@ -267,9 +252,27 @@ class Current(object, metaclass=type):
         """
         ...
     @property
+    def card(self) -> "metaflow.plugins.cards.component_serializer.CardComponentCollector":
+        """
+        (only in the presence of the @card decorator)
+        
+        The `@card` decorator makes the cards available through the `current.card`
+        object. If multiple `@card` decorators are present, you can add an `ID` to
+        distinguish between them using `@card(id=ID)` as the decorator. You will then
+        be able to access that specific card using `current.card[ID].
+        
+        Methods available are `append` and `extend`
+        
+        Returns
+        -------
+        CardComponentCollector
+            The or one of the cards attached to this step.
+        """
+        ...
+    @property
     def trigger(self) -> "metaflow.events.Trigger":
         """
-        (only in the presence of the @trigger_on_finish, or @trigger decorators)
+        (only in the presence of the @trigger, or @trigger_on_finish decorators)
         
         Returns `Trigger` if the current run is triggered by an event
         

metaflow-stubs/mflog/mflog.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.651165                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.014112                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/multicore_utils.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.589356                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.949043                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/parameters.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.590772                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.950574                                        #
 ##################################################################################
 
 from __future__ import annotations
@@ -10,8 +10,8 @@ import typing
 if typing.TYPE_CHECKING:
     import metaflow.parameters
     import metaflow.exception
-    import typing
     import metaflow._vendor.click.types
+    import typing
 
 class ParameterFieldFailed(metaflow.exception.MetaflowException, metaclass=type):
     def __init__(self, name, field):
@@ -31,6 +31,9 @@ class MetaflowException(Exception, metaclass=type):
     ...
 
 class ParameterContext(tuple, metaclass=type):
+    """
+    ParameterContext(flow_name, user_name, parameter_name, logger, ds_type)
+    """
     @staticmethod
     def __new__(_cls, flow_name: str, user_name: str, parameter_name: str, logger: typing.Callable[..., None], ds_type: str):
         """
@@ -71,6 +74,14 @@ class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
     ...
 
 class DeployTimeField(object, metaclass=type):
+    """
+    This a wrapper object for a user-defined function that is called
+    at deploy time to populate fields in a Parameter. The wrapper
+    is needed to make Click show the actual value returned by the
+    function instead of a function pointer in its help text. Also, this
+    object curries the context argument for the function, and pretty
+    prints any exceptions that occur during evaluation.
+    """
     def __init__(self, parameter_name, parameter_type, field, fun, return_str = True, print_representation = None):
         ...
     def __call__(self, deploy_time = False):
@@ -91,6 +102,18 @@ def set_parameter_context(flow_name, echo, datastore):
     ...
 
 class DelayedEvaluationParameter(object, metaclass=type):
+    """
+    This is a very simple wrapper to allow parameter "conversion" to be delayed until
+    the `_set_constants` function in FlowSpec. Typically, parameters are converted
+    by click when the command line option is processed. For some parameters, like
+    IncludeFile, this is too early as it would mean we would trigger the upload
+    of the file too early. If a parameter converts to a DelayedEvaluationParameter
+    object through the usual click mechanisms, `_set_constants` knows to invoke the
+    __call__ method on that DelayedEvaluationParameter; in that case, the __call__
+    method is invoked without any parameter. The return_str parameter will be used
+    by schedulers when they need to convert DelayedEvaluationParameters to a
+    string to store them
+    """
     def __init__(self, name, field, fun):
         ...
     def __call__(self, return_str = False):
@@ -98,6 +121,49 @@ class DelayedEvaluationParameter(object, metaclass=type):
     ...
 
 class Parameter(object, metaclass=type):
+    """
+    Defines a parameter for a flow.
+    
+    Parameters must be instantiated as class variables in flow classes, e.g.
+    ```
+    class MyFlow(FlowSpec):
+        param = Parameter('myparam')
+    ```
+    in this case, the parameter is specified on the command line as
+    ```
+    python myflow.py run --myparam=5
+    ```
+    and its value is accessible through a read-only artifact like this:
+    ```
+    print(self.param == 5)
+    ```
+    Note that the user-visible parameter name, `myparam` above, can be
+    different from the artifact name, `param` above.
+    
+    The parameter value is converted to a Python type based on the `type`
+    argument or to match the type of `default`, if it is set.
+    
+    Parameters
+    ----------
+    name : str
+        User-visible parameter name.
+    default : str or float or int or bool or `JSONType` or a function.
+        Default value for the parameter. Use a special `JSONType` class to
+        indicate that the value must be a valid JSON object. A function
+        implies that the parameter corresponds to a *deploy-time parameter*.
+        The type of the default value is used as the parameter `type`.
+    type : Type, default None
+        If `default` is not specified, define the parameter type. Specify
+        one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
+        to the type of `default` or `str` if none specified.
+    help : str, optional
+        Help text to show in `run --help`.
+    required : bool, default False
+        Require that the user specified a value for the parameter.
+        `required=True` implies that the `default` is not used.
+    show_default : bool, default True
+        If True, show the default value in the help text.
+    """
     def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
         ...
     def __repr__(self):

metaflow-stubs/plugins/__init__.pyi

@@ -1,19 +1,23 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.605182                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.965641                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.unbounded_foreach
     import metaflow.plugins.cards.card_modules.card
+    import metaflow.unbounded_foreach
 
 CLIS_DESC: list
 
 class InternalTestUnboundedForeachInput(metaflow.unbounded_foreach.UnboundedForeachInput, metaclass=type):
+    """
+    Test class that wraps around values (any iterator) and simulates an
+    unbounded-foreach instead of a bounded foreach.
+    """
     def __init__(self, iterable):
         ...
     def __iter__(self):
@@ -194,6 +198,9 @@ class TestTimeoutCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, met
     ...
 
 class TestRefreshCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, metaclass=type):
+    """
+    This card takes no components and helps test the `current.card.refresh(data)` interface.
+    """
     def render(self, task) -> str:
         ...
     def render_runtime(self, task, data):
@@ -203,6 +210,10 @@ class TestRefreshCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, met
     ...
 
 class TestRefreshComponentCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, metaclass=type):
+    """
+    This card takes components and helps test the `current.card.components["A"].update()`
+    interface
+    """
     def __init__(self, options = {}, components = [], graph = None):
         ...
     def render(self, task) -> str:

metaflow-stubs/plugins/airflow/__init__.pyi

@@ -1,7 +1,7 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.631865                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:39.995730                                        #
 ##################################################################################
 
 from __future__ import annotations

metaflow-stubs/plugins/airflow/airflow.pyi

@@ -1,15 +1,15 @@
 ##################################################################################
 #                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.22                                                            #
-# Generated on 2024-09-20T00:45:49.670762                                        #
+# MF version: 2.12.23                                                            #
+# Generated on 2024-10-01T14:32:40.048837                                        #
 ##################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.exception
     import metaflow.metaflow_current
+    import metaflow.exception
     import metaflow._vendor.click.types
 
 current: metaflow.metaflow_current.Current
@@ -75,6 +75,18 @@ SERVICE_INTERNAL_URL: None
 AZURE_KEY_VAULT_PREFIX: None
 
 class DelayedEvaluationParameter(object, metaclass=type):
+    """
+    This is a very simple wrapper to allow parameter "conversion" to be delayed until
+    the `_set_constants` function in FlowSpec. Typically, parameters are converted
+    by click when the command line option is processed. For some parameters, like
+    IncludeFile, this is too early as it would mean we would trigger the upload
+    of the file too early. If a parameter converts to a DelayedEvaluationParameter
+    object through the usual click mechanisms, `_set_constants` knows to invoke the
+    __call__ method on that DelayedEvaluationParameter; in that case, the __call__
+    method is invoked without any parameter. The return_str parameter will be used
+    by schedulers when they need to convert DelayedEvaluationParameters to a
+    string to store them
+    """
     def __init__(self, name, field, fun):
         ...
     def __call__(self, return_str = False):