metaflow-stubs 2.12.28__py2.py3-none-any.whl → 2.12.29__py2.py3-none-any.whl

metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi

@@ -1,12 +1,13 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.544350                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.629488                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
 
+
 class GcpDefaultClientProvider(object, metaclass=type):
     @staticmethod
     def get_gs_storage_client(*args, **kwargs):

metaflow-stubs/plugins/gcp/gs_utils.pyi

@@ -1,27 +1,15 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.544927                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.629882                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
-import typing
-if typing.TYPE_CHECKING:
-    import metaflow.exception
 
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
-
-class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
-    ...
-
-class MetaflowGSPackageError(metaflow.exception.MetaflowException, metaclass=type):
-    ...
+from ...exception import MetaflowException as MetaflowException
+from ...exception import MetaflowInternalError as MetaflowInternalError
+from .gs_exceptions import MetaflowGSPackageError as MetaflowGSPackageError
 
 def parse_gs_full_path(gs_uri):
     ...

metaflow-stubs/plugins/gcp/includefile_support.pyi

@@ -1,24 +1,14 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.476903                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.630409                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
-import typing
-if typing.TYPE_CHECKING:
-    import metaflow.exception
 
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
-
-class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
-    ...
+from ...exception import MetaflowException as MetaflowException
+from ...exception import MetaflowInternalError as MetaflowInternalError
 
 class GS(object, metaclass=type):
     @classmethod

metaflow-stubs/plugins/kubernetes/__init__.pyi

@@ -1,9 +1,17 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.483016                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.587611                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
 
+from . import kubernetes_jobsets as kubernetes_jobsets
+from . import kubernetes_job as kubernetes_job
+from . import kubernetes_client as kubernetes_client
+from . import kubernetes as kubernetes
+from . import kubernetes_decorator as kubernetes_decorator
+from . import kube_utils as kube_utils
+from . import kubernetes_cli as kubernetes_cli
+

metaflow-stubs/plugins/kubernetes/kube_utils.pyi

@@ -1,17 +1,13 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.540763                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.628577                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
-import typing
-if typing.TYPE_CHECKING:
-    import metaflow.exception
 
-class CommandException(metaflow.exception.MetaflowException, metaclass=type):
-    ...
+from ...exception import CommandException as CommandException
 
 def parse_cli_options(flow_name, run_id, user, my_runs, echo):
     ...

metaflow-stubs/plugins/kubernetes/kubernetes.pyi

@@ -1,24 +1,19 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.539325                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.627542                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
 import typing
+import metaflow
 if typing.TYPE_CHECKING:
-    import metaflow.metaflow_current
     import metaflow.exception
 
-current: metaflow.metaflow_current.Current
-
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
+from ...metaflow_current import current as current
+from ...exception import MetaflowException as MetaflowException
+from .kubernetes_client import KubernetesClient as KubernetesClient
 
 ARGO_EVENTS_EVENT: None
 
@@ -90,21 +85,6 @@ UBF_TASK: str
 
 BASH_SAVE_LOGS: str
 
-class KubernetesClient(object, metaclass=type):
-    def __init__(self):
-        ...
-    def get(self):
-        ...
-    def list(self, flow_name, run_id, user):
-        ...
-    def kill_pods(self, flow_name, run_id, user, echo):
-        ...
-    def jobset(self, **kwargs):
-        ...
-    def job(self, **kwargs):
-        ...
-    ...
-
 LOGS_DIR: str
 
 STDOUT_FILE: str

metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi

@@ -1,56 +1,26 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.542151                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.628924                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
-import typing
-if typing.TYPE_CHECKING:
-    import metaflow.decorators
-    import metaflow._vendor.click.types
-    import metaflow.exception
 
-def parse_cli_options(flow_name, run_id, user, my_runs, echo):
-    ...
-
-class KubernetesClient(object, metaclass=type):
-    def __init__(self):
-        ...
-    def get(self):
-        ...
-    def list(self, flow_name, run_id, user):
-        ...
-    def kill_pods(self, flow_name, run_id, user, echo):
-        ...
-    def jobset(self, **kwargs):
-        ...
-    def job(self, **kwargs):
-        ...
-    ...
-
-class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
-    def convert(self, value, param, ctx):
-        ...
-    def __str__(self):
-        ...
-    def __repr__(self):
-        ...
-    ...
+from .kube_utils import parse_cli_options as parse_cli_options
+from .kubernetes_client import KubernetesClient as KubernetesClient
+from ...parameters import JSONTypeClass as JSONTypeClass
+from ..._vendor import click as click
+from ...exception import MetaflowException as MetaflowException
+from ...metadata_provider.util import sync_local_metadata_from_datastore as sync_local_metadata_from_datastore
+from .kubernetes import Kubernetes as Kubernetes
+from .kubernetes import KubernetesException as KubernetesException
+from .kubernetes import KubernetesKilledException as KubernetesKilledException
+from .kubernetes import parse_kube_keyvalue_list as parse_kube_keyvalue_list
+from .kubernetes_decorator import KubernetesDecorator as KubernetesDecorator
 
 METAFLOW_EXIT_DISALLOW_RETRY: int
 
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
-
-def sync_local_metadata_from_datastore(metadata_local_dir, task_ds):
-    ...
-
 DATASTORE_LOCAL_DIR: str
 
 KUBERNETES_LABELS: str
@@ -61,112 +31,3 @@ UBF_CONTROL: str
 
 UBF_TASK: str
 
-class Kubernetes(object, metaclass=type):
-    def __init__(self, datastore, metadata, environment):
-        ...
-    def launch_job(self, **kwargs):
-        ...
-    def create_jobset(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, docker_image, docker_image_pull_policy, step_cli = None, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, num_parallel = None):
-        ...
-    def create_job_object(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, step_cli, docker_image, docker_image_pull_policy, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, name_pattern = None):
-        ...
-    def create_k8sjob(self, job):
-        ...
-    def wait(self, stdout_location, stderr_location, echo = None):
-        ...
-    ...
-
-class KubernetesException(metaflow.exception.MetaflowException, metaclass=type):
-    ...
-
-class KubernetesKilledException(metaflow.exception.MetaflowException, metaclass=type):
-    ...
-
-def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True):
-    ...
-
-class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
-    """
-    Specifies that this step should execute on Kubernetes.
-    
-    Parameters
-    ----------
-    cpu : int, default 1
-        Number of CPUs required for this step. If `@resources` is
-        also present, the maximum value from all decorators is used.
-    memory : int, default 4096
-        Memory size (in MB) required for this step. If
-        `@resources` is also present, the maximum value from all decorators is
-        used.
-    disk : int, default 10240
-        Disk size (in MB) required for this step. If
-        `@resources` is also present, the maximum value from all decorators is
-        used.
-    image : str, optional, default None
-        Docker image to use when launching on Kubernetes. If not specified, and
-        METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
-        not, a default Docker image mapping to the current version of Python is used.
-    image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
-        If given, the imagePullPolicy to be applied to the Docker image of the step.
-    service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
-        Kubernetes service account to use when launching pod in Kubernetes.
-    secrets : List[str], optional, default None
-        Kubernetes secrets to use when launching pod in Kubernetes. These
-        secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
-        in Metaflow configuration.
-    node_selector: Union[Dict[str,str], str], optional, default None
-        Kubernetes node selector(s) to apply to the pod running the task.
-        Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
-        or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
-    namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
-        Kubernetes namespace to use when launching pod in Kubernetes.
-    gpu : int, optional, default None
-        Number of GPUs required for this step. A value of zero implies that
-        the scheduled node should not have GPUs.
-    gpu_vendor : str, default KUBERNETES_GPU_VENDOR
-        The vendor of the GPUs to be used for this step.
-    tolerations : List[str], default []
-        The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
-        Kubernetes tolerations to use when launching pod in Kubernetes.
-    use_tmpfs : bool, default False
-        This enables an explicit tmpfs mount for this step.
-    tmpfs_tempdir : bool, default True
-        sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
-    tmpfs_size : int, optional, default: None
-        The value for the size (in MiB) of the tmpfs mount for this step.
-        This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
-        memory allocated for this step.
-    tmpfs_path : str, optional, default /metaflow_temp
-        Path to tmpfs mount for this step.
-    persistent_volume_claims : Dict[str, str], optional, default None
-        A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
-        volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
-    shared_memory: int, optional
-        Shared memory size (in MiB) required for this step
-    port: int, optional
-        Port number to specify in the Kubernetes job object
-    compute_pool : str, optional, default None
-        Compute pool to be used for for this step.
-        If not specified, any accessible compute pool within the perimeter is used.
-    hostname_resolution_timeout: int, default 10 * 60
-        Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
-        Only applicable when @parallel is used.
-    """
-    def __init__(self, attributes = None, statically_defined = False):
-        ...
-    def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
-        ...
-    def package_init(self, flow, step_name, environment):
-        ...
-    def runtime_init(self, flow, graph, package, run_id):
-        ...
-    def runtime_task_created(self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context):
-        ...
-    def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
-        ...
-    def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_retries, ubf_context, inputs):
-        ...
-    def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_retries):
-        ...
-    ...
-

metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi

@@ -1,85 +1,22 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.537970                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.626229                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
+import metaflow
 import typing
 if typing.TYPE_CHECKING:
     import metaflow.exception
 
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
+from ...exception import MetaflowException as MetaflowException
+from .kubernetes_job import KubernetesJob as KubernetesJob
+from .kubernetes_jobsets import KubernetesJobSet as KubernetesJobSet
 
 KUBERNETES_NAMESPACE: str
 
-class KubernetesJob(object, metaclass=type):
-    def __init__(self, client, **kwargs):
-        ...
-    def create_job_spec(self):
-        ...
-    def create(self):
-        ...
-    def execute(self):
-        ...
-    def step_name(self, step_name):
-        ...
-    def namespace(self, namespace):
-        ...
-    def name(self, name):
-        ...
-    def command(self, command):
-        ...
-    def image(self, image):
-        ...
-    def cpu(self, cpu):
-        ...
-    def memory(self, mem):
-        ...
-    def environment_variable(self, name, value):
-        ...
-    def label(self, name, value):
-        ...
-    def annotation(self, name, value):
-        ...
-    ...
-
-class KubernetesJobSet(object, metaclass=type):
-    def __init__(self, client, name = None, namespace = None, num_parallel = None, **kwargs):
-        ...
-    @property
-    def jobset_control_addr(self):
-        ...
-    @property
-    def worker(self):
-        ...
-    @property
-    def control(self):
-        ...
-    def environment_variable_from_selector(self, name, label_value):
-        ...
-    def environment_variables_from_selectors(self, env_dict):
-        ...
-    def environment_variable(self, name, value):
-        ...
-    def label(self, name, value):
-        ...
-    def annotation(self, name, value):
-        ...
-    def secret(self, name):
-        ...
-    def dump(self):
-        ...
-    def execute(self):
-        ...
-    ...
-
 CLIENT_REFRESH_INTERVAL_SECONDS: int
 
 class KubernetesClientException(metaflow.exception.MetaflowException, metaclass=type):

metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi

@@ -1,50 +1,26 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.28                                                            #
-# Generated on 2024-11-01T10:21:04.540493                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.29                                                                                #
+# Generated on 2024-11-07T22:19:34.628373                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
+import metaflow
 import typing
 if typing.TYPE_CHECKING:
     import metaflow.decorators
-    import metaflow.metaflow_current
-    import metaflow.exception
 
-current: metaflow.metaflow_current.Current
-
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
-
-class MetaDatum(tuple, metaclass=type):
-    """
-    MetaDatum(field, value, type, tags)
-    """
-    @staticmethod
-    def __new__(_cls, field, value, type, tags):
-        """
-        Create new instance of MetaDatum(field, value, type, tags)
-        """
-        ...
-    def __repr__(self):
-        """
-        Return a nicely formatted representation string
-        """
-        ...
-    def __getnewargs__(self):
-        """
-        Return self as a plain tuple.  Used by copy and pickle.
-        """
-        ...
-    ...
-
-def sync_local_metadata_to_datastore(metadata_local_dir, task_ds):
-    ...
+from ...metaflow_current import current as current
+from ...exception import MetaflowException as MetaflowException
+from ...metadata_provider.metadata import MetaDatum as MetaDatum
+from ...metadata_provider.util import sync_local_metadata_to_datastore as sync_local_metadata_to_datastore
+from ..resources_decorator import ResourcesDecorator as ResourcesDecorator
+from ..timeout_decorator import get_run_time_limit_for_task as get_run_time_limit_for_task
+from ..aws.aws_utils import get_docker_registry as get_docker_registry
+from ..aws.aws_utils import get_ec2_instance_metadata as get_ec2_instance_metadata
+from .kubernetes import KubernetesException as KubernetesException
+from .kubernetes import parse_kube_keyvalue_list as parse_kube_keyvalue_list
 
 DATASTORE_LOCAL_DIR: str
 
@@ -78,108 +54,8 @@ KUBERNETES_SHARED_MEMORY: None
 
 KUBERNETES_TOLERATIONS: str
 
-class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
-    """
-    Specifies the resources needed when executing this step.
-    
-    Use `@resources` to specify the resource requirements
-    independently of the specific compute layer (`@batch`, `@kubernetes`).
-    
-    You can choose the compute layer on the command line by executing e.g.
-    ```
-    python myflow.py run --with batch
-    ```
-    or
-    ```
-    python myflow.py run --with kubernetes
-    ```
-    which executes the flow on the desired system using the
-    requirements specified in `@resources`.
-    
-    Parameters
-    ----------
-    cpu : int, default 1
-        Number of CPUs required for this step.
-    gpu : int, optional, default None
-        Number of GPUs required for this step.
-    disk : int, optional, default None
-        Disk size (in MB) required for this step. Only applies on Kubernetes.
-    memory : int, default 4096
-        Memory size (in MB) required for this step.
-    shared_memory : int, optional, default None
-        The value for the size (in MiB) of the /dev/shm volume for this step.
-        This parameter maps to the `--shm-size` option in Docker.
-    """
-    ...
-
-def get_run_time_limit_for_task(step_decos):
-    ...
-
 UBF_CONTROL: str
 
-def get_docker_registry(image_uri):
-    """
-    Explanation:
-        (.+?(?:[:.].+?)\/)? - [GROUP 0] REGISTRY
-            .+?                  - A registry must start with at least one character
-            (?:[:.].+?)\/       - A registry must have ":" or "." and end with "/"
-            ?                    - Make a registry optional
-        (.*?)                - [GROUP 1] REPOSITORY
-            .*?                  - Get repository name until separator
-        (?:[@:])?            - SEPARATOR
-            ?:                   - Don't capture separator
-            [@:]                 - The separator must be either "@" or ":"
-            ?                    - The separator is optional
-        ((?<=[@:]).*)?       - [GROUP 2] TAG / DIGEST
-            (?<=[@:])            - A tag / digest must be preceded by "@" or ":"
-            .*                   - Capture rest of tag / digest
-            ?                    - A tag / digest is optional
-    Examples:
-        image
-            - None
-            - image
-            - None
-        example/image
-            - None
-            - example/image
-            - None
-        example/image:tag
-            - None
-            - example/image
-            - tag
-        example.domain.com/example/image:tag
-            - example.domain.com/
-            - example/image
-            - tag
-        123.123.123.123:123/example/image:tag
-            - 123.123.123.123:123/
-            - example/image
-            - tag
-        example.domain.com/example/image@sha256:45b23dee0
-            - example.domain.com/
-            - example/image
-            - sha256:45b23dee0
-    """
-    ...
-
-def get_ec2_instance_metadata():
-    """
-    Fetches the EC2 instance metadata through AWS instance metadata service
-    
-    Returns either an empty dictionary, or one with the keys
-        - ec2-instance-id
-        - ec2-instance-type
-        - ec2-region
-        - ec2-availability-zone
-    """
-    ...
-
-class KubernetesException(metaflow.exception.MetaflowException, metaclass=type):
-    ...
-
-def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True):
-    ...
-
 class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
     """
     Specifies that this step should execute on Kubernetes.
@@ -211,8 +87,9 @@ class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
         in Metaflow configuration.
     node_selector: Union[Dict[str,str], str], optional, default None
         Kubernetes node selector(s) to apply to the pod running the task.
-        Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
-        or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
+        Can be passed in as a comma separated string of values e.g.
+        'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
+        {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
     namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
         Kubernetes namespace to use when launching pod in Kubernetes.
     gpu : int, optional, default None