ob-metaflow-stubs 2.11.4.1__py2.py3-none-any.whl

metaflow-stubs/plugins/argo/argo_workflows_cli.pyi

@@ -0,0 +1,180 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.971565                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.metaflow_current
+    import metaflow.exception
+    import metaflow.decorators
+    import metaflow.parameters
+
+JSONType: metaflow.parameters.JSONTypeClass
+
+current: metaflow.metaflow_current.Current
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+ARGO_WORKFLOWS_UI_URL: None
+
+KUBERNETES_NAMESPACE: str
+
+SERVICE_VERSION_CHECK: bool
+
+UI_URL: None
+
+def load_token(token_prefix):
+    ...
+
+def new_token(token_prefix, prev_token = None):
+    ...
+
+def store_token(token_prefix, token):
+    ...
+
+class EnvironmentDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
+        ...
+    ...
+
+class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    def __init__(self, attributes = None, statically_defined = False):
+        ...
+    def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
+        ...
+    def package_init(self, flow, step_name, environment):
+        ...
+    def runtime_init(self, flow, graph, package, run_id):
+        ...
+    def runtime_task_created(self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context):
+        ...
+    def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
+        ...
+    def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_retries, ubf_context, inputs):
+        ...
+    def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_retries):
+        ...
+    ...
+
+def validate_tags(tags, existing_tags = None):
+    """
+    Raises MetaflowTaggingError if invalid based on these rules:
+    
+    Tag set size is too large. But it's OK if tag set is not larger
+    than an existing tag set (if provided).
+    
+    Then, we validate each tag.  See validate_tag()
+    """
+    ...
+
+class ArgoWorkflows(object, metaclass=type):
+    def __init__(self, name, graph, flow, code_package_sha, code_package_url, production_token, metadata, flow_datastore, environment, event_logger, monitor, tags = None, namespace = None, username = None, max_workers = None, workflow_timeout = None, workflow_priority = None, auto_emit_argo_events = False, notify_on_error = False, notify_on_success = False, notify_slack_webhook_url = None, notify_pager_duty_integration_key = None):
+        ...
+    def __str__(self):
+        ...
+    def deploy(self):
+        ...
+    @staticmethod
+    def list_templates(flow_name, all = False):
+        ...
+    @staticmethod
+    def delete(name):
+        ...
+    @classmethod
+    def terminate(cls, flow_name, name):
+        ...
+    @staticmethod
+    def get_workflow_status(flow_name, name):
+        ...
+    @staticmethod
+    def suspend(name):
+        ...
+    @staticmethod
+    def unsuspend(name):
+        ...
+    @classmethod
+    def trigger(cls, name, parameters = None):
+        ...
+    def schedule(self):
+        ...
+    def trigger_explanation(self):
+        ...
+    @classmethod
+    def get_existing_deployment(cls, name):
+        ...
+    @classmethod
+    def get_execution(cls, name):
+        ...
+    def list_to_prose(self, items, singular):
+        ...
+    ...
+
+class IncorrectProductionToken(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class RunIdMismatch(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class IncorrectMetadataServiceVersion(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class ArgoWorkflowsNameTooLong(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class UnsupportedPythonVersion(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+def check_python_version(obj):
+    ...
+
+def check_metadata_service_version(obj):
+    ...
+
+def resolve_workflow_name(obj, name):
+    ...
+
+def make_flow(obj, token, name, tags, namespace, max_workers, workflow_timeout, workflow_priority, auto_emit_argo_events, notify_on_error, notify_on_success, notify_slack_webhook_url, notify_pager_duty_integration_key):
+    ...
+
+def resolve_token(name, token_prefix, obj, authorize, given_token, generate_new_token, is_project):
+    ...
+
+def validate_token(name, token_prefix, authorize, instructions_fn = None):
+    """
+    Validate that the production token matches that of the deployed flow.
+    In case both the user and token do not match, raises an error.
+    Optionally outputs instructions on token usage via the provided instruction_fn(flow_name, prev_user)
+    """
+    ...
+
+def validate_run_id(workflow_name, token_prefix, authorize, run_id, instructions_fn = None):
+    """
+    Validates that a run_id adheres to the Argo Workflows naming rules, and
+    that it belongs to the current flow (accounting for project branch as well).
+    """
+    ...
+
+def sanitize_for_argo(text):
+    """
+    Sanitizes a string so it does not contain characters that are not permitted in Argo Workflow resource names.
+    """
+    ...
+
+def remap_status(status):
+    """
+    Group similar Argo Workflow statuses together in order to have similar output to step functions statuses.
+    """
+    ...
+

metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi

@@ -0,0 +1,169 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.961868                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.metaflow_current
+    import metaflow
+    import metaflow.events
+    import metaflow.decorators
+
+current: metaflow.metaflow_current.Current
+
+class Trigger(object, metaclass=type):
+    def __init__(self, _meta = None):
+        ...
+    @classmethod
+    def from_runs(cls, run_objs: typing.List["metaflow.Run"]):
+        ...
+    @property
+    def event(self) -> typing.Optional[metaflow.events.MetaflowEvent]:
+        """
+        The `MetaflowEvent` object corresponding to the triggering event.
+        
+        If multiple events triggered the run, this property is the latest event.
+        
+        Returns
+        -------
+        MetaflowEvent, optional
+            The latest event that triggered the run, if applicable.
+        """
+        ...
+    @property
+    def events(self) -> typing.Optional[typing.List[metaflow.events.MetaflowEvent]]:
+        """
+        The list of `MetaflowEvent` objects correspondings to all the triggering events.
+        
+        Returns
+        -------
+        List[MetaflowEvent], optional
+            List of all events that triggered the run
+        """
+        ...
+    @property
+    def run(self) -> typing.Optional["metaflow.Run"]:
+        """
+        The corresponding `Run` object if the triggering event is a Metaflow run.
+        
+        In case multiple runs triggered the run, this property is the latest run.
+        Returns `None` if none of the triggering events are a `Run`.
+        
+        Returns
+        -------
+        Run, optional
+            Latest Run that triggered this run, if applicable.
+        """
+        ...
+    @property
+    def runs(self) -> typing.Optional[typing.List["metaflow.Run"]]:
+        """
+        The list of `Run` objects in the triggering events.
+        Returns `None` if none of the triggering events are `Run` objects.
+        
+        Returns
+        -------
+        List[Run], optional
+            List of runs that triggered this run, if applicable.
+        """
+        ...
+    def __getitem__(self, key: str) -> typing.Union["metaflow.Run", metaflow.events.MetaflowEvent]:
+        """
+        If triggering events are runs, `key` corresponds to the flow name of the triggering run.
+        Otherwise, `key` corresponds to the event name and a `MetaflowEvent` object is returned.
+        
+        Returns
+        -------
+        Union[Run, MetaflowEvent]
+            `Run` object if triggered by a run. Otherwise returns a `MetaflowEvent`.
+        """
+        ...
+    def __iter__(self):
+        ...
+    def __contains__(self, ident: str) -> bool:
+        ...
+    ...
+
+class MetaDatum(tuple, metaclass=type):
+    @staticmethod
+    def __new__(_cls, field, value, type, tags):
+        """
+        Create new instance of MetaDatum(field, value, type, tags)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
+ARGO_EVENTS_WEBHOOK_URL: None
+
+class ArgoEvent(object, metaclass=type):
+    def __init__(self, name, url = None, payload = None, access_token = None):
+        ...
+    def add_to_payload(self, key, value):
+        """
+        Add a key-value pair in the payload. This is typically used to set parameters
+        of triggered flows. Often, `key` is the parameter name you want to set to
+        `value`. Overrides any existing value of `key`.
+        
+        Parameters
+        ----------
+        key : str
+            Key
+        value : str
+            Value
+        """
+        ...
+    def safe_publish(self, payload = None, ignore_errors = True):
+        """
+        Publishes an event when called inside a deployed workflow. Outside a deployed workflow
+        this function does nothing.
+        
+        Use this function inside flows to create events safely. As this function is a no-op
+        for local runs, you can safely call it during local development without causing unintended
+        side-effects. It takes effect only when deployed on Argo Workflows.
+        
+        Parameters
+        ----------
+        payload : dict
+            Additional key-value pairs to add to the payload.
+        ignore_errors : bool, default True
+            If True, events are created on a best effort basis - errors are silently ignored.
+        """
+        ...
+    def publish(self, payload = None, force = True, ignore_errors = True):
+        """
+        Publishes an event.
+        
+        Note that the function returns immediately after the event has been sent. It
+        does not wait for flows to start, nor it guarantees that any flows will start.
+        
+        Parameters
+        ----------
+        payload : dict
+            Additional key-value pairs to add to the payload.
+        ignore_errors : bool, default True
+            If True, events are created on a best effort basis - errors are silently ignored.
+        """
+        ...
+    ...
+
+class ArgoWorkflowsInternalDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_user_code_retries, ubf_context, inputs):
+        ...
+    def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_user_code_retries):
+        ...
+    ...
+

metaflow-stubs/plugins/aws/__init__.pyi

@@ -0,0 +1,9 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.928423                                        #
+##################################################################################
+
+from __future__ import annotations
+
+

metaflow-stubs/plugins/aws/aws_client.pyi

@@ -0,0 +1,22 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.956839                                        #
+##################################################################################
+
+from __future__ import annotations
+
+
+cached_aws_sandbox_creds: None
+
+cached_provider_class: None
+
+class Boto3ClientProvider(object, metaclass=type):
+    @staticmethod
+    def get_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
+        ...
+    ...
+
+def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
+    ...
+

metaflow-stubs/plugins/aws/aws_utils.pyi

@@ -0,0 +1,93 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.956196                                        #
+##################################################################################
+
+from __future__ import annotations
+
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+MAX_MEMORY_PER_TASK: None
+
+MAX_CPU_PER_TASK: None
+
+def get_ec2_instance_metadata():
+    """
+    Fetches the EC2 instance metadata through AWS instance metadata service
+    
+    Returns either an empty dictionary, or one with the keys
+        - ec2-instance-id
+        - ec2-instance-type
+        - ec2-region
+        - ec2-availability-zone
+    """
+    ...
+
+def get_docker_registry(image_uri):
+    """
+    Explanation:
+        (.+?(?:[:.].+?)\/)? - [GROUP 0] REGISTRY
+            .+?                  - A registry must start with at least one character
+            (?:[:.].+?)\/       - A registry must have ":" or "." and end with "/"
+            ?                    - Make a registry optional
+        (.*?)                - [GROUP 1] REPOSITORY
+            .*?                  - Get repository name until separator
+        (?:[@:])?            - SEPARATOR
+            ?:                   - Don't capture separator
+            [@:]                 - The separator must be either "@" or ":"
+            ?                    - The separator is optional
+        ((?<=[@:]).*)?       - [GROUP 2] TAG / DIGEST
+            (?<=[@:])            - A tag / digest must be preceded by "@" or ":"
+            .*                   - Capture rest of tag / digest
+            ?                    - A tag / digest is optional
+    Examples:
+        image
+            - None
+            - image
+            - None
+        example/image
+            - None
+            - example/image
+            - None
+        example/image:tag
+            - None
+            - example/image
+            - tag
+        example.domain.com/example/image:tag
+            - example.domain.com/
+            - example/image
+            - tag
+        123.123.123.123:123/example/image:tag
+            - 123.123.123.123:123/
+            - example/image
+            - tag
+        example.domain.com/example/image@sha256:45b23dee0
+            - example.domain.com/
+            - example/image
+            - sha256:45b23dee0
+    """
+    ...
+
+def compute_resource_attributes(decos, compute_deco, step_name, resource_defaults):
+    """
+    Compute resource values taking into account defaults, the values specified
+    in the compute decorator (like @batch or @kubernetes) directly, and
+    resources specified via @resources decorator.
+    
+    Returns a dictionary of resource attr -> value (str).
+    """
+    ...
+
+def sanitize_batch_tag(key, value):
+    """
+    Sanitize a key and value for use as a Batch tag.
+    """
+    ...
+

metaflow-stubs/plugins/aws/batch/__init__.pyi

@@ -0,0 +1,9 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.956310                                        #
+##################################################################################
+
+from __future__ import annotations
+
+

metaflow-stubs/plugins/aws/batch/batch.pyi

@@ -0,0 +1,120 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:44.002020                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+
+class S3Tail(object, metaclass=type):
+    def __init__(self, s3url):
+        ...
+    def reset_client(self, hard_reset = False):
+        ...
+    def clone(self, s3url):
+        ...
+    @property
+    def bytes_read(self):
+        ...
+    @property
+    def tail(self):
+        ...
+    def __iter__(self):
+        ...
+    def _make_range_request(self, *args, **kwargs):
+        ...
+    ...
+
+def sanitize_batch_tag(key, value):
+    """
+    Sanitize a key and value for use as a Batch tag.
+    """
+    ...
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+OTEL_ENDPOINT: None
+
+SERVICE_INTERNAL_URL: None
+
+DATATOOLS_S3ROOT: None
+
+DATASTORE_SYSROOT_S3: None
+
+DEFAULT_METADATA: str
+
+SERVICE_HEADERS: dict
+
+BATCH_EMIT_TAGS: bool
+
+CARD_S3ROOT: None
+
+S3_ENDPOINT_URL: None
+
+DEFAULT_SECRETS_BACKEND_TYPE: None
+
+AWS_SECRETS_MANAGER_DEFAULT_REGION: None
+
+S3_SERVER_SIDE_ENCRYPTION: None
+
+BASH_SAVE_LOGS: str
+
+class BatchClient(object, metaclass=type):
+    def __init__(self):
+        ...
+    def active_job_queues(self):
+        ...
+    def unfinished_jobs(self):
+        ...
+    def describe_jobs(self, job_ids):
+        ...
+    def describe_job_queue(self, job_queue):
+        ...
+    def job(self):
+        ...
+    def attach_job(self, job_id):
+        ...
+    def region(self):
+        ...
+    ...
+
+LOGS_DIR: str
+
+STDOUT_FILE: str
+
+STDERR_FILE: str
+
+STDOUT_PATH: str
+
+STDERR_PATH: str
+
+class BatchException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class BatchKilledException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class Batch(object, metaclass=type):
+    def __init__(self, metadata, environment):
+        ...
+    def list_jobs(self, flow_name, run_id, user, echo):
+        ...
+    def kill_jobs(self, flow_name, run_id, user, echo):
+        ...
+    def create_job(self, step_name, step_cli, task_spec, code_package_sha, code_package_url, code_package_ds, image, queue, iam_role = None, execution_role = None, cpu = None, gpu = None, memory = None, run_time_limit = None, shared_memory = None, max_swap = None, swappiness = None, inferentia = None, efa = None, env = {}, attrs = {}, host_volumes = None, efs_volumes = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, num_parallel = 0):
+        ...
+    def launch_job(self, step_name, step_cli, task_spec, code_package_sha, code_package_url, code_package_ds, image, queue, iam_role = None, execution_role = None, cpu = None, gpu = None, memory = None, run_time_limit = None, shared_memory = None, max_swap = None, swappiness = None, inferentia = None, efa = None, host_volumes = None, efs_volumes = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, num_parallel = 0, env = {}, attrs = {}):
+        ...
+    def wait(self, stdout_location, stderr_location, echo = None):
+        ...
+    ...
+

metaflow-stubs/plugins/aws/batch/batch_cli.pyi

@@ -0,0 +1,42 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:44.003931                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+
+class CommandException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+METAFLOW_EXIT_DISALLOW_RETRY: int
+
+def sync_local_metadata_from_datastore(metadata_local_dir, task_ds):
+    ...
+
+DATASTORE_LOCAL_DIR: str
+
+TASK_LOG_SOURCE: str
+
+class Batch(object, metaclass=type):
+    def __init__(self, metadata, environment):
+        ...
+    def list_jobs(self, flow_name, run_id, user, echo):
+        ...
+    def kill_jobs(self, flow_name, run_id, user, echo):
+        ...
+    def create_job(self, step_name, step_cli, task_spec, code_package_sha, code_package_url, code_package_ds, image, queue, iam_role = None, execution_role = None, cpu = None, gpu = None, memory = None, run_time_limit = None, shared_memory = None, max_swap = None, swappiness = None, inferentia = None, efa = None, env = {}, attrs = {}, host_volumes = None, efs_volumes = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, num_parallel = 0):
+        ...
+    def launch_job(self, step_name, step_cli, task_spec, code_package_sha, code_package_url, code_package_ds, image, queue, iam_role = None, execution_role = None, cpu = None, gpu = None, memory = None, run_time_limit = None, shared_memory = None, max_swap = None, swappiness = None, inferentia = None, efa = None, host_volumes = None, efs_volumes = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, num_parallel = 0, env = {}, attrs = {}):
+        ...
+    def wait(self, stdout_location, stderr_location, echo = None):
+        ...
+    ...
+
+class BatchKilledException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+