ob-metaflow-stubs 2.11.4.1__py2.py3-none-any.whl

metaflow-stubs/plugins/aws/step_functions/step_functions.pyi

@@ -0,0 +1,220 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:44.008833                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+EVENTS_SFN_ACCESS_IAM_ROLE: None
+
+S3_ENDPOINT_URL: None
+
+SFN_DYNAMO_DB_TABLE: None
+
+SFN_EXECUTION_LOG_GROUP_ARN: None
+
+SFN_IAM_ROLE: None
+
+SFN_S3_DISTRIBUTED_MAP_OUTPUT_PATH: None
+
+def deploy_time_eval(value):
+    ...
+
+class Batch(object, metaclass=type):
+    def __init__(self, metadata, environment):
+        ...
+    def list_jobs(self, flow_name, run_id, user, echo):
+        ...
+    def kill_jobs(self, flow_name, run_id, user, echo):
+        ...
+    def create_job(self, step_name, step_cli, task_spec, code_package_sha, code_package_url, code_package_ds, image, queue, iam_role = None, execution_role = None, cpu = None, gpu = None, memory = None, run_time_limit = None, shared_memory = None, max_swap = None, swappiness = None, inferentia = None, efa = None, env = {}, attrs = {}, host_volumes = None, efs_volumes = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, num_parallel = 0):
+        ...
+    def launch_job(self, step_name, step_cli, task_spec, code_package_sha, code_package_url, code_package_ds, image, queue, iam_role = None, execution_role = None, cpu = None, gpu = None, memory = None, run_time_limit = None, shared_memory = None, max_swap = None, swappiness = None, inferentia = None, efa = None, host_volumes = None, efs_volumes = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, num_parallel = 0, env = {}, attrs = {}):
+        ...
+    def wait(self, stdout_location, stderr_location, echo = None):
+        ...
+    ...
+
+class EventBridgeClient(object, metaclass=type):
+    def __init__(self, name):
+        ...
+    def cron(self, cron):
+        ...
+    def role_arn(self, role_arn):
+        ...
+    def state_machine_arn(self, state_machine_arn):
+        ...
+    def schedule(self):
+        ...
+    def delete(self):
+        ...
+    ...
+
+class StepFunctionsClient(object, metaclass=type):
+    def __init__(self):
+        ...
+    def search(self, name):
+        ...
+    def push(self, name, definition, role_arn, log_execution_history):
+        ...
+    def get(self, name):
+        ...
+    def trigger(self, state_machine_arn, input):
+        ...
+    def list_executions(self, state_machine_arn, states):
+        ...
+    def terminate_execution(self, execution_arn):
+        ...
+    def get_state_machine_arn(self, name):
+        ...
+    def delete(self, name):
+        ...
+    ...
+
+class StepFunctionsException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class StepFunctionsSchedulingException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class StepFunctions(object, metaclass=type):
+    def __init__(self, name, graph, flow, code_package_sha, code_package_url, production_token, metadata, flow_datastore, environment, event_logger, monitor, tags = None, namespace = None, username = None, max_workers = None, workflow_timeout = None, is_project = False, use_distributed_map = False):
+        ...
+    def to_json(self):
+        ...
+    def trigger_explanation(self):
+        ...
+    def deploy(self, log_execution_history):
+        ...
+    def schedule(self):
+        ...
+    @classmethod
+    def delete(cls, name):
+        ...
+    @classmethod
+    def terminate(cls, flow_name, name):
+        ...
+    @classmethod
+    def trigger(cls, name, parameters):
+        ...
+    @classmethod
+    def list(cls, name, states):
+        ...
+    @classmethod
+    def get_existing_deployment(cls, name):
+        ...
+    @classmethod
+    def get_execution(cls, state_machine_name, name):
+        ...
+    ...
+
+class Workflow(object, metaclass=type):
+    def __init__(self, name):
+        ...
+    def mode(self, mode):
+        ...
+    def start_at(self, start_at):
+        ...
+    def add_state(self, state):
+        ...
+    def timeout_seconds(self, timeout_seconds):
+        ...
+    def to_json(self, pretty = False):
+        ...
+    ...
+
+class State(object, metaclass=type):
+    def __init__(self, name):
+        ...
+    def resource(self, resource):
+        ...
+    def next(self, state):
+        ...
+    def end(self):
+        ...
+    def parameter(self, name, value):
+        ...
+    def output_path(self, output_path):
+        ...
+    def result_path(self, result_path):
+        ...
+    def result_selector(self, name, value):
+        ...
+    def retry_strategy(self, retry_strategy):
+        ...
+    def batch(self, job):
+        ...
+    def dynamo_db(self, table_name, primary_key, values):
+        ...
+    ...
+
+class Pass(object, metaclass=type):
+    def __init__(self, name):
+        ...
+    def end(self):
+        ...
+    def parameter(self, name, value):
+        ...
+    def output_path(self, output_path):
+        ...
+    ...
+
+class Parallel(object, metaclass=type):
+    def __init__(self, name):
+        ...
+    def branch(self, workflow):
+        ...
+    def next(self, state):
+        ...
+    def output_path(self, output_path):
+        ...
+    def result_path(self, result_path):
+        ...
+    ...
+
+class Map(object, metaclass=type):
+    def __init__(self, name):
+        ...
+    def iterator(self, workflow):
+        ...
+    def next(self, state):
+        ...
+    def items_path(self, items_path):
+        ...
+    def parameter(self, name, value):
+        ...
+    def max_concurrency(self, max_concurrency):
+        ...
+    def output_path(self, output_path):
+        ...
+    def result_path(self, result_path):
+        ...
+    def item_reader(self, item_reader):
+        ...
+    def result_writer(self, bucket, prefix):
+        ...
+    ...
+
+class JSONItemReader(object, metaclass=type):
+    def __init__(self):
+        ...
+    def resource(self, resource):
+        ...
+    def parameter(self, name, value):
+        ...
+    def output_path(self, output_path):
+        ...
+    ...
+

metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi

@@ -0,0 +1,139 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:44.010735                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.metaflow_current
+    import metaflow.exception
+    import metaflow.decorators
+    import metaflow.parameters
+
+JSONType: metaflow.parameters.JSONTypeClass
+
+current: metaflow.metaflow_current.Current
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+SERVICE_VERSION_CHECK: bool
+
+SFN_STATE_MACHINE_PREFIX: None
+
+UI_URL: None
+
+class BatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    def __init__(self, attributes = None, statically_defined = False):
+        ...
+    def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
+        ...
+    def runtime_init(self, flow, graph, package, run_id):
+        ...
+    def runtime_task_created(self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context):
+        ...
+    def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
+        ...
+    def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_retries, ubf_context, inputs):
+        ...
+    def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_retries):
+        ...
+    ...
+
+def validate_tags(tags, existing_tags = None):
+    """
+    Raises MetaflowTaggingError if invalid based on these rules:
+    
+    Tag set size is too large. But it's OK if tag set is not larger
+    than an existing tag set (if provided).
+    
+    Then, we validate each tag.  See validate_tag()
+    """
+    ...
+
+def load_token(token_prefix):
+    ...
+
+def new_token(token_prefix, prev_token = None):
+    ...
+
+def store_token(token_prefix, token):
+    ...
+
+class StepFunctions(object, metaclass=type):
+    def __init__(self, name, graph, flow, code_package_sha, code_package_url, production_token, metadata, flow_datastore, environment, event_logger, monitor, tags = None, namespace = None, username = None, max_workers = None, workflow_timeout = None, is_project = False, use_distributed_map = False):
+        ...
+    def to_json(self):
+        ...
+    def trigger_explanation(self):
+        ...
+    def deploy(self, log_execution_history):
+        ...
+    def schedule(self):
+        ...
+    @classmethod
+    def delete(cls, name):
+        ...
+    @classmethod
+    def terminate(cls, flow_name, name):
+        ...
+    @classmethod
+    def trigger(cls, name, parameters):
+        ...
+    @classmethod
+    def list(cls, name, states):
+        ...
+    @classmethod
+    def get_existing_deployment(cls, name):
+        ...
+    @classmethod
+    def get_execution(cls, state_machine_name, name):
+        ...
+    ...
+
+class IncorrectProductionToken(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class RunIdMismatch(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class IncorrectMetadataServiceVersion(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class StepFunctionsStateMachineNameTooLong(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+def check_metadata_service_version(obj):
+    ...
+
+def resolve_state_machine_name(obj, name):
+    ...
+
+def make_flow(obj, token, name, tags, namespace, max_workers, workflow_timeout, is_project, use_distributed_map):
+    ...
+
+def resolve_token(name, token_prefix, obj, authorize, given_token, generate_new_token, is_project):
+    ...
+
+def validate_run_id(state_machine_name, token_prefix, authorize, run_id, instructions_fn = None):
+    ...
+
+def validate_token(name, token_prefix, authorize, instruction_fn = None):
+    """
+    Validate that the production token matches that of the deployed flow.
+    
+    In case both the user and token do not match, raises an error.
+    Optionally outputs instructions on token usage via the provided instruction_fn(flow_name, prev_user)
+    """
+    ...
+

metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi

@@ -0,0 +1,36 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:44.006031                                        #
+##################################################################################
+
+from __future__ import annotations
+
+
+AWS_SANDBOX_ENABLED: bool
+
+AWS_SANDBOX_REGION: None
+
+SFN_EXECUTION_LOG_GROUP_ARN: None
+
+class StepFunctionsClient(object, metaclass=type):
+    def __init__(self):
+        ...
+    def search(self, name):
+        ...
+    def push(self, name, definition, role_arn, log_execution_history):
+        ...
+    def get(self, name):
+        ...
+    def trigger(self, state_machine_arn, input):
+        ...
+    def list_executions(self, state_machine_arn, states):
+        ...
+    def terminate_execution(self, execution_arn):
+        ...
+    def get_state_machine_arn(self, name):
+        ...
+    def delete(self, name):
+        ...
+    ...
+

metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi

@@ -0,0 +1,49 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:44.004739                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.decorators
+
+class MetaDatum(tuple, metaclass=type):
+    @staticmethod
+    def __new__(_cls, field, value, type, tags):
+        """
+        Create new instance of MetaDatum(field, value, type, tags)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
+class DynamoDbClient(object, metaclass=type):
+    def __init__(self):
+        ...
+    def save_foreach_cardinality(self, foreach_split_task_id, foreach_cardinality, ttl):
+        ...
+    def save_parent_task_id_for_foreach_join(self, foreach_split_task_id, foreach_join_parent_task_id):
+        ...
+    def get_parent_task_ids_for_foreach_join(self, foreach_split_task_id):
+        ...
+    ...
+
+class StepFunctionsInternalDecorator(metaflow.decorators.StepDecorator, metaclass=type):
+    def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_user_code_retries, ubf_context, inputs):
+        ...
+    def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_user_code_retries):
+        ...
+    ...
+

metaflow-stubs/plugins/azure/__init__.pyi

@@ -0,0 +1,9 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.934212                                        #
+##################################################################################
+
+from __future__ import annotations
+
+

metaflow-stubs/plugins/azure/azure_credential.pyi

@@ -0,0 +1,28 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.994653                                        #
+##################################################################################
+
+from __future__ import annotations
+
+
+class AzureDefaultClientProvider(object, metaclass=type):
+    @staticmethod
+    def create_cacheable_azure_credential(*args, **kwargs):
+        """
+        azure.identity.DefaultAzureCredential is not readily cacheable in a dictionary
+        because it does not have a content based hash and equality implementations.
+        
+        We implement a subclass CacheableDefaultAzureCredential to add them.
+        
+        We need this because credentials will be part of the cache key in _ClientCache.
+        """
+        ...
+    ...
+
+cached_provider_class: None
+
+def create_cacheable_azure_credential():
+    ...
+

metaflow-stubs/plugins/azure/azure_exceptions.pyi

@@ -0,0 +1,28 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.994415                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+class MetaflowAzureAuthenticationError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class MetaflowAzureResourceError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class MetaflowAzurePackageError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+

metaflow-stubs/plugins/azure/azure_utils.pyi

@@ -0,0 +1,76 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.995237                                        #
+##################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+
+class MetaflowAzureAuthenticationError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class MetaflowAzureResourceError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class MetaflowAzurePackageError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+def create_cacheable_azure_credential():
+    ...
+
+def check_azure_deps(func):
+    """
+    The decorated function checks Azure dependencies (as needed for Azure storage backend). This includes
+    various Azure SDK packages, as well as a Python version of >3.6
+    
+    We also tune some warning and logging configurations to reduce excessive log lines from Azure SDK.
+    """
+    ...
+
+def parse_azure_full_path(blob_full_uri):
+    """
+    Parse an Azure Blob Storage path str into a tuple (container_name, blob).
+    
+    Expected format is: <container_name>/<blob>
+    
+    This is sometimes used to parse an Azure sys root, in which case:
+    
+    - <container_name> is the Azure Blob Storage container name
+    - <blob> is effectively a blob_prefix, a subpath within the container in which blobs will live
+    
+    Blob may be None, if input looks like <container_name>. I.e. no slashes present.
+    
+    We take a strict validation approach, doing no implicit string manipulations on
+    the user's behalf.  Path manipulations by themselves are complicated enough without
+    adding magic.
+    
+    We provide clear error messages so the user knows exactly how to fix any validation error.
+    """
+    ...
+
+def process_exception(*args, **kwargs):
+    ...
+
+def handle_exceptions(func):
+    """
+    This is a decorator leveraging the logic from process_exception()
+    """
+    ...
+
+def create_static_token_credential(*args, **kwargs):
+    ...
+

metaflow-stubs/plugins/azure/blob_service_client_factory.pyi

@@ -0,0 +1,52 @@
+##################################################################################
+#                       Auto-generated Metaflow stub file                        #
+# MF version: 2.11.4.1                                                           #
+# Generated on 2024-02-28T05:30:43.995696                                        #
+##################################################################################
+
+from __future__ import annotations
+
+
+class MetaflowException(Exception, metaclass=type):
+    def __init__(self, msg = "", lineno = None):
+        ...
+    def __str__(self):
+        ...
+    ...
+
+AZURE_STORAGE_BLOB_SERVICE_ENDPOINT: None
+
+def check_azure_deps(func):
+    """
+    The decorated function checks Azure dependencies (as needed for Azure storage backend). This includes
+    various Azure SDK packages, as well as a Python version of >3.6
+    
+    We also tune some warning and logging configurations to reduce excessive log lines from Azure SDK.
+    """
+    ...
+
+def create_cacheable_azure_credential():
+    ...
+
+AZURE_CLIENT_CONNECTION_DATA_BLOCK_SIZE: int
+
+AZURE_CLIENT_MAX_SINGLE_GET_SIZE_MB: int
+
+AZURE_CLIENT_MAX_SINGLE_PUT_SIZE_MB: int
+
+AZURE_CLIENT_MAX_CHUNK_GET_SIZE_MB: int
+
+BYTES_IN_MB: int
+
+def get_azure_blob_service_client(credential = None, credential_is_cacheable = False, max_single_get_size = 33554432, max_single_put_size = 67108864, max_chunk_get_size = 16777216, connection_data_block_size = 262144):
+    """
+    Returns a azure.storage.blob.BlobServiceClient.
+    
+    The value adds are:
+    - connection caching (see _ClientCache)
+    - auto storage account URL detection
+    - auto credential handling (pull SAS token from environment, OR DefaultAzureCredential)
+    - sensible default values for Azure SDK tunables
+    """
+    ...
+