metaflow-stubs 2.12.27__py2.py3-none-any.whl → 2.12.29__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. metaflow-stubs/__init__.pyi +217 -2921
  2. metaflow-stubs/cards.pyi +19 -473
  3. metaflow-stubs/cli.pyi +17 -81
  4. metaflow-stubs/client/__init__.pyi +19 -1113
  5. metaflow-stubs/client/core.pyi +19 -159
  6. metaflow-stubs/client/filecache.pyi +8 -12
  7. metaflow-stubs/clone_util.pyi +6 -26
  8. metaflow-stubs/events.pyi +7 -6
  9. metaflow-stubs/exception.pyi +8 -6
  10. metaflow-stubs/flowspec.pyi +22 -106
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +16 -564
  13. metaflow-stubs/info_file.pyi +6 -5
  14. metaflow-stubs/metadata_provider/__init__.pyi +16 -0
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
  16. metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
  17. metaflow-stubs/metadata_provider/util.pyi +19 -0
  18. metaflow-stubs/metaflow_config.pyi +8 -11
  19. metaflow-stubs/metaflow_current.pyi +8 -7
  20. metaflow-stubs/mflog/__init__.pyi +6 -0
  21. metaflow-stubs/mflog/mflog.pyi +52 -5
  22. metaflow-stubs/multicore_utils.pyi +6 -5
  23. metaflow-stubs/parameters.pyi +13 -23
  24. metaflow-stubs/plugins/__init__.pyi +51 -163
  25. metaflow-stubs/plugins/airflow/__init__.pyi +12 -5
  26. metaflow-stubs/plugins/airflow/airflow.pyi +19 -130
  27. metaflow-stubs/plugins/airflow/airflow_cli.pyi +17 -136
  28. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +7 -26
  29. metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
  30. metaflow-stubs/plugins/airflow/exception.pyi +7 -11
  31. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
  32. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
  33. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
  34. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
  35. metaflow-stubs/plugins/argo/__init__.pyi +12 -5
  36. metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
  37. metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
  38. metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -121
  39. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
  40. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +13 -405
  41. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +65 -322
  42. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
  43. metaflow-stubs/plugins/aws/__init__.pyi +11 -5
  44. metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
  45. metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
  46. metaflow-stubs/plugins/aws/batch/__init__.pyi +10 -5
  47. metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
  48. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +10 -31
  49. metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
  50. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
  51. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
  52. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
  53. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +15 -5
  54. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +6 -5
  55. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
  56. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
  57. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
  58. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
  59. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +19 -175
  60. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +8 -37
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +53 -290
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
  64. metaflow-stubs/plugins/azure/__init__.pyi +12 -7
  65. metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
  66. metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
  67. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
  68. metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
  69. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
  70. metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
  71. metaflow-stubs/plugins/cards/__init__.pyi +15 -5
  72. metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
  73. metaflow-stubs/plugins/cards/card_client.pyi +14 -76
  74. metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +14 -81
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
  80. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +12 -73
  81. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +6 -61
  82. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +6 -5
  83. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +8 -45
  84. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +7 -6
  85. metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
  86. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
  87. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
  88. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
  89. metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
  90. metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
  91. metaflow-stubs/plugins/cards/exception.pyi +7 -11
  92. metaflow-stubs/plugins/catch_decorator.pyi +9 -29
  93. metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
  94. metaflow-stubs/plugins/datatools/local.pyi +7 -11
  95. metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
  96. metaflow-stubs/plugins/datatools/s3/s3.pyi +16 -264
  97. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
  98. metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
  99. metaflow-stubs/plugins/debug_logger.pyi +7 -5
  100. metaflow-stubs/plugins/debug_monitor.pyi +7 -5
  101. metaflow-stubs/plugins/environment_decorator.pyi +7 -5
  102. metaflow-stubs/plugins/events_decorator.pyi +8 -14
  103. metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
  104. metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
  105. metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
  106. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
  107. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
  108. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
  109. metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
  110. metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
  111. metaflow-stubs/plugins/kubernetes/__init__.pyi +13 -5
  112. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
  113. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +9 -29
  114. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
  115. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +9 -72
  116. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
  117. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +8 -41
  118. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
  119. metaflow-stubs/plugins/logs_cli.pyi +10 -9
  120. metaflow-stubs/plugins/package_cli.pyi +7 -5
  121. metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
  122. metaflow-stubs/plugins/project_decorator.pyi +8 -14
  123. metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
  124. metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
  125. metaflow-stubs/plugins/pypi/conda_environment.pyi +12 -18
  126. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
  127. metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
  128. metaflow-stubs/plugins/pypi/utils.pyi +7 -11
  129. metaflow-stubs/plugins/resources_decorator.pyi +7 -5
  130. metaflow-stubs/plugins/retry_decorator.pyi +7 -11
  131. metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
  132. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +8 -13
  133. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
  134. metaflow-stubs/plugins/storage_executor.pyi +6 -11
  135. metaflow-stubs/plugins/tag_cli.pyi +14 -396
  136. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +9 -34
  137. metaflow-stubs/plugins/timeout_decorator.pyi +7 -11
  138. metaflow-stubs/procpoll.pyi +7 -5
  139. metaflow-stubs/pylint_wrapper.pyi +7 -11
  140. metaflow-stubs/runner/__init__.pyi +13 -5
  141. metaflow-stubs/runner/deployer.pyi +102 -210
  142. metaflow-stubs/runner/deployer_impl.pyi +87 -0
  143. metaflow-stubs/runner/metaflow_runner.pyi +24 -508
  144. metaflow-stubs/runner/nbdeploy.pyi +16 -60
  145. metaflow-stubs/runner/nbrun.pyi +11 -148
  146. metaflow-stubs/runner/subprocess_manager.pyi +9 -10
  147. metaflow-stubs/runner/utils.pyi +44 -9
  148. metaflow-stubs/system/__init__.pyi +9 -107
  149. metaflow-stubs/system/system_logger.pyi +6 -15
  150. metaflow-stubs/system/system_monitor.pyi +6 -16
  151. metaflow-stubs/tagging_util.pyi +6 -10
  152. metaflow-stubs/tuple_util.pyi +6 -5
  153. metaflow-stubs/version.pyi +6 -5
  154. {metaflow_stubs-2.12.27.dist-info → metaflow_stubs-2.12.29.dist-info}/METADATA +2 -2
  155. metaflow_stubs-2.12.29.dist-info/RECORD +158 -0
  156. {metaflow_stubs-2.12.27.dist-info → metaflow_stubs-2.12.29.dist-info}/WHEEL +1 -1
  157. metaflow-stubs/metadata/util.pyi +0 -18
  158. metaflow_stubs-2.12.27.dist-info/RECORD +0 -152
  159. {metaflow_stubs-2.12.27.dist-info → metaflow_stubs-2.12.29.dist-info}/top_level.txt +0 -0
@@ -1,36 +1,26 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.979045 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.622880 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
- if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
- import metaflow.metaflow_current
13
- import metaflow._vendor.click.types
14
9
 
15
- current: metaflow.metaflow_current.Current
16
-
17
- class MetaflowException(Exception, metaclass=type):
18
- def __init__(self, msg = "", lineno = None):
19
- ...
20
- def __str__(self):
21
- ...
22
- ...
23
-
24
- class FilePathClass(metaflow._vendor.click.types.ParamType, metaclass=type):
25
- def __init__(self, is_text, encoding):
26
- ...
27
- def convert(self, value, param, ctx):
28
- ...
29
- def __str__(self):
30
- ...
31
- def __repr__(self):
32
- ...
33
- ...
10
+ from ...metaflow_current import current as current
11
+ from ...exception import MetaflowException as MetaflowException
12
+ from ...includefile import FilePathClass as FilePathClass
13
+ from ...parameters import DelayedEvaluationParameter as DelayedEvaluationParameter
14
+ from ...parameters import JSONTypeClass as JSONTypeClass
15
+ from ...parameters import deploy_time_eval as deploy_time_eval
16
+ from ..cards.card_modules import chevron as chevron
17
+ from ..kubernetes.kubernetes import Kubernetes as Kubernetes
18
+ from ..timeout_decorator import get_run_time_limit_for_task as get_run_time_limit_for_task
19
+ from . import airflow_utils as airflow_utils
20
+ from .airflow_utils import AIRFLOW_MACROS as AIRFLOW_MACROS
21
+ from .airflow_utils import AirflowTask as AirflowTask
22
+ from .airflow_utils import Workflow as Workflow
23
+ from .exception import AirflowException as AirflowException
34
24
 
35
25
  AIRFLOW_KUBERNETES_CONN_ID: None
36
26
 
@@ -74,109 +64,8 @@ SERVICE_INTERNAL_URL: None
74
64
 
75
65
  AZURE_KEY_VAULT_PREFIX: None
76
66
 
77
- class DelayedEvaluationParameter(object, metaclass=type):
78
- """
79
- This is a very simple wrapper to allow parameter "conversion" to be delayed until
80
- the `_set_constants` function in FlowSpec. Typically, parameters are converted
81
- by click when the command line option is processed. For some parameters, like
82
- IncludeFile, this is too early as it would mean we would trigger the upload
83
- of the file too early. If a parameter converts to a DelayedEvaluationParameter
84
- object through the usual click mechanisms, `_set_constants` knows to invoke the
85
- __call__ method on that DelayedEvaluationParameter; in that case, the __call__
86
- method is invoked without any parameter. The return_str parameter will be used
87
- by schedulers when they need to convert DelayedEvaluationParameters to a
88
- string to store them
89
- """
90
- def __init__(self, name, field, fun):
91
- ...
92
- def __call__(self, return_str = False):
93
- ...
94
- ...
95
-
96
- class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
97
- def convert(self, value, param, ctx):
98
- ...
99
- def __str__(self):
100
- ...
101
- def __repr__(self):
102
- ...
103
- ...
104
-
105
- def deploy_time_eval(value):
106
- ...
107
-
108
- class Kubernetes(object, metaclass=type):
109
- def __init__(self, datastore, metadata, environment):
110
- ...
111
- def launch_job(self, **kwargs):
112
- ...
113
- def create_jobset(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, docker_image, docker_image_pull_policy, step_cli = None, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, num_parallel = None):
114
- ...
115
- def create_job_object(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, step_cli, docker_image, docker_image_pull_policy, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, name_pattern = None):
116
- ...
117
- def create_k8sjob(self, job):
118
- ...
119
- def wait(self, stdout_location, stderr_location, echo = None):
120
- ...
121
- ...
122
-
123
- def get_run_time_limit_for_task(step_decos):
124
- ...
125
-
126
- class AIRFLOW_MACROS(object, metaclass=type):
127
- @classmethod
128
- def create_task_id(cls, is_foreach):
129
- ...
130
- @classmethod
131
- def pathspec(cls, flowname, is_foreach = False):
132
- ...
133
- ...
134
-
135
67
  TASK_ID_XCOM_KEY: str
136
68
 
137
- class AirflowTask(object, metaclass=type):
138
- def __init__(self, name, operator_type = "kubernetes", flow_name = None, is_mapper_node = False, flow_contains_foreach = False):
139
- ...
140
- @property
141
- def is_mapper_node(self):
142
- ...
143
- def set_operator_args(self, **kwargs):
144
- ...
145
- def to_dict(self):
146
- ...
147
- @classmethod
148
- def from_dict(cls, task_dict, flow_name = None, flow_contains_foreach = False):
149
- ...
150
- def to_task(self):
151
- ...
152
- ...
153
-
154
- class Workflow(object, metaclass=type):
155
- def __init__(self, file_path = None, graph_structure = None, metadata = None, **kwargs):
156
- ...
157
- def set_parameters(self, params):
158
- ...
159
- def add_state(self, state):
160
- ...
161
- def to_dict(self):
162
- ...
163
- def to_json(self):
164
- ...
165
- @classmethod
166
- def from_dict(cls, data_dict):
167
- ...
168
- @classmethod
169
- def from_json(cls, json_string):
170
- ...
171
- def compile(self):
172
- ...
173
- ...
174
-
175
- class AirflowException(metaflow.exception.MetaflowException, metaclass=type):
176
- def __init__(self, msg):
177
- ...
178
- ...
179
-
180
69
  SUPPORTED_SENSORS: list
181
70
 
182
71
  AIRFLOW_DEPLOY_TEMPLATE_FILE: str
@@ -1,146 +1,27 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.980359 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.623352 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
12
  import metaflow.exception
12
- import metaflow.metaflow_current
13
- import metaflow.decorators
14
13
 
15
- current: metaflow.metaflow_current.Current
16
-
17
- class MetaflowException(Exception, metaclass=type):
18
- def __init__(self, msg = "", lineno = None):
19
- ...
20
- def __str__(self):
21
- ...
22
- ...
23
-
24
- class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
25
- ...
26
-
27
- def load_token(token_prefix):
28
- ...
29
-
30
- def new_token(token_prefix, prev_token = None):
31
- ...
32
-
33
- def store_token(token_prefix, token):
34
- ...
35
-
36
- class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
37
- """
38
- Specifies that this step should execute on Kubernetes.
39
-
40
- Parameters
41
- ----------
42
- cpu : int, default 1
43
- Number of CPUs required for this step. If `@resources` is
44
- also present, the maximum value from all decorators is used.
45
- memory : int, default 4096
46
- Memory size (in MB) required for this step. If
47
- `@resources` is also present, the maximum value from all decorators is
48
- used.
49
- disk : int, default 10240
50
- Disk size (in MB) required for this step. If
51
- `@resources` is also present, the maximum value from all decorators is
52
- used.
53
- image : str, optional, default None
54
- Docker image to use when launching on Kubernetes. If not specified, and
55
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
56
- not, a default Docker image mapping to the current version of Python is used.
57
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
58
- If given, the imagePullPolicy to be applied to the Docker image of the step.
59
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
60
- Kubernetes service account to use when launching pod in Kubernetes.
61
- secrets : List[str], optional, default None
62
- Kubernetes secrets to use when launching pod in Kubernetes. These
63
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
64
- in Metaflow configuration.
65
- node_selector: Union[Dict[str,str], str], optional, default None
66
- Kubernetes node selector(s) to apply to the pod running the task.
67
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
68
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
69
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
70
- Kubernetes namespace to use when launching pod in Kubernetes.
71
- gpu : int, optional, default None
72
- Number of GPUs required for this step. A value of zero implies that
73
- the scheduled node should not have GPUs.
74
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
75
- The vendor of the GPUs to be used for this step.
76
- tolerations : List[str], default []
77
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
78
- Kubernetes tolerations to use when launching pod in Kubernetes.
79
- use_tmpfs : bool, default False
80
- This enables an explicit tmpfs mount for this step.
81
- tmpfs_tempdir : bool, default True
82
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
83
- tmpfs_size : int, optional, default: None
84
- The value for the size (in MiB) of the tmpfs mount for this step.
85
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
86
- memory allocated for this step.
87
- tmpfs_path : str, optional, default /metaflow_temp
88
- Path to tmpfs mount for this step.
89
- persistent_volume_claims : Dict[str, str], optional, default None
90
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
91
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
92
- shared_memory: int, optional
93
- Shared memory size (in MiB) required for this step
94
- port: int, optional
95
- Port number to specify in the Kubernetes job object
96
- compute_pool : str, optional, default None
97
- Compute pool to be used for for this step.
98
- If not specified, any accessible compute pool within the perimeter is used.
99
- hostname_resolution_timeout: int, default 10 * 60
100
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
101
- Only applicable when @parallel is used.
102
- """
103
- def __init__(self, attributes = None, statically_defined = False):
104
- ...
105
- def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
106
- ...
107
- def package_init(self, flow, step_name, environment):
108
- ...
109
- def runtime_init(self, flow, graph, package, run_id):
110
- ...
111
- def runtime_task_created(self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context):
112
- ...
113
- def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
114
- ...
115
- def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_retries, ubf_context, inputs):
116
- ...
117
- def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_retries):
118
- ...
119
- ...
120
-
121
- class Airflow(object, metaclass=type):
122
- def __init__(self, name, graph, flow, code_package_sha, code_package_url, metadata, flow_datastore, environment, event_logger, monitor, production_token, tags = None, namespace = None, username = None, max_workers = None, worker_pool = None, description = None, file_path = None, workflow_timeout = None, is_paused_upon_creation = True):
123
- ...
124
- @classmethod
125
- def get_existing_deployment(cls, name, flow_datastore):
126
- ...
127
- @classmethod
128
- def get_token_path(cls, name):
129
- ...
130
- @classmethod
131
- def save_deployment_token(cls, owner, name, token, flow_datastore):
132
- ...
133
- def compile(self):
134
- ...
135
- ...
136
-
137
- class AirflowException(metaflow.exception.MetaflowException, metaclass=type):
138
- def __init__(self, msg):
139
- ...
140
- ...
141
-
142
- class NotSupportedException(metaflow.exception.MetaflowException, metaclass=type):
143
- ...
14
+ from ...metaflow_current import current as current
15
+ from ..._vendor import click as click
16
+ from ...exception import MetaflowException as MetaflowException
17
+ from ...exception import MetaflowInternalError as MetaflowInternalError
18
+ from ..aws.step_functions.production_token import load_token as load_token
19
+ from ..aws.step_functions.production_token import new_token as new_token
20
+ from ..aws.step_functions.production_token import store_token as store_token
21
+ from ..kubernetes.kubernetes_decorator import KubernetesDecorator as KubernetesDecorator
22
+ from .airflow import Airflow as Airflow
23
+ from .exception import AirflowException as AirflowException
24
+ from .exception import NotSupportedException as NotSupportedException
144
25
 
145
26
  class IncorrectProductionToken(metaflow.exception.MetaflowException, metaclass=type):
146
27
  ...
@@ -1,36 +1,17 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.976516 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.621553 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
12
  import metaflow.decorators
12
13
 
13
- class MetaDatum(tuple, metaclass=type):
14
- """
15
- MetaDatum(field, value, type, tags)
16
- """
17
- @staticmethod
18
- def __new__(_cls, field, value, type, tags):
19
- """
20
- Create new instance of MetaDatum(field, value, type, tags)
21
- """
22
- ...
23
- def __repr__(self):
24
- """
25
- Return a nicely formatted representation string
26
- """
27
- ...
28
- def __getnewargs__(self):
29
- """
30
- Return self as a plain tuple. Used by copy and pickle.
31
- """
32
- ...
33
- ...
14
+ from ...metadata_provider.metadata import MetaDatum as MetaDatum
34
15
 
35
16
  TASK_ID_XCOM_KEY: str
36
17
 
@@ -1,12 +1,13 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.976048 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.621209 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
 
10
+
10
11
  TASK_ID_XCOM_KEY: str
11
12
 
12
13
  FOREACH_CARDINALITY_XCOM_KEY: str
@@ -98,7 +99,7 @@ def get_metaflow_kubernetes_operator():
98
99
  ...
99
100
 
100
101
  class AirflowTask(object, metaclass=type):
101
- def __init__(self, name, operator_type = "kubernetes", flow_name = None, is_mapper_node = False, flow_contains_foreach = False):
102
+ def __init__(self, name, operator_type = 'kubernetes', flow_name = None, is_mapper_node = False, flow_contains_foreach = False):
102
103
  ...
103
104
  @property
104
105
  def is_mapper_node(self):
@@ -1,21 +1,17 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.976779 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.621757 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
12
  import metaflow.exception
12
13
 
13
- class MetaflowException(Exception, metaclass=type):
14
- def __init__(self, msg = "", lineno = None):
15
- ...
16
- def __str__(self):
17
- ...
18
- ...
14
+ from ...exception import MetaflowException as MetaflowException
19
15
 
20
16
  class AirflowException(metaflow.exception.MetaflowException, metaclass=type):
21
17
  def __init__(self, msg):
@@ -1,104 +1,17 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.977120 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.622039 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
- if typing.TYPE_CHECKING:
11
- import metaflow.plugins.airflow.sensors.base_sensor
12
9
 
13
- class ExternalTaskSensorDecorator(metaflow.plugins.airflow.sensors.base_sensor.AirflowSensorDecorator, metaclass=type):
14
- """
15
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
16
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
17
-
18
- Parameters
19
- ----------
20
- timeout : int
21
- Time, in seconds before the task times out and fails. (Default: 3600)
22
- poke_interval : int
23
- Time in seconds that the job should wait in between each try. (Default: 60)
24
- mode : str
25
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
26
- exponential_backoff : bool
27
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
28
- pool : str
29
- the slot pool this task should run in,
30
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
31
- soft_fail : bool
32
- Set to true to mark the task as SKIPPED on failure. (Default: False)
33
- name : str
34
- Name of the sensor on Airflow
35
- description : str
36
- Description of sensor in the Airflow UI
37
- external_dag_id : str
38
- The dag_id that contains the task you want to wait for.
39
- external_task_ids : List[str]
40
- The list of task_ids that you want to wait for.
41
- If None (default value) the sensor waits for the DAG. (Default: None)
42
- allowed_states : List[str]
43
- Iterable of allowed states, (Default: ['success'])
44
- failed_states : List[str]
45
- Iterable of failed or dis-allowed states. (Default: None)
46
- execution_delta : datetime.timedelta
47
- time difference with the previous execution to look at,
48
- the default is the same logical date as the current task or DAG. (Default: None)
49
- check_existence: bool
50
- Set to True to check if the external task exists or check if
51
- the DAG to wait for exists. (Default: True)
52
- """
53
- def serialize_operator_args(self):
54
- ...
55
- def validate(self, flow):
56
- ...
57
- ...
58
-
59
- class S3KeySensorDecorator(metaflow.plugins.airflow.sensors.base_sensor.AirflowSensorDecorator, metaclass=type):
60
- """
61
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
62
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
63
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
64
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
65
- starts only after all sensors finish.
66
-
67
- Parameters
68
- ----------
69
- timeout : int
70
- Time, in seconds before the task times out and fails. (Default: 3600)
71
- poke_interval : int
72
- Time in seconds that the job should wait in between each try. (Default: 60)
73
- mode : str
74
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
75
- exponential_backoff : bool
76
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
77
- pool : str
78
- the slot pool this task should run in,
79
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
80
- soft_fail : bool
81
- Set to true to mark the task as SKIPPED on failure. (Default: False)
82
- name : str
83
- Name of the sensor on Airflow
84
- description : str
85
- Description of sensor in the Airflow UI
86
- bucket_key : Union[str, List[str]]
87
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
88
- When it's specified as a full s3:// url, please leave `bucket_name` as None
89
- bucket_name : str
90
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
91
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
92
- wildcard_match : bool
93
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
94
- aws_conn_id : str
95
- a reference to the s3 connection on Airflow. (Default: None)
96
- verify : bool
97
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
98
- """
99
- def validate(self, flow):
100
- ...
101
- ...
10
+ from . import base_sensor as base_sensor
11
+ from . import external_task_sensor as external_task_sensor
12
+ from .external_task_sensor import ExternalTaskSensorDecorator as ExternalTaskSensorDecorator
13
+ from . import s3_sensor as s3_sensor
14
+ from .s3_sensor import S3KeySensorDecorator as S3KeySensorDecorator
102
15
 
103
16
  SUPPORTED_SENSORS: list
104
17
 
@@ -1,40 +1,19 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.999442 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.644264 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
12
  import metaflow.decorators
13
13
 
14
- class AirflowException(metaflow.exception.MetaflowException, metaclass=type):
15
- def __init__(self, msg):
16
- ...
17
- ...
18
-
19
- class AirflowTask(object, metaclass=type):
20
- def __init__(self, name, operator_type = "kubernetes", flow_name = None, is_mapper_node = False, flow_contains_foreach = False):
21
- ...
22
- @property
23
- def is_mapper_node(self):
24
- ...
25
- def set_operator_args(self, **kwargs):
26
- ...
27
- def to_dict(self):
28
- ...
29
- @classmethod
30
- def from_dict(cls, task_dict, flow_name = None, flow_contains_foreach = False):
31
- ...
32
- def to_task(self):
33
- ...
34
- ...
35
-
36
- def id_creator(val, hash_len):
37
- ...
14
+ from ..exception import AirflowException as AirflowException
15
+ from ..airflow_utils import AirflowTask as AirflowTask
16
+ from ..airflow_utils import id_creator as id_creator
38
17
 
39
18
  TASK_ID_HASH_LEN: int
40
19
 
@@ -1,50 +1,19 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27 #
4
- # Generated on 2024-10-24T19:27:17.999939 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.29 #
4
+ # Generated on 2024-11-07T22:19:34.644568 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
12
  import metaflow.plugins.airflow.sensors.base_sensor
12
- import metaflow.exception
13
- import metaflow.decorators
14
13
 
15
- class AirflowSensorDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
16
- """
17
- Base class for all Airflow sensor decorators.
18
- """
19
- def __init__(self, *args, **kwargs):
20
- ...
21
- def serialize_operator_args(self):
22
- """
23
- Subclasses will parse the decorator arguments to
24
- Airflow task serializable arguments.
25
- """
26
- ...
27
- def create_task(self):
28
- ...
29
- def validate(self, flow):
30
- """
31
- Validate if the arguments for the sensor are correct.
32
- """
33
- ...
34
- def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
35
- ...
36
- ...
37
-
38
- class SensorNames(object, metaclass=type):
39
- @classmethod
40
- def get_supported_sensors(cls):
41
- ...
42
- ...
43
-
44
- class AirflowException(metaflow.exception.MetaflowException, metaclass=type):
45
- def __init__(self, msg):
46
- ...
47
- ...
14
+ from .base_sensor import AirflowSensorDecorator as AirflowSensorDecorator
15
+ from ..airflow_utils import SensorNames as SensorNames
16
+ from ..exception import AirflowException as AirflowException
48
17
 
49
18
  AIRFLOW_STATES: dict
50
19