ob-metaflow-stubs 6.0.3.103rc3__py2.py3-none-any.whl → 6.0.3.105__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +836 -498
  2. metaflow-stubs/cards.pyi +211 -5
  3. metaflow-stubs/cli.pyi +23 -3
  4. metaflow-stubs/client/__init__.pyi +129 -4
  5. metaflow-stubs/client/core.pyi +227 -7
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +5 -2
  8. metaflow-stubs/events.pyi +21 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +71 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +157 -4
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +8 -2
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +39 -36
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +70 -4
  21. metaflow-stubs/plugins/__init__.pyi +13 -2
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +45 -3
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +45 -3
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +18 -5
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +33 -5
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +103 -3
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +62 -4
  61. metaflow-stubs/plugins/cards/card_client.pyi +33 -2
  62. metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +42 -3
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +36 -3
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
  74. metaflow-stubs/plugins/cards/exception.pyi +8 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +20 -3
  76. metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +16 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +82 -5
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +10 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +106 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +23 -2
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +12 -3
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +100 -3
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +4 -4
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +59 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +33 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +21 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +6 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +36 -5
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +9 -2
  120. metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +71 -132
  126. metaflow-stubs/runner/metaflow_runner.pyi +117 -9
  127. metaflow-stubs/runner/nbdeploy.pyi +66 -2
  128. metaflow-stubs/runner/nbrun.pyi +79 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +16 -4
  130. metaflow-stubs/runner/utils.pyi +32 -2
  131. metaflow-stubs/system/__init__.pyi +3 -3
  132. metaflow-stubs/system/system_logger.pyi +2 -2
  133. metaflow-stubs/system/system_monitor.pyi +3 -3
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.105.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.103rc3.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/top_level.txt +0 -0
@@ -1,16 +1,15 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.598152 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.697512 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.graph
12
- import metaflow.runner.subprocess_manager
13
11
  import metaflow.runner.deployer
12
+ import metaflow.graph
14
13
 
15
14
  class ArgoWorkflows(object, metaclass=type):
16
15
  def __init__(self, name, graph: metaflow.graph.FlowGraph, flow, code_package_sha, code_package_url, production_token, metadata, flow_datastore, environment, event_logger, monitor, tags = None, namespace = None, username = None, max_workers = None, workflow_timeout = None, workflow_priority = None, auto_emit_argo_events = False, notify_on_error = False, notify_on_success = False, notify_slack_webhook_url = None, notify_pager_duty_integration_key = None, enable_heartbeat_daemon = True, enable_error_msg_capture = False):
@@ -55,6 +54,30 @@ class ArgoWorkflows(object, metaclass=type):
55
54
  ...
56
55
 
57
56
  class DeployerImpl(object, metaclass=type):
57
+ """
58
+ Base class for deployer implementations. Each implementation should define a TYPE
59
+ class variable that matches the name of the CLI group.
60
+
61
+ Parameters
62
+ ----------
63
+ flow_file : str
64
+ Path to the flow file to deploy.
65
+ show_output : bool, default True
66
+ Show the 'stdout' and 'stderr' to the console by default.
67
+ profile : Optional[str], default None
68
+ Metaflow profile to use for the deployment. If not specified, the default
69
+ profile is used.
70
+ env : Optional[Dict], default None
71
+ Additional environment variables to set for the deployment.
72
+ cwd : Optional[str], default None
73
+ The directory to run the subprocess in; if not specified, the current
74
+ directory is used.
75
+ file_read_timeout : int, default 3600
76
+ The timeout until which we try to read the deployer attribute file.
77
+ **kwargs : Any
78
+ Additional arguments that you would pass to `python myflow.py` before
79
+ the deployment command.
80
+ """
58
81
  def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
59
82
  ...
60
83
  def __enter__(self) -> metaflow.runner.deployer.DeployerImpl:
@@ -93,11 +116,25 @@ class DeployerImpl(object, metaclass=type):
93
116
  ...
94
117
 
95
118
  class DeployedFlow(object, metaclass=type):
119
+ """
120
+ DeployedFlow class represents a flow that has been deployed.
121
+
122
+ Parameters
123
+ ----------
124
+ deployer : DeployerImpl
125
+ Instance of the deployer implementation.
126
+ """
96
127
  def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl):
97
128
  ...
98
129
  ...
99
130
 
100
131
  class TriggeredRun(object, metaclass=type):
132
+ """
133
+ TriggeredRun class represents a run that has been triggered on a production orchestrator.
134
+
135
+ Only when the `start` task starts running, the `run` object corresponding to the run
136
+ becomes available.
137
+ """
101
138
  def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl, content: str):
102
139
  ...
103
140
  def wait_for_run(self, timeout = None):
@@ -158,7 +195,7 @@ def get_lower_level_group(api, top_level_kwargs: typing.Dict, _type: typing.Opti
158
195
  """
159
196
  ...
160
197
 
161
- def handle_timeout(tfp_runner_attribute, command_obj: metaflow.runner.subprocess_manager.CommandManager, file_read_timeout: int):
198
+ def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
162
199
  """
163
200
  Handle the timeout for a running subprocess command that reads a file
164
201
  and raises an error with appropriate logs if a TimeoutError occurs.
@@ -293,6 +330,14 @@ def trigger(instance: metaflow.runner.deployer.DeployedFlow, **kwargs):
293
330
  ...
294
331
 
295
332
  class ArgoWorkflowsDeployer(metaflow.runner.deployer.DeployerImpl, metaclass=type):
333
+ """
334
+ Deployer implementation for Argo Workflows.
335
+
336
+ Attributes
337
+ ----------
338
+ TYPE : ClassVar[Optional[str]]
339
+ The type of the deployer, which is "argo-workflows".
340
+ """
296
341
  def __init__(self, deployer_kwargs, **kwargs):
297
342
  """
298
343
  Initialize the ArgoWorkflowsDeployer.
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.578880 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.668521 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.609641 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.705483 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.609178 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.705054 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.609432 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.705271 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.628132 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.720857 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.627110 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.719844 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,26 +1,60 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.628857 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.721653 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
+ import metaflow.exception
11
12
  import metaflow.metaflow_current
12
13
  import metaflow.decorators
13
- import metaflow.exception
14
14
 
15
15
  current: metaflow.metaflow_current.Current
16
16
 
17
17
  class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
18
+ """
19
+ Specifies the resources needed when executing this step.
20
+
21
+ Use `@resources` to specify the resource requirements
22
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
23
+
24
+ You can choose the compute layer on the command line by executing e.g.
25
+ ```
26
+ python myflow.py run --with batch
27
+ ```
28
+ or
29
+ ```
30
+ python myflow.py run --with kubernetes
31
+ ```
32
+ which executes the flow on the desired system using the
33
+ requirements specified in `@resources`.
34
+
35
+ Parameters
36
+ ----------
37
+ cpu : int, default 1
38
+ Number of CPUs required for this step.
39
+ gpu : int, default 0
40
+ Number of GPUs required for this step.
41
+ disk : int, optional, default None
42
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
43
+ memory : int, default 4096
44
+ Memory size (in MB) required for this step.
45
+ shared_memory : int, optional, default None
46
+ The value for the size (in MiB) of the /dev/shm volume for this step.
47
+ This parameter maps to the `--shm-size` option in Docker.
48
+ """
18
49
  ...
19
50
 
20
51
  def get_run_time_limit_for_task(step_decos):
21
52
  ...
22
53
 
23
54
  class MetaDatum(tuple, metaclass=type):
55
+ """
56
+ MetaDatum(field, value, type, tags)
57
+ """
24
58
  @staticmethod
25
59
  def __new__(_cls, field, value, type, tags):
26
60
  """
@@ -127,6 +161,72 @@ def get_ec2_instance_metadata():
127
161
  ...
128
162
 
129
163
  class BatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
164
+ """
165
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
166
+
167
+ Parameters
168
+ ----------
169
+ cpu : int, default 1
170
+ Number of CPUs required for this step. If `@resources` is
171
+ also present, the maximum value from all decorators is used.
172
+ gpu : int, default 0
173
+ Number of GPUs required for this step. If `@resources` is
174
+ also present, the maximum value from all decorators is used.
175
+ memory : int, default 4096
176
+ Memory size (in MB) required for this step. If
177
+ `@resources` is also present, the maximum value from all decorators is
178
+ used.
179
+ image : str, optional, default None
180
+ Docker image to use when launching on AWS Batch. If not specified, and
181
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
182
+ not, a default Docker image mapping to the current version of Python is used.
183
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
184
+ AWS Batch Job Queue to submit the job to.
185
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
186
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
187
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
188
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
189
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
190
+ shared_memory : int, optional, default None
191
+ The value for the size (in MiB) of the /dev/shm volume for this step.
192
+ This parameter maps to the `--shm-size` option in Docker.
193
+ max_swap : int, optional, default None
194
+ The total amount of swap memory (in MiB) a container can use for this
195
+ step. This parameter is translated to the `--memory-swap` option in
196
+ Docker where the value is the sum of the container memory plus the
197
+ `max_swap` value.
198
+ swappiness : int, optional, default None
199
+ This allows you to tune memory swappiness behavior for this step.
200
+ A swappiness value of 0 causes swapping not to happen unless absolutely
201
+ necessary. A swappiness value of 100 causes pages to be swapped very
202
+ aggressively. Accepted values are whole numbers between 0 and 100.
203
+ use_tmpfs : bool, default False
204
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
205
+ not available on Fargate compute environments
206
+ tmpfs_tempdir : bool, default True
207
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
208
+ tmpfs_size : int, optional, default None
209
+ The value for the size (in MiB) of the tmpfs mount for this step.
210
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
211
+ memory allocated for this step.
212
+ tmpfs_path : str, optional, default None
213
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
214
+ inferentia : int, default 0
215
+ Number of Inferentia chips required for this step.
216
+ trainium : int, default None
217
+ Alias for inferentia. Use only one of the two.
218
+ efa : int, default 0
219
+ Number of elastic fabric adapter network devices to attach to container
220
+ ephemeral_storage : int, default None
221
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
222
+ This is only relevant for Fargate compute environments
223
+ log_driver: str, optional, default None
224
+ The log driver to use for the Amazon ECS container.
225
+ log_options: List[str], optional, default None
226
+ List of strings containing options for the chosen log driver. The configurable values
227
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
228
+ Example: [`awslogs-group:aws/batch/job`]
229
+ """
130
230
  def __init__(self, attributes = None, statically_defined = False):
131
231
  ...
132
232
  def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.609743 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.705584 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.629251 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.722089 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,8 +9,8 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.plugins.secrets
12
- import abc
13
12
  import metaflow.exception
13
+ import abc
14
14
 
15
15
  class MetaflowException(Exception, metaclass=type):
16
16
  def __init__(self, msg = "", lineno = None):
@@ -31,15 +31,27 @@ class SecretsProvider(abc.ABC, metaclass=abc.ABCMeta):
31
31
  ...
32
32
 
33
33
  class MetaflowAWSSecretsManagerBadResponse(metaflow.exception.MetaflowException, metaclass=type):
34
+ """
35
+ Raised when the response from AWS Secrets Manager is not valid in some way
36
+ """
34
37
  ...
35
38
 
36
39
  class MetaflowAWSSecretsManagerDuplicateKey(metaflow.exception.MetaflowException, metaclass=type):
40
+ """
41
+ Raised when the response from AWS Secrets Manager contains duplicate keys
42
+ """
37
43
  ...
38
44
 
39
45
  class MetaflowAWSSecretsManagerJSONParseError(metaflow.exception.MetaflowException, metaclass=type):
46
+ """
47
+ Raised when the SecretString response from AWS Secrets Manager is not valid JSON
48
+ """
40
49
  ...
41
50
 
42
51
  class MetaflowAWSSecretsManagerNotJSONObject(metaflow.exception.MetaflowException, metaclass=type):
52
+ """
53
+ Raised when the SecretString response from AWS Secrets Manager is not valid JSON object (dictionary)
54
+ """
43
55
  ...
44
56
 
45
57
  class AwsSecretsManagerSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.609314 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.705169 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.622359 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.714473 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.621530 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.713656 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.622056 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.714084 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -11,6 +11,25 @@ if typing.TYPE_CHECKING:
11
11
  import metaflow.decorators
12
12
 
13
13
  class ScheduleDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
14
+ """
15
+ Specifies the times when the flow should be run when running on a
16
+ production scheduler.
17
+
18
+ Parameters
19
+ ----------
20
+ hourly : bool, default False
21
+ Run the workflow hourly.
22
+ daily : bool, default True
23
+ Run the workflow daily.
24
+ weekly : bool, default False
25
+ Run the workflow weekly.
26
+ cron : str, optional, default None
27
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
28
+ specified by this expression.
29
+ timezone : str, optional, default None
30
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
31
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
32
+ """
14
33
  def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
15
34
  ...
16
35
  ...
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.624688 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.717344 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.622676 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.715042 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,14 +1,13 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.625908 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.718626 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.runner.subprocess_manager
12
11
  import metaflow.runner.deployer
13
12
 
14
13
  class StepFunctions(object, metaclass=type):
@@ -43,6 +42,30 @@ class StepFunctions(object, metaclass=type):
43
42
  ...
44
43
 
45
44
  class DeployerImpl(object, metaclass=type):
45
+ """
46
+ Base class for deployer implementations. Each implementation should define a TYPE
47
+ class variable that matches the name of the CLI group.
48
+
49
+ Parameters
50
+ ----------
51
+ flow_file : str
52
+ Path to the flow file to deploy.
53
+ show_output : bool, default True
54
+ Show the 'stdout' and 'stderr' to the console by default.
55
+ profile : Optional[str], default None
56
+ Metaflow profile to use for the deployment. If not specified, the default
57
+ profile is used.
58
+ env : Optional[Dict], default None
59
+ Additional environment variables to set for the deployment.
60
+ cwd : Optional[str], default None
61
+ The directory to run the subprocess in; if not specified, the current
62
+ directory is used.
63
+ file_read_timeout : int, default 3600
64
+ The timeout until which we try to read the deployer attribute file.
65
+ **kwargs : Any
66
+ Additional arguments that you would pass to `python myflow.py` before
67
+ the deployment command.
68
+ """
46
69
  def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
47
70
  ...
48
71
  def __enter__(self) -> metaflow.runner.deployer.DeployerImpl:
@@ -81,11 +104,25 @@ class DeployerImpl(object, metaclass=type):
81
104
  ...
82
105
 
83
106
  class DeployedFlow(object, metaclass=type):
107
+ """
108
+ DeployedFlow class represents a flow that has been deployed.
109
+
110
+ Parameters
111
+ ----------
112
+ deployer : DeployerImpl
113
+ Instance of the deployer implementation.
114
+ """
84
115
  def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl):
85
116
  ...
86
117
  ...
87
118
 
88
119
  class TriggeredRun(object, metaclass=type):
120
+ """
121
+ TriggeredRun class represents a run that has been triggered on a production orchestrator.
122
+
123
+ Only when the `start` task starts running, the `run` object corresponding to the run
124
+ becomes available.
125
+ """
89
126
  def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl, content: str):
90
127
  ...
91
128
  def wait_for_run(self, timeout = None):
@@ -146,7 +183,7 @@ def get_lower_level_group(api, top_level_kwargs: typing.Dict, _type: typing.Opti
146
183
  """
147
184
  ...
148
185
 
149
- def handle_timeout(tfp_runner_attribute, command_obj: metaflow.runner.subprocess_manager.CommandManager, file_read_timeout: int):
186
+ def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
150
187
  """
151
188
  Handle the timeout for a running subprocess command that reads a file
152
189
  and raises an error with appropriate logs if a TimeoutError occurs.
@@ -261,6 +298,14 @@ def trigger(instance: metaflow.runner.deployer.DeployedFlow, **kwargs):
261
298
  ...
262
299
 
263
300
  class StepFunctionsDeployer(metaflow.runner.deployer.DeployerImpl, metaclass=type):
301
+ """
302
+ Deployer implementation for AWS Step Functions.
303
+
304
+ Attributes
305
+ ----------
306
+ TYPE : ClassVar[Optional[str]]
307
+ The type of the deployer, which is "step-functions".
308
+ """
264
309
  def __init__(self, deployer_kwargs, **kwargs):
265
310
  """
266
311
  Initialize the StepFunctionsDeployer.
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.581295 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.673195 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.619799 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.711946 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.620011 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.712162 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:42:44.621183 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.713333 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,8 +9,8 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.plugins.secrets
12
- import abc
13
12
  import metaflow.exception
13
+ import abc
14
14
 
15
15
  class SecretsProvider(abc.ABC, metaclass=abc.ABCMeta):
16
16
  def get_secret_as_dict(self, secret_id, options = {}, role = None) -> typing.Dict[str, str]:
@@ -34,21 +34,39 @@ def create_cacheable_azure_credential():
34
34
  ...
35
35
 
36
36
  class MetaflowAzureKeyVaultBadVault(metaflow.exception.MetaflowException, metaclass=type):
37
+ """
38
+ Raised when the secretid is fully qualified but does not have the right key vault domain
39
+ """
37
40
  ...
38
41
 
39
42
  class MetaflowAzureKeyVaultBadSecretType(metaflow.exception.MetaflowException, metaclass=type):
43
+ """
44
+ Raised when the secret type is anything except secrets
45
+ """
40
46
  ...
41
47
 
42
48
  class MetaflowAzureKeyVaultBadSecretPath(metaflow.exception.MetaflowException, metaclass=type):
49
+ """
50
+ Raised when the secret path does not match to expected length
51
+ """
43
52
  ...
44
53
 
45
54
  class MetaflowAzureKeyVaultBadSecretName(metaflow.exception.MetaflowException, metaclass=type):
55
+ """
56
+ Raised when the secret name does not match expected pattern
57
+ """
46
58
  ...
47
59
 
48
60
  class MetaflowAzureKeyVaultBadSecretVersion(metaflow.exception.MetaflowException, metaclass=type):
61
+ """
62
+ Raised when the secret version does not match expected pattern
63
+ """
49
64
  ...
50
65
 
51
66
  class MetaflowAzureKeyVaultBadSecret(metaflow.exception.MetaflowException, metaclass=type):
67
+ """
68
+ Raised when the secret does not match supported patterns in Metaflow
69
+ """
52
70
  ...
53
71
 
54
72
  class AzureKeyVaultSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):