metaflow-stubs 2.12.22__py2.py3-none-any.whl → 2.12.24__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. metaflow-stubs/__init__.pyi +805 -414
  2. metaflow-stubs/cards.pyi +211 -5
  3. metaflow-stubs/cli.pyi +23 -3
  4. metaflow-stubs/client/__init__.pyi +128 -3
  5. metaflow-stubs/client/core.pyi +226 -6
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +5 -2
  8. metaflow-stubs/events.pyi +21 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +71 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +158 -5
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +9 -3
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +26 -23
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +70 -4
  21. metaflow-stubs/plugins/__init__.pyi +14 -3
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow.pyi +15 -3
  24. metaflow-stubs/plugins/airflow/airflow_cli.pyi +66 -3
  25. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +5 -2
  26. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
  29. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +6 -3
  30. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +46 -4
  31. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +46 -4
  32. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
  35. metaflow-stubs/plugins/argo/argo_workflows.pyi +9 -5
  36. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
  37. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +34 -6
  38. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +49 -4
  39. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  46. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +103 -3
  47. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +16 -4
  49. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +70 -4
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +5 -2
  58. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
  59. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  61. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  62. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +22 -4
  63. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  64. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  65. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  66. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_cli.pyi +61 -3
  68. metaflow-stubs/plugins/cards/card_client.pyi +34 -3
  69. metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
  70. metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
  71. metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
  72. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
  73. metaflow-stubs/plugins/cards/card_modules/basic.pyi +42 -3
  74. metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
  81. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
  82. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
  84. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  85. metaflow-stubs/plugins/cards/component_serializer.pyi +57 -4
  86. metaflow-stubs/plugins/cards/exception.pyi +8 -2
  87. metaflow-stubs/plugins/catch_decorator.pyi +20 -3
  88. metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
  89. metaflow-stubs/plugins/datatools/local.pyi +16 -2
  90. metaflow-stubs/plugins/datatools/s3/__init__.pyi +73 -4
  91. metaflow-stubs/plugins/datatools/s3/s3.pyi +82 -5
  92. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  94. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  95. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  96. metaflow-stubs/plugins/environment_decorator.pyi +10 -2
  97. metaflow-stubs/plugins/events_decorator.pyi +106 -2
  98. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
  100. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +16 -4
  102. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  104. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  106. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
  109. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +66 -3
  110. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +100 -3
  112. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
  114. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  115. metaflow-stubs/plugins/package_cli.pyi +2 -2
  116. metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
  117. metaflow-stubs/plugins/project_decorator.pyi +59 -2
  118. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  119. metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
  120. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  121. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
  122. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  123. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  124. metaflow-stubs/plugins/resources_decorator.pyi +33 -2
  125. metaflow-stubs/plugins/retry_decorator.pyi +21 -2
  126. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  128. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
  129. metaflow-stubs/plugins/storage_executor.pyi +6 -2
  130. metaflow-stubs/plugins/tag_cli.pyi +36 -5
  131. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
  132. metaflow-stubs/plugins/timeout_decorator.pyi +24 -3
  133. metaflow-stubs/procpoll.pyi +2 -2
  134. metaflow-stubs/pylint_wrapper.pyi +2 -2
  135. metaflow-stubs/runner/__init__.pyi +2 -2
  136. metaflow-stubs/runner/deployer.pyi +70 -131
  137. metaflow-stubs/runner/metaflow_runner.pyi +118 -10
  138. metaflow-stubs/runner/nbdeploy.pyi +66 -2
  139. metaflow-stubs/runner/nbrun.pyi +79 -2
  140. metaflow-stubs/runner/subprocess_manager.pyi +16 -4
  141. metaflow-stubs/runner/utils.pyi +32 -2
  142. metaflow-stubs/system/__init__.pyi +3 -3
  143. metaflow-stubs/system/system_logger.pyi +3 -3
  144. metaflow-stubs/system/system_monitor.pyi +3 -3
  145. metaflow-stubs/tagging_util.pyi +2 -2
  146. metaflow-stubs/tuple_util.pyi +2 -2
  147. metaflow-stubs/version.pyi +2 -2
  148. {metaflow_stubs-2.12.22.dist-info → metaflow_stubs-2.12.24.dist-info}/METADATA +2 -2
  149. metaflow_stubs-2.12.24.dist-info/RECORD +152 -0
  150. metaflow_stubs-2.12.22.dist-info/RECORD +0 -152
  151. {metaflow_stubs-2.12.22.dist-info → metaflow_stubs-2.12.24.dist-info}/WHEEL +0 -0
  152. {metaflow_stubs-2.12.22.dist-info → metaflow_stubs-2.12.24.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,28 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.654256 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.019831 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
+ import metaflow.metaflow_current
12
+ import metaflow.datastore.inputs
13
+ import typing
14
+ import metaflow.decorators
11
15
  import metaflow.flowspec
12
16
  import metaflow.events
13
- import metaflow.decorators
14
- import metaflow.metaflow_current
15
17
  import metaflow
16
18
  import metaflow.graph
17
- import metaflow.datastore.inputs
18
- import typing
19
19
 
20
20
  current: metaflow.metaflow_current.Current
21
21
 
22
22
  class Trigger(object, metaclass=type):
23
+ """
24
+ Defines a container of event triggers' metadata.
25
+ """
23
26
  def __init__(self, _meta = None):
24
27
  ...
25
28
  @classmethod
@@ -93,6 +96,9 @@ class Trigger(object, metaclass=type):
93
96
  ...
94
97
 
95
98
  class MetaDatum(tuple, metaclass=type):
99
+ """
100
+ MetaDatum(field, value, type, tags)
101
+ """
96
102
  @staticmethod
97
103
  def __new__(_cls, field, value, type, tags):
98
104
  """
@@ -114,6 +120,14 @@ class MetaDatum(tuple, metaclass=type):
114
120
  ARGO_EVENTS_WEBHOOK_URL: None
115
121
 
116
122
  class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
123
+ """
124
+ Main class from which all Flows should inherit.
125
+
126
+ Attributes
127
+ ----------
128
+ index
129
+ input
130
+ """
117
131
  def __init__(self, use_cli = True):
118
132
  """
119
133
  Construct a FlowSpec
@@ -335,6 +349,20 @@ class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
335
349
  ...
336
350
 
337
351
  class ArgoEvent(object, metaclass=type):
352
+ """
353
+ ArgoEvent is a small event, a message, that can be published to Argo Workflows. The
354
+ event will eventually start all flows which have been previously deployed with `@trigger`
355
+ to wait for this particular named event.
356
+
357
+ Parameters
358
+ ----------
359
+ name : str,
360
+ Name of the event
361
+ url : str, optional
362
+ Override the event endpoint from `ARGO_EVENTS_WEBHOOK_URL`.
363
+ payload : Dict, optional
364
+ A set of key-value pairs delivered in this event. Used to set parameters of triggered flows.
365
+ """
338
366
  def __init__(self, name, url = None, payload = None, access_token = None):
339
367
  ...
340
368
  def add_to_payload(self, key, value):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.662603 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.028409 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,7 +9,6 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.runner.deployer
12
- import metaflow.runner.subprocess_manager
13
12
  import metaflow.graph
14
13
 
15
14
  class ArgoWorkflows(object, metaclass=type):
@@ -55,6 +54,30 @@ class ArgoWorkflows(object, metaclass=type):
55
54
  ...
56
55
 
57
56
  class DeployerImpl(object, metaclass=type):
57
+ """
58
+ Base class for deployer implementations. Each implementation should define a TYPE
59
+ class variable that matches the name of the CLI group.
60
+
61
+ Parameters
62
+ ----------
63
+ flow_file : str
64
+ Path to the flow file to deploy.
65
+ show_output : bool, default True
66
+ Show the 'stdout' and 'stderr' to the console by default.
67
+ profile : Optional[str], default None
68
+ Metaflow profile to use for the deployment. If not specified, the default
69
+ profile is used.
70
+ env : Optional[Dict], default None
71
+ Additional environment variables to set for the deployment.
72
+ cwd : Optional[str], default None
73
+ The directory to run the subprocess in; if not specified, the current
74
+ directory is used.
75
+ file_read_timeout : int, default 3600
76
+ The timeout until which we try to read the deployer attribute file.
77
+ **kwargs : Any
78
+ Additional arguments that you would pass to `python myflow.py` before
79
+ the deployment command.
80
+ """
58
81
  def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
59
82
  ...
60
83
  def __enter__(self) -> metaflow.runner.deployer.DeployerImpl:
@@ -93,11 +116,25 @@ class DeployerImpl(object, metaclass=type):
93
116
  ...
94
117
 
95
118
  class DeployedFlow(object, metaclass=type):
119
+ """
120
+ DeployedFlow class represents a flow that has been deployed.
121
+
122
+ Parameters
123
+ ----------
124
+ deployer : DeployerImpl
125
+ Instance of the deployer implementation.
126
+ """
96
127
  def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl):
97
128
  ...
98
129
  ...
99
130
 
100
131
  class TriggeredRun(object, metaclass=type):
132
+ """
133
+ TriggeredRun class represents a run that has been triggered on a production orchestrator.
134
+
135
+ Only when the `start` task starts running, the `run` object corresponding to the run
136
+ becomes available.
137
+ """
101
138
  def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl, content: str):
102
139
  ...
103
140
  def wait_for_run(self, timeout = None):
@@ -158,7 +195,7 @@ def get_lower_level_group(api, top_level_kwargs: typing.Dict, _type: typing.Opti
158
195
  """
159
196
  ...
160
197
 
161
- def handle_timeout(tfp_runner_attribute, command_obj: metaflow.runner.subprocess_manager.CommandManager, file_read_timeout: int):
198
+ def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
162
199
  """
163
200
  Handle the timeout for a running subprocess command that reads a file
164
201
  and raises an error with appropriate logs if a TimeoutError occurs.
@@ -293,6 +330,14 @@ def trigger(instance: metaflow.runner.deployer.DeployedFlow, **kwargs):
293
330
  ...
294
331
 
295
332
  class ArgoWorkflowsDeployer(metaflow.runner.deployer.DeployerImpl, metaclass=type):
333
+ """
334
+ Deployer implementation for Argo Workflows.
335
+
336
+ Attributes
337
+ ----------
338
+ TYPE : ClassVar[Optional[str]]
339
+ The type of the deployer, which is "argo-workflows".
340
+ """
296
341
  def __init__(self, deployer_kwargs, **kwargs):
297
342
  """
298
343
  Initialize the ArgoWorkflowsDeployer.
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.630443 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:46.998923 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.652126 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.050343 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.651870 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.050063 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.651436 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.049661 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.708727 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.079224 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.710325 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.080883 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.707484 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.077940 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,26 +1,60 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.709616 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.080166 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
11
  import metaflow.metaflow_current
13
12
  import metaflow.decorators
13
+ import metaflow.exception
14
14
 
15
15
  current: metaflow.metaflow_current.Current
16
16
 
17
17
  class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
18
+ """
19
+ Specifies the resources needed when executing this step.
20
+
21
+ Use `@resources` to specify the resource requirements
22
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
23
+
24
+ You can choose the compute layer on the command line by executing e.g.
25
+ ```
26
+ python myflow.py run --with batch
27
+ ```
28
+ or
29
+ ```
30
+ python myflow.py run --with kubernetes
31
+ ```
32
+ which executes the flow on the desired system using the
33
+ requirements specified in `@resources`.
34
+
35
+ Parameters
36
+ ----------
37
+ cpu : int, default 1
38
+ Number of CPUs required for this step.
39
+ gpu : int, default 0
40
+ Number of GPUs required for this step.
41
+ disk : int, optional, default None
42
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
43
+ memory : int, default 4096
44
+ Memory size (in MB) required for this step.
45
+ shared_memory : int, optional, default None
46
+ The value for the size (in MiB) of the /dev/shm volume for this step.
47
+ This parameter maps to the `--shm-size` option in Docker.
48
+ """
18
49
  ...
19
50
 
20
51
  def get_run_time_limit_for_task(step_decos):
21
52
  ...
22
53
 
23
54
  class MetaDatum(tuple, metaclass=type):
55
+ """
56
+ MetaDatum(field, value, type, tags)
57
+ """
24
58
  @staticmethod
25
59
  def __new__(_cls, field, value, type, tags):
26
60
  """
@@ -127,6 +161,72 @@ def get_ec2_instance_metadata():
127
161
  ...
128
162
 
129
163
  class BatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
164
+ """
165
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
166
+
167
+ Parameters
168
+ ----------
169
+ cpu : int, default 1
170
+ Number of CPUs required for this step. If `@resources` is
171
+ also present, the maximum value from all decorators is used.
172
+ gpu : int, default 0
173
+ Number of GPUs required for this step. If `@resources` is
174
+ also present, the maximum value from all decorators is used.
175
+ memory : int, default 4096
176
+ Memory size (in MB) required for this step. If
177
+ `@resources` is also present, the maximum value from all decorators is
178
+ used.
179
+ image : str, optional, default None
180
+ Docker image to use when launching on AWS Batch. If not specified, and
181
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
182
+ not, a default Docker image mapping to the current version of Python is used.
183
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
184
+ AWS Batch Job Queue to submit the job to.
185
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
186
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
187
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
188
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
189
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
190
+ shared_memory : int, optional, default None
191
+ The value for the size (in MiB) of the /dev/shm volume for this step.
192
+ This parameter maps to the `--shm-size` option in Docker.
193
+ max_swap : int, optional, default None
194
+ The total amount of swap memory (in MiB) a container can use for this
195
+ step. This parameter is translated to the `--memory-swap` option in
196
+ Docker where the value is the sum of the container memory plus the
197
+ `max_swap` value.
198
+ swappiness : int, optional, default None
199
+ This allows you to tune memory swappiness behavior for this step.
200
+ A swappiness value of 0 causes swapping not to happen unless absolutely
201
+ necessary. A swappiness value of 100 causes pages to be swapped very
202
+ aggressively. Accepted values are whole numbers between 0 and 100.
203
+ use_tmpfs : bool, default False
204
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
205
+ not available on Fargate compute environments
206
+ tmpfs_tempdir : bool, default True
207
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
208
+ tmpfs_size : int, optional, default None
209
+ The value for the size (in MiB) of the tmpfs mount for this step.
210
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
211
+ memory allocated for this step.
212
+ tmpfs_path : str, optional, default None
213
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
214
+ inferentia : int, default 0
215
+ Number of Inferentia chips required for this step.
216
+ trainium : int, default None
217
+ Alias for inferentia. Use only one of the two.
218
+ efa : int, default 0
219
+ Number of elastic fabric adapter network devices to attach to container
220
+ ephemeral_storage : int, default None
221
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
222
+ This is only relevant for Fargate compute environments
223
+ log_driver: str, optional, default None
224
+ The log driver to use for the Amazon ECS container.
225
+ log_options: List[str], optional, default None
226
+ List of strings containing options for the chosen log driver. The configurable values
227
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
228
+ Example: [`awslogs-group:aws/batch/job`]
229
+ """
130
230
  def __init__(self, attributes = None, statically_defined = False):
131
231
  ...
132
232
  def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.652243 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.050466 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,16 +1,16 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.710794 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.081391 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import abc
12
- import metaflow.plugins.secrets
13
11
  import metaflow.exception
12
+ import metaflow.plugins.secrets
13
+ import abc
14
14
 
15
15
  class MetaflowException(Exception, metaclass=type):
16
16
  def __init__(self, msg = "", lineno = None):
@@ -31,15 +31,27 @@ class SecretsProvider(abc.ABC, metaclass=abc.ABCMeta):
31
31
  ...
32
32
 
33
33
  class MetaflowAWSSecretsManagerBadResponse(metaflow.exception.MetaflowException, metaclass=type):
34
+ """
35
+ Raised when the response from AWS Secrets Manager is not valid in some way
36
+ """
34
37
  ...
35
38
 
36
39
  class MetaflowAWSSecretsManagerDuplicateKey(metaflow.exception.MetaflowException, metaclass=type):
40
+ """
41
+ Raised when the response from AWS Secrets Manager contains duplicate keys
42
+ """
37
43
  ...
38
44
 
39
45
  class MetaflowAWSSecretsManagerJSONParseError(metaflow.exception.MetaflowException, metaclass=type):
46
+ """
47
+ Raised when the SecretString response from AWS Secrets Manager is not valid JSON
48
+ """
40
49
  ...
41
50
 
42
51
  class MetaflowAWSSecretsManagerNotJSONObject(metaflow.exception.MetaflowException, metaclass=type):
52
+ """
53
+ Raised when the SecretString response from AWS Secrets Manager is not valid JSON object (dictionary)
54
+ """
43
55
  ...
44
56
 
45
57
  class AwsSecretsManagerSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.651323 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.049542 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.698884 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.069087 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.699859 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.070133 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.704298 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.066591 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.699521 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.069806 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -11,6 +11,25 @@ if typing.TYPE_CHECKING:
11
11
  import metaflow.decorators
12
12
 
13
13
  class ScheduleDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
14
+ """
15
+ Specifies the times when the flow should be run when running on a
16
+ production scheduler.
17
+
18
+ Parameters
19
+ ----------
20
+ hourly : bool, default False
21
+ Run the workflow hourly.
22
+ daily : bool, default True
23
+ Run the workflow daily.
24
+ weekly : bool, default False
25
+ Run the workflow weekly.
26
+ cron : str, optional, default None
27
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
28
+ specified by this expression.
29
+ timezone : str, optional, default None
30
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
31
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
32
+ """
14
33
  def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
15
34
  ...
16
35
  ...
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.702548 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.072838 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,17 +1,17 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.706007 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.076367 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
11
  import metaflow.metaflow_current
13
- import metaflow.parameters
14
12
  import metaflow.decorators
13
+ import metaflow.parameters
14
+ import metaflow.exception
15
15
 
16
16
  JSONType: metaflow.parameters.JSONTypeClass
17
17
 
@@ -54,6 +54,72 @@ SFN_STATE_MACHINE_PREFIX: None
54
54
  UI_URL: None
55
55
 
56
56
  class BatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
57
+ """
58
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
59
+
60
+ Parameters
61
+ ----------
62
+ cpu : int, default 1
63
+ Number of CPUs required for this step. If `@resources` is
64
+ also present, the maximum value from all decorators is used.
65
+ gpu : int, default 0
66
+ Number of GPUs required for this step. If `@resources` is
67
+ also present, the maximum value from all decorators is used.
68
+ memory : int, default 4096
69
+ Memory size (in MB) required for this step. If
70
+ `@resources` is also present, the maximum value from all decorators is
71
+ used.
72
+ image : str, optional, default None
73
+ Docker image to use when launching on AWS Batch. If not specified, and
74
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
75
+ not, a default Docker image mapping to the current version of Python is used.
76
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
77
+ AWS Batch Job Queue to submit the job to.
78
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
79
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
80
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
81
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
82
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
83
+ shared_memory : int, optional, default None
84
+ The value for the size (in MiB) of the /dev/shm volume for this step.
85
+ This parameter maps to the `--shm-size` option in Docker.
86
+ max_swap : int, optional, default None
87
+ The total amount of swap memory (in MiB) a container can use for this
88
+ step. This parameter is translated to the `--memory-swap` option in
89
+ Docker where the value is the sum of the container memory plus the
90
+ `max_swap` value.
91
+ swappiness : int, optional, default None
92
+ This allows you to tune memory swappiness behavior for this step.
93
+ A swappiness value of 0 causes swapping not to happen unless absolutely
94
+ necessary. A swappiness value of 100 causes pages to be swapped very
95
+ aggressively. Accepted values are whole numbers between 0 and 100.
96
+ use_tmpfs : bool, default False
97
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
98
+ not available on Fargate compute environments
99
+ tmpfs_tempdir : bool, default True
100
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
101
+ tmpfs_size : int, optional, default None
102
+ The value for the size (in MiB) of the tmpfs mount for this step.
103
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
104
+ memory allocated for this step.
105
+ tmpfs_path : str, optional, default None
106
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
107
+ inferentia : int, default 0
108
+ Number of Inferentia chips required for this step.
109
+ trainium : int, default None
110
+ Alias for inferentia. Use only one of the two.
111
+ efa : int, default 0
112
+ Number of elastic fabric adapter network devices to attach to container
113
+ ephemeral_storage : int, default None
114
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
115
+ This is only relevant for Fargate compute environments
116
+ log_driver: str, optional, default None
117
+ The log driver to use for the Amazon ECS container.
118
+ log_options: List[str], optional, default None
119
+ List of strings containing options for the chosen log driver. The configurable values
120
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
121
+ Example: [`awslogs-group:aws/batch/job`]
122
+ """
57
123
  def __init__(self, attributes = None, statically_defined = False):
58
124
  ...
59
125
  def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22 #
4
- # Generated on 2024-09-20T00:45:49.700225 #
3
+ # MF version: 2.12.24 #
4
+ # Generated on 2024-10-04T11:37:47.070513 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations