ob-metaflow-stubs 6.0.3.117__py2.py3-none-any.whl → 6.0.3.118__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (175) hide show
  1. metaflow-stubs/__init__.pyi +319 -2739
  2. metaflow-stubs/cards.pyi +19 -473
  3. metaflow-stubs/cli.pyi +17 -81
  4. metaflow-stubs/client/__init__.pyi +19 -1113
  5. metaflow-stubs/client/core.pyi +18 -158
  6. metaflow-stubs/client/filecache.pyi +8 -12
  7. metaflow-stubs/clone_util.pyi +6 -26
  8. metaflow-stubs/events.pyi +6 -5
  9. metaflow-stubs/exception.pyi +8 -6
  10. metaflow-stubs/flowspec.pyi +22 -106
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +16 -564
  13. metaflow-stubs/info_file.pyi +6 -5
  14. metaflow-stubs/metadata_provider/__init__.pyi +16 -0
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
  16. metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
  17. metaflow-stubs/metadata_provider/util.pyi +19 -0
  18. metaflow-stubs/metaflow_config.pyi +8 -13
  19. metaflow-stubs/metaflow_current.pyi +134 -133
  20. metaflow-stubs/mf_extensions/__init__.pyi +6 -0
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +6 -0
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +6 -0
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +6 -0
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +6 -0
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +129 -0
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +26 -0
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +156 -0
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +25 -0
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +111 -0
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +6 -0
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +188 -0
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +30 -0
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +28 -0
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +19 -0
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +115 -0
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +40 -0
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +6 -0
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +71 -0
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +56 -0
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +67 -0
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +49 -0
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +6 -0
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +6 -0
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +130 -0
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +42 -0
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +6 -0
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +17 -0
  48. metaflow-stubs/mflog/__init__.pyi +6 -0
  49. metaflow-stubs/mflog/mflog.pyi +52 -5
  50. metaflow-stubs/multicore_utils.pyi +6 -5
  51. metaflow-stubs/parameters.pyi +13 -23
  52. metaflow-stubs/plugins/__init__.pyi +52 -165
  53. metaflow-stubs/plugins/airflow/__init__.pyi +9 -5
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
  55. metaflow-stubs/plugins/airflow/exception.pyi +7 -11
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
  60. metaflow-stubs/plugins/argo/__init__.pyi +12 -5
  61. metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
  62. metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +18 -130
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +12 -404
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +63 -448
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
  68. metaflow-stubs/plugins/aws/__init__.pyi +11 -5
  69. metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -5
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -5
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +52 -292
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
  85. metaflow-stubs/plugins/azure/__init__.pyi +12 -7
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
  92. metaflow-stubs/plugins/cards/__init__.pyi +15 -5
  93. metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
  94. metaflow-stubs/plugins/cards/card_client.pyi +13 -75
  95. metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +13 -81
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
  107. metaflow-stubs/plugins/cards/exception.pyi +7 -11
  108. metaflow-stubs/plugins/catch_decorator.pyi +10 -30
  109. metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
  110. metaflow-stubs/plugins/datatools/local.pyi +7 -11
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +16 -264
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
  115. metaflow-stubs/plugins/debug_logger.pyi +7 -5
  116. metaflow-stubs/plugins/debug_monitor.pyi +7 -5
  117. metaflow-stubs/plugins/environment_decorator.pyi +7 -5
  118. metaflow-stubs/plugins/events_decorator.pyi +8 -14
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
  121. metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +12 -5
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +10 -38
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +7 -11
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
  134. metaflow-stubs/plugins/logs_cli.pyi +10 -9
  135. metaflow-stubs/plugins/package_cli.pyi +7 -5
  136. metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
  137. metaflow-stubs/plugins/perimeters.pyi +6 -11
  138. metaflow-stubs/plugins/project_decorator.pyi +8 -14
  139. metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +14 -20
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
  144. metaflow-stubs/plugins/pypi/utils.pyi +7 -11
  145. metaflow-stubs/plugins/resources_decorator.pyi +7 -5
  146. metaflow-stubs/plugins/retry_decorator.pyi +7 -11
  147. metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +8 -13
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
  150. metaflow-stubs/plugins/storage_executor.pyi +6 -11
  151. metaflow-stubs/plugins/tag_cli.pyi +14 -396
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +8 -33
  153. metaflow-stubs/plugins/timeout_decorator.pyi +8 -12
  154. metaflow-stubs/procpoll.pyi +7 -5
  155. metaflow-stubs/profilers/__init__.pyi +7 -11
  156. metaflow-stubs/pylint_wrapper.pyi +7 -11
  157. metaflow-stubs/runner/__init__.pyi +13 -5
  158. metaflow-stubs/runner/deployer.pyi +99 -223
  159. metaflow-stubs/runner/deployer_impl.pyi +87 -0
  160. metaflow-stubs/runner/metaflow_runner.pyi +24 -508
  161. metaflow-stubs/runner/nbdeploy.pyi +16 -60
  162. metaflow-stubs/runner/nbrun.pyi +11 -148
  163. metaflow-stubs/runner/subprocess_manager.pyi +9 -10
  164. metaflow-stubs/runner/utils.pyi +44 -9
  165. metaflow-stubs/system/__init__.pyi +9 -107
  166. metaflow-stubs/system/system_logger.pyi +6 -15
  167. metaflow-stubs/system/system_monitor.pyi +6 -16
  168. metaflow-stubs/tagging_util.pyi +6 -10
  169. metaflow-stubs/tuple_util.pyi +6 -5
  170. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.118.dist-info/RECORD +174 -0
  172. metaflow-stubs/metadata/util.pyi +0 -18
  173. ob_metaflow_stubs-6.0.3.117.dist-info/RECORD +0 -140
  174. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/WHEEL +0 -0
  175. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/top_level.txt +0 -0
@@ -1,443 +1,71 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.340728 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.932934 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.flowspec
12
11
  import datetime
13
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
14
- import metaflow.client.core
15
- import metaflow.parameters
16
- import metaflow._vendor.click.types
17
- import metaflow.events
18
- import metaflow.metaflow_current
19
- import metaflow.runner.metaflow_runner
20
- import metaflow.datastore.inputs
21
12
  import typing
22
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
14
  StepFlag = typing.NewType("StepFlag", bool)
24
15
 
25
- EXT_PKG: str
26
-
27
- def parallel_imap_unordered(func: typing.Callable[[typing.Any], typing.Any], iterable: typing.Iterable[typing.Any], max_parallel: typing.Optional[int] = None, dir: typing.Optional[str] = None) -> typing.Iterator[typing.Any]:
28
- """
29
- Parallelizes execution of a function using multiprocessing. The result
30
- order is not guaranteed.
31
-
32
- Parameters
33
- ----------
34
- func : Callable[[Any], Any]
35
- Function taking a single argument and returning a result
36
- iterable : Iterable[Any]
37
- Iterable over arguments to pass to fun
38
- max_parallel int, optional, default None
39
- Maximum parallelism. If not specified, uses the number of CPUs
40
- dir : str, optional, default None
41
- If specified, directory where temporary files are created
42
-
43
- Yields
44
- ------
45
- Any
46
- One result from calling func on one argument
47
- """
48
- ...
49
-
50
- def parallel_map(func: typing.Callable[[typing.Any], typing.Any], iterable: typing.Iterable[typing.Any], max_parallel: typing.Optional[int] = None, dir: typing.Optional[str] = None) -> typing.List[typing.Any]:
51
- """
52
- Parallelizes execution of a function using multiprocessing. The result
53
- order is that of the arguments in `iterable`
54
-
55
- Parameters
56
- ----------
57
- func : Callable[[Any], Any]
58
- Function taking a single argument and returning a result
59
- iterable : Iterable[Any]
60
- Iterable over arguments to pass to fun
61
- max_parallel int, optional, default None
62
- Maximum parallelism. If not specified, uses the number of CPUs
63
- dir : str, optional, default None
64
- If specified, directory where temporary files are created
65
-
66
- Returns
67
- -------
68
- List[Any]
69
- Results. The items in the list are in the same order as the items
70
- in `iterable`.
71
- """
72
- ...
73
-
74
- current: metaflow.metaflow_current.Current
75
-
76
- def metadata(ms: str) -> str:
77
- """
78
- Switch Metadata provider.
79
-
80
- This call has a global effect. Selecting the local metadata will,
81
- for example, not allow access to information stored in remote
82
- metadata providers.
83
-
84
- Note that you don't typically have to call this function directly. Usually
85
- the metadata provider is set through the Metaflow configuration file. If you
86
- need to switch between multiple providers, you can use the `METAFLOW_PROFILE`
87
- environment variable to switch between configurations.
88
-
89
- Parameters
90
- ----------
91
- ms : str
92
- Can be a path (selects local metadata), a URL starting with http (selects
93
- the service metadata) or an explicit specification <metadata_type>@<info>; as an
94
- example, you can specify local@<path> or service@<url>.
95
-
96
- Returns
97
- -------
98
- str
99
- The description of the metadata selected (equivalent to the result of
100
- get_metadata()).
101
- """
102
- ...
16
+ from . import info_file as info_file
17
+ from . import exception as exception
18
+ from . import metaflow_config as metaflow_config
19
+ from . import multicore_utils as multicore_utils
20
+ from .multicore_utils import parallel_imap_unordered as parallel_imap_unordered
21
+ from .multicore_utils import parallel_map as parallel_map
22
+ from . import metaflow_current as metaflow_current
23
+ from .metaflow_current import current as current
24
+ from . import parameters as parameters
25
+ from . import tagging_util as tagging_util
26
+ from . import metadata_provider as metadata_provider
27
+ from . import flowspec as flowspec
28
+ from .flowspec import FlowSpec as FlowSpec
29
+ from .parameters import Parameter as Parameter
30
+ from .parameters import JSONTypeClass as JSONTypeClass
31
+ from .parameters import JSONType as JSONType
32
+ from . import tuple_util as tuple_util
33
+ from . import cards as cards
34
+ from . import events as events
35
+ from . import runner as runner
36
+ from . import plugins as plugins
37
+ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
38
+ from . import includefile as includefile
39
+ from .includefile import IncludeFile as IncludeFile
40
+ from . import client as client
41
+ from .client.core import namespace as namespace
42
+ from .client.core import get_namespace as get_namespace
43
+ from .client.core import default_namespace as default_namespace
44
+ from .client.core import metadata as metadata
45
+ from .client.core import get_metadata as get_metadata
46
+ from .client.core import default_metadata as default_metadata
47
+ from .client.core import Metaflow as Metaflow
48
+ from .client.core import Flow as Flow
49
+ from .client.core import Run as Run
50
+ from .client.core import Step as Step
51
+ from .client.core import Task as Task
52
+ from .client.core import DataArtifact as DataArtifact
53
+ from .runner.metaflow_runner import Runner as Runner
54
+ from .runner.nbrun import NBRunner as NBRunner
55
+ from .runner.deployer import Deployer as Deployer
56
+ from .runner.deployer import DeployedFlow as DeployedFlow
57
+ from .runner.nbdeploy import NBDeployer as NBDeployer
58
+ from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.final_api import Checkpoint as Checkpoint
59
+ from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures import load_model as load_model
60
+ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_aws_client as get_aws_client
61
+ from . import system as system
62
+ from . import pylint_wrapper as pylint_wrapper
63
+ from . import procpoll as procpoll
64
+ from . import clone_util as clone_util
65
+ from . import cli as cli
66
+ from . import profilers as profilers
103
67
 
104
- class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
105
- """
106
- Main class from which all Flows should inherit.
107
-
108
- Attributes
109
- ----------
110
- index
111
- input
112
- """
113
- def __init__(self, use_cli = True):
114
- """
115
- Construct a FlowSpec
116
-
117
- Parameters
118
- ----------
119
- use_cli : bool, default True
120
- Set to True if the flow is invoked from __main__ or the command line
121
- """
122
- ...
123
- @property
124
- def script_name(self) -> str:
125
- """
126
- [Legacy function - do not use. Use `current` instead]
127
-
128
- Returns the name of the script containing the flow
129
-
130
- Returns
131
- -------
132
- str
133
- A string containing the name of the script
134
- """
135
- ...
136
- def __iter__(self):
137
- """
138
- [Legacy function - do not use]
139
-
140
- Iterate over all steps in the Flow
141
-
142
- Returns
143
- -------
144
- Iterator[graph.DAGNode]
145
- Iterator over the steps in the flow
146
- """
147
- ...
148
- def __getattr__(self, name: str):
149
- ...
150
- def cmd(self, cmdline, input = {}, output = []):
151
- """
152
- [Legacy function - do not use]
153
- """
154
- ...
155
- @property
156
- def index(self) -> typing.Optional[int]:
157
- """
158
- The index of this foreach branch.
159
-
160
- In a foreach step, multiple instances of this step (tasks) will be executed,
161
- one for each element in the foreach. This property returns the zero based index
162
- of the current task. If this is not a foreach step, this returns None.
163
-
164
- If you need to know the indices of the parent tasks in a nested foreach, use
165
- `FlowSpec.foreach_stack`.
166
-
167
- Returns
168
- -------
169
- int, optional
170
- Index of the task in a foreach step.
171
- """
172
- ...
173
- @property
174
- def input(self) -> typing.Optional[typing.Any]:
175
- """
176
- The value of the foreach artifact in this foreach branch.
177
-
178
- In a foreach step, multiple instances of this step (tasks) will be executed,
179
- one for each element in the foreach. This property returns the element passed
180
- to the current task. If this is not a foreach step, this returns None.
181
-
182
- If you need to know the values of the parent tasks in a nested foreach, use
183
- `FlowSpec.foreach_stack`.
184
-
185
- Returns
186
- -------
187
- object, optional
188
- Input passed to the foreach task.
189
- """
190
- ...
191
- def foreach_stack(self) -> typing.Optional[typing.List[typing.Tuple[int, int, typing.Any]]]:
192
- """
193
- Returns the current stack of foreach indexes and values for the current step.
194
-
195
- Use this information to understand what data is being processed in the current
196
- foreach branch. For example, considering the following code:
197
- ```
198
- @step
199
- def root(self):
200
- self.split_1 = ['a', 'b', 'c']
201
- self.next(self.nest_1, foreach='split_1')
202
-
203
- @step
204
- def nest_1(self):
205
- self.split_2 = ['d', 'e', 'f', 'g']
206
- self.next(self.nest_2, foreach='split_2'):
207
-
208
- @step
209
- def nest_2(self):
210
- foo = self.foreach_stack()
211
- ```
212
-
213
- `foo` will take the following values in the various tasks for nest_2:
214
- ```
215
- [(0, 3, 'a'), (0, 4, 'd')]
216
- [(0, 3, 'a'), (1, 4, 'e')]
217
- ...
218
- [(0, 3, 'a'), (3, 4, 'g')]
219
- [(1, 3, 'b'), (0, 4, 'd')]
220
- ...
221
- ```
222
- where each tuple corresponds to:
223
-
224
- - The index of the task for that level of the loop.
225
- - The number of splits for that level of the loop.
226
- - The value for that level of the loop.
227
-
228
- Note that the last tuple returned in a task corresponds to:
229
-
230
- - 1st element: value returned by `self.index`.
231
- - 3rd element: value returned by `self.input`.
232
-
233
- Returns
234
- -------
235
- List[Tuple[int, int, Any]]
236
- An array describing the current stack of foreach steps.
237
- """
238
- ...
239
- def merge_artifacts(self, inputs: metaflow.datastore.inputs.Inputs, exclude: typing.Optional[typing.List[str]] = None, include: typing.Optional[typing.List[str]] = None):
240
- """
241
- Helper function for merging artifacts in a join step.
242
-
243
- This function takes all the artifacts coming from the branches of a
244
- join point and assigns them to self in the calling step. Only artifacts
245
- not set in the current step are considered. If, for a given artifact, different
246
- values are present on the incoming edges, an error will be thrown and the artifacts
247
- that conflict will be reported.
248
-
249
- As a few examples, in the simple graph: A splitting into B and C and joining in D:
250
- ```
251
- A:
252
- self.x = 5
253
- self.y = 6
254
- B:
255
- self.b_var = 1
256
- self.x = from_b
257
- C:
258
- self.x = from_c
259
-
260
- D:
261
- merge_artifacts(inputs)
262
- ```
263
- In D, the following artifacts are set:
264
- - `y` (value: 6), `b_var` (value: 1)
265
- - if `from_b` and `from_c` are the same, `x` will be accessible and have value `from_b`
266
- - if `from_b` and `from_c` are different, an error will be thrown. To prevent this error,
267
- you need to manually set `self.x` in D to a merged value (for example the max) prior to
268
- calling `merge_artifacts`.
269
-
270
- Parameters
271
- ----------
272
- inputs : Inputs
273
- Incoming steps to the join point.
274
- exclude : List[str], optional, default None
275
- If specified, do not consider merging artifacts with a name in `exclude`.
276
- Cannot specify if `include` is also specified.
277
- include : List[str], optional, default None
278
- If specified, only merge artifacts specified. Cannot specify if `exclude` is
279
- also specified.
280
-
281
- Raises
282
- ------
283
- MetaflowException
284
- This exception is thrown if this is not called in a join step.
285
- UnhandledInMergeArtifactsException
286
- This exception is thrown in case of unresolved conflicts.
287
- MissingInMergeArtifactsException
288
- This exception is thrown in case an artifact specified in `include` cannot
289
- be found.
290
- """
291
- ...
292
- def next(self, *dsts: typing.Callable[..., None], **kwargs):
293
- """
294
- Indicates the next step to execute after this step has completed.
295
-
296
- This statement should appear as the last statement of each step, except
297
- the end step.
298
-
299
- There are several valid formats to specify the next step:
300
-
301
- - Straight-line connection: `self.next(self.next_step)` where `next_step` is a method in
302
- the current class decorated with the `@step` decorator.
303
-
304
- - Static fan-out connection: `self.next(self.step1, self.step2, ...)` where `stepX` are
305
- methods in the current class decorated with the `@step` decorator.
306
-
307
- - Foreach branch:
308
- ```
309
- self.next(self.foreach_step, foreach='foreach_iterator')
310
- ```
311
- In this situation, `foreach_step` is a method in the current class decorated with the
312
- `@step` decorator and `foreach_iterator` is a variable name in the current class that
313
- evaluates to an iterator. A task will be launched for each value in the iterator and
314
- each task will execute the code specified by the step `foreach_step`.
315
-
316
- Parameters
317
- ----------
318
- dsts : Callable[..., None]
319
- One or more methods annotated with `@step`.
320
-
321
- Raises
322
- ------
323
- InvalidNextException
324
- Raised if the format of the arguments does not match one of the ones given above.
325
- """
326
- ...
327
- def __str__(self):
328
- ...
329
- def __getstate__(self):
330
- ...
331
- ...
332
-
333
- class Parameter(object, metaclass=type):
334
- """
335
- Defines a parameter for a flow.
336
-
337
- Parameters must be instantiated as class variables in flow classes, e.g.
338
- ```
339
- class MyFlow(FlowSpec):
340
- param = Parameter('myparam')
341
- ```
342
- in this case, the parameter is specified on the command line as
343
- ```
344
- python myflow.py run --myparam=5
345
- ```
346
- and its value is accessible through a read-only artifact like this:
347
- ```
348
- print(self.param == 5)
349
- ```
350
- Note that the user-visible parameter name, `myparam` above, can be
351
- different from the artifact name, `param` above.
352
-
353
- The parameter value is converted to a Python type based on the `type`
354
- argument or to match the type of `default`, if it is set.
355
-
356
- Parameters
357
- ----------
358
- name : str
359
- User-visible parameter name.
360
- default : str or float or int or bool or `JSONType` or a function.
361
- Default value for the parameter. Use a special `JSONType` class to
362
- indicate that the value must be a valid JSON object. A function
363
- implies that the parameter corresponds to a *deploy-time parameter*.
364
- The type of the default value is used as the parameter `type`.
365
- type : Type, default None
366
- If `default` is not specified, define the parameter type. Specify
367
- one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
368
- to the type of `default` or `str` if none specified.
369
- help : str, optional
370
- Help text to show in `run --help`.
371
- required : bool, default False
372
- Require that the user specified a value for the parameter.
373
- `required=True` implies that the `default` is not used.
374
- show_default : bool, default True
375
- If True, show the default value in the help text.
376
- """
377
- def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], metaflow.parameters.JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
378
- ...
379
- def __repr__(self):
380
- ...
381
- def __str__(self):
382
- ...
383
- def option_kwargs(self, deploy_mode):
384
- ...
385
- def load_parameter(self, v):
386
- ...
387
- @property
388
- def is_string_type(self):
389
- ...
390
- def __getitem__(self, x):
391
- ...
392
- ...
393
-
394
- class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
395
- def convert(self, value, param, ctx):
396
- ...
397
- def __str__(self):
398
- ...
399
- def __repr__(self):
400
- ...
401
- ...
402
-
403
- JSONType: metaflow.parameters.JSONTypeClass
404
-
405
- def S3(*args, **kwargs):
406
- ...
407
-
408
- class IncludeFile(metaflow.parameters.Parameter, metaclass=type):
409
- """
410
- Includes a local file as a parameter for the flow.
411
-
412
- `IncludeFile` behaves like `Parameter` except that it reads its value from a file instead of
413
- the command line. The user provides a path to a file on the command line. The file contents
414
- are saved as a read-only artifact which is available in all steps of the flow.
415
-
416
- Parameters
417
- ----------
418
- name : str
419
- User-visible parameter name.
420
- default : Union[str, Callable[ParameterContext, str]]
421
- Default path to a local file. A function
422
- implies that the parameter corresponds to a *deploy-time parameter*.
423
- is_text : bool, default True
424
- Convert the file contents to a string using the provided `encoding`.
425
- If False, the artifact is stored in `bytes`.
426
- encoding : str, optional, default 'utf-8'
427
- Use this encoding to decode the file contexts if `is_text=True`.
428
- required : bool, default False
429
- Require that the user specified a value for the parameter.
430
- `required=True` implies that the `default` is not used.
431
- help : str, optional
432
- Help text to show in `run --help`.
433
- show_default : bool, default True
434
- If True, show the default value in the help text.
435
- """
436
- def __init__(self, name: str, required: bool = False, is_text: bool = True, encoding: str = "utf-8", help: typing.Optional[str] = None, **kwargs: typing.Dict[str, str]):
437
- ...
438
- def load_parameter(self, v):
439
- ...
440
- ...
68
+ EXT_PKG: str
441
69
 
442
70
  @typing.overload
443
71
  def step(f: typing.Callable[[FlowSpecDerived], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
@@ -513,204 +141,190 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
513
141
  ...
514
142
 
515
143
  @typing.overload
516
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
144
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
517
145
  """
518
- Enables loading / saving of models within a step.
519
-
520
-
521
- Parameters
522
- ----------
523
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
524
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
525
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
526
- - `current.checkpoint`
527
- - `current.model`
528
- - `current.huggingface_hub`
529
-
530
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
531
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
532
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
533
-
534
- temp_dir_root : str, default: None
535
- The root directory under which `current.model.loaded` will store loaded models
536
-
537
-
146
+ Specifies the Conda environment for the step.
538
147
 
148
+ Information in this decorator will augment any
149
+ attributes set in the `@conda_base` flow-level decorator. Hence,
150
+ you can use `@conda_base` to set packages required by all
151
+ steps and use `@conda` to specify step-specific overrides.
539
152
  """
540
153
  ...
541
154
 
542
155
  @typing.overload
543
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
156
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
544
157
  ...
545
158
 
546
159
  @typing.overload
547
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
160
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
548
161
  ...
549
162
 
550
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
163
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
551
164
  """
552
- Enables loading / saving of models within a step.
553
-
554
-
555
- Parameters
556
- ----------
557
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
558
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
559
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
560
- - `current.checkpoint`
561
- - `current.model`
562
- - `current.huggingface_hub`
563
-
564
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
565
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
566
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
567
-
568
- temp_dir_root : str, default: None
569
- The root directory under which `current.model.loaded` will store loaded models
570
-
571
-
165
+ Specifies the Conda environment for the step.
572
166
 
167
+ Information in this decorator will augment any
168
+ attributes set in the `@conda_base` flow-level decorator. Hence,
169
+ you can use `@conda_base` to set packages required by all
170
+ steps and use `@conda` to specify step-specific overrides.
573
171
  """
574
172
  ...
575
173
 
576
174
  @typing.overload
577
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
175
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
578
176
  """
579
- Specifies a timeout for your step.
580
-
581
- This decorator is useful if this step may hang indefinitely.
582
-
583
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
584
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
585
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
586
-
587
- Note that all the values specified in parameters are added together so if you specify
588
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
177
+ Specifies the PyPI packages for the step.
589
178
 
590
- Parameters
591
- ----------
592
- seconds : int, default 0
593
- Number of seconds to wait prior to timing out.
594
- minutes : int, default 0
595
- Number of minutes to wait prior to timing out.
596
- hours : int, default 0
597
- Number of hours to wait prior to timing out.
179
+ Information in this decorator will augment any
180
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
181
+ you can use `@pypi_base` to set packages required by all
182
+ steps and use `@pypi` to specify step-specific overrides.
598
183
  """
599
184
  ...
600
185
 
601
186
  @typing.overload
602
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
187
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
603
188
  ...
604
189
 
605
190
  @typing.overload
606
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
191
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
607
192
  ...
608
193
 
609
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
194
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
610
195
  """
611
- Specifies a timeout for your step.
612
-
613
- This decorator is useful if this step may hang indefinitely.
614
-
615
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
616
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
617
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
618
-
619
- Note that all the values specified in parameters are added together so if you specify
620
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
196
+ Specifies the PyPI packages for the step.
621
197
 
622
- Parameters
623
- ----------
624
- seconds : int, default 0
625
- Number of seconds to wait prior to timing out.
626
- minutes : int, default 0
627
- Number of minutes to wait prior to timing out.
628
- hours : int, default 0
629
- Number of hours to wait prior to timing out.
198
+ Information in this decorator will augment any
199
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
200
+ you can use `@pypi_base` to set packages required by all
201
+ steps and use `@pypi` to specify step-specific overrides.
202
+ """
203
+ ...
204
+
205
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
206
+ """
207
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
630
208
  """
631
209
  ...
632
210
 
633
211
  @typing.overload
634
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
212
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
635
213
  """
636
- Specifies secrets to be retrieved and injected as environment variables prior to
637
- the execution of a step.
214
+ Decorator prototype for all step decorators. This function gets specialized
215
+ and imported for all decorators types by _import_plugin_decorators().
216
+ """
217
+ ...
218
+
219
+ @typing.overload
220
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
221
+ ...
222
+
223
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
224
+ """
225
+ Decorator prototype for all step decorators. This function gets specialized
226
+ and imported for all decorators types by _import_plugin_decorators().
227
+ """
228
+ ...
229
+
230
+ @typing.overload
231
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
232
+ """
233
+ Specifies that the step will success under all circumstances.
638
234
 
639
- Parameters
640
- ----------
641
- sources : List[Union[str, Dict[str, Any]]], default: []
642
- List of secret specs, defining how the secrets are to be retrieved
235
+ The decorator will create an optional artifact, specified by `var`, which
236
+ contains the exception raised. You can use it to detect the presence
237
+ of errors, indicating that all happy-path artifacts produced by the step
238
+ are missing.
643
239
  """
644
240
  ...
645
241
 
646
242
  @typing.overload
647
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
243
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
648
244
  ...
649
245
 
650
246
  @typing.overload
651
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
247
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
652
248
  ...
653
249
 
654
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
250
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
655
251
  """
656
- Specifies secrets to be retrieved and injected as environment variables prior to
657
- the execution of a step.
252
+ Specifies that the step will success under all circumstances.
658
253
 
659
- Parameters
660
- ----------
661
- sources : List[Union[str, Dict[str, Any]]], default: []
662
- List of secret specs, defining how the secrets are to be retrieved
254
+ The decorator will create an optional artifact, specified by `var`, which
255
+ contains the exception raised. You can use it to detect the presence
256
+ of errors, indicating that all happy-path artifacts produced by the step
257
+ are missing.
663
258
  """
664
259
  ...
665
260
 
666
261
  @typing.overload
667
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
262
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
668
263
  """
669
- Creates a human-readable report, a Metaflow Card, after this step completes.
670
-
671
- Note that you may add multiple `@card` decorators in a step with different parameters.
672
-
673
- Parameters
674
- ----------
675
- type : str, default 'default'
676
- Card type.
677
- id : str, optional, default None
678
- If multiple cards are present, use this id to identify this card.
679
- options : Dict[str, Any], default {}
680
- Options passed to the card. The contents depend on the card type.
681
- timeout : int, default 45
682
- Interrupt reporting if it takes more than this many seconds.
264
+ Specifies the resources needed when executing this step.
683
265
 
266
+ Use `@resources` to specify the resource requirements
267
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
684
268
 
269
+ You can choose the compute layer on the command line by executing e.g.
270
+ ```
271
+ python myflow.py run --with batch
272
+ ```
273
+ or
274
+ ```
275
+ python myflow.py run --with kubernetes
276
+ ```
277
+ which executes the flow on the desired system using the
278
+ requirements specified in `@resources`.
685
279
  """
686
280
  ...
687
281
 
688
282
  @typing.overload
689
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
283
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
690
284
  ...
691
285
 
692
286
  @typing.overload
693
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
287
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
694
288
  ...
695
289
 
696
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
290
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
697
291
  """
698
- Creates a human-readable report, a Metaflow Card, after this step completes.
699
-
700
- Note that you may add multiple `@card` decorators in a step with different parameters.
701
-
702
- Parameters
703
- ----------
704
- type : str, default 'default'
705
- Card type.
706
- id : str, optional, default None
707
- If multiple cards are present, use this id to identify this card.
708
- options : Dict[str, Any], default {}
709
- Options passed to the card. The contents depend on the card type.
710
- timeout : int, default 45
711
- Interrupt reporting if it takes more than this many seconds.
292
+ Specifies the resources needed when executing this step.
712
293
 
294
+ Use `@resources` to specify the resource requirements
295
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
713
296
 
297
+ You can choose the compute layer on the command line by executing e.g.
298
+ ```
299
+ python myflow.py run --with batch
300
+ ```
301
+ or
302
+ ```
303
+ python myflow.py run --with kubernetes
304
+ ```
305
+ which executes the flow on the desired system using the
306
+ requirements specified in `@resources`.
307
+ """
308
+ ...
309
+
310
+ @typing.overload
311
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
312
+ """
313
+ Specifies environment variables to be set prior to the execution of a step.
314
+ """
315
+ ...
316
+
317
+ @typing.overload
318
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
319
+ ...
320
+
321
+ @typing.overload
322
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
323
+ ...
324
+
325
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
326
+ """
327
+ Specifies environment variables to be set prior to the execution of a step.
714
328
  """
715
329
  ...
716
330
 
@@ -727,13 +341,6 @@ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callabl
727
341
  This can be used in conjunction with the `@catch` decorator. The `@catch`
728
342
  decorator will execute a no-op task after all retries have been exhausted,
729
343
  ensuring that the flow execution can continue.
730
-
731
- Parameters
732
- ----------
733
- times : int, default 3
734
- Number of times to retry this task.
735
- minutes_between_retries : int, default 2
736
- Number of minutes between retries.
737
344
  """
738
345
  ...
739
346
 
@@ -757,32 +364,50 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
757
364
  This can be used in conjunction with the `@catch` decorator. The `@catch`
758
365
  decorator will execute a no-op task after all retries have been exhausted,
759
366
  ensuring that the flow execution can continue.
760
-
761
- Parameters
762
- ----------
763
- times : int, default 3
764
- Number of times to retry this task.
765
- minutes_between_retries : int, default 2
766
- Number of minutes between retries.
767
367
  """
768
368
  ...
769
369
 
770
370
  @typing.overload
771
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
371
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
772
372
  """
773
- Decorator prototype for all step decorators. This function gets specialized
774
- and imported for all decorators types by _import_plugin_decorators().
373
+ Specifies secrets to be retrieved and injected as environment variables prior to
374
+ the execution of a step.
775
375
  """
776
376
  ...
777
377
 
778
378
  @typing.overload
779
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
379
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
780
380
  ...
781
381
 
782
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
382
+ @typing.overload
383
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
384
+ ...
385
+
386
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
783
387
  """
784
- Decorator prototype for all step decorators. This function gets specialized
785
- and imported for all decorators types by _import_plugin_decorators().
388
+ Specifies secrets to be retrieved and injected as environment variables prior to
389
+ the execution of a step.
390
+ """
391
+ ...
392
+
393
+ @typing.overload
394
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
395
+ """
396
+ Enables loading / saving of models within a step.
397
+ """
398
+ ...
399
+
400
+ @typing.overload
401
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
402
+ ...
403
+
404
+ @typing.overload
405
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
406
+ ...
407
+
408
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
409
+ """
410
+ Enables loading / saving of models within a step.
786
411
  """
787
412
  ...
788
413
 
@@ -803,30 +428,16 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
803
428
  """
804
429
  ...
805
430
 
431
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
432
+ """
433
+ Specifies that this step should execute on Kubernetes.
434
+ """
435
+ ...
436
+
806
437
  @typing.overload
807
- def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
438
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
808
439
  """
809
440
  Enables checkpointing for a step.
810
-
811
-
812
- Parameters
813
- ----------
814
- load_policy : str, default: "fresh"
815
- The policy for loading the checkpoint. The following policies are supported:
816
- - "eager": Loads the the latest available checkpoint within the namespace.
817
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
818
- will be loaded at the start of the task.
819
- - "none": Do not load any checkpoint
820
- - "fresh": Loads the lastest checkpoint created within the running Task.
821
- This mode helps loading checkpoints across various retry attempts of the same task.
822
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
823
- created within the task will be loaded when the task is retries execution on failure.
824
-
825
- temp_dir_root : str, default: None
826
- The root directory under which `current.checkpoint.directory` will be created.
827
-
828
-
829
-
830
441
  """
831
442
  ...
832
443
 
@@ -838,384 +449,185 @@ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.
838
449
  def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
839
450
  ...
840
451
 
841
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
452
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
842
453
  """
843
454
  Enables checkpointing for a step.
455
+ """
456
+ ...
457
+
458
+ @typing.overload
459
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
460
+ """
461
+ Specifies a timeout for your step.
844
462
 
463
+ This decorator is useful if this step may hang indefinitely.
845
464
 
846
- Parameters
847
- ----------
848
- load_policy : str, default: "fresh"
849
- The policy for loading the checkpoint. The following policies are supported:
850
- - "eager": Loads the the latest available checkpoint within the namespace.
851
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
852
- will be loaded at the start of the task.
853
- - "none": Do not load any checkpoint
854
- - "fresh": Loads the lastest checkpoint created within the running Task.
855
- This mode helps loading checkpoints across various retry attempts of the same task.
856
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
857
- created within the task will be loaded when the task is retries execution on failure.
465
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
466
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
467
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
858
468
 
859
- temp_dir_root : str, default: None
860
- The root directory under which `current.checkpoint.directory` will be created.
469
+ Note that all the values specified in parameters are added together so if you specify
470
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
471
+ """
472
+ ...
473
+
474
+ @typing.overload
475
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
476
+ ...
477
+
478
+ @typing.overload
479
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
480
+ ...
481
+
482
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
483
+ """
484
+ Specifies a timeout for your step.
861
485
 
486
+ This decorator is useful if this step may hang indefinitely.
862
487
 
488
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
489
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
490
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
863
491
 
492
+ Note that all the values specified in parameters are added together so if you specify
493
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
864
494
  """
865
495
  ...
866
496
 
867
497
  @typing.overload
868
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
498
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
869
499
  """
870
- Specifies the Conda environment for the step.
871
-
872
- Information in this decorator will augment any
873
- attributes set in the `@conda_base` flow-level decorator. Hence,
874
- you can use `@conda_base` to set packages required by all
875
- steps and use `@conda` to specify step-specific overrides.
500
+ Creates a human-readable report, a Metaflow Card, after this step completes.
876
501
 
877
- Parameters
878
- ----------
879
- packages : Dict[str, str], default {}
880
- Packages to use for this step. The key is the name of the package
881
- and the value is the version to use.
882
- libraries : Dict[str, str], default {}
883
- Supported for backward compatibility. When used with packages, packages will take precedence.
884
- python : str, optional, default None
885
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
886
- that the version used will correspond to the version of the Python interpreter used to start the run.
887
- disabled : bool, default False
888
- If set to True, disables @conda.
502
+ Note that you may add multiple `@card` decorators in a step with different parameters.
889
503
  """
890
504
  ...
891
505
 
892
506
  @typing.overload
893
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
507
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
894
508
  ...
895
509
 
896
510
  @typing.overload
897
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
511
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
898
512
  ...
899
513
 
900
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
514
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
901
515
  """
902
- Specifies the Conda environment for the step.
903
-
904
- Information in this decorator will augment any
905
- attributes set in the `@conda_base` flow-level decorator. Hence,
906
- you can use `@conda_base` to set packages required by all
907
- steps and use `@conda` to specify step-specific overrides.
516
+ Creates a human-readable report, a Metaflow Card, after this step completes.
908
517
 
909
- Parameters
910
- ----------
911
- packages : Dict[str, str], default {}
912
- Packages to use for this step. The key is the name of the package
913
- and the value is the version to use.
914
- libraries : Dict[str, str], default {}
915
- Supported for backward compatibility. When used with packages, packages will take precedence.
916
- python : str, optional, default None
917
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
918
- that the version used will correspond to the version of the Python interpreter used to start the run.
919
- disabled : bool, default False
920
- If set to True, disables @conda.
518
+ Note that you may add multiple `@card` decorators in a step with different parameters.
921
519
  """
922
520
  ...
923
521
 
924
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
522
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
925
523
  """
926
- Decorator that helps cache, version and store models/datasets from huggingface hub.
927
-
928
- Parameters
929
- ----------
930
- temp_dir_root : str, optional
931
- The root directory that will hold the temporary directory where objects will be downloaded.
524
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
932
525
 
933
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
934
- The list of models to load.
526
+ User code call
527
+ -----------
528
+ @nim(
529
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
530
+ backend='managed'
531
+ )
935
532
 
533
+ Valid backend options
534
+ ---------------------
535
+ - 'managed': Outerbounds selects a compute provider based on the model.
536
+ - 🚧 'dataplane': Run in your account.
936
537
 
538
+ Valid model options
539
+ ----------------
540
+ - 'meta/llama3-8b-instruct': 8B parameter model
541
+ - 'meta/llama3-70b-instruct': 70B parameter model
542
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
937
543
  """
938
544
  ...
939
545
 
940
546
  @typing.overload
941
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
547
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
942
548
  """
943
- Specifies the resources needed when executing this step.
944
-
945
- Use `@resources` to specify the resource requirements
946
- independently of the specific compute layer (`@batch`, `@kubernetes`).
947
-
948
- You can choose the compute layer on the command line by executing e.g.
949
- ```
950
- python myflow.py run --with batch
951
- ```
952
- or
953
- ```
954
- python myflow.py run --with kubernetes
955
- ```
956
- which executes the flow on the desired system using the
957
- requirements specified in `@resources`.
549
+ Specifies the Conda environment for all steps of the flow.
958
550
 
959
- Parameters
960
- ----------
961
- cpu : int, default 1
962
- Number of CPUs required for this step.
963
- gpu : int, optional, default None
964
- Number of GPUs required for this step.
965
- disk : int, optional, default None
966
- Disk size (in MB) required for this step. Only applies on Kubernetes.
967
- memory : int, default 4096
968
- Memory size (in MB) required for this step.
969
- shared_memory : int, optional, default None
970
- The value for the size (in MiB) of the /dev/shm volume for this step.
971
- This parameter maps to the `--shm-size` option in Docker.
551
+ Use `@conda_base` to set common libraries required by all
552
+ steps and use `@conda` to specify step-specific additions.
972
553
  """
973
554
  ...
974
555
 
975
556
  @typing.overload
976
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
977
- ...
978
-
979
- @typing.overload
980
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
557
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
981
558
  ...
982
559
 
983
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
560
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
984
561
  """
985
- Specifies the resources needed when executing this step.
986
-
987
- Use `@resources` to specify the resource requirements
988
- independently of the specific compute layer (`@batch`, `@kubernetes`).
989
-
990
- You can choose the compute layer on the command line by executing e.g.
991
- ```
992
- python myflow.py run --with batch
993
- ```
994
- or
995
- ```
996
- python myflow.py run --with kubernetes
997
- ```
998
- which executes the flow on the desired system using the
999
- requirements specified in `@resources`.
562
+ Specifies the Conda environment for all steps of the flow.
1000
563
 
1001
- Parameters
1002
- ----------
1003
- cpu : int, default 1
1004
- Number of CPUs required for this step.
1005
- gpu : int, optional, default None
1006
- Number of GPUs required for this step.
1007
- disk : int, optional, default None
1008
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1009
- memory : int, default 4096
1010
- Memory size (in MB) required for this step.
1011
- shared_memory : int, optional, default None
1012
- The value for the size (in MiB) of the /dev/shm volume for this step.
1013
- This parameter maps to the `--shm-size` option in Docker.
564
+ Use `@conda_base` to set common libraries required by all
565
+ steps and use `@conda` to specify step-specific additions.
1014
566
  """
1015
567
  ...
1016
568
 
1017
569
  @typing.overload
1018
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
570
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1019
571
  """
1020
- Specifies environment variables to be set prior to the execution of a step.
1021
-
1022
- Parameters
1023
- ----------
1024
- vars : Dict[str, str], default {}
1025
- Dictionary of environment variables to set.
572
+ Specifies the times when the flow should be run when running on a
573
+ production scheduler.
1026
574
  """
1027
575
  ...
1028
576
 
1029
577
  @typing.overload
1030
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
578
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1031
579
  ...
1032
580
 
1033
- @typing.overload
1034
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
581
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
582
+ """
583
+ Specifies the times when the flow should be run when running on a
584
+ production scheduler.
585
+ """
1035
586
  ...
1036
587
 
1037
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
588
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1038
589
  """
1039
- Specifies environment variables to be set prior to the execution of a step.
590
+ Specifies what flows belong to the same project.
1040
591
 
1041
- Parameters
1042
- ----------
1043
- vars : Dict[str, str], default {}
1044
- Dictionary of environment variables to set.
592
+ A project-specific namespace is created for all flows that
593
+ use the same `@project(name)`.
1045
594
  """
1046
595
  ...
1047
596
 
1048
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
597
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1049
598
  """
1050
- Specifies that this step should execute on Kubernetes.
1051
-
1052
- Parameters
1053
- ----------
1054
- cpu : int, default 1
1055
- Number of CPUs required for this step. If `@resources` is
1056
- also present, the maximum value from all decorators is used.
1057
- memory : int, default 4096
1058
- Memory size (in MB) required for this step. If
1059
- `@resources` is also present, the maximum value from all decorators is
1060
- used.
1061
- disk : int, default 10240
1062
- Disk size (in MB) required for this step. If
1063
- `@resources` is also present, the maximum value from all decorators is
1064
- used.
1065
- image : str, optional, default None
1066
- Docker image to use when launching on Kubernetes. If not specified, and
1067
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1068
- not, a default Docker image mapping to the current version of Python is used.
1069
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1070
- If given, the imagePullPolicy to be applied to the Docker image of the step.
1071
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1072
- Kubernetes service account to use when launching pod in Kubernetes.
1073
- secrets : List[str], optional, default None
1074
- Kubernetes secrets to use when launching pod in Kubernetes. These
1075
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1076
- in Metaflow configuration.
1077
- node_selector: Union[Dict[str,str], str], optional, default None
1078
- Kubernetes node selector(s) to apply to the pod running the task.
1079
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
1080
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
1081
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1082
- Kubernetes namespace to use when launching pod in Kubernetes.
1083
- gpu : int, optional, default None
1084
- Number of GPUs required for this step. A value of zero implies that
1085
- the scheduled node should not have GPUs.
1086
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1087
- The vendor of the GPUs to be used for this step.
1088
- tolerations : List[str], default []
1089
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1090
- Kubernetes tolerations to use when launching pod in Kubernetes.
1091
- use_tmpfs : bool, default False
1092
- This enables an explicit tmpfs mount for this step.
1093
- tmpfs_tempdir : bool, default True
1094
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1095
- tmpfs_size : int, optional, default: None
1096
- The value for the size (in MiB) of the tmpfs mount for this step.
1097
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1098
- memory allocated for this step.
1099
- tmpfs_path : str, optional, default /metaflow_temp
1100
- Path to tmpfs mount for this step.
1101
- persistent_volume_claims : Dict[str, str], optional, default None
1102
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1103
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1104
- shared_memory: int, optional
1105
- Shared memory size (in MiB) required for this step
1106
- port: int, optional
1107
- Port number to specify in the Kubernetes job object
1108
- compute_pool : str, optional, default None
1109
- Compute pool to be used for for this step.
1110
- If not specified, any accessible compute pool within the perimeter is used.
1111
- hostname_resolution_timeout: int, default 10 * 60
1112
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1113
- Only applicable when @parallel is used.
599
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
600
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1114
601
  """
1115
602
  ...
1116
603
 
1117
604
  @typing.overload
1118
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
605
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1119
606
  """
1120
- Specifies that the step will success under all circumstances.
1121
-
1122
- The decorator will create an optional artifact, specified by `var`, which
1123
- contains the exception raised. You can use it to detect the presence
1124
- of errors, indicating that all happy-path artifacts produced by the step
1125
- are missing.
607
+ Specifies the PyPI packages for all steps of the flow.
1126
608
 
1127
- Parameters
1128
- ----------
1129
- var : str, optional, default None
1130
- Name of the artifact in which to store the caught exception.
1131
- If not specified, the exception is not stored.
1132
- print_exception : bool, default True
1133
- Determines whether or not the exception is printed to
1134
- stdout when caught.
609
+ Use `@pypi_base` to set common packages required by all
610
+ steps and use `@pypi` to specify step-specific overrides.
1135
611
  """
1136
612
  ...
1137
613
 
1138
614
  @typing.overload
1139
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1140
- ...
1141
-
1142
- @typing.overload
1143
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
615
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1144
616
  ...
1145
617
 
1146
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
618
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1147
619
  """
1148
- Specifies that the step will success under all circumstances.
1149
-
1150
- The decorator will create an optional artifact, specified by `var`, which
1151
- contains the exception raised. You can use it to detect the presence
1152
- of errors, indicating that all happy-path artifacts produced by the step
1153
- are missing.
620
+ Specifies the PyPI packages for all steps of the flow.
1154
621
 
1155
- Parameters
1156
- ----------
1157
- var : str, optional, default None
1158
- Name of the artifact in which to store the caught exception.
1159
- If not specified, the exception is not stored.
1160
- print_exception : bool, default True
1161
- Determines whether or not the exception is printed to
1162
- stdout when caught.
622
+ Use `@pypi_base` to set common packages required by all
623
+ steps and use `@pypi` to specify step-specific overrides.
1163
624
  """
1164
625
  ...
1165
626
 
1166
627
  @typing.overload
1167
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
628
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1168
629
  """
1169
- Specifies the PyPI packages for the step.
1170
-
1171
- Information in this decorator will augment any
1172
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1173
- you can use `@pypi_base` to set packages required by all
1174
- steps and use `@pypi` to specify step-specific overrides.
1175
-
1176
- Parameters
1177
- ----------
1178
- packages : Dict[str, str], default: {}
1179
- Packages to use for this step. The key is the name of the package
1180
- and the value is the version to use.
1181
- python : str, optional, default: None
1182
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1183
- that the version used will correspond to the version of the Python interpreter used to start the run.
1184
- """
1185
- ...
1186
-
1187
- @typing.overload
1188
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1189
- ...
1190
-
1191
- @typing.overload
1192
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1193
- ...
1194
-
1195
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1196
- """
1197
- Specifies the PyPI packages for the step.
1198
-
1199
- Information in this decorator will augment any
1200
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1201
- you can use `@pypi_base` to set packages required by all
1202
- steps and use `@pypi` to specify step-specific overrides.
1203
-
1204
- Parameters
1205
- ----------
1206
- packages : Dict[str, str], default: {}
1207
- Packages to use for this step. The key is the name of the package
1208
- and the value is the version to use.
1209
- python : str, optional, default: None
1210
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1211
- that the version used will correspond to the version of the Python interpreter used to start the run.
1212
- """
1213
- ...
1214
-
1215
- @typing.overload
1216
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1217
- """
1218
- Specifies the flow(s) that this flow depends on.
630
+ Specifies the flow(s) that this flow depends on.
1219
631
 
1220
632
  ```
1221
633
  @trigger_on_finish(flow='FooFlow')
@@ -1248,17 +660,6 @@ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] =
1248
660
  - `user.bob`
1249
661
  - `test.my_experiment`
1250
662
  - `prod.staging`
1251
-
1252
- Parameters
1253
- ----------
1254
- flow : Union[str, Dict[str, str]], optional, default None
1255
- Upstream flow dependency for this flow.
1256
- flows : List[Union[str, Dict[str, str]]], default []
1257
- Upstream flow dependencies for this flow.
1258
- options : Dict[str, Any], default {}
1259
- Backend-specific configuration for tuning eventing behavior.
1260
-
1261
-
1262
663
  """
1263
664
  ...
1264
665
 
@@ -1301,17 +702,16 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1301
702
  - `user.bob`
1302
703
  - `test.my_experiment`
1303
704
  - `prod.staging`
1304
-
1305
- Parameters
1306
- ----------
1307
- flow : Union[str, Dict[str, str]], optional, default None
1308
- Upstream flow dependency for this flow.
1309
- flows : List[Union[str, Dict[str, str]]], default []
1310
- Upstream flow dependencies for this flow.
1311
- options : Dict[str, Any], default {}
1312
- Backend-specific configuration for tuning eventing behavior.
1313
-
1314
-
705
+ """
706
+ ...
707
+
708
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
709
+ """
710
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
711
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
712
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
713
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
714
+ starts only after all sensors finish.
1315
715
  """
1316
716
  ...
1317
717
 
@@ -1347,17 +747,6 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1347
747
  ```
1348
748
  @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1349
749
  ```
1350
-
1351
- Parameters
1352
- ----------
1353
- event : Union[str, Dict[str, Any]], optional, default None
1354
- Event dependency for this flow.
1355
- events : List[Union[str, Dict[str, Any]]], default []
1356
- Events dependency for this flow.
1357
- options : Dict[str, Any], default {}
1358
- Backend-specific configuration for tuning eventing behavior.
1359
-
1360
-
1361
750
  """
1362
751
  ...
1363
752
 
@@ -1396,1817 +785,8 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1396
785
  ```
1397
786
  @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1398
787
  ```
1399
-
1400
- Parameters
1401
- ----------
1402
- event : Union[str, Dict[str, Any]], optional, default None
1403
- Event dependency for this flow.
1404
- events : List[Union[str, Dict[str, Any]]], default []
1405
- Events dependency for this flow.
1406
- options : Dict[str, Any], default {}
1407
- Backend-specific configuration for tuning eventing behavior.
1408
-
1409
-
1410
- """
1411
- ...
1412
-
1413
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1414
- """
1415
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1416
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1417
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1418
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1419
- starts only after all sensors finish.
1420
-
1421
- Parameters
1422
- ----------
1423
- timeout : int
1424
- Time, in seconds before the task times out and fails. (Default: 3600)
1425
- poke_interval : int
1426
- Time in seconds that the job should wait in between each try. (Default: 60)
1427
- mode : str
1428
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1429
- exponential_backoff : bool
1430
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1431
- pool : str
1432
- the slot pool this task should run in,
1433
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1434
- soft_fail : bool
1435
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1436
- name : str
1437
- Name of the sensor on Airflow
1438
- description : str
1439
- Description of sensor in the Airflow UI
1440
- bucket_key : Union[str, List[str]]
1441
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1442
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1443
- bucket_name : str
1444
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1445
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1446
- wildcard_match : bool
1447
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1448
- aws_conn_id : str
1449
- a reference to the s3 connection on Airflow. (Default: None)
1450
- verify : bool
1451
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1452
788
  """
1453
789
  ...
1454
790
 
1455
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1456
- """
1457
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1458
-
1459
- User code call
1460
- -----------
1461
- @nim(
1462
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1463
- backend='managed'
1464
- )
1465
-
1466
- Valid backend options
1467
- ---------------------
1468
- - 'managed': Outerbounds selects a compute provider based on the model.
1469
- - 🚧 'dataplane': Run in your account.
1470
-
1471
- Valid model options
1472
- ----------------
1473
- - 'meta/llama3-8b-instruct': 8B parameter model
1474
- - 'meta/llama3-70b-instruct': 70B parameter model
1475
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1476
-
1477
- Parameters
1478
- ----------
1479
- models: list[NIM]
1480
- List of NIM containers running models in sidecars.
1481
- backend: str
1482
- Compute provider to run the NIM container.
1483
- """
1484
- ...
1485
-
1486
- @typing.overload
1487
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1488
- """
1489
- Specifies the Conda environment for all steps of the flow.
1490
-
1491
- Use `@conda_base` to set common libraries required by all
1492
- steps and use `@conda` to specify step-specific additions.
1493
-
1494
- Parameters
1495
- ----------
1496
- packages : Dict[str, str], default {}
1497
- Packages to use for this flow. The key is the name of the package
1498
- and the value is the version to use.
1499
- libraries : Dict[str, str], default {}
1500
- Supported for backward compatibility. When used with packages, packages will take precedence.
1501
- python : str, optional, default None
1502
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1503
- that the version used will correspond to the version of the Python interpreter used to start the run.
1504
- disabled : bool, default False
1505
- If set to True, disables Conda.
1506
- """
1507
- ...
1508
-
1509
- @typing.overload
1510
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1511
- ...
1512
-
1513
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1514
- """
1515
- Specifies the Conda environment for all steps of the flow.
1516
-
1517
- Use `@conda_base` to set common libraries required by all
1518
- steps and use `@conda` to specify step-specific additions.
1519
-
1520
- Parameters
1521
- ----------
1522
- packages : Dict[str, str], default {}
1523
- Packages to use for this flow. The key is the name of the package
1524
- and the value is the version to use.
1525
- libraries : Dict[str, str], default {}
1526
- Supported for backward compatibility. When used with packages, packages will take precedence.
1527
- python : str, optional, default None
1528
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1529
- that the version used will correspond to the version of the Python interpreter used to start the run.
1530
- disabled : bool, default False
1531
- If set to True, disables Conda.
1532
- """
1533
- ...
1534
-
1535
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1536
- """
1537
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1538
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1539
-
1540
- Parameters
1541
- ----------
1542
- timeout : int
1543
- Time, in seconds before the task times out and fails. (Default: 3600)
1544
- poke_interval : int
1545
- Time in seconds that the job should wait in between each try. (Default: 60)
1546
- mode : str
1547
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1548
- exponential_backoff : bool
1549
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1550
- pool : str
1551
- the slot pool this task should run in,
1552
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1553
- soft_fail : bool
1554
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1555
- name : str
1556
- Name of the sensor on Airflow
1557
- description : str
1558
- Description of sensor in the Airflow UI
1559
- external_dag_id : str
1560
- The dag_id that contains the task you want to wait for.
1561
- external_task_ids : List[str]
1562
- The list of task_ids that you want to wait for.
1563
- If None (default value) the sensor waits for the DAG. (Default: None)
1564
- allowed_states : List[str]
1565
- Iterable of allowed states, (Default: ['success'])
1566
- failed_states : List[str]
1567
- Iterable of failed or dis-allowed states. (Default: None)
1568
- execution_delta : datetime.timedelta
1569
- time difference with the previous execution to look at,
1570
- the default is the same logical date as the current task or DAG. (Default: None)
1571
- check_existence: bool
1572
- Set to True to check if the external task exists or check if
1573
- the DAG to wait for exists. (Default: True)
1574
- """
1575
- ...
1576
-
1577
- @typing.overload
1578
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1579
- """
1580
- Specifies the PyPI packages for all steps of the flow.
1581
-
1582
- Use `@pypi_base` to set common packages required by all
1583
- steps and use `@pypi` to specify step-specific overrides.
1584
- Parameters
1585
- ----------
1586
- packages : Dict[str, str], default: {}
1587
- Packages to use for this flow. The key is the name of the package
1588
- and the value is the version to use.
1589
- python : str, optional, default: None
1590
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1591
- that the version used will correspond to the version of the Python interpreter used to start the run.
1592
- """
1593
- ...
1594
-
1595
- @typing.overload
1596
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1597
- ...
1598
-
1599
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1600
- """
1601
- Specifies the PyPI packages for all steps of the flow.
1602
-
1603
- Use `@pypi_base` to set common packages required by all
1604
- steps and use `@pypi` to specify step-specific overrides.
1605
- Parameters
1606
- ----------
1607
- packages : Dict[str, str], default: {}
1608
- Packages to use for this flow. The key is the name of the package
1609
- and the value is the version to use.
1610
- python : str, optional, default: None
1611
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1612
- that the version used will correspond to the version of the Python interpreter used to start the run.
1613
- """
1614
- ...
1615
-
1616
- @typing.overload
1617
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1618
- """
1619
- Specifies the times when the flow should be run when running on a
1620
- production scheduler.
1621
-
1622
- Parameters
1623
- ----------
1624
- hourly : bool, default False
1625
- Run the workflow hourly.
1626
- daily : bool, default True
1627
- Run the workflow daily.
1628
- weekly : bool, default False
1629
- Run the workflow weekly.
1630
- cron : str, optional, default None
1631
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1632
- specified by this expression.
1633
- timezone : str, optional, default None
1634
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1635
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1636
- """
1637
- ...
1638
-
1639
- @typing.overload
1640
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1641
- ...
1642
-
1643
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1644
- """
1645
- Specifies the times when the flow should be run when running on a
1646
- production scheduler.
1647
-
1648
- Parameters
1649
- ----------
1650
- hourly : bool, default False
1651
- Run the workflow hourly.
1652
- daily : bool, default True
1653
- Run the workflow daily.
1654
- weekly : bool, default False
1655
- Run the workflow weekly.
1656
- cron : str, optional, default None
1657
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1658
- specified by this expression.
1659
- timezone : str, optional, default None
1660
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1661
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1662
- """
1663
- ...
1664
-
1665
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1666
- """
1667
- Specifies what flows belong to the same project.
1668
-
1669
- A project-specific namespace is created for all flows that
1670
- use the same `@project(name)`.
1671
-
1672
- Parameters
1673
- ----------
1674
- name : str
1675
- Project name. Make sure that the name is unique amongst all
1676
- projects that use the same production scheduler. The name may
1677
- contain only lowercase alphanumeric characters and underscores.
1678
-
1679
-
1680
- """
1681
- ...
1682
-
1683
- def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1684
- """
1685
- Switch namespace to the one provided.
1686
-
1687
- This call has a global effect. No objects outside this namespace
1688
- will be accessible. To access all objects regardless of namespaces,
1689
- pass None to this call.
1690
-
1691
- Parameters
1692
- ----------
1693
- ns : str, optional
1694
- Namespace to switch to or None to ignore namespaces.
1695
-
1696
- Returns
1697
- -------
1698
- str, optional
1699
- Namespace set (result of get_namespace()).
1700
- """
1701
- ...
1702
-
1703
- def get_namespace() -> typing.Optional[str]:
1704
- """
1705
- Return the current namespace that is currently being used to filter objects.
1706
-
1707
- The namespace is a tag associated with all objects in Metaflow.
1708
-
1709
- Returns
1710
- -------
1711
- str, optional
1712
- The current namespace used to filter objects.
1713
- """
1714
- ...
1715
-
1716
- def default_namespace() -> str:
1717
- """
1718
- Resets the namespace used to filter objects to the default one, i.e. the one that was
1719
- used prior to any `namespace` calls.
1720
-
1721
- Returns
1722
- -------
1723
- str
1724
- The result of get_namespace() after the namespace has been reset.
1725
- """
1726
- ...
1727
-
1728
- def get_metadata() -> str:
1729
- """
1730
- Returns the current Metadata provider.
1731
-
1732
- If this is not set explicitly using `metadata`, the default value is
1733
- determined through the Metaflow configuration. You can use this call to
1734
- check that your configuration is set up properly.
1735
-
1736
- If multiple configuration profiles are present, this call returns the one
1737
- selected through the `METAFLOW_PROFILE` environment variable.
1738
-
1739
- Returns
1740
- -------
1741
- str
1742
- Information about the Metadata provider currently selected. This information typically
1743
- returns provider specific information (like URL for remote providers or local paths for
1744
- local providers).
1745
- """
1746
- ...
1747
-
1748
- def default_metadata() -> str:
1749
- """
1750
- Resets the Metadata provider to the default value, that is, to the value
1751
- that was used prior to any `metadata` calls.
1752
-
1753
- Returns
1754
- -------
1755
- str
1756
- The result of get_metadata() after resetting the provider.
1757
- """
1758
- ...
1759
-
1760
- class Metaflow(object, metaclass=type):
1761
- """
1762
- Entry point to all objects in the Metaflow universe.
1763
-
1764
- This object can be used to list all the flows present either through the explicit property
1765
- or by iterating over this object.
1766
-
1767
- Attributes
1768
- ----------
1769
- flows : List[Flow]
1770
- Returns the list of all `Flow` objects known to this metadata provider. Note that only
1771
- flows present in the current namespace will be returned. A `Flow` is present in a namespace
1772
- if it has at least one run in the namespace.
1773
- """
1774
- def __init__(self, _current_metadata: typing.Optional[str] = None):
1775
- ...
1776
- @property
1777
- def flows(self) -> typing.List[metaflow.client.core.Flow]:
1778
- """
1779
- Returns a list of all the flows present.
1780
-
1781
- Only flows present in the set namespace are returned. A flow is present in a namespace if
1782
- it has at least one run that is in the namespace.
1783
-
1784
- Returns
1785
- -------
1786
- List[Flow]
1787
- List of all flows present.
1788
- """
1789
- ...
1790
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Flow]:
1791
- """
1792
- Iterator over all flows present.
1793
-
1794
- Only flows present in the set namespace are returned. A flow is present in a
1795
- namespace if it has at least one run that is in the namespace.
1796
-
1797
- Yields
1798
- -------
1799
- Flow
1800
- A Flow present in the Metaflow universe.
1801
- """
1802
- ...
1803
- def __str__(self) -> str:
1804
- ...
1805
- def __getitem__(self, name: str) -> metaflow.client.core.Flow:
1806
- """
1807
- Returns a specific flow by name.
1808
-
1809
- The flow will only be returned if it is present in the current namespace.
1810
-
1811
- Parameters
1812
- ----------
1813
- name : str
1814
- Name of the Flow
1815
-
1816
- Returns
1817
- -------
1818
- Flow
1819
- Flow with the given name.
1820
- """
1821
- ...
1822
- ...
1823
-
1824
- class Flow(metaflow.client.core.MetaflowObject, metaclass=type):
1825
- """
1826
- A Flow represents all existing flows with a certain name, in other words,
1827
- classes derived from `FlowSpec`. A container of `Run` objects.
1828
-
1829
- Attributes
1830
- ----------
1831
- latest_run : Run
1832
- Latest `Run` (in progress or completed, successfully or not) of this flow.
1833
- latest_successful_run : Run
1834
- Latest successfully completed `Run` of this flow.
1835
- """
1836
- def __init__(self, *args, **kwargs):
1837
- ...
1838
- @property
1839
- def latest_run(self) -> typing.Optional[metaflow.client.core.Run]:
1840
- """
1841
- Returns the latest run (either in progress or completed) of this flow.
1842
-
1843
- Note that an in-progress run may be returned by this call. Use latest_successful_run
1844
- to get an object representing a completed successful run.
1845
-
1846
- Returns
1847
- -------
1848
- Run, optional
1849
- Latest run of this flow
1850
- """
1851
- ...
1852
- @property
1853
- def latest_successful_run(self) -> typing.Optional[metaflow.client.core.Run]:
1854
- """
1855
- Returns the latest successful run of this flow.
1856
-
1857
- Returns
1858
- -------
1859
- Run, optional
1860
- Latest successful run of this flow
1861
- """
1862
- ...
1863
- def runs(self, *tags: str) -> typing.Iterator[metaflow.client.core.Run]:
1864
- """
1865
- Returns an iterator over all `Run`s of this flow.
1866
-
1867
- An optional filter is available that allows you to filter on tags.
1868
- If multiple tags are specified, only runs that have all the
1869
- specified tags are returned.
1870
-
1871
- Parameters
1872
- ----------
1873
- tags : str
1874
- Tags to match.
1875
-
1876
- Yields
1877
- ------
1878
- Run
1879
- `Run` objects in this flow.
1880
- """
1881
- ...
1882
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Task]:
1883
- """
1884
- Iterate over all children Run of this Flow.
1885
-
1886
- Note that only runs in the current namespace are returned unless
1887
- _namespace_check is False
1888
-
1889
- Yields
1890
- ------
1891
- Run
1892
- A Run in this Flow
1893
- """
1894
- ...
1895
- def __getitem__(self, run_id: str) -> metaflow.client.core.Run:
1896
- """
1897
- Returns the Run object with the run ID 'run_id'
1898
-
1899
- Parameters
1900
- ----------
1901
- run_id : str
1902
- Run OD
1903
-
1904
- Returns
1905
- -------
1906
- Run
1907
- Run for this run ID in this Flow
1908
-
1909
- Raises
1910
- ------
1911
- KeyError
1912
- If the run_id does not identify a valid Run object
1913
- """
1914
- ...
1915
- def __getstate__(self):
1916
- ...
1917
- def __setstate__(self, state):
1918
- ...
1919
- ...
1920
-
1921
- class Run(metaflow.client.core.MetaflowObject, metaclass=type):
1922
- """
1923
- A `Run` represents an execution of a `Flow`. It is a container of `Step`s.
1924
-
1925
- Attributes
1926
- ----------
1927
- data : MetaflowData
1928
- a shortcut to run['end'].task.data, i.e. data produced by this run.
1929
- successful : bool
1930
- True if the run completed successfully.
1931
- finished : bool
1932
- True if the run completed.
1933
- finished_at : datetime
1934
- Time this run finished.
1935
- code : MetaflowCode
1936
- Code package for this run (if present). See `MetaflowCode`.
1937
- trigger : MetaflowTrigger
1938
- Information about event(s) that triggered this run (if present). See `MetaflowTrigger`.
1939
- end_task : Task
1940
- `Task` for the end step (if it is present already).
1941
- """
1942
- def steps(self, *tags: str) -> typing.Iterator[metaflow.client.core.Step]:
1943
- """
1944
- [Legacy function - do not use]
1945
-
1946
- Returns an iterator over all `Step` objects in the step. This is an alias
1947
- to iterating the object itself, i.e.
1948
- ```
1949
- list(Run(...)) == list(Run(...).steps())
1950
- ```
1951
-
1952
- Parameters
1953
- ----------
1954
- tags : str
1955
- No op (legacy functionality)
1956
-
1957
- Yields
1958
- ------
1959
- Step
1960
- `Step` objects in this run.
1961
- """
1962
- ...
1963
- @property
1964
- def code(self) -> typing.Optional[metaflow.client.core.MetaflowCode]:
1965
- """
1966
- Returns the MetaflowCode object for this run, if present.
1967
- Code is packed if atleast one `Step` runs remotely, else None is returned.
1968
-
1969
- Returns
1970
- -------
1971
- MetaflowCode, optional
1972
- Code package for this run
1973
- """
1974
- ...
1975
- @property
1976
- def data(self) -> typing.Optional[metaflow.client.core.MetaflowData]:
1977
- """
1978
- Returns a container of data artifacts produced by this run.
1979
-
1980
- You can access data produced by this run as follows:
1981
- ```
1982
- print(run.data.my_var)
1983
- ```
1984
- This is a shorthand for `run['end'].task.data`. If the 'end' step has not yet
1985
- executed, returns None.
1986
-
1987
- Returns
1988
- -------
1989
- MetaflowData, optional
1990
- Container of all artifacts produced by this task
1991
- """
1992
- ...
1993
- @property
1994
- def successful(self) -> bool:
1995
- """
1996
- Indicates whether or not the run completed successfully.
1997
-
1998
- A run is successful if its 'end' step is successful.
1999
-
2000
- Returns
2001
- -------
2002
- bool
2003
- True if the run completed successfully and False otherwise
2004
- """
2005
- ...
2006
- @property
2007
- def finished(self) -> bool:
2008
- """
2009
- Indicates whether or not the run completed.
2010
-
2011
- A run completed if its 'end' step completed.
2012
-
2013
- Returns
2014
- -------
2015
- bool
2016
- True if the run completed and False otherwise
2017
- """
2018
- ...
2019
- @property
2020
- def finished_at(self) -> typing.Optional[datetime.datetime]:
2021
- """
2022
- Returns the datetime object of when the run finished (successfully or not).
2023
-
2024
- The completion time of a run is the same as the completion time of its 'end' step.
2025
- If the 'end' step has not completed, returns None.
2026
-
2027
- Returns
2028
- -------
2029
- datetime, optional
2030
- Datetime of when the run finished
2031
- """
2032
- ...
2033
- @property
2034
- def end_task(self) -> typing.Optional[metaflow.client.core.Task]:
2035
- """
2036
- Returns the Task corresponding to the 'end' step.
2037
-
2038
- This returns None if the end step does not yet exist.
2039
-
2040
- Returns
2041
- -------
2042
- Task, optional
2043
- The 'end' task
2044
- """
2045
- ...
2046
- def add_tag(self, tag: str):
2047
- """
2048
- Add a tag to this `Run`.
2049
-
2050
- Note that if the tag is already a system tag, it is not added as a user tag,
2051
- and no error is thrown.
2052
-
2053
- Parameters
2054
- ----------
2055
- tag : str
2056
- Tag to add.
2057
- """
2058
- ...
2059
- def add_tags(self, tags: typing.Iterable[str]):
2060
- """
2061
- Add one or more tags to this `Run`.
2062
-
2063
- Note that if any tag is already a system tag, it is not added as a user tag
2064
- and no error is thrown.
2065
-
2066
- Parameters
2067
- ----------
2068
- tags : Iterable[str]
2069
- Tags to add.
2070
- """
2071
- ...
2072
- def remove_tag(self, tag: str):
2073
- """
2074
- Remove one tag from this `Run`.
2075
-
2076
- Removing a system tag is an error. Removing a non-existent
2077
- user tag is a no-op.
2078
-
2079
- Parameters
2080
- ----------
2081
- tag : str
2082
- Tag to remove.
2083
- """
2084
- ...
2085
- def remove_tags(self, tags: typing.Iterable[str]):
2086
- """
2087
- Remove one or more tags to this `Run`.
2088
-
2089
- Removing a system tag will result in an error. Removing a non-existent
2090
- user tag is a no-op.
2091
-
2092
- Parameters
2093
- ----------
2094
- tags : Iterable[str]
2095
- Tags to remove.
2096
- """
2097
- ...
2098
- def replace_tag(self, tag_to_remove: str, tag_to_add: str):
2099
- """
2100
- Remove a tag and add a tag atomically. Removal is done first.
2101
- The rules for `Run.add_tag` and `Run.remove_tag` also apply here.
2102
-
2103
- Parameters
2104
- ----------
2105
- tag_to_remove : str
2106
- Tag to remove.
2107
- tag_to_add : str
2108
- Tag to add.
2109
- """
2110
- ...
2111
- def replace_tags(self, tags_to_remove: typing.Iterable[str], tags_to_add: typing.Iterable[str]):
2112
- """
2113
- Remove and add tags atomically; the removal is done first.
2114
- The rules for `Run.add_tag` and `Run.remove_tag` also apply here.
2115
-
2116
- Parameters
2117
- ----------
2118
- tags_to_remove : Iterable[str]
2119
- Tags to remove.
2120
- tags_to_add : Iterable[str]
2121
- Tags to add.
2122
- """
2123
- ...
2124
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Step]:
2125
- """
2126
- Iterate over all children Step of this Run
2127
-
2128
- Yields
2129
- ------
2130
- Step
2131
- A Step in this Run
2132
- """
2133
- ...
2134
- def __getitem__(self, name: str) -> metaflow.client.core.Step:
2135
- """
2136
- Returns the Step object with the step name 'name'
2137
-
2138
- Parameters
2139
- ----------
2140
- name : str
2141
- Step name
2142
-
2143
- Returns
2144
- -------
2145
- Step
2146
- Step for this step name in this Run
2147
-
2148
- Raises
2149
- ------
2150
- KeyError
2151
- If the name does not identify a valid Step object
2152
- """
2153
- ...
2154
- def __getstate__(self):
2155
- ...
2156
- def __setstate__(self, state):
2157
- ...
2158
- @property
2159
- def trigger(self) -> typing.Optional[metaflow.events.Trigger]:
2160
- """
2161
- Returns a container of events that triggered this run.
2162
-
2163
- This returns None if the run was not triggered by any events.
2164
-
2165
- Returns
2166
- -------
2167
- Trigger, optional
2168
- Container of triggering events
2169
- """
2170
- ...
2171
- ...
2172
-
2173
- class Step(metaflow.client.core.MetaflowObject, metaclass=type):
2174
- """
2175
- A `Step` represents a user-defined step, that is, a method annotated with the `@step` decorator.
2176
-
2177
- It contains `Task` objects associated with the step, that is, all executions of the
2178
- `Step`. The step may contain multiple `Task`s in the case of a foreach step.
2179
-
2180
- Attributes
2181
- ----------
2182
- task : Task
2183
- The first `Task` object in this step. This is a shortcut for retrieving the only
2184
- task contained in a non-foreach step.
2185
- finished_at : datetime
2186
- Time when the latest `Task` of this step finished. Note that in the case of foreaches,
2187
- this time may change during execution of the step.
2188
- environment_info : Dict[str, Any]
2189
- Information about the execution environment.
2190
- """
2191
- @property
2192
- def task(self) -> typing.Optional[metaflow.client.core.Task]:
2193
- """
2194
- Returns a Task object belonging to this step.
2195
-
2196
- This is useful when the step only contains one task (a linear step for example).
2197
-
2198
- Returns
2199
- -------
2200
- Task
2201
- A task in the step
2202
- """
2203
- ...
2204
- def tasks(self, *tags: str) -> typing.Iterable[metaflow.client.core.Task]:
2205
- """
2206
- [Legacy function - do not use]
2207
-
2208
- Returns an iterator over all `Task` objects in the step. This is an alias
2209
- to iterating the object itself, i.e.
2210
- ```
2211
- list(Step(...)) == list(Step(...).tasks())
2212
- ```
2213
-
2214
- Parameters
2215
- ----------
2216
- tags : str
2217
- No op (legacy functionality)
2218
-
2219
- Yields
2220
- ------
2221
- Task
2222
- `Task` objects in this step.
2223
- """
2224
- ...
2225
- @property
2226
- def control_task(self) -> typing.Optional[metaflow.client.core.Task]:
2227
- """
2228
- [Unpublished API - use with caution!]
2229
-
2230
- Returns a Control Task object belonging to this step.
2231
- This is useful when the step only contains one control task.
2232
-
2233
- Returns
2234
- -------
2235
- Task
2236
- A control task in the step
2237
- """
2238
- ...
2239
- def control_tasks(self, *tags: str) -> typing.Iterator[metaflow.client.core.Task]:
2240
- """
2241
- [Unpublished API - use with caution!]
2242
-
2243
- Returns an iterator over all the control tasks in the step.
2244
- An optional filter is available that allows you to filter on tags. The
2245
- control tasks returned if the filter is specified will contain all the
2246
- tags specified.
2247
- Parameters
2248
- ----------
2249
- tags : str
2250
- Tags to match
2251
-
2252
- Yields
2253
- ------
2254
- Task
2255
- Control Task objects for this step
2256
- """
2257
- ...
2258
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Task]:
2259
- """
2260
- Iterate over all children Task of this Step
2261
-
2262
- Yields
2263
- ------
2264
- Task
2265
- A Task in this Step
2266
- """
2267
- ...
2268
- def __getitem__(self, task_id: str) -> metaflow.client.core.Task:
2269
- """
2270
- Returns the Task object with the task ID 'task_id'
2271
-
2272
- Parameters
2273
- ----------
2274
- task_id : str
2275
- Task ID
2276
-
2277
- Returns
2278
- -------
2279
- Task
2280
- Task for this task ID in this Step
2281
-
2282
- Raises
2283
- ------
2284
- KeyError
2285
- If the task_id does not identify a valid Task object
2286
- """
2287
- ...
2288
- def __getstate__(self):
2289
- ...
2290
- def __setstate__(self, state):
2291
- ...
2292
- @property
2293
- def finished_at(self) -> typing.Optional[datetime.datetime]:
2294
- """
2295
- Returns the datetime object of when the step finished (successfully or not).
2296
-
2297
- A step is considered finished when all the tasks that belong to it have
2298
- finished. This call will return None if the step has not finished
2299
-
2300
- Returns
2301
- -------
2302
- datetime
2303
- Datetime of when the step finished
2304
- """
2305
- ...
2306
- @property
2307
- def environment_info(self) -> typing.Optional[typing.Dict[str, typing.Any]]:
2308
- """
2309
- Returns information about the environment that was used to execute this step. As an
2310
- example, if the Conda environment is selected, this will return information about the
2311
- dependencies that were used in the environment.
2312
-
2313
- This environment information is only available for steps that have tasks
2314
- for which the code package has been saved.
2315
-
2316
- Returns
2317
- -------
2318
- Dict[str, Any], optional
2319
- Dictionary describing the environment
2320
- """
2321
- ...
2322
- ...
2323
-
2324
- class Task(metaflow.client.core.MetaflowObject, metaclass=type):
2325
- """
2326
- A `Task` represents an execution of a `Step`.
2327
-
2328
- It contains all `DataArtifact` objects produced by the task as
2329
- well as metadata related to execution.
2330
-
2331
- Note that the `@retry` decorator may cause multiple attempts of
2332
- the task to be present. Usually you want the latest attempt, which
2333
- is what instantiating a `Task` object returns by default. If
2334
- you need to e.g. retrieve logs from a failed attempt, you can
2335
- explicitly get information about a specific attempt by using the
2336
- following syntax when creating a task:
2337
-
2338
- `Task('flow/run/step/task', attempt=<attempt>)`
2339
-
2340
- where `attempt=0` corresponds to the first attempt etc.
2341
-
2342
- Attributes
2343
- ----------
2344
- metadata : List[Metadata]
2345
- List of all metadata events associated with the task.
2346
- metadata_dict : Dict[str, str]
2347
- A condensed version of `metadata`: A dictionary where keys
2348
- are names of metadata events and values the latest corresponding event.
2349
- data : MetaflowData
2350
- Container of all data artifacts produced by this task. Note that this
2351
- call downloads all data locally, so it can be slower than accessing
2352
- artifacts individually. See `MetaflowData` for more information.
2353
- artifacts : MetaflowArtifacts
2354
- Container of `DataArtifact` objects produced by this task.
2355
- successful : bool
2356
- True if the task completed successfully.
2357
- finished : bool
2358
- True if the task completed.
2359
- exception : object
2360
- Exception raised by this task if there was one.
2361
- finished_at : datetime
2362
- Time this task finished.
2363
- runtime_name : str
2364
- Runtime this task was executed on.
2365
- stdout : str
2366
- Standard output for the task execution.
2367
- stderr : str
2368
- Standard error output for the task execution.
2369
- code : MetaflowCode
2370
- Code package for this task (if present). See `MetaflowCode`.
2371
- environment_info : Dict[str, str]
2372
- Information about the execution environment.
2373
- """
2374
- def __init__(self, *args, **kwargs):
2375
- ...
2376
- @property
2377
- def metadata(self) -> typing.List[metaflow.client.core.Metadata]:
2378
- """
2379
- Metadata events produced by this task across all attempts of the task
2380
- *except* if you selected a specific task attempt.
2381
-
2382
- Note that Metadata is different from tags.
2383
-
2384
- Returns
2385
- -------
2386
- List[Metadata]
2387
- Metadata produced by this task
2388
- """
2389
- ...
2390
- @property
2391
- def metadata_dict(self) -> typing.Dict[str, str]:
2392
- """
2393
- Dictionary mapping metadata names (keys) and their associated values.
2394
-
2395
- Note that unlike the metadata() method, this call will only return the latest
2396
- metadata for a given name. For example, if a task executes multiple times (retries),
2397
- the same metadata name will be generated multiple times (one for each execution of the
2398
- task). The metadata() method returns all those metadata elements whereas this call will
2399
- return the metadata associated with the latest execution of the task.
2400
-
2401
- Returns
2402
- -------
2403
- Dict[str, str]
2404
- Dictionary mapping metadata name with value
2405
- """
2406
- ...
2407
- @property
2408
- def index(self) -> typing.Optional[int]:
2409
- """
2410
- Returns the index of the innermost foreach loop if this task is run inside at least
2411
- one foreach.
2412
-
2413
- The index is what distinguishes the various tasks inside a given step.
2414
- This call returns None if this task was not run in a foreach loop.
2415
-
2416
- Returns
2417
- -------
2418
- int, optional
2419
- Index in the innermost loop for this task
2420
- """
2421
- ...
2422
- @property
2423
- def data(self) -> metaflow.client.core.MetaflowData:
2424
- """
2425
- Returns a container of data artifacts produced by this task.
2426
-
2427
- You can access data produced by this task as follows:
2428
- ```
2429
- print(task.data.my_var)
2430
- ```
2431
-
2432
- Returns
2433
- -------
2434
- MetaflowData
2435
- Container of all artifacts produced by this task
2436
- """
2437
- ...
2438
- @property
2439
- def artifacts(self) -> typing.NamedTuple:
2440
- """
2441
- Returns a container of DataArtifacts produced by this task.
2442
-
2443
- You can access each DataArtifact by name like so:
2444
- ```
2445
- print(task.artifacts.my_var)
2446
- ```
2447
- This method differs from data() because it returns DataArtifact objects
2448
- (which contain additional metadata) as opposed to just the data.
2449
-
2450
- Returns
2451
- -------
2452
- MetaflowArtifacts
2453
- Container of all DataArtifacts produced by this task
2454
- """
2455
- ...
2456
- @property
2457
- def successful(self) -> bool:
2458
- """
2459
- Indicates whether or not the task completed successfully.
2460
-
2461
- This information is always about the latest task to have completed (in case
2462
- of retries).
2463
-
2464
- Returns
2465
- -------
2466
- bool
2467
- True if the task completed successfully and False otherwise
2468
- """
2469
- ...
2470
- @property
2471
- def finished(self) -> bool:
2472
- """
2473
- Indicates whether or not the task completed.
2474
-
2475
- This information is always about the latest task to have completed (in case
2476
- of retries).
2477
-
2478
- Returns
2479
- -------
2480
- bool
2481
- True if the task completed and False otherwise
2482
- """
2483
- ...
2484
- @property
2485
- def exception(self) -> typing.Optional[typing.Any]:
2486
- """
2487
- Returns the exception that caused the task to fail, if any.
2488
-
2489
- This information is always about the latest task to have completed (in case
2490
- of retries). If successful() returns False and finished() returns True,
2491
- this method can help determine what went wrong.
2492
-
2493
- Returns
2494
- -------
2495
- object
2496
- Exception raised by the task or None if not applicable
2497
- """
2498
- ...
2499
- @property
2500
- def finished_at(self) -> typing.Optional[datetime.datetime]:
2501
- """
2502
- Returns the datetime object of when the task finished (successfully or not).
2503
-
2504
- This information is always about the latest task to have completed (in case
2505
- of retries). This call will return None if the task is not finished.
2506
-
2507
- Returns
2508
- -------
2509
- datetime
2510
- Datetime of when the task finished
2511
- """
2512
- ...
2513
- @property
2514
- def runtime_name(self) -> typing.Optional[str]:
2515
- """
2516
- Returns the name of the runtime this task executed on.
2517
-
2518
-
2519
- Returns
2520
- -------
2521
- str
2522
- Name of the runtime this task executed on
2523
- """
2524
- ...
2525
- @property
2526
- def stdout(self) -> str:
2527
- """
2528
- Returns the full standard out of this task.
2529
-
2530
- If you specify a specific attempt for this task, it will return the
2531
- standard out for that attempt. If you do not specify an attempt,
2532
- this will return the current standard out for the latest *started*
2533
- attempt of the task. In both cases, multiple calls to this
2534
- method will return the most up-to-date log (so if an attempt is not
2535
- done, each call will fetch the latest log).
2536
-
2537
- Returns
2538
- -------
2539
- str
2540
- Standard output of this task
2541
- """
2542
- ...
2543
- @property
2544
- def stdout_size(self) -> int:
2545
- """
2546
- Returns the size of the stdout log of this task.
2547
-
2548
- Similar to `stdout`, the size returned is the latest size of the log
2549
- (so for a running attempt, this value will increase as the task produces
2550
- more output).
2551
-
2552
- Returns
2553
- -------
2554
- int
2555
- Size of the stdout log content (in bytes)
2556
- """
2557
- ...
2558
- @property
2559
- def stderr(self) -> str:
2560
- """
2561
- Returns the full standard error of this task.
2562
-
2563
- If you specify a specific attempt for this task, it will return the
2564
- standard error for that attempt. If you do not specify an attempt,
2565
- this will return the current standard error for the latest *started*
2566
- attempt. In both cases, multiple calls to this
2567
- method will return the most up-to-date log (so if an attempt is not
2568
- done, each call will fetch the latest log).
2569
-
2570
- Returns
2571
- -------
2572
- str
2573
- Standard error of this task
2574
- """
2575
- ...
2576
- @property
2577
- def stderr_size(self) -> int:
2578
- """
2579
- Returns the size of the stderr log of this task.
2580
-
2581
- Similar to `stderr`, the size returned is the latest size of the log
2582
- (so for a running attempt, this value will increase as the task produces
2583
- more output).
2584
-
2585
- Returns
2586
- -------
2587
- int
2588
- Size of the stderr log content (in bytes)
2589
- """
2590
- ...
2591
- @property
2592
- def current_attempt(self) -> int:
2593
- """
2594
- Get the relevant attempt for this Task.
2595
-
2596
- Returns the specific attempt used when
2597
- initializing the instance, or the latest *started* attempt for the Task.
2598
-
2599
- Returns
2600
- -------
2601
- int
2602
- attempt id for this task object
2603
- """
2604
- ...
2605
- @property
2606
- def code(self) -> typing.Optional[metaflow.client.core.MetaflowCode]:
2607
- """
2608
- Returns the MetaflowCode object for this task, if present.
2609
-
2610
- Not all tasks save their code so this call may return None in those cases.
2611
-
2612
- Returns
2613
- -------
2614
- MetaflowCode
2615
- Code package for this task
2616
- """
2617
- ...
2618
- @property
2619
- def environment_info(self) -> typing.Dict[str, typing.Any]:
2620
- """
2621
- Returns information about the environment that was used to execute this task. As an
2622
- example, if the Conda environment is selected, this will return information about the
2623
- dependencies that were used in the environment.
2624
-
2625
- This environment information is only available for tasks that have a code package.
2626
-
2627
- Returns
2628
- -------
2629
- Dict
2630
- Dictionary describing the environment
2631
- """
2632
- ...
2633
- def loglines(self, stream: str, as_unicode: bool = True, meta_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Iterator[typing.Tuple[datetime.datetime, str]]:
2634
- """
2635
- Return an iterator over (utc_timestamp, logline) tuples.
2636
-
2637
- Parameters
2638
- ----------
2639
- stream : str
2640
- Either 'stdout' or 'stderr'.
2641
- as_unicode : bool, default: True
2642
- If as_unicode=False, each logline is returned as a byte object. Otherwise,
2643
- it is returned as a (unicode) string.
2644
-
2645
- Yields
2646
- ------
2647
- Tuple[datetime, str]
2648
- Tuple of timestamp, logline pairs.
2649
- """
2650
- ...
2651
- def __iter__(self) -> typing.Iterator[metaflow.client.core.DataArtifact]:
2652
- """
2653
- Iterate over all children DataArtifact of this Task
2654
-
2655
- Yields
2656
- ------
2657
- DataArtifact
2658
- A DataArtifact in this Step
2659
- """
2660
- ...
2661
- def __getitem__(self, name: str) -> metaflow.client.core.DataArtifact:
2662
- """
2663
- Returns the DataArtifact object with the artifact name 'name'
2664
-
2665
- Parameters
2666
- ----------
2667
- name : str
2668
- Data artifact name
2669
-
2670
- Returns
2671
- -------
2672
- DataArtifact
2673
- DataArtifact for this artifact name in this task
2674
-
2675
- Raises
2676
- ------
2677
- KeyError
2678
- If the name does not identify a valid DataArtifact object
2679
- """
2680
- ...
2681
- def __getstate__(self):
2682
- ...
2683
- def __setstate__(self, state):
2684
- ...
2685
- ...
2686
-
2687
- class DataArtifact(metaflow.client.core.MetaflowObject, metaclass=type):
2688
- """
2689
- A single data artifact and associated metadata. Note that this object does
2690
- not contain other objects as it is the leaf object in the hierarchy.
2691
-
2692
- Attributes
2693
- ----------
2694
- data : object
2695
- The data contained in this artifact, that is, the object produced during
2696
- execution of this run.
2697
- sha : string
2698
- A unique ID of this artifact.
2699
- finished_at : datetime
2700
- Corresponds roughly to the `Task.finished_at` time of the parent `Task`.
2701
- An alias for `DataArtifact.created_at`.
2702
- """
2703
- @property
2704
- def data(self) -> typing.Any:
2705
- """
2706
- Unpickled representation of the data contained in this artifact.
2707
-
2708
- Returns
2709
- -------
2710
- object
2711
- Object contained in this artifact
2712
- """
2713
- ...
2714
- @property
2715
- def size(self) -> int:
2716
- """
2717
- Returns the size (in bytes) of the pickled object representing this
2718
- DataArtifact
2719
-
2720
- Returns
2721
- -------
2722
- int
2723
- size of the pickled representation of data artifact (in bytes)
2724
- """
2725
- ...
2726
- @property
2727
- def sha(self) -> str:
2728
- """
2729
- Unique identifier for this artifact.
2730
-
2731
- This is a unique hash of the artifact (historically SHA1 hash)
2732
-
2733
- Returns
2734
- -------
2735
- str
2736
- Hash of this artifact
2737
- """
2738
- ...
2739
- @property
2740
- def finished_at(self) -> datetime.datetime:
2741
- """
2742
- Creation time for this artifact.
2743
-
2744
- Alias for created_at.
2745
-
2746
- Returns
2747
- -------
2748
- datetime
2749
- Creation time
2750
- """
2751
- ...
2752
- def __getstate__(self):
2753
- ...
2754
- def __setstate__(self, state):
2755
- ...
2756
- ...
2757
-
2758
- class Runner(object, metaclass=type):
2759
- """
2760
- Metaflow's Runner API that presents a programmatic interface
2761
- to run flows and perform other operations either synchronously or asynchronously.
2762
- The class expects a path to the flow file along with optional arguments
2763
- that match top-level options on the command-line.
2764
-
2765
- This class works as a context manager, calling `cleanup()` to remove
2766
- temporary files at exit.
2767
-
2768
- Example:
2769
- ```python
2770
- with Runner('slowflow.py', pylint=False) as runner:
2771
- result = runner.run(alpha=5, tags=["abc", "def"], max_workers=5)
2772
- print(result.run.finished)
2773
- ```
2774
-
2775
- Parameters
2776
- ----------
2777
- flow_file : str
2778
- Path to the flow file to run
2779
- show_output : bool, default True
2780
- Show the 'stdout' and 'stderr' to the console by default,
2781
- Only applicable for synchronous 'run' and 'resume' functions.
2782
- profile : Optional[str], default None
2783
- Metaflow profile to use to run this run. If not specified, the default
2784
- profile is used (or the one already set using `METAFLOW_PROFILE`)
2785
- env : Optional[Dict], default None
2786
- Additional environment variables to set for the Run. This overrides the
2787
- environment set for this process.
2788
- cwd : Optional[str], default None
2789
- The directory to run the subprocess in; if not specified, the current
2790
- directory is used.
2791
- file_read_timeout : int, default 3600
2792
- The timeout until which we try to read the runner attribute file.
2793
- **kwargs : Any
2794
- Additional arguments that you would pass to `python myflow.py` before
2795
- the `run` command.
2796
- """
2797
- def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
2798
- ...
2799
- def __enter__(self) -> metaflow.runner.metaflow_runner.Runner:
2800
- ...
2801
- def __aenter__(self) -> metaflow.runner.metaflow_runner.Runner:
2802
- ...
2803
- def _Runner__get_executing_run(self, tfp_runner_attribute, command_obj):
2804
- ...
2805
- def run(self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun:
2806
- """
2807
- Blocking execution of the run. This method will wait until
2808
- the run has completed execution.
2809
-
2810
- Parameters
2811
- ----------
2812
- **kwargs : Any
2813
- Additional arguments that you would pass to `python myflow.py` after
2814
- the `run` command, in particular, any parameters accepted by the flow.
2815
-
2816
- Returns
2817
- -------
2818
- ExecutingRun
2819
- ExecutingRun containing the results of the run.
2820
- """
2821
- ...
2822
- def resume(self, **kwargs):
2823
- """
2824
- Blocking resume execution of the run.
2825
- This method will wait until the resumed run has completed execution.
2826
-
2827
- Parameters
2828
- ----------
2829
- **kwargs : Any
2830
- Additional arguments that you would pass to `python ./myflow.py` after
2831
- the `resume` command.
2832
-
2833
- Returns
2834
- -------
2835
- ExecutingRun
2836
- ExecutingRun containing the results of the resumed run.
2837
- """
2838
- ...
2839
- def async_run(self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun:
2840
- """
2841
- Non-blocking execution of the run. This method will return as soon as the
2842
- run has launched.
2843
-
2844
- Note that this method is asynchronous and needs to be `await`ed.
2845
-
2846
- Parameters
2847
- ----------
2848
- **kwargs : Any
2849
- Additional arguments that you would pass to `python myflow.py` after
2850
- the `run` command, in particular, any parameters accepted by the flow.
2851
-
2852
- Returns
2853
- -------
2854
- ExecutingRun
2855
- ExecutingRun representing the run that was started.
2856
- """
2857
- ...
2858
- def async_resume(self, **kwargs):
2859
- """
2860
- Non-blocking resume execution of the run.
2861
- This method will return as soon as the resume has launched.
2862
-
2863
- Note that this method is asynchronous and needs to be `await`ed.
2864
-
2865
- Parameters
2866
- ----------
2867
- **kwargs : Any
2868
- Additional arguments that you would pass to `python myflow.py` after
2869
- the `resume` command.
2870
-
2871
- Returns
2872
- -------
2873
- ExecutingRun
2874
- ExecutingRun representing the resumed run that was started.
2875
- """
2876
- ...
2877
- def __exit__(self, exc_type, exc_value, traceback):
2878
- ...
2879
- def __aexit__(self, exc_type, exc_value, traceback):
2880
- ...
2881
- def cleanup(self):
2882
- """
2883
- Delete any temporary files created during execution.
2884
- """
2885
- ...
2886
- ...
2887
-
2888
- class NBRunner(object, metaclass=type):
2889
- """
2890
- A wrapper over `Runner` for executing flows defined in a Jupyter
2891
- notebook cell.
2892
-
2893
- Instantiate this class on the last line of a notebook cell where
2894
- a `flow` is defined. In contrast to `Runner`, this class is not
2895
- meant to be used in a context manager. Instead, use a blocking helper
2896
- function like `nbrun` (which calls `cleanup()` internally) or call
2897
- `cleanup()` explictly when using non-blocking APIs.
2898
-
2899
- ```python
2900
- run = NBRunner(FlowName).nbrun()
2901
- ```
2902
-
2903
- Parameters
2904
- ----------
2905
- flow : FlowSpec
2906
- Flow defined in the same cell
2907
- show_output : bool, default True
2908
- Show the 'stdout' and 'stderr' to the console by default,
2909
- Only applicable for synchronous 'run' and 'resume' functions.
2910
- profile : Optional[str], default None
2911
- Metaflow profile to use to run this run. If not specified, the default
2912
- profile is used (or the one already set using `METAFLOW_PROFILE`)
2913
- env : Optional[Dict], default None
2914
- Additional environment variables to set for the Run. This overrides the
2915
- environment set for this process.
2916
- base_dir : Optional[str], default None
2917
- The directory to run the subprocess in; if not specified, the current
2918
- working directory is used.
2919
- file_read_timeout : int, default 3600
2920
- The timeout until which we try to read the runner attribute file.
2921
- **kwargs : Any
2922
- Additional arguments that you would pass to `python myflow.py` before
2923
- the `run` command.
2924
- """
2925
- def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
2926
- ...
2927
- def nbrun(self, **kwargs):
2928
- """
2929
- Blocking execution of the run. This method will wait until
2930
- the run has completed execution.
2931
-
2932
- Note that in contrast to `run`, this method returns a
2933
- `metaflow.Run` object directly and calls `cleanup()` internally
2934
- to support a common notebook pattern of executing a flow and
2935
- retrieving its results immediately.
2936
-
2937
- Parameters
2938
- ----------
2939
- **kwargs : Any
2940
- Additional arguments that you would pass to `python myflow.py` after
2941
- the `run` command, in particular, any parameters accepted by the flow.
2942
-
2943
- Returns
2944
- -------
2945
- Run
2946
- A `metaflow.Run` object representing the finished run.
2947
- """
2948
- ...
2949
- def nbresume(self, **kwargs):
2950
- """
2951
- Blocking resuming of a run. This method will wait until
2952
- the resumed run has completed execution.
2953
-
2954
- Note that in contrast to `resume`, this method returns a
2955
- `metaflow.Run` object directly and calls `cleanup()` internally
2956
- to support a common notebook pattern of executing a flow and
2957
- retrieving its results immediately.
2958
-
2959
- Parameters
2960
- ----------
2961
- **kwargs : Any
2962
- Additional arguments that you would pass to `python myflow.py` after
2963
- the `resume` command.
2964
-
2965
- Returns
2966
- -------
2967
- Run
2968
- A `metaflow.Run` object representing the resumed run.
2969
- """
2970
- ...
2971
- def run(self, **kwargs):
2972
- """
2973
- Runs the flow.
2974
- """
2975
- ...
2976
- def resume(self, **kwargs):
2977
- """
2978
- Resumes the flow.
2979
- """
2980
- ...
2981
- def async_run(self, **kwargs):
2982
- """
2983
- Non-blocking execution of the run. This method will return as soon as the
2984
- run has launched. This method is equivalent to `Runner.async_run`.
2985
-
2986
- Note that this method is asynchronous and needs to be `await`ed.
2987
-
2988
-
2989
- Parameters
2990
- ----------
2991
- **kwargs : Any
2992
- Additional arguments that you would pass to `python myflow.py` after
2993
- the `run` command, in particular, any parameters accepted by the flow.
2994
-
2995
- Returns
2996
- -------
2997
- ExecutingRun
2998
- ExecutingRun representing the run that was started.
2999
- """
3000
- ...
3001
- def async_resume(self, **kwargs):
3002
- """
3003
- Non-blocking execution of the run. This method will return as soon as the
3004
- run has launched. This method is equivalent to `Runner.async_resume`.
3005
-
3006
- Note that this method is asynchronous and needs to be `await`ed.
3007
-
3008
- Parameters
3009
- ----------
3010
- **kwargs : Any
3011
- Additional arguments that you would pass to `python myflow.py` after
3012
- the `run` command, in particular, any parameters accepted by the flow.
3013
-
3014
- Returns
3015
- -------
3016
- ExecutingRun
3017
- ExecutingRun representing the run that was started.
3018
- """
3019
- ...
3020
- def cleanup(self):
3021
- """
3022
- Delete any temporary files created during execution.
3023
-
3024
- Call this method after using `async_run` or `async_resume`. You don't
3025
- have to call this after `nbrun` or `nbresume`.
3026
- """
3027
- ...
3028
- ...
3029
-
3030
- class Deployer(object, metaclass=type):
3031
- """
3032
- Use the `Deployer` class to configure and access one of the production
3033
- orchestrators supported by Metaflow.
3034
-
3035
- Parameters
3036
- ----------
3037
- flow_file : str
3038
- Path to the flow file to deploy.
3039
- show_output : bool, default True
3040
- Show the 'stdout' and 'stderr' to the console by default.
3041
- profile : Optional[str], default None
3042
- Metaflow profile to use for the deployment. If not specified, the default
3043
- profile is used.
3044
- env : Optional[Dict[str, str]], default None
3045
- Additional environment variables to set for the deployment.
3046
- cwd : Optional[str], default None
3047
- The directory to run the subprocess in; if not specified, the current
3048
- directory is used.
3049
- file_read_timeout : int, default 3600
3050
- The timeout until which we try to read the deployer attribute file.
3051
- **kwargs : Any
3052
- Additional arguments that you would pass to `python myflow.py` before
3053
- the deployment command.
3054
- """
3055
- def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
3056
- ...
3057
- def _Deployer__make_function(self, deployer_class):
3058
- """
3059
- Create a function for the given deployer class.
3060
-
3061
- Parameters
3062
- ----------
3063
- deployer_class : Type[DeployerImpl]
3064
- Deployer implementation class.
3065
-
3066
- Returns
3067
- -------
3068
- Callable
3069
- Function that initializes and returns an instance of the deployer class.
3070
- """
3071
- ...
3072
- ...
3073
-
3074
- class NBDeployer(object, metaclass=type):
3075
- """
3076
- A wrapper over `Deployer` for deploying flows defined in a Jupyter
3077
- notebook cell.
3078
-
3079
- Instantiate this class on the last line of a notebook cell where
3080
- a `flow` is defined. In contrast to `Deployer`, this class is not
3081
- meant to be used in a context manager.
3082
-
3083
- ```python
3084
- deployer = NBDeployer(FlowName)
3085
- ar = deployer.argo_workflows(name="madhur")
3086
- ar_obj = ar.create()
3087
- result = ar_obj.trigger(alpha=300)
3088
- print(result.status)
3089
- print(result.run)
3090
- result.terminate()
3091
- ```
3092
-
3093
- Parameters
3094
- ----------
3095
- flow : FlowSpec
3096
- Flow defined in the same cell
3097
- show_output : bool, default True
3098
- Show the 'stdout' and 'stderr' to the console by default,
3099
- profile : Optional[str], default None
3100
- Metaflow profile to use to deploy this run. If not specified, the default
3101
- profile is used (or the one already set using `METAFLOW_PROFILE`)
3102
- env : Optional[Dict[str, str]], default None
3103
- Additional environment variables to set. This overrides the
3104
- environment set for this process.
3105
- base_dir : Optional[str], default None
3106
- The directory to run the subprocess in; if not specified, the current
3107
- working directory is used.
3108
- **kwargs : Any
3109
- Additional arguments that you would pass to `python myflow.py` i.e. options
3110
- listed in `python myflow.py --help`
3111
- """
3112
- def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
3113
- ...
3114
- def cleanup(self):
3115
- """
3116
- Delete any temporary files created during execution.
3117
- """
3118
- ...
3119
- ...
3120
-
3121
791
  pkg_name: str
3122
792
 
3123
- class Checkpoint(object, metaclass=type):
3124
- def __init__(self, temp_dir_root = None, init_dir = False):
3125
- ...
3126
- @property
3127
- def directory(self):
3128
- ...
3129
- def save(self, path = None, metadata = None, latest = True, name = "mfchckpt", storage_format = "files"):
3130
- """
3131
- Saves the checkpoint to the datastore
3132
-
3133
- Parameters
3134
- ----------
3135
- path : Optional[Union[str, os.PathLike]], default: None
3136
- The path to save the checkpoint. Accepts a file path or a directory path.
3137
- - If a directory path is provided, all the contents within that directory will be saved.
3138
- When a checkpoint is reloaded during task retries, `the current.checkpoint.directory` will
3139
- contain the contents of this directory.
3140
- - If a file path is provided, the file will be directly saved to the datastore (with the same filename).
3141
- When the checkpoint is reloaded during task retries, the file with the same name will be available in the
3142
- `current.checkpoint.directory`.
3143
- - If no path is provided then the `Checkpoint.directory` will be saved as the checkpoint.
3144
-
3145
- name : Optional[str], default: "mfchckpt"
3146
- The name of the checkpoint.
3147
-
3148
- metadata : Optional[Dict], default: {}
3149
- Any metadata that needs to be saved with the checkpoint.
3150
-
3151
- latest : bool, default: True
3152
- If True, the checkpoint will be marked as the latest checkpoint.
3153
- This helps determine if the checkpoint gets loaded when the task restarts.
3154
-
3155
- storage_format : str, default: files
3156
- If `tar`, the contents of the directory will be tarred before saving to the datastore.
3157
- If `files`, saves directory directly to the datastore.
3158
- """
3159
- ...
3160
- def __enter__(self):
3161
- ...
3162
- def __exit__(self, exc_type, exc_val, exc_tb):
3163
- ...
3164
- def list(self, name: typing.Optional[str] = None, task: typing.Union["metaflow.Task", str, None] = None, attempt: typing.Union[int, str, None] = None, as_dict: bool = True, within_task: bool = True) -> typing.Iterable[typing.Union[typing.Dict, metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]]:
3165
- """
3166
- lists the checkpoints in the datastore based on the Task.
3167
- It will always be task scoped.
3168
-
3169
- Usage:
3170
- ------
3171
-
3172
- ```python
3173
-
3174
- Checkpoint().list(name="best") # lists checkpoints in the current task with the name "best"
3175
- Checkpoint().list(task="anotherflow/somerunid/somestep/sometask", name="best") # Identical as the above one but
3176
- Checkpoint().list() # lists all the checkpoints in the current task
3177
-
3178
- ```
3179
-
3180
- Parameters
3181
- ----------
3182
-
3183
- - `name`:
3184
- - name of the checkpoint to filter for
3185
- - `task`:
3186
- - Task object outside the one that is currently set in the `Checkpoint` object; Can be a pathspec string.
3187
- - `attempt`:
3188
- - attempt number of the task (optional filter. If none, then lists all checkpoints from all attempts)
3189
- """
3190
- ...
3191
- def load(self, reference: typing.Union[str, typing.Dict, metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact], path: typing.Optional[str] = None):
3192
- """
3193
- loads a checkpoint reference from the datastore. (resembles a read op)
3194
-
3195
- Parameters
3196
- ----------
3197
-
3198
- `reference` :
3199
- - can be a string, dict or a CheckpointArtifact object:
3200
- - string: a string reference to the checkpoint (checkpoint key)
3201
- - dict: a dictionary reference to the checkpoint
3202
- - CheckpointArtifact: a CheckpointArtifact object reference to the checkpoint
3203
- """
3204
- ...
3205
- ...
3206
-
3207
- def load_model(reference: typing.Union[str, metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.MetaflowDataArtifactReference, dict], path: str):
3208
- ...
3209
-
3210
- def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
3211
- ...
3212
-