ob-metaflow-stubs 6.0.3.116__py2.py3-none-any.whl → 6.0.3.118__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (175) hide show
  1. metaflow-stubs/__init__.pyi +353 -2770
  2. metaflow-stubs/cards.pyi +19 -473
  3. metaflow-stubs/cli.pyi +17 -81
  4. metaflow-stubs/client/__init__.pyi +19 -1113
  5. metaflow-stubs/client/core.pyi +18 -158
  6. metaflow-stubs/client/filecache.pyi +7 -11
  7. metaflow-stubs/clone_util.pyi +6 -26
  8. metaflow-stubs/events.pyi +6 -5
  9. metaflow-stubs/exception.pyi +8 -6
  10. metaflow-stubs/flowspec.pyi +22 -106
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +16 -564
  13. metaflow-stubs/info_file.pyi +6 -5
  14. metaflow-stubs/metadata_provider/__init__.pyi +16 -0
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
  16. metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
  17. metaflow-stubs/metadata_provider/util.pyi +19 -0
  18. metaflow-stubs/metaflow_config.pyi +8 -13
  19. metaflow-stubs/metaflow_current.pyi +126 -89
  20. metaflow-stubs/mf_extensions/__init__.pyi +6 -0
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +6 -0
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +6 -0
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +6 -0
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +6 -0
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +129 -0
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +26 -0
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +156 -0
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +25 -0
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +111 -0
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +6 -0
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +188 -0
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +30 -0
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +28 -0
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +19 -0
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +115 -0
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +40 -0
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +6 -0
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +71 -0
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +56 -0
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +67 -0
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +49 -0
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +6 -0
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +6 -0
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +130 -0
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +42 -0
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +6 -0
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +17 -0
  48. metaflow-stubs/mflog/__init__.pyi +6 -0
  49. metaflow-stubs/mflog/mflog.pyi +52 -5
  50. metaflow-stubs/multicore_utils.pyi +6 -5
  51. metaflow-stubs/parameters.pyi +13 -23
  52. metaflow-stubs/plugins/__init__.pyi +52 -165
  53. metaflow-stubs/plugins/airflow/__init__.pyi +9 -5
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
  55. metaflow-stubs/plugins/airflow/exception.pyi +7 -11
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
  60. metaflow-stubs/plugins/argo/__init__.pyi +12 -5
  61. metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
  62. metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +18 -130
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +12 -404
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +63 -448
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
  68. metaflow-stubs/plugins/aws/__init__.pyi +11 -5
  69. metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -5
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -5
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +52 -292
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
  85. metaflow-stubs/plugins/azure/__init__.pyi +12 -7
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
  92. metaflow-stubs/plugins/cards/__init__.pyi +15 -5
  93. metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
  94. metaflow-stubs/plugins/cards/card_client.pyi +14 -76
  95. metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +13 -81
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
  107. metaflow-stubs/plugins/cards/exception.pyi +7 -11
  108. metaflow-stubs/plugins/catch_decorator.pyi +10 -30
  109. metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
  110. metaflow-stubs/plugins/datatools/local.pyi +7 -11
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +15 -263
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
  115. metaflow-stubs/plugins/debug_logger.pyi +7 -5
  116. metaflow-stubs/plugins/debug_monitor.pyi +7 -5
  117. metaflow-stubs/plugins/environment_decorator.pyi +7 -5
  118. metaflow-stubs/plugins/events_decorator.pyi +8 -14
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
  121. metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +12 -5
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +10 -38
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +7 -11
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
  134. metaflow-stubs/plugins/logs_cli.pyi +10 -9
  135. metaflow-stubs/plugins/package_cli.pyi +7 -5
  136. metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
  137. metaflow-stubs/plugins/perimeters.pyi +6 -11
  138. metaflow-stubs/plugins/project_decorator.pyi +8 -14
  139. metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +13 -19
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
  144. metaflow-stubs/plugins/pypi/utils.pyi +7 -11
  145. metaflow-stubs/plugins/resources_decorator.pyi +7 -5
  146. metaflow-stubs/plugins/retry_decorator.pyi +7 -11
  147. metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +9 -14
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
  150. metaflow-stubs/plugins/storage_executor.pyi +6 -11
  151. metaflow-stubs/plugins/tag_cli.pyi +14 -396
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +8 -33
  153. metaflow-stubs/plugins/timeout_decorator.pyi +7 -11
  154. metaflow-stubs/procpoll.pyi +7 -5
  155. metaflow-stubs/profilers/__init__.pyi +7 -11
  156. metaflow-stubs/pylint_wrapper.pyi +7 -11
  157. metaflow-stubs/runner/__init__.pyi +13 -5
  158. metaflow-stubs/runner/deployer.pyi +99 -223
  159. metaflow-stubs/runner/deployer_impl.pyi +87 -0
  160. metaflow-stubs/runner/metaflow_runner.pyi +24 -508
  161. metaflow-stubs/runner/nbdeploy.pyi +16 -60
  162. metaflow-stubs/runner/nbrun.pyi +11 -148
  163. metaflow-stubs/runner/subprocess_manager.pyi +9 -10
  164. metaflow-stubs/runner/utils.pyi +44 -9
  165. metaflow-stubs/system/__init__.pyi +9 -107
  166. metaflow-stubs/system/system_logger.pyi +7 -16
  167. metaflow-stubs/system/system_monitor.pyi +6 -16
  168. metaflow-stubs/tagging_util.pyi +6 -10
  169. metaflow-stubs/tuple_util.pyi +6 -5
  170. {ob_metaflow_stubs-6.0.3.116.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.118.dist-info/RECORD +174 -0
  172. {ob_metaflow_stubs-6.0.3.116.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/WHEEL +1 -1
  173. metaflow-stubs/metadata/util.pyi +0 -18
  174. ob_metaflow_stubs-6.0.3.116.dist-info/RECORD +0 -140
  175. {ob_metaflow_stubs-6.0.3.116.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/top_level.txt +0 -0
@@ -1,443 +1,71 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.1);ob(v1) #
4
- # Generated on 2024-11-04T20:32:12.972782 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.932934 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.datastore.inputs
12
- import typing
13
- import metaflow.flowspec
14
- import metaflow.metaflow_current
15
- import metaflow.runner.metaflow_runner
16
11
  import datetime
17
- import metaflow.client.core
18
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
19
- import metaflow.events
20
- import metaflow.parameters
21
- import metaflow._vendor.click.types
12
+ import typing
22
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
14
  StepFlag = typing.NewType("StepFlag", bool)
24
15
 
25
- EXT_PKG: str
26
-
27
- def parallel_imap_unordered(func: typing.Callable[[typing.Any], typing.Any], iterable: typing.Iterable[typing.Any], max_parallel: typing.Optional[int] = None, dir: typing.Optional[str] = None) -> typing.Iterator[typing.Any]:
28
- """
29
- Parallelizes execution of a function using multiprocessing. The result
30
- order is not guaranteed.
31
-
32
- Parameters
33
- ----------
34
- func : Callable[[Any], Any]
35
- Function taking a single argument and returning a result
36
- iterable : Iterable[Any]
37
- Iterable over arguments to pass to fun
38
- max_parallel int, optional, default None
39
- Maximum parallelism. If not specified, uses the number of CPUs
40
- dir : str, optional, default None
41
- If specified, directory where temporary files are created
42
-
43
- Yields
44
- ------
45
- Any
46
- One result from calling func on one argument
47
- """
48
- ...
49
-
50
- def parallel_map(func: typing.Callable[[typing.Any], typing.Any], iterable: typing.Iterable[typing.Any], max_parallel: typing.Optional[int] = None, dir: typing.Optional[str] = None) -> typing.List[typing.Any]:
51
- """
52
- Parallelizes execution of a function using multiprocessing. The result
53
- order is that of the arguments in `iterable`
54
-
55
- Parameters
56
- ----------
57
- func : Callable[[Any], Any]
58
- Function taking a single argument and returning a result
59
- iterable : Iterable[Any]
60
- Iterable over arguments to pass to fun
61
- max_parallel int, optional, default None
62
- Maximum parallelism. If not specified, uses the number of CPUs
63
- dir : str, optional, default None
64
- If specified, directory where temporary files are created
65
-
66
- Returns
67
- -------
68
- List[Any]
69
- Results. The items in the list are in the same order as the items
70
- in `iterable`.
71
- """
72
- ...
73
-
74
- current: metaflow.metaflow_current.Current
75
-
76
- def metadata(ms: str) -> str:
77
- """
78
- Switch Metadata provider.
79
-
80
- This call has a global effect. Selecting the local metadata will,
81
- for example, not allow access to information stored in remote
82
- metadata providers.
83
-
84
- Note that you don't typically have to call this function directly. Usually
85
- the metadata provider is set through the Metaflow configuration file. If you
86
- need to switch between multiple providers, you can use the `METAFLOW_PROFILE`
87
- environment variable to switch between configurations.
88
-
89
- Parameters
90
- ----------
91
- ms : str
92
- Can be a path (selects local metadata), a URL starting with http (selects
93
- the service metadata) or an explicit specification <metadata_type>@<info>; as an
94
- example, you can specify local@<path> or service@<url>.
95
-
96
- Returns
97
- -------
98
- str
99
- The description of the metadata selected (equivalent to the result of
100
- get_metadata()).
101
- """
102
- ...
16
+ from . import info_file as info_file
17
+ from . import exception as exception
18
+ from . import metaflow_config as metaflow_config
19
+ from . import multicore_utils as multicore_utils
20
+ from .multicore_utils import parallel_imap_unordered as parallel_imap_unordered
21
+ from .multicore_utils import parallel_map as parallel_map
22
+ from . import metaflow_current as metaflow_current
23
+ from .metaflow_current import current as current
24
+ from . import parameters as parameters
25
+ from . import tagging_util as tagging_util
26
+ from . import metadata_provider as metadata_provider
27
+ from . import flowspec as flowspec
28
+ from .flowspec import FlowSpec as FlowSpec
29
+ from .parameters import Parameter as Parameter
30
+ from .parameters import JSONTypeClass as JSONTypeClass
31
+ from .parameters import JSONType as JSONType
32
+ from . import tuple_util as tuple_util
33
+ from . import cards as cards
34
+ from . import events as events
35
+ from . import runner as runner
36
+ from . import plugins as plugins
37
+ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
38
+ from . import includefile as includefile
39
+ from .includefile import IncludeFile as IncludeFile
40
+ from . import client as client
41
+ from .client.core import namespace as namespace
42
+ from .client.core import get_namespace as get_namespace
43
+ from .client.core import default_namespace as default_namespace
44
+ from .client.core import metadata as metadata
45
+ from .client.core import get_metadata as get_metadata
46
+ from .client.core import default_metadata as default_metadata
47
+ from .client.core import Metaflow as Metaflow
48
+ from .client.core import Flow as Flow
49
+ from .client.core import Run as Run
50
+ from .client.core import Step as Step
51
+ from .client.core import Task as Task
52
+ from .client.core import DataArtifact as DataArtifact
53
+ from .runner.metaflow_runner import Runner as Runner
54
+ from .runner.nbrun import NBRunner as NBRunner
55
+ from .runner.deployer import Deployer as Deployer
56
+ from .runner.deployer import DeployedFlow as DeployedFlow
57
+ from .runner.nbdeploy import NBDeployer as NBDeployer
58
+ from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.final_api import Checkpoint as Checkpoint
59
+ from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures import load_model as load_model
60
+ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_aws_client as get_aws_client
61
+ from . import system as system
62
+ from . import pylint_wrapper as pylint_wrapper
63
+ from . import procpoll as procpoll
64
+ from . import clone_util as clone_util
65
+ from . import cli as cli
66
+ from . import profilers as profilers
103
67
 
104
- class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
105
- """
106
- Main class from which all Flows should inherit.
107
-
108
- Attributes
109
- ----------
110
- index
111
- input
112
- """
113
- def __init__(self, use_cli = True):
114
- """
115
- Construct a FlowSpec
116
-
117
- Parameters
118
- ----------
119
- use_cli : bool, default True
120
- Set to True if the flow is invoked from __main__ or the command line
121
- """
122
- ...
123
- @property
124
- def script_name(self) -> str:
125
- """
126
- [Legacy function - do not use. Use `current` instead]
127
-
128
- Returns the name of the script containing the flow
129
-
130
- Returns
131
- -------
132
- str
133
- A string containing the name of the script
134
- """
135
- ...
136
- def __iter__(self):
137
- """
138
- [Legacy function - do not use]
139
-
140
- Iterate over all steps in the Flow
141
-
142
- Returns
143
- -------
144
- Iterator[graph.DAGNode]
145
- Iterator over the steps in the flow
146
- """
147
- ...
148
- def __getattr__(self, name: str):
149
- ...
150
- def cmd(self, cmdline, input = {}, output = []):
151
- """
152
- [Legacy function - do not use]
153
- """
154
- ...
155
- @property
156
- def index(self) -> typing.Optional[int]:
157
- """
158
- The index of this foreach branch.
159
-
160
- In a foreach step, multiple instances of this step (tasks) will be executed,
161
- one for each element in the foreach. This property returns the zero based index
162
- of the current task. If this is not a foreach step, this returns None.
163
-
164
- If you need to know the indices of the parent tasks in a nested foreach, use
165
- `FlowSpec.foreach_stack`.
166
-
167
- Returns
168
- -------
169
- int, optional
170
- Index of the task in a foreach step.
171
- """
172
- ...
173
- @property
174
- def input(self) -> typing.Optional[typing.Any]:
175
- """
176
- The value of the foreach artifact in this foreach branch.
177
-
178
- In a foreach step, multiple instances of this step (tasks) will be executed,
179
- one for each element in the foreach. This property returns the element passed
180
- to the current task. If this is not a foreach step, this returns None.
181
-
182
- If you need to know the values of the parent tasks in a nested foreach, use
183
- `FlowSpec.foreach_stack`.
184
-
185
- Returns
186
- -------
187
- object, optional
188
- Input passed to the foreach task.
189
- """
190
- ...
191
- def foreach_stack(self) -> typing.Optional[typing.List[typing.Tuple[int, int, typing.Any]]]:
192
- """
193
- Returns the current stack of foreach indexes and values for the current step.
194
-
195
- Use this information to understand what data is being processed in the current
196
- foreach branch. For example, considering the following code:
197
- ```
198
- @step
199
- def root(self):
200
- self.split_1 = ['a', 'b', 'c']
201
- self.next(self.nest_1, foreach='split_1')
202
-
203
- @step
204
- def nest_1(self):
205
- self.split_2 = ['d', 'e', 'f', 'g']
206
- self.next(self.nest_2, foreach='split_2'):
207
-
208
- @step
209
- def nest_2(self):
210
- foo = self.foreach_stack()
211
- ```
212
-
213
- `foo` will take the following values in the various tasks for nest_2:
214
- ```
215
- [(0, 3, 'a'), (0, 4, 'd')]
216
- [(0, 3, 'a'), (1, 4, 'e')]
217
- ...
218
- [(0, 3, 'a'), (3, 4, 'g')]
219
- [(1, 3, 'b'), (0, 4, 'd')]
220
- ...
221
- ```
222
- where each tuple corresponds to:
223
-
224
- - The index of the task for that level of the loop.
225
- - The number of splits for that level of the loop.
226
- - The value for that level of the loop.
227
-
228
- Note that the last tuple returned in a task corresponds to:
229
-
230
- - 1st element: value returned by `self.index`.
231
- - 3rd element: value returned by `self.input`.
232
-
233
- Returns
234
- -------
235
- List[Tuple[int, int, Any]]
236
- An array describing the current stack of foreach steps.
237
- """
238
- ...
239
- def merge_artifacts(self, inputs: metaflow.datastore.inputs.Inputs, exclude: typing.Optional[typing.List[str]] = None, include: typing.Optional[typing.List[str]] = None):
240
- """
241
- Helper function for merging artifacts in a join step.
242
-
243
- This function takes all the artifacts coming from the branches of a
244
- join point and assigns them to self in the calling step. Only artifacts
245
- not set in the current step are considered. If, for a given artifact, different
246
- values are present on the incoming edges, an error will be thrown and the artifacts
247
- that conflict will be reported.
248
-
249
- As a few examples, in the simple graph: A splitting into B and C and joining in D:
250
- ```
251
- A:
252
- self.x = 5
253
- self.y = 6
254
- B:
255
- self.b_var = 1
256
- self.x = from_b
257
- C:
258
- self.x = from_c
259
-
260
- D:
261
- merge_artifacts(inputs)
262
- ```
263
- In D, the following artifacts are set:
264
- - `y` (value: 6), `b_var` (value: 1)
265
- - if `from_b` and `from_c` are the same, `x` will be accessible and have value `from_b`
266
- - if `from_b` and `from_c` are different, an error will be thrown. To prevent this error,
267
- you need to manually set `self.x` in D to a merged value (for example the max) prior to
268
- calling `merge_artifacts`.
269
-
270
- Parameters
271
- ----------
272
- inputs : Inputs
273
- Incoming steps to the join point.
274
- exclude : List[str], optional, default None
275
- If specified, do not consider merging artifacts with a name in `exclude`.
276
- Cannot specify if `include` is also specified.
277
- include : List[str], optional, default None
278
- If specified, only merge artifacts specified. Cannot specify if `exclude` is
279
- also specified.
280
-
281
- Raises
282
- ------
283
- MetaflowException
284
- This exception is thrown if this is not called in a join step.
285
- UnhandledInMergeArtifactsException
286
- This exception is thrown in case of unresolved conflicts.
287
- MissingInMergeArtifactsException
288
- This exception is thrown in case an artifact specified in `include` cannot
289
- be found.
290
- """
291
- ...
292
- def next(self, *dsts: typing.Callable[..., None], **kwargs):
293
- """
294
- Indicates the next step to execute after this step has completed.
295
-
296
- This statement should appear as the last statement of each step, except
297
- the end step.
298
-
299
- There are several valid formats to specify the next step:
300
-
301
- - Straight-line connection: `self.next(self.next_step)` where `next_step` is a method in
302
- the current class decorated with the `@step` decorator.
303
-
304
- - Static fan-out connection: `self.next(self.step1, self.step2, ...)` where `stepX` are
305
- methods in the current class decorated with the `@step` decorator.
306
-
307
- - Foreach branch:
308
- ```
309
- self.next(self.foreach_step, foreach='foreach_iterator')
310
- ```
311
- In this situation, `foreach_step` is a method in the current class decorated with the
312
- `@step` decorator and `foreach_iterator` is a variable name in the current class that
313
- evaluates to an iterator. A task will be launched for each value in the iterator and
314
- each task will execute the code specified by the step `foreach_step`.
315
-
316
- Parameters
317
- ----------
318
- dsts : Callable[..., None]
319
- One or more methods annotated with `@step`.
320
-
321
- Raises
322
- ------
323
- InvalidNextException
324
- Raised if the format of the arguments does not match one of the ones given above.
325
- """
326
- ...
327
- def __str__(self):
328
- ...
329
- def __getstate__(self):
330
- ...
331
- ...
332
-
333
- class Parameter(object, metaclass=type):
334
- """
335
- Defines a parameter for a flow.
336
-
337
- Parameters must be instantiated as class variables in flow classes, e.g.
338
- ```
339
- class MyFlow(FlowSpec):
340
- param = Parameter('myparam')
341
- ```
342
- in this case, the parameter is specified on the command line as
343
- ```
344
- python myflow.py run --myparam=5
345
- ```
346
- and its value is accessible through a read-only artifact like this:
347
- ```
348
- print(self.param == 5)
349
- ```
350
- Note that the user-visible parameter name, `myparam` above, can be
351
- different from the artifact name, `param` above.
352
-
353
- The parameter value is converted to a Python type based on the `type`
354
- argument or to match the type of `default`, if it is set.
355
-
356
- Parameters
357
- ----------
358
- name : str
359
- User-visible parameter name.
360
- default : str or float or int or bool or `JSONType` or a function.
361
- Default value for the parameter. Use a special `JSONType` class to
362
- indicate that the value must be a valid JSON object. A function
363
- implies that the parameter corresponds to a *deploy-time parameter*.
364
- The type of the default value is used as the parameter `type`.
365
- type : Type, default None
366
- If `default` is not specified, define the parameter type. Specify
367
- one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
368
- to the type of `default` or `str` if none specified.
369
- help : str, optional
370
- Help text to show in `run --help`.
371
- required : bool, default False
372
- Require that the user specified a value for the parameter.
373
- `required=True` implies that the `default` is not used.
374
- show_default : bool, default True
375
- If True, show the default value in the help text.
376
- """
377
- def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], metaflow.parameters.JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
378
- ...
379
- def __repr__(self):
380
- ...
381
- def __str__(self):
382
- ...
383
- def option_kwargs(self, deploy_mode):
384
- ...
385
- def load_parameter(self, v):
386
- ...
387
- @property
388
- def is_string_type(self):
389
- ...
390
- def __getitem__(self, x):
391
- ...
392
- ...
393
-
394
- class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
395
- def convert(self, value, param, ctx):
396
- ...
397
- def __str__(self):
398
- ...
399
- def __repr__(self):
400
- ...
401
- ...
402
-
403
- JSONType: metaflow.parameters.JSONTypeClass
404
-
405
- def S3(*args, **kwargs):
406
- ...
407
-
408
- class IncludeFile(metaflow.parameters.Parameter, metaclass=type):
409
- """
410
- Includes a local file as a parameter for the flow.
411
-
412
- `IncludeFile` behaves like `Parameter` except that it reads its value from a file instead of
413
- the command line. The user provides a path to a file on the command line. The file contents
414
- are saved as a read-only artifact which is available in all steps of the flow.
415
-
416
- Parameters
417
- ----------
418
- name : str
419
- User-visible parameter name.
420
- default : Union[str, Callable[ParameterContext, str]]
421
- Default path to a local file. A function
422
- implies that the parameter corresponds to a *deploy-time parameter*.
423
- is_text : bool, default True
424
- Convert the file contents to a string using the provided `encoding`.
425
- If False, the artifact is stored in `bytes`.
426
- encoding : str, optional, default 'utf-8'
427
- Use this encoding to decode the file contexts if `is_text=True`.
428
- required : bool, default False
429
- Require that the user specified a value for the parameter.
430
- `required=True` implies that the `default` is not used.
431
- help : str, optional
432
- Help text to show in `run --help`.
433
- show_default : bool, default True
434
- If True, show the default value in the help text.
435
- """
436
- def __init__(self, name: str, required: bool = False, is_text: bool = True, encoding: str = "utf-8", help: typing.Optional[str] = None, **kwargs: typing.Dict[str, str]):
437
- ...
438
- def load_parameter(self, v):
439
- ...
440
- ...
68
+ EXT_PKG: str
441
69
 
442
70
  @typing.overload
443
71
  def step(f: typing.Callable[[FlowSpecDerived], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
@@ -513,163 +141,120 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
513
141
  ...
514
142
 
515
143
  @typing.overload
516
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
144
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
517
145
  """
518
- Specifies the number of times the task corresponding
519
- to a step needs to be retried.
520
-
521
- This decorator is useful for handling transient errors, such as networking issues.
522
- If your task contains operations that can't be retried safely, e.g. database updates,
523
- it is advisable to annotate it with `@retry(times=0)`.
524
-
525
- This can be used in conjunction with the `@catch` decorator. The `@catch`
526
- decorator will execute a no-op task after all retries have been exhausted,
527
- ensuring that the flow execution can continue.
146
+ Specifies the Conda environment for the step.
528
147
 
529
- Parameters
530
- ----------
531
- times : int, default 3
532
- Number of times to retry this task.
533
- minutes_between_retries : int, default 2
534
- Number of minutes between retries.
148
+ Information in this decorator will augment any
149
+ attributes set in the `@conda_base` flow-level decorator. Hence,
150
+ you can use `@conda_base` to set packages required by all
151
+ steps and use `@conda` to specify step-specific overrides.
535
152
  """
536
153
  ...
537
154
 
538
155
  @typing.overload
539
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
156
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
540
157
  ...
541
158
 
542
159
  @typing.overload
543
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
160
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
544
161
  ...
545
162
 
546
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
163
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
547
164
  """
548
- Specifies the number of times the task corresponding
549
- to a step needs to be retried.
550
-
551
- This decorator is useful for handling transient errors, such as networking issues.
552
- If your task contains operations that can't be retried safely, e.g. database updates,
553
- it is advisable to annotate it with `@retry(times=0)`.
554
-
555
- This can be used in conjunction with the `@catch` decorator. The `@catch`
556
- decorator will execute a no-op task after all retries have been exhausted,
557
- ensuring that the flow execution can continue.
165
+ Specifies the Conda environment for the step.
558
166
 
559
- Parameters
560
- ----------
561
- times : int, default 3
562
- Number of times to retry this task.
563
- minutes_between_retries : int, default 2
564
- Number of minutes between retries.
167
+ Information in this decorator will augment any
168
+ attributes set in the `@conda_base` flow-level decorator. Hence,
169
+ you can use `@conda_base` to set packages required by all
170
+ steps and use `@conda` to specify step-specific overrides.
565
171
  """
566
172
  ...
567
173
 
568
174
  @typing.overload
569
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
175
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
570
176
  """
571
- Creates a human-readable report, a Metaflow Card, after this step completes.
572
-
573
- Note that you may add multiple `@card` decorators in a step with different parameters.
574
-
575
- Parameters
576
- ----------
577
- type : str, default 'default'
578
- Card type.
579
- id : str, optional, default None
580
- If multiple cards are present, use this id to identify this card.
581
- options : Dict[str, Any], default {}
582
- Options passed to the card. The contents depend on the card type.
583
- timeout : int, default 45
584
- Interrupt reporting if it takes more than this many seconds.
585
-
177
+ Specifies the PyPI packages for the step.
586
178
 
179
+ Information in this decorator will augment any
180
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
181
+ you can use `@pypi_base` to set packages required by all
182
+ steps and use `@pypi` to specify step-specific overrides.
587
183
  """
588
184
  ...
589
185
 
590
186
  @typing.overload
591
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
187
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
592
188
  ...
593
189
 
594
190
  @typing.overload
595
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
191
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
596
192
  ...
597
193
 
598
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
194
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
599
195
  """
600
- Creates a human-readable report, a Metaflow Card, after this step completes.
601
-
602
- Note that you may add multiple `@card` decorators in a step with different parameters.
603
-
604
- Parameters
605
- ----------
606
- type : str, default 'default'
607
- Card type.
608
- id : str, optional, default None
609
- If multiple cards are present, use this id to identify this card.
610
- options : Dict[str, Any], default {}
611
- Options passed to the card. The contents depend on the card type.
612
- timeout : int, default 45
613
- Interrupt reporting if it takes more than this many seconds.
614
-
196
+ Specifies the PyPI packages for the step.
615
197
 
198
+ Information in this decorator will augment any
199
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
200
+ you can use `@pypi_base` to set packages required by all
201
+ steps and use `@pypi` to specify step-specific overrides.
202
+ """
203
+ ...
204
+
205
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
206
+ """
207
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
616
208
  """
617
209
  ...
618
210
 
619
211
  @typing.overload
620
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
212
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
621
213
  """
622
- Specifies the Conda environment for the step.
623
-
624
- Information in this decorator will augment any
625
- attributes set in the `@conda_base` flow-level decorator. Hence,
626
- you can use `@conda_base` to set packages required by all
627
- steps and use `@conda` to specify step-specific overrides.
214
+ Decorator prototype for all step decorators. This function gets specialized
215
+ and imported for all decorators types by _import_plugin_decorators().
216
+ """
217
+ ...
218
+
219
+ @typing.overload
220
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
221
+ ...
222
+
223
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
224
+ """
225
+ Decorator prototype for all step decorators. This function gets specialized
226
+ and imported for all decorators types by _import_plugin_decorators().
227
+ """
228
+ ...
229
+
230
+ @typing.overload
231
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
232
+ """
233
+ Specifies that the step will success under all circumstances.
628
234
 
629
- Parameters
630
- ----------
631
- packages : Dict[str, str], default {}
632
- Packages to use for this step. The key is the name of the package
633
- and the value is the version to use.
634
- libraries : Dict[str, str], default {}
635
- Supported for backward compatibility. When used with packages, packages will take precedence.
636
- python : str, optional, default None
637
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
638
- that the version used will correspond to the version of the Python interpreter used to start the run.
639
- disabled : bool, default False
640
- If set to True, disables @conda.
235
+ The decorator will create an optional artifact, specified by `var`, which
236
+ contains the exception raised. You can use it to detect the presence
237
+ of errors, indicating that all happy-path artifacts produced by the step
238
+ are missing.
641
239
  """
642
240
  ...
643
241
 
644
242
  @typing.overload
645
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
243
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
646
244
  ...
647
245
 
648
246
  @typing.overload
649
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
247
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
650
248
  ...
651
249
 
652
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
250
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
653
251
  """
654
- Specifies the Conda environment for the step.
655
-
656
- Information in this decorator will augment any
657
- attributes set in the `@conda_base` flow-level decorator. Hence,
658
- you can use `@conda_base` to set packages required by all
659
- steps and use `@conda` to specify step-specific overrides.
252
+ Specifies that the step will success under all circumstances.
660
253
 
661
- Parameters
662
- ----------
663
- packages : Dict[str, str], default {}
664
- Packages to use for this step. The key is the name of the package
665
- and the value is the version to use.
666
- libraries : Dict[str, str], default {}
667
- Supported for backward compatibility. When used with packages, packages will take precedence.
668
- python : str, optional, default None
669
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
670
- that the version used will correspond to the version of the Python interpreter used to start the run.
671
- disabled : bool, default False
672
- If set to True, disables @conda.
254
+ The decorator will create an optional artifact, specified by `var`, which
255
+ contains the exception raised. You can use it to detect the presence
256
+ of errors, indicating that all happy-path artifacts produced by the step
257
+ are missing.
673
258
  """
674
259
  ...
675
260
 
@@ -691,20 +276,6 @@ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Op
691
276
  ```
692
277
  which executes the flow on the desired system using the
693
278
  requirements specified in `@resources`.
694
-
695
- Parameters
696
- ----------
697
- cpu : int, default 1
698
- Number of CPUs required for this step.
699
- gpu : int, optional, default None
700
- Number of GPUs required for this step.
701
- disk : int, optional, default None
702
- Disk size (in MB) required for this step. Only applies on Kubernetes.
703
- memory : int, default 4096
704
- Memory size (in MB) required for this step.
705
- shared_memory : int, optional, default None
706
- The value for the size (in MiB) of the /dev/shm volume for this step.
707
- This parameter maps to the `--shm-size` option in Docker.
708
279
  """
709
280
  ...
710
281
 
@@ -733,574 +304,404 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
733
304
  ```
734
305
  which executes the flow on the desired system using the
735
306
  requirements specified in `@resources`.
736
-
737
- Parameters
738
- ----------
739
- cpu : int, default 1
740
- Number of CPUs required for this step.
741
- gpu : int, optional, default None
742
- Number of GPUs required for this step.
743
- disk : int, optional, default None
744
- Disk size (in MB) required for this step. Only applies on Kubernetes.
745
- memory : int, default 4096
746
- Memory size (in MB) required for this step.
747
- shared_memory : int, optional, default None
748
- The value for the size (in MiB) of the /dev/shm volume for this step.
749
- This parameter maps to the `--shm-size` option in Docker.
750
307
  """
751
308
  ...
752
309
 
753
310
  @typing.overload
754
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
311
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
755
312
  """
756
- Internal decorator to support Fast bakery
313
+ Specifies environment variables to be set prior to the execution of a step.
757
314
  """
758
315
  ...
759
316
 
760
317
  @typing.overload
761
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
318
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
762
319
  ...
763
320
 
764
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
321
+ @typing.overload
322
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
323
+ ...
324
+
325
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
765
326
  """
766
- Internal decorator to support Fast bakery
327
+ Specifies environment variables to be set prior to the execution of a step.
767
328
  """
768
329
  ...
769
330
 
770
331
  @typing.overload
771
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
332
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
772
333
  """
773
- Specifies a timeout for your step.
774
-
775
- This decorator is useful if this step may hang indefinitely.
776
-
777
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
778
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
779
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
334
+ Specifies the number of times the task corresponding
335
+ to a step needs to be retried.
780
336
 
781
- Note that all the values specified in parameters are added together so if you specify
782
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
337
+ This decorator is useful for handling transient errors, such as networking issues.
338
+ If your task contains operations that can't be retried safely, e.g. database updates,
339
+ it is advisable to annotate it with `@retry(times=0)`.
783
340
 
784
- Parameters
785
- ----------
786
- seconds : int, default 0
787
- Number of seconds to wait prior to timing out.
788
- minutes : int, default 0
789
- Number of minutes to wait prior to timing out.
790
- hours : int, default 0
791
- Number of hours to wait prior to timing out.
341
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
342
+ decorator will execute a no-op task after all retries have been exhausted,
343
+ ensuring that the flow execution can continue.
792
344
  """
793
345
  ...
794
346
 
795
347
  @typing.overload
796
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
348
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
797
349
  ...
798
350
 
799
351
  @typing.overload
800
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
352
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
801
353
  ...
802
354
 
803
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
355
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
804
356
  """
805
- Specifies a timeout for your step.
806
-
807
- This decorator is useful if this step may hang indefinitely.
808
-
809
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
810
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
811
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
357
+ Specifies the number of times the task corresponding
358
+ to a step needs to be retried.
812
359
 
813
- Note that all the values specified in parameters are added together so if you specify
814
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
360
+ This decorator is useful for handling transient errors, such as networking issues.
361
+ If your task contains operations that can't be retried safely, e.g. database updates,
362
+ it is advisable to annotate it with `@retry(times=0)`.
815
363
 
816
- Parameters
817
- ----------
818
- seconds : int, default 0
819
- Number of seconds to wait prior to timing out.
820
- minutes : int, default 0
821
- Number of minutes to wait prior to timing out.
822
- hours : int, default 0
823
- Number of hours to wait prior to timing out.
364
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
365
+ decorator will execute a no-op task after all retries have been exhausted,
366
+ ensuring that the flow execution can continue.
824
367
  """
825
368
  ...
826
369
 
827
370
  @typing.overload
828
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
371
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
829
372
  """
830
- Enables loading / saving of models within a step.
831
-
832
-
833
- Parameters
834
- ----------
835
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
836
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
837
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
838
- - `current.checkpoint`
839
- - `current.model`
840
- - `current.huggingface_hub`
841
-
842
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
843
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
844
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
845
-
846
- temp_dir_root : str, default: None
847
- The root directory under which `current.model.loaded` will store loaded models
848
-
849
-
850
-
373
+ Specifies secrets to be retrieved and injected as environment variables prior to
374
+ the execution of a step.
851
375
  """
852
376
  ...
853
377
 
854
378
  @typing.overload
855
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
379
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
856
380
  ...
857
381
 
858
382
  @typing.overload
859
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
383
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
860
384
  ...
861
385
 
862
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
386
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
863
387
  """
864
- Enables loading / saving of models within a step.
865
-
866
-
867
- Parameters
868
- ----------
869
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
870
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
871
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
872
- - `current.checkpoint`
873
- - `current.model`
874
- - `current.huggingface_hub`
875
-
876
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
877
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
878
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
879
-
880
- temp_dir_root : str, default: None
881
- The root directory under which `current.model.loaded` will store loaded models
882
-
883
-
884
-
388
+ Specifies secrets to be retrieved and injected as environment variables prior to
389
+ the execution of a step.
885
390
  """
886
391
  ...
887
392
 
888
393
  @typing.overload
889
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
394
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
890
395
  """
891
- Specifies that the step will success under all circumstances.
892
-
893
- The decorator will create an optional artifact, specified by `var`, which
894
- contains the exception raised. You can use it to detect the presence
895
- of errors, indicating that all happy-path artifacts produced by the step
896
- are missing.
897
-
898
- Parameters
899
- ----------
900
- var : str, optional, default None
901
- Name of the artifact in which to store the caught exception.
902
- If not specified, the exception is not stored.
903
- print_exception : bool, default True
904
- Determines whether or not the exception is printed to
905
- stdout when caught.
396
+ Enables loading / saving of models within a step.
906
397
  """
907
398
  ...
908
399
 
909
400
  @typing.overload
910
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
401
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
911
402
  ...
912
403
 
913
404
  @typing.overload
914
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
405
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
915
406
  ...
916
407
 
917
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
408
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
918
409
  """
919
- Specifies that the step will success under all circumstances.
920
-
921
- The decorator will create an optional artifact, specified by `var`, which
922
- contains the exception raised. You can use it to detect the presence
923
- of errors, indicating that all happy-path artifacts produced by the step
924
- are missing.
925
-
926
- Parameters
927
- ----------
928
- var : str, optional, default None
929
- Name of the artifact in which to store the caught exception.
930
- If not specified, the exception is not stored.
931
- print_exception : bool, default True
932
- Determines whether or not the exception is printed to
933
- stdout when caught.
410
+ Enables loading / saving of models within a step.
934
411
  """
935
412
  ...
936
413
 
937
414
  @typing.overload
938
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
415
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
939
416
  """
940
- Decorator prototype for all step decorators. This function gets specialized
941
- and imported for all decorators types by _import_plugin_decorators().
417
+ Internal decorator to support Fast bakery
942
418
  """
943
419
  ...
944
420
 
945
421
  @typing.overload
946
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
422
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
947
423
  ...
948
424
 
949
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
425
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
950
426
  """
951
- Decorator prototype for all step decorators. This function gets specialized
952
- and imported for all decorators types by _import_plugin_decorators().
427
+ Internal decorator to support Fast bakery
953
428
  """
954
429
  ...
955
430
 
956
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
431
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
957
432
  """
958
433
  Specifies that this step should execute on Kubernetes.
959
-
960
- Parameters
961
- ----------
962
- cpu : int, default 1
963
- Number of CPUs required for this step. If `@resources` is
964
- also present, the maximum value from all decorators is used.
965
- memory : int, default 4096
966
- Memory size (in MB) required for this step. If
967
- `@resources` is also present, the maximum value from all decorators is
968
- used.
969
- disk : int, default 10240
970
- Disk size (in MB) required for this step. If
971
- `@resources` is also present, the maximum value from all decorators is
972
- used.
973
- image : str, optional, default None
974
- Docker image to use when launching on Kubernetes. If not specified, and
975
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
976
- not, a default Docker image mapping to the current version of Python is used.
977
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
978
- If given, the imagePullPolicy to be applied to the Docker image of the step.
979
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
980
- Kubernetes service account to use when launching pod in Kubernetes.
981
- secrets : List[str], optional, default None
982
- Kubernetes secrets to use when launching pod in Kubernetes. These
983
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
984
- in Metaflow configuration.
985
- node_selector: Union[Dict[str,str], str], optional, default None
986
- Kubernetes node selector(s) to apply to the pod running the task.
987
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
988
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
989
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
990
- Kubernetes namespace to use when launching pod in Kubernetes.
991
- gpu : int, optional, default None
992
- Number of GPUs required for this step. A value of zero implies that
993
- the scheduled node should not have GPUs.
994
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
995
- The vendor of the GPUs to be used for this step.
996
- tolerations : List[str], default []
997
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
998
- Kubernetes tolerations to use when launching pod in Kubernetes.
999
- use_tmpfs : bool, default False
1000
- This enables an explicit tmpfs mount for this step.
1001
- tmpfs_tempdir : bool, default True
1002
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1003
- tmpfs_size : int, optional, default: None
1004
- The value for the size (in MiB) of the tmpfs mount for this step.
1005
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1006
- memory allocated for this step.
1007
- tmpfs_path : str, optional, default /metaflow_temp
1008
- Path to tmpfs mount for this step.
1009
- persistent_volume_claims : Dict[str, str], optional, default None
1010
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1011
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1012
- shared_memory: int, optional
1013
- Shared memory size (in MiB) required for this step
1014
- port: int, optional
1015
- Port number to specify in the Kubernetes job object
1016
- compute_pool : str, optional, default None
1017
- Compute pool to be used for for this step.
1018
- If not specified, any accessible compute pool within the perimeter is used.
1019
- hostname_resolution_timeout: int, default 10 * 60
1020
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1021
- Only applicable when @parallel is used.
1022
434
  """
1023
435
  ...
1024
436
 
1025
437
  @typing.overload
1026
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
438
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1027
439
  """
1028
- Specifies the PyPI packages for the step.
1029
-
1030
- Information in this decorator will augment any
1031
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1032
- you can use `@pypi_base` to set packages required by all
1033
- steps and use `@pypi` to specify step-specific overrides.
1034
-
1035
- Parameters
1036
- ----------
1037
- packages : Dict[str, str], default: {}
1038
- Packages to use for this step. The key is the name of the package
1039
- and the value is the version to use.
1040
- python : str, optional, default: None
1041
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1042
- that the version used will correspond to the version of the Python interpreter used to start the run.
440
+ Enables checkpointing for a step.
1043
441
  """
1044
442
  ...
1045
443
 
1046
444
  @typing.overload
1047
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
445
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1048
446
  ...
1049
447
 
1050
448
  @typing.overload
1051
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
449
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1052
450
  ...
1053
451
 
1054
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
452
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
1055
453
  """
1056
- Specifies the PyPI packages for the step.
1057
-
1058
- Information in this decorator will augment any
1059
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1060
- you can use `@pypi_base` to set packages required by all
1061
- steps and use `@pypi` to specify step-specific overrides.
1062
-
1063
- Parameters
1064
- ----------
1065
- packages : Dict[str, str], default: {}
1066
- Packages to use for this step. The key is the name of the package
1067
- and the value is the version to use.
1068
- python : str, optional, default: None
1069
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1070
- that the version used will correspond to the version of the Python interpreter used to start the run.
454
+ Enables checkpointing for a step.
1071
455
  """
1072
456
  ...
1073
457
 
1074
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
458
+ @typing.overload
459
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1075
460
  """
1076
- Decorator that helps cache, version and store models/datasets from huggingface hub.
461
+ Specifies a timeout for your step.
1077
462
 
1078
- Parameters
1079
- ----------
1080
- temp_dir_root : str, optional
1081
- The root directory that will hold the temporary directory where objects will be downloaded.
463
+ This decorator is useful if this step may hang indefinitely.
1082
464
 
465
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
466
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
467
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1083
468
 
469
+ Note that all the values specified in parameters are added together so if you specify
470
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1084
471
  """
1085
472
  ...
1086
473
 
1087
474
  @typing.overload
1088
- def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
475
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
476
+ ...
477
+
478
+ @typing.overload
479
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
480
+ ...
481
+
482
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1089
483
  """
1090
- Enables checkpointing for a step.
1091
-
1092
-
1093
- Parameters
1094
- ----------
1095
- load_policy : str, default: "fresh"
1096
- The policy for loading the checkpoint. The following policies are supported:
1097
- - "eager": Loads the the latest available checkpoint within the namespace.
1098
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1099
- will be loaded at the start of the task.
1100
- - "none": Do not load any checkpoint
1101
- - "fresh": Loads the lastest checkpoint created within the running Task.
1102
- This mode helps loading checkpoints across various retry attempts of the same task.
1103
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1104
- created within the task will be loaded when the task is retries execution on failure.
484
+ Specifies a timeout for your step.
1105
485
 
1106
- temp_dir_root : str, default: None
1107
- The root directory under which `current.checkpoint.directory` will be created.
486
+ This decorator is useful if this step may hang indefinitely.
1108
487
 
488
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
489
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
490
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1109
491
 
492
+ Note that all the values specified in parameters are added together so if you specify
493
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
494
+ """
495
+ ...
496
+
497
+ @typing.overload
498
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
499
+ """
500
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1110
501
 
502
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1111
503
  """
1112
504
  ...
1113
505
 
1114
506
  @typing.overload
1115
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
507
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1116
508
  ...
1117
509
 
1118
510
  @typing.overload
1119
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
511
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1120
512
  ...
1121
513
 
1122
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
514
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1123
515
  """
1124
- Enables checkpointing for a step.
1125
-
516
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1126
517
 
1127
- Parameters
1128
- ----------
1129
- load_policy : str, default: "fresh"
1130
- The policy for loading the checkpoint. The following policies are supported:
1131
- - "eager": Loads the the latest available checkpoint within the namespace.
1132
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1133
- will be loaded at the start of the task.
1134
- - "none": Do not load any checkpoint
1135
- - "fresh": Loads the lastest checkpoint created within the running Task.
1136
- This mode helps loading checkpoints across various retry attempts of the same task.
1137
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1138
- created within the task will be loaded when the task is retries execution on failure.
518
+ Note that you may add multiple `@card` decorators in a step with different parameters.
519
+ """
520
+ ...
521
+
522
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
523
+ """
524
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1139
525
 
1140
- temp_dir_root : str, default: None
1141
- The root directory under which `current.checkpoint.directory` will be created.
526
+ User code call
527
+ -----------
528
+ @nim(
529
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
530
+ backend='managed'
531
+ )
1142
532
 
533
+ Valid backend options
534
+ ---------------------
535
+ - 'managed': Outerbounds selects a compute provider based on the model.
536
+ - 🚧 'dataplane': Run in your account.
1143
537
 
538
+ Valid model options
539
+ ----------------
540
+ - 'meta/llama3-8b-instruct': 8B parameter model
541
+ - 'meta/llama3-70b-instruct': 70B parameter model
542
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
543
+ """
544
+ ...
545
+
546
+ @typing.overload
547
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
548
+ """
549
+ Specifies the Conda environment for all steps of the flow.
1144
550
 
551
+ Use `@conda_base` to set common libraries required by all
552
+ steps and use `@conda` to specify step-specific additions.
1145
553
  """
1146
554
  ...
1147
555
 
1148
556
  @typing.overload
1149
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
557
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
558
+ ...
559
+
560
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1150
561
  """
1151
- Specifies secrets to be retrieved and injected as environment variables prior to
1152
- the execution of a step.
562
+ Specifies the Conda environment for all steps of the flow.
1153
563
 
1154
- Parameters
1155
- ----------
1156
- sources : List[Union[str, Dict[str, Any]]], default: []
1157
- List of secret specs, defining how the secrets are to be retrieved
564
+ Use `@conda_base` to set common libraries required by all
565
+ steps and use `@conda` to specify step-specific additions.
1158
566
  """
1159
567
  ...
1160
568
 
1161
569
  @typing.overload
1162
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
570
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
571
+ """
572
+ Specifies the times when the flow should be run when running on a
573
+ production scheduler.
574
+ """
1163
575
  ...
1164
576
 
1165
577
  @typing.overload
1166
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
578
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1167
579
  ...
1168
580
 
1169
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
581
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1170
582
  """
1171
- Specifies secrets to be retrieved and injected as environment variables prior to
1172
- the execution of a step.
1173
-
1174
- Parameters
1175
- ----------
1176
- sources : List[Union[str, Dict[str, Any]]], default: []
1177
- List of secret specs, defining how the secrets are to be retrieved
583
+ Specifies the times when the flow should be run when running on a
584
+ production scheduler.
1178
585
  """
1179
586
  ...
1180
587
 
1181
- @typing.overload
1182
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
588
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1183
589
  """
1184
- Specifies environment variables to be set prior to the execution of a step.
590
+ Specifies what flows belong to the same project.
1185
591
 
1186
- Parameters
1187
- ----------
1188
- vars : Dict[str, str], default {}
1189
- Dictionary of environment variables to set.
592
+ A project-specific namespace is created for all flows that
593
+ use the same `@project(name)`.
594
+ """
595
+ ...
596
+
597
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
598
+ """
599
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
600
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1190
601
  """
1191
602
  ...
1192
603
 
1193
604
  @typing.overload
1194
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
605
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
606
+ """
607
+ Specifies the PyPI packages for all steps of the flow.
608
+
609
+ Use `@pypi_base` to set common packages required by all
610
+ steps and use `@pypi` to specify step-specific overrides.
611
+ """
1195
612
  ...
1196
613
 
1197
614
  @typing.overload
1198
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
615
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1199
616
  ...
1200
617
 
1201
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
618
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1202
619
  """
1203
- Specifies environment variables to be set prior to the execution of a step.
620
+ Specifies the PyPI packages for all steps of the flow.
1204
621
 
1205
- Parameters
1206
- ----------
1207
- vars : Dict[str, str], default {}
1208
- Dictionary of environment variables to set.
622
+ Use `@pypi_base` to set common packages required by all
623
+ steps and use `@pypi` to specify step-specific overrides.
1209
624
  """
1210
625
  ...
1211
626
 
1212
627
  @typing.overload
1213
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
628
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1214
629
  """
1215
- Specifies the event(s) that this flow depends on.
630
+ Specifies the flow(s) that this flow depends on.
1216
631
 
1217
632
  ```
1218
- @trigger(event='foo')
633
+ @trigger_on_finish(flow='FooFlow')
1219
634
  ```
1220
635
  or
1221
636
  ```
1222
- @trigger(events=['foo', 'bar'])
637
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1223
638
  ```
639
+ This decorator respects the @project decorator and triggers the flow
640
+ when upstream runs within the same namespace complete successfully
1224
641
 
1225
- Additionally, you can specify the parameter mappings
1226
- to map event payload to Metaflow parameters for the flow.
642
+ Additionally, you can specify project aware upstream flow dependencies
643
+ by specifying the fully qualified project_flow_name.
1227
644
  ```
1228
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
645
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1229
646
  ```
1230
647
  or
1231
648
  ```
1232
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1233
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
649
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1234
650
  ```
1235
651
 
1236
- 'parameters' can also be a list of strings and tuples like so:
1237
- ```
1238
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1239
- ```
1240
- This is equivalent to:
652
+ You can also specify just the project or project branch (other values will be
653
+ inferred from the current project or project branch):
1241
654
  ```
1242
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
655
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1243
656
  ```
1244
657
 
1245
- Parameters
1246
- ----------
1247
- event : Union[str, Dict[str, Any]], optional, default None
1248
- Event dependency for this flow.
1249
- events : List[Union[str, Dict[str, Any]]], default []
1250
- Events dependency for this flow.
1251
- options : Dict[str, Any], default {}
1252
- Backend-specific configuration for tuning eventing behavior.
1253
-
1254
-
658
+ Note that `branch` is typically one of:
659
+ - `prod`
660
+ - `user.bob`
661
+ - `test.my_experiment`
662
+ - `prod.staging`
1255
663
  """
1256
664
  ...
1257
665
 
1258
666
  @typing.overload
1259
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
667
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1260
668
  ...
1261
669
 
1262
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
670
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1263
671
  """
1264
- Specifies the event(s) that this flow depends on.
672
+ Specifies the flow(s) that this flow depends on.
1265
673
 
1266
674
  ```
1267
- @trigger(event='foo')
675
+ @trigger_on_finish(flow='FooFlow')
1268
676
  ```
1269
677
  or
1270
678
  ```
1271
- @trigger(events=['foo', 'bar'])
679
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1272
680
  ```
681
+ This decorator respects the @project decorator and triggers the flow
682
+ when upstream runs within the same namespace complete successfully
1273
683
 
1274
- Additionally, you can specify the parameter mappings
1275
- to map event payload to Metaflow parameters for the flow.
684
+ Additionally, you can specify project aware upstream flow dependencies
685
+ by specifying the fully qualified project_flow_name.
1276
686
  ```
1277
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
687
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1278
688
  ```
1279
689
  or
1280
690
  ```
1281
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1282
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
691
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1283
692
  ```
1284
693
 
1285
- 'parameters' can also be a list of strings and tuples like so:
1286
- ```
1287
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1288
- ```
1289
- This is equivalent to:
694
+ You can also specify just the project or project branch (other values will be
695
+ inferred from the current project or project branch):
1290
696
  ```
1291
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
697
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1292
698
  ```
1293
699
 
1294
- Parameters
1295
- ----------
1296
- event : Union[str, Dict[str, Any]], optional, default None
1297
- Event dependency for this flow.
1298
- events : List[Union[str, Dict[str, Any]]], default []
1299
- Events dependency for this flow.
1300
- options : Dict[str, Any], default {}
1301
- Backend-specific configuration for tuning eventing behavior.
1302
-
1303
-
700
+ Note that `branch` is typically one of:
701
+ - `prod`
702
+ - `user.bob`
703
+ - `test.my_experiment`
704
+ - `prod.staging`
1304
705
  """
1305
706
  ...
1306
707
 
@@ -1311,1899 +712,81 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1311
712
  and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1312
713
  added as a flow decorators. Adding more than one decorator will ensure that `start` step
1313
714
  starts only after all sensors finish.
1314
-
1315
- Parameters
1316
- ----------
1317
- timeout : int
1318
- Time, in seconds before the task times out and fails. (Default: 3600)
1319
- poke_interval : int
1320
- Time in seconds that the job should wait in between each try. (Default: 60)
1321
- mode : str
1322
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1323
- exponential_backoff : bool
1324
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1325
- pool : str
1326
- the slot pool this task should run in,
1327
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1328
- soft_fail : bool
1329
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1330
- name : str
1331
- Name of the sensor on Airflow
1332
- description : str
1333
- Description of sensor in the Airflow UI
1334
- bucket_key : Union[str, List[str]]
1335
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1336
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1337
- bucket_name : str
1338
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1339
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1340
- wildcard_match : bool
1341
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1342
- aws_conn_id : str
1343
- a reference to the s3 connection on Airflow. (Default: None)
1344
- verify : bool
1345
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1346
- """
1347
- ...
1348
-
1349
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1350
- """
1351
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1352
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1353
-
1354
- Parameters
1355
- ----------
1356
- timeout : int
1357
- Time, in seconds before the task times out and fails. (Default: 3600)
1358
- poke_interval : int
1359
- Time in seconds that the job should wait in between each try. (Default: 60)
1360
- mode : str
1361
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1362
- exponential_backoff : bool
1363
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1364
- pool : str
1365
- the slot pool this task should run in,
1366
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1367
- soft_fail : bool
1368
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1369
- name : str
1370
- Name of the sensor on Airflow
1371
- description : str
1372
- Description of sensor in the Airflow UI
1373
- external_dag_id : str
1374
- The dag_id that contains the task you want to wait for.
1375
- external_task_ids : List[str]
1376
- The list of task_ids that you want to wait for.
1377
- If None (default value) the sensor waits for the DAG. (Default: None)
1378
- allowed_states : List[str]
1379
- Iterable of allowed states, (Default: ['success'])
1380
- failed_states : List[str]
1381
- Iterable of failed or dis-allowed states. (Default: None)
1382
- execution_delta : datetime.timedelta
1383
- time difference with the previous execution to look at,
1384
- the default is the same logical date as the current task or DAG. (Default: None)
1385
- check_existence: bool
1386
- Set to True to check if the external task exists or check if
1387
- the DAG to wait for exists. (Default: True)
1388
- """
1389
- ...
1390
-
1391
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1392
- """
1393
- Specifies what flows belong to the same project.
1394
-
1395
- A project-specific namespace is created for all flows that
1396
- use the same `@project(name)`.
1397
-
1398
- Parameters
1399
- ----------
1400
- name : str
1401
- Project name. Make sure that the name is unique amongst all
1402
- projects that use the same production scheduler. The name may
1403
- contain only lowercase alphanumeric characters and underscores.
1404
-
1405
-
1406
715
  """
1407
716
  ...
1408
717
 
1409
718
  @typing.overload
1410
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
719
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1411
720
  """
1412
- Specifies the flow(s) that this flow depends on.
721
+ Specifies the event(s) that this flow depends on.
1413
722
 
1414
723
  ```
1415
- @trigger_on_finish(flow='FooFlow')
724
+ @trigger(event='foo')
1416
725
  ```
1417
726
  or
1418
727
  ```
1419
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
728
+ @trigger(events=['foo', 'bar'])
1420
729
  ```
1421
- This decorator respects the @project decorator and triggers the flow
1422
- when upstream runs within the same namespace complete successfully
1423
730
 
1424
- Additionally, you can specify project aware upstream flow dependencies
1425
- by specifying the fully qualified project_flow_name.
731
+ Additionally, you can specify the parameter mappings
732
+ to map event payload to Metaflow parameters for the flow.
1426
733
  ```
1427
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
734
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1428
735
  ```
1429
736
  or
1430
737
  ```
1431
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
738
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
739
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1432
740
  ```
1433
741
 
1434
- You can also specify just the project or project branch (other values will be
1435
- inferred from the current project or project branch):
742
+ 'parameters' can also be a list of strings and tuples like so:
1436
743
  ```
1437
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
744
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
745
+ ```
746
+ This is equivalent to:
747
+ ```
748
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1438
749
  ```
1439
-
1440
- Note that `branch` is typically one of:
1441
- - `prod`
1442
- - `user.bob`
1443
- - `test.my_experiment`
1444
- - `prod.staging`
1445
-
1446
- Parameters
1447
- ----------
1448
- flow : Union[str, Dict[str, str]], optional, default None
1449
- Upstream flow dependency for this flow.
1450
- flows : List[Union[str, Dict[str, str]]], default []
1451
- Upstream flow dependencies for this flow.
1452
- options : Dict[str, Any], default {}
1453
- Backend-specific configuration for tuning eventing behavior.
1454
-
1455
-
1456
750
  """
1457
751
  ...
1458
752
 
1459
753
  @typing.overload
1460
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
754
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1461
755
  ...
1462
756
 
1463
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
757
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1464
758
  """
1465
- Specifies the flow(s) that this flow depends on.
759
+ Specifies the event(s) that this flow depends on.
1466
760
 
1467
761
  ```
1468
- @trigger_on_finish(flow='FooFlow')
762
+ @trigger(event='foo')
1469
763
  ```
1470
764
  or
1471
765
  ```
1472
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
766
+ @trigger(events=['foo', 'bar'])
1473
767
  ```
1474
- This decorator respects the @project decorator and triggers the flow
1475
- when upstream runs within the same namespace complete successfully
1476
768
 
1477
- Additionally, you can specify project aware upstream flow dependencies
1478
- by specifying the fully qualified project_flow_name.
769
+ Additionally, you can specify the parameter mappings
770
+ to map event payload to Metaflow parameters for the flow.
1479
771
  ```
1480
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
772
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1481
773
  ```
1482
774
  or
1483
775
  ```
1484
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
776
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
777
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1485
778
  ```
1486
779
 
1487
- You can also specify just the project or project branch (other values will be
1488
- inferred from the current project or project branch):
1489
- ```
1490
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
780
+ 'parameters' can also be a list of strings and tuples like so:
1491
781
  ```
1492
-
1493
- Note that `branch` is typically one of:
1494
- - `prod`
1495
- - `user.bob`
1496
- - `test.my_experiment`
1497
- - `prod.staging`
1498
-
1499
- Parameters
1500
- ----------
1501
- flow : Union[str, Dict[str, str]], optional, default None
1502
- Upstream flow dependency for this flow.
1503
- flows : List[Union[str, Dict[str, str]]], default []
1504
- Upstream flow dependencies for this flow.
1505
- options : Dict[str, Any], default {}
1506
- Backend-specific configuration for tuning eventing behavior.
1507
-
1508
-
1509
- """
1510
- ...
1511
-
1512
- @typing.overload
1513
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1514
- """
1515
- Specifies the times when the flow should be run when running on a
1516
- production scheduler.
1517
-
1518
- Parameters
1519
- ----------
1520
- hourly : bool, default False
1521
- Run the workflow hourly.
1522
- daily : bool, default True
1523
- Run the workflow daily.
1524
- weekly : bool, default False
1525
- Run the workflow weekly.
1526
- cron : str, optional, default None
1527
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1528
- specified by this expression.
1529
- timezone : str, optional, default None
1530
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1531
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1532
- """
1533
- ...
1534
-
1535
- @typing.overload
1536
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1537
- ...
1538
-
1539
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1540
- """
1541
- Specifies the times when the flow should be run when running on a
1542
- production scheduler.
1543
-
1544
- Parameters
1545
- ----------
1546
- hourly : bool, default False
1547
- Run the workflow hourly.
1548
- daily : bool, default True
1549
- Run the workflow daily.
1550
- weekly : bool, default False
1551
- Run the workflow weekly.
1552
- cron : str, optional, default None
1553
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1554
- specified by this expression.
1555
- timezone : str, optional, default None
1556
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1557
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1558
- """
1559
- ...
1560
-
1561
- @typing.overload
1562
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1563
- """
1564
- Specifies the Conda environment for all steps of the flow.
1565
-
1566
- Use `@conda_base` to set common libraries required by all
1567
- steps and use `@conda` to specify step-specific additions.
1568
-
1569
- Parameters
1570
- ----------
1571
- packages : Dict[str, str], default {}
1572
- Packages to use for this flow. The key is the name of the package
1573
- and the value is the version to use.
1574
- libraries : Dict[str, str], default {}
1575
- Supported for backward compatibility. When used with packages, packages will take precedence.
1576
- python : str, optional, default None
1577
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1578
- that the version used will correspond to the version of the Python interpreter used to start the run.
1579
- disabled : bool, default False
1580
- If set to True, disables Conda.
1581
- """
1582
- ...
1583
-
1584
- @typing.overload
1585
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1586
- ...
1587
-
1588
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1589
- """
1590
- Specifies the Conda environment for all steps of the flow.
1591
-
1592
- Use `@conda_base` to set common libraries required by all
1593
- steps and use `@conda` to specify step-specific additions.
1594
-
1595
- Parameters
1596
- ----------
1597
- packages : Dict[str, str], default {}
1598
- Packages to use for this flow. The key is the name of the package
1599
- and the value is the version to use.
1600
- libraries : Dict[str, str], default {}
1601
- Supported for backward compatibility. When used with packages, packages will take precedence.
1602
- python : str, optional, default None
1603
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1604
- that the version used will correspond to the version of the Python interpreter used to start the run.
1605
- disabled : bool, default False
1606
- If set to True, disables Conda.
1607
- """
1608
- ...
1609
-
1610
- @typing.overload
1611
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1612
- """
1613
- Specifies the PyPI packages for all steps of the flow.
1614
-
1615
- Use `@pypi_base` to set common packages required by all
1616
- steps and use `@pypi` to specify step-specific overrides.
1617
- Parameters
1618
- ----------
1619
- packages : Dict[str, str], default: {}
1620
- Packages to use for this flow. The key is the name of the package
1621
- and the value is the version to use.
1622
- python : str, optional, default: None
1623
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1624
- that the version used will correspond to the version of the Python interpreter used to start the run.
1625
- """
1626
- ...
1627
-
1628
- @typing.overload
1629
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1630
- ...
1631
-
1632
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1633
- """
1634
- Specifies the PyPI packages for all steps of the flow.
1635
-
1636
- Use `@pypi_base` to set common packages required by all
1637
- steps and use `@pypi` to specify step-specific overrides.
1638
- Parameters
1639
- ----------
1640
- packages : Dict[str, str], default: {}
1641
- Packages to use for this flow. The key is the name of the package
1642
- and the value is the version to use.
1643
- python : str, optional, default: None
1644
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1645
- that the version used will correspond to the version of the Python interpreter used to start the run.
1646
- """
1647
- ...
1648
-
1649
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1650
- """
1651
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1652
-
1653
- User code call
1654
- -----------
1655
- @nim(
1656
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1657
- backend='managed'
1658
- )
1659
-
1660
- Valid backend options
1661
- ---------------------
1662
- - 'managed': Outerbounds selects a compute provider based on the model.
1663
- - 🚧 'dataplane': Run in your account.
1664
-
1665
- Valid model options
1666
- ----------------
1667
- - 'meta/llama3-8b-instruct': 8B parameter model
1668
- - 'meta/llama3-70b-instruct': 70B parameter model
1669
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1670
-
1671
- Parameters
1672
- ----------
1673
- models: list[NIM]
1674
- List of NIM containers running models in sidecars.
1675
- backend: str
1676
- Compute provider to run the NIM container.
1677
- """
1678
- ...
1679
-
1680
- def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1681
- """
1682
- Switch namespace to the one provided.
1683
-
1684
- This call has a global effect. No objects outside this namespace
1685
- will be accessible. To access all objects regardless of namespaces,
1686
- pass None to this call.
1687
-
1688
- Parameters
1689
- ----------
1690
- ns : str, optional
1691
- Namespace to switch to or None to ignore namespaces.
1692
-
1693
- Returns
1694
- -------
1695
- str, optional
1696
- Namespace set (result of get_namespace()).
1697
- """
1698
- ...
1699
-
1700
- def get_namespace() -> typing.Optional[str]:
1701
- """
1702
- Return the current namespace that is currently being used to filter objects.
1703
-
1704
- The namespace is a tag associated with all objects in Metaflow.
1705
-
1706
- Returns
1707
- -------
1708
- str, optional
1709
- The current namespace used to filter objects.
1710
- """
1711
- ...
1712
-
1713
- def default_namespace() -> str:
1714
- """
1715
- Resets the namespace used to filter objects to the default one, i.e. the one that was
1716
- used prior to any `namespace` calls.
1717
-
1718
- Returns
1719
- -------
1720
- str
1721
- The result of get_namespace() after the namespace has been reset.
1722
- """
1723
- ...
1724
-
1725
- def get_metadata() -> str:
1726
- """
1727
- Returns the current Metadata provider.
1728
-
1729
- If this is not set explicitly using `metadata`, the default value is
1730
- determined through the Metaflow configuration. You can use this call to
1731
- check that your configuration is set up properly.
1732
-
1733
- If multiple configuration profiles are present, this call returns the one
1734
- selected through the `METAFLOW_PROFILE` environment variable.
1735
-
1736
- Returns
1737
- -------
1738
- str
1739
- Information about the Metadata provider currently selected. This information typically
1740
- returns provider specific information (like URL for remote providers or local paths for
1741
- local providers).
1742
- """
1743
- ...
1744
-
1745
- def default_metadata() -> str:
1746
- """
1747
- Resets the Metadata provider to the default value, that is, to the value
1748
- that was used prior to any `metadata` calls.
1749
-
1750
- Returns
1751
- -------
1752
- str
1753
- The result of get_metadata() after resetting the provider.
1754
- """
1755
- ...
1756
-
1757
- class Metaflow(object, metaclass=type):
1758
- """
1759
- Entry point to all objects in the Metaflow universe.
1760
-
1761
- This object can be used to list all the flows present either through the explicit property
1762
- or by iterating over this object.
1763
-
1764
- Attributes
1765
- ----------
1766
- flows : List[Flow]
1767
- Returns the list of all `Flow` objects known to this metadata provider. Note that only
1768
- flows present in the current namespace will be returned. A `Flow` is present in a namespace
1769
- if it has at least one run in the namespace.
1770
- """
1771
- def __init__(self, _current_metadata: typing.Optional[str] = None):
1772
- ...
1773
- @property
1774
- def flows(self) -> typing.List[metaflow.client.core.Flow]:
1775
- """
1776
- Returns a list of all the flows present.
1777
-
1778
- Only flows present in the set namespace are returned. A flow is present in a namespace if
1779
- it has at least one run that is in the namespace.
1780
-
1781
- Returns
1782
- -------
1783
- List[Flow]
1784
- List of all flows present.
1785
- """
1786
- ...
1787
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Flow]:
1788
- """
1789
- Iterator over all flows present.
1790
-
1791
- Only flows present in the set namespace are returned. A flow is present in a
1792
- namespace if it has at least one run that is in the namespace.
1793
-
1794
- Yields
1795
- -------
1796
- Flow
1797
- A Flow present in the Metaflow universe.
1798
- """
1799
- ...
1800
- def __str__(self) -> str:
1801
- ...
1802
- def __getitem__(self, name: str) -> metaflow.client.core.Flow:
1803
- """
1804
- Returns a specific flow by name.
1805
-
1806
- The flow will only be returned if it is present in the current namespace.
1807
-
1808
- Parameters
1809
- ----------
1810
- name : str
1811
- Name of the Flow
1812
-
1813
- Returns
1814
- -------
1815
- Flow
1816
- Flow with the given name.
1817
- """
1818
- ...
1819
- ...
1820
-
1821
- class Flow(metaflow.client.core.MetaflowObject, metaclass=type):
1822
- """
1823
- A Flow represents all existing flows with a certain name, in other words,
1824
- classes derived from `FlowSpec`. A container of `Run` objects.
1825
-
1826
- Attributes
1827
- ----------
1828
- latest_run : Run
1829
- Latest `Run` (in progress or completed, successfully or not) of this flow.
1830
- latest_successful_run : Run
1831
- Latest successfully completed `Run` of this flow.
1832
- """
1833
- def __init__(self, *args, **kwargs):
1834
- ...
1835
- @property
1836
- def latest_run(self) -> typing.Optional[metaflow.client.core.Run]:
1837
- """
1838
- Returns the latest run (either in progress or completed) of this flow.
1839
-
1840
- Note that an in-progress run may be returned by this call. Use latest_successful_run
1841
- to get an object representing a completed successful run.
1842
-
1843
- Returns
1844
- -------
1845
- Run, optional
1846
- Latest run of this flow
1847
- """
1848
- ...
1849
- @property
1850
- def latest_successful_run(self) -> typing.Optional[metaflow.client.core.Run]:
1851
- """
1852
- Returns the latest successful run of this flow.
1853
-
1854
- Returns
1855
- -------
1856
- Run, optional
1857
- Latest successful run of this flow
1858
- """
1859
- ...
1860
- def runs(self, *tags: str) -> typing.Iterator[metaflow.client.core.Run]:
1861
- """
1862
- Returns an iterator over all `Run`s of this flow.
1863
-
1864
- An optional filter is available that allows you to filter on tags.
1865
- If multiple tags are specified, only runs that have all the
1866
- specified tags are returned.
1867
-
1868
- Parameters
1869
- ----------
1870
- tags : str
1871
- Tags to match.
1872
-
1873
- Yields
1874
- ------
1875
- Run
1876
- `Run` objects in this flow.
1877
- """
1878
- ...
1879
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Task]:
1880
- """
1881
- Iterate over all children Run of this Flow.
1882
-
1883
- Note that only runs in the current namespace are returned unless
1884
- _namespace_check is False
1885
-
1886
- Yields
1887
- ------
1888
- Run
1889
- A Run in this Flow
1890
- """
1891
- ...
1892
- def __getitem__(self, run_id: str) -> metaflow.client.core.Run:
1893
- """
1894
- Returns the Run object with the run ID 'run_id'
1895
-
1896
- Parameters
1897
- ----------
1898
- run_id : str
1899
- Run OD
1900
-
1901
- Returns
1902
- -------
1903
- Run
1904
- Run for this run ID in this Flow
1905
-
1906
- Raises
1907
- ------
1908
- KeyError
1909
- If the run_id does not identify a valid Run object
1910
- """
1911
- ...
1912
- def __getstate__(self):
1913
- ...
1914
- def __setstate__(self, state):
1915
- ...
1916
- ...
1917
-
1918
- class Run(metaflow.client.core.MetaflowObject, metaclass=type):
1919
- """
1920
- A `Run` represents an execution of a `Flow`. It is a container of `Step`s.
1921
-
1922
- Attributes
1923
- ----------
1924
- data : MetaflowData
1925
- a shortcut to run['end'].task.data, i.e. data produced by this run.
1926
- successful : bool
1927
- True if the run completed successfully.
1928
- finished : bool
1929
- True if the run completed.
1930
- finished_at : datetime
1931
- Time this run finished.
1932
- code : MetaflowCode
1933
- Code package for this run (if present). See `MetaflowCode`.
1934
- trigger : MetaflowTrigger
1935
- Information about event(s) that triggered this run (if present). See `MetaflowTrigger`.
1936
- end_task : Task
1937
- `Task` for the end step (if it is present already).
1938
- """
1939
- def steps(self, *tags: str) -> typing.Iterator[metaflow.client.core.Step]:
1940
- """
1941
- [Legacy function - do not use]
1942
-
1943
- Returns an iterator over all `Step` objects in the step. This is an alias
1944
- to iterating the object itself, i.e.
1945
- ```
1946
- list(Run(...)) == list(Run(...).steps())
1947
- ```
1948
-
1949
- Parameters
1950
- ----------
1951
- tags : str
1952
- No op (legacy functionality)
1953
-
1954
- Yields
1955
- ------
1956
- Step
1957
- `Step` objects in this run.
1958
- """
1959
- ...
1960
- @property
1961
- def code(self) -> typing.Optional[metaflow.client.core.MetaflowCode]:
1962
- """
1963
- Returns the MetaflowCode object for this run, if present.
1964
- Code is packed if atleast one `Step` runs remotely, else None is returned.
1965
-
1966
- Returns
1967
- -------
1968
- MetaflowCode, optional
1969
- Code package for this run
1970
- """
1971
- ...
1972
- @property
1973
- def data(self) -> typing.Optional[metaflow.client.core.MetaflowData]:
1974
- """
1975
- Returns a container of data artifacts produced by this run.
1976
-
1977
- You can access data produced by this run as follows:
1978
- ```
1979
- print(run.data.my_var)
1980
- ```
1981
- This is a shorthand for `run['end'].task.data`. If the 'end' step has not yet
1982
- executed, returns None.
1983
-
1984
- Returns
1985
- -------
1986
- MetaflowData, optional
1987
- Container of all artifacts produced by this task
1988
- """
1989
- ...
1990
- @property
1991
- def successful(self) -> bool:
1992
- """
1993
- Indicates whether or not the run completed successfully.
1994
-
1995
- A run is successful if its 'end' step is successful.
1996
-
1997
- Returns
1998
- -------
1999
- bool
2000
- True if the run completed successfully and False otherwise
2001
- """
2002
- ...
2003
- @property
2004
- def finished(self) -> bool:
2005
- """
2006
- Indicates whether or not the run completed.
2007
-
2008
- A run completed if its 'end' step completed.
2009
-
2010
- Returns
2011
- -------
2012
- bool
2013
- True if the run completed and False otherwise
2014
- """
2015
- ...
2016
- @property
2017
- def finished_at(self) -> typing.Optional[datetime.datetime]:
2018
- """
2019
- Returns the datetime object of when the run finished (successfully or not).
2020
-
2021
- The completion time of a run is the same as the completion time of its 'end' step.
2022
- If the 'end' step has not completed, returns None.
2023
-
2024
- Returns
2025
- -------
2026
- datetime, optional
2027
- Datetime of when the run finished
2028
- """
2029
- ...
2030
- @property
2031
- def end_task(self) -> typing.Optional[metaflow.client.core.Task]:
2032
- """
2033
- Returns the Task corresponding to the 'end' step.
2034
-
2035
- This returns None if the end step does not yet exist.
2036
-
2037
- Returns
2038
- -------
2039
- Task, optional
2040
- The 'end' task
2041
- """
2042
- ...
2043
- def add_tag(self, tag: str):
2044
- """
2045
- Add a tag to this `Run`.
2046
-
2047
- Note that if the tag is already a system tag, it is not added as a user tag,
2048
- and no error is thrown.
2049
-
2050
- Parameters
2051
- ----------
2052
- tag : str
2053
- Tag to add.
2054
- """
2055
- ...
2056
- def add_tags(self, tags: typing.Iterable[str]):
2057
- """
2058
- Add one or more tags to this `Run`.
2059
-
2060
- Note that if any tag is already a system tag, it is not added as a user tag
2061
- and no error is thrown.
2062
-
2063
- Parameters
2064
- ----------
2065
- tags : Iterable[str]
2066
- Tags to add.
2067
- """
2068
- ...
2069
- def remove_tag(self, tag: str):
2070
- """
2071
- Remove one tag from this `Run`.
2072
-
2073
- Removing a system tag is an error. Removing a non-existent
2074
- user tag is a no-op.
2075
-
2076
- Parameters
2077
- ----------
2078
- tag : str
2079
- Tag to remove.
2080
- """
2081
- ...
2082
- def remove_tags(self, tags: typing.Iterable[str]):
2083
- """
2084
- Remove one or more tags to this `Run`.
2085
-
2086
- Removing a system tag will result in an error. Removing a non-existent
2087
- user tag is a no-op.
2088
-
2089
- Parameters
2090
- ----------
2091
- tags : Iterable[str]
2092
- Tags to remove.
2093
- """
2094
- ...
2095
- def replace_tag(self, tag_to_remove: str, tag_to_add: str):
2096
- """
2097
- Remove a tag and add a tag atomically. Removal is done first.
2098
- The rules for `Run.add_tag` and `Run.remove_tag` also apply here.
2099
-
2100
- Parameters
2101
- ----------
2102
- tag_to_remove : str
2103
- Tag to remove.
2104
- tag_to_add : str
2105
- Tag to add.
2106
- """
2107
- ...
2108
- def replace_tags(self, tags_to_remove: typing.Iterable[str], tags_to_add: typing.Iterable[str]):
2109
- """
2110
- Remove and add tags atomically; the removal is done first.
2111
- The rules for `Run.add_tag` and `Run.remove_tag` also apply here.
2112
-
2113
- Parameters
2114
- ----------
2115
- tags_to_remove : Iterable[str]
2116
- Tags to remove.
2117
- tags_to_add : Iterable[str]
2118
- Tags to add.
2119
- """
2120
- ...
2121
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Step]:
2122
- """
2123
- Iterate over all children Step of this Run
2124
-
2125
- Yields
2126
- ------
2127
- Step
2128
- A Step in this Run
2129
- """
2130
- ...
2131
- def __getitem__(self, name: str) -> metaflow.client.core.Step:
2132
- """
2133
- Returns the Step object with the step name 'name'
2134
-
2135
- Parameters
2136
- ----------
2137
- name : str
2138
- Step name
2139
-
2140
- Returns
2141
- -------
2142
- Step
2143
- Step for this step name in this Run
2144
-
2145
- Raises
2146
- ------
2147
- KeyError
2148
- If the name does not identify a valid Step object
2149
- """
2150
- ...
2151
- def __getstate__(self):
2152
- ...
2153
- def __setstate__(self, state):
2154
- ...
2155
- @property
2156
- def trigger(self) -> typing.Optional[metaflow.events.Trigger]:
2157
- """
2158
- Returns a container of events that triggered this run.
2159
-
2160
- This returns None if the run was not triggered by any events.
2161
-
2162
- Returns
2163
- -------
2164
- Trigger, optional
2165
- Container of triggering events
2166
- """
2167
- ...
2168
- ...
2169
-
2170
- class Step(metaflow.client.core.MetaflowObject, metaclass=type):
2171
- """
2172
- A `Step` represents a user-defined step, that is, a method annotated with the `@step` decorator.
2173
-
2174
- It contains `Task` objects associated with the step, that is, all executions of the
2175
- `Step`. The step may contain multiple `Task`s in the case of a foreach step.
2176
-
2177
- Attributes
2178
- ----------
2179
- task : Task
2180
- The first `Task` object in this step. This is a shortcut for retrieving the only
2181
- task contained in a non-foreach step.
2182
- finished_at : datetime
2183
- Time when the latest `Task` of this step finished. Note that in the case of foreaches,
2184
- this time may change during execution of the step.
2185
- environment_info : Dict[str, Any]
2186
- Information about the execution environment.
2187
- """
2188
- @property
2189
- def task(self) -> typing.Optional[metaflow.client.core.Task]:
2190
- """
2191
- Returns a Task object belonging to this step.
2192
-
2193
- This is useful when the step only contains one task (a linear step for example).
2194
-
2195
- Returns
2196
- -------
2197
- Task
2198
- A task in the step
2199
- """
2200
- ...
2201
- def tasks(self, *tags: str) -> typing.Iterable[metaflow.client.core.Task]:
2202
- """
2203
- [Legacy function - do not use]
2204
-
2205
- Returns an iterator over all `Task` objects in the step. This is an alias
2206
- to iterating the object itself, i.e.
2207
- ```
2208
- list(Step(...)) == list(Step(...).tasks())
2209
- ```
2210
-
2211
- Parameters
2212
- ----------
2213
- tags : str
2214
- No op (legacy functionality)
2215
-
2216
- Yields
2217
- ------
2218
- Task
2219
- `Task` objects in this step.
2220
- """
2221
- ...
2222
- @property
2223
- def control_task(self) -> typing.Optional[metaflow.client.core.Task]:
2224
- """
2225
- [Unpublished API - use with caution!]
2226
-
2227
- Returns a Control Task object belonging to this step.
2228
- This is useful when the step only contains one control task.
2229
-
2230
- Returns
2231
- -------
2232
- Task
2233
- A control task in the step
2234
- """
2235
- ...
2236
- def control_tasks(self, *tags: str) -> typing.Iterator[metaflow.client.core.Task]:
2237
- """
2238
- [Unpublished API - use with caution!]
2239
-
2240
- Returns an iterator over all the control tasks in the step.
2241
- An optional filter is available that allows you to filter on tags. The
2242
- control tasks returned if the filter is specified will contain all the
2243
- tags specified.
2244
- Parameters
2245
- ----------
2246
- tags : str
2247
- Tags to match
2248
-
2249
- Yields
2250
- ------
2251
- Task
2252
- Control Task objects for this step
2253
- """
2254
- ...
2255
- def __iter__(self) -> typing.Iterator[metaflow.client.core.Task]:
2256
- """
2257
- Iterate over all children Task of this Step
2258
-
2259
- Yields
2260
- ------
2261
- Task
2262
- A Task in this Step
2263
- """
2264
- ...
2265
- def __getitem__(self, task_id: str) -> metaflow.client.core.Task:
2266
- """
2267
- Returns the Task object with the task ID 'task_id'
2268
-
2269
- Parameters
2270
- ----------
2271
- task_id : str
2272
- Task ID
2273
-
2274
- Returns
2275
- -------
2276
- Task
2277
- Task for this task ID in this Step
2278
-
2279
- Raises
2280
- ------
2281
- KeyError
2282
- If the task_id does not identify a valid Task object
2283
- """
2284
- ...
2285
- def __getstate__(self):
2286
- ...
2287
- def __setstate__(self, state):
2288
- ...
2289
- @property
2290
- def finished_at(self) -> typing.Optional[datetime.datetime]:
2291
- """
2292
- Returns the datetime object of when the step finished (successfully or not).
2293
-
2294
- A step is considered finished when all the tasks that belong to it have
2295
- finished. This call will return None if the step has not finished
2296
-
2297
- Returns
2298
- -------
2299
- datetime
2300
- Datetime of when the step finished
2301
- """
2302
- ...
2303
- @property
2304
- def environment_info(self) -> typing.Optional[typing.Dict[str, typing.Any]]:
2305
- """
2306
- Returns information about the environment that was used to execute this step. As an
2307
- example, if the Conda environment is selected, this will return information about the
2308
- dependencies that were used in the environment.
2309
-
2310
- This environment information is only available for steps that have tasks
2311
- for which the code package has been saved.
2312
-
2313
- Returns
2314
- -------
2315
- Dict[str, Any], optional
2316
- Dictionary describing the environment
2317
- """
2318
- ...
2319
- ...
2320
-
2321
- class Task(metaflow.client.core.MetaflowObject, metaclass=type):
2322
- """
2323
- A `Task` represents an execution of a `Step`.
2324
-
2325
- It contains all `DataArtifact` objects produced by the task as
2326
- well as metadata related to execution.
2327
-
2328
- Note that the `@retry` decorator may cause multiple attempts of
2329
- the task to be present. Usually you want the latest attempt, which
2330
- is what instantiating a `Task` object returns by default. If
2331
- you need to e.g. retrieve logs from a failed attempt, you can
2332
- explicitly get information about a specific attempt by using the
2333
- following syntax when creating a task:
2334
-
2335
- `Task('flow/run/step/task', attempt=<attempt>)`
2336
-
2337
- where `attempt=0` corresponds to the first attempt etc.
2338
-
2339
- Attributes
2340
- ----------
2341
- metadata : List[Metadata]
2342
- List of all metadata events associated with the task.
2343
- metadata_dict : Dict[str, str]
2344
- A condensed version of `metadata`: A dictionary where keys
2345
- are names of metadata events and values the latest corresponding event.
2346
- data : MetaflowData
2347
- Container of all data artifacts produced by this task. Note that this
2348
- call downloads all data locally, so it can be slower than accessing
2349
- artifacts individually. See `MetaflowData` for more information.
2350
- artifacts : MetaflowArtifacts
2351
- Container of `DataArtifact` objects produced by this task.
2352
- successful : bool
2353
- True if the task completed successfully.
2354
- finished : bool
2355
- True if the task completed.
2356
- exception : object
2357
- Exception raised by this task if there was one.
2358
- finished_at : datetime
2359
- Time this task finished.
2360
- runtime_name : str
2361
- Runtime this task was executed on.
2362
- stdout : str
2363
- Standard output for the task execution.
2364
- stderr : str
2365
- Standard error output for the task execution.
2366
- code : MetaflowCode
2367
- Code package for this task (if present). See `MetaflowCode`.
2368
- environment_info : Dict[str, str]
2369
- Information about the execution environment.
2370
- """
2371
- def __init__(self, *args, **kwargs):
2372
- ...
2373
- @property
2374
- def metadata(self) -> typing.List[metaflow.client.core.Metadata]:
2375
- """
2376
- Metadata events produced by this task across all attempts of the task
2377
- *except* if you selected a specific task attempt.
2378
-
2379
- Note that Metadata is different from tags.
2380
-
2381
- Returns
2382
- -------
2383
- List[Metadata]
2384
- Metadata produced by this task
2385
- """
2386
- ...
2387
- @property
2388
- def metadata_dict(self) -> typing.Dict[str, str]:
2389
- """
2390
- Dictionary mapping metadata names (keys) and their associated values.
2391
-
2392
- Note that unlike the metadata() method, this call will only return the latest
2393
- metadata for a given name. For example, if a task executes multiple times (retries),
2394
- the same metadata name will be generated multiple times (one for each execution of the
2395
- task). The metadata() method returns all those metadata elements whereas this call will
2396
- return the metadata associated with the latest execution of the task.
2397
-
2398
- Returns
2399
- -------
2400
- Dict[str, str]
2401
- Dictionary mapping metadata name with value
2402
- """
2403
- ...
2404
- @property
2405
- def index(self) -> typing.Optional[int]:
2406
- """
2407
- Returns the index of the innermost foreach loop if this task is run inside at least
2408
- one foreach.
2409
-
2410
- The index is what distinguishes the various tasks inside a given step.
2411
- This call returns None if this task was not run in a foreach loop.
2412
-
2413
- Returns
2414
- -------
2415
- int, optional
2416
- Index in the innermost loop for this task
2417
- """
2418
- ...
2419
- @property
2420
- def data(self) -> metaflow.client.core.MetaflowData:
2421
- """
2422
- Returns a container of data artifacts produced by this task.
2423
-
2424
- You can access data produced by this task as follows:
2425
- ```
2426
- print(task.data.my_var)
2427
- ```
2428
-
2429
- Returns
2430
- -------
2431
- MetaflowData
2432
- Container of all artifacts produced by this task
2433
- """
2434
- ...
2435
- @property
2436
- def artifacts(self) -> typing.NamedTuple:
2437
- """
2438
- Returns a container of DataArtifacts produced by this task.
2439
-
2440
- You can access each DataArtifact by name like so:
2441
- ```
2442
- print(task.artifacts.my_var)
2443
- ```
2444
- This method differs from data() because it returns DataArtifact objects
2445
- (which contain additional metadata) as opposed to just the data.
2446
-
2447
- Returns
2448
- -------
2449
- MetaflowArtifacts
2450
- Container of all DataArtifacts produced by this task
2451
- """
2452
- ...
2453
- @property
2454
- def successful(self) -> bool:
2455
- """
2456
- Indicates whether or not the task completed successfully.
2457
-
2458
- This information is always about the latest task to have completed (in case
2459
- of retries).
2460
-
2461
- Returns
2462
- -------
2463
- bool
2464
- True if the task completed successfully and False otherwise
2465
- """
2466
- ...
2467
- @property
2468
- def finished(self) -> bool:
2469
- """
2470
- Indicates whether or not the task completed.
2471
-
2472
- This information is always about the latest task to have completed (in case
2473
- of retries).
2474
-
2475
- Returns
2476
- -------
2477
- bool
2478
- True if the task completed and False otherwise
2479
- """
2480
- ...
2481
- @property
2482
- def exception(self) -> typing.Optional[typing.Any]:
2483
- """
2484
- Returns the exception that caused the task to fail, if any.
2485
-
2486
- This information is always about the latest task to have completed (in case
2487
- of retries). If successful() returns False and finished() returns True,
2488
- this method can help determine what went wrong.
2489
-
2490
- Returns
2491
- -------
2492
- object
2493
- Exception raised by the task or None if not applicable
2494
- """
2495
- ...
2496
- @property
2497
- def finished_at(self) -> typing.Optional[datetime.datetime]:
2498
- """
2499
- Returns the datetime object of when the task finished (successfully or not).
2500
-
2501
- This information is always about the latest task to have completed (in case
2502
- of retries). This call will return None if the task is not finished.
2503
-
2504
- Returns
2505
- -------
2506
- datetime
2507
- Datetime of when the task finished
2508
- """
2509
- ...
2510
- @property
2511
- def runtime_name(self) -> typing.Optional[str]:
2512
- """
2513
- Returns the name of the runtime this task executed on.
2514
-
2515
-
2516
- Returns
2517
- -------
2518
- str
2519
- Name of the runtime this task executed on
2520
- """
2521
- ...
2522
- @property
2523
- def stdout(self) -> str:
2524
- """
2525
- Returns the full standard out of this task.
2526
-
2527
- If you specify a specific attempt for this task, it will return the
2528
- standard out for that attempt. If you do not specify an attempt,
2529
- this will return the current standard out for the latest *started*
2530
- attempt of the task. In both cases, multiple calls to this
2531
- method will return the most up-to-date log (so if an attempt is not
2532
- done, each call will fetch the latest log).
2533
-
2534
- Returns
2535
- -------
2536
- str
2537
- Standard output of this task
2538
- """
2539
- ...
2540
- @property
2541
- def stdout_size(self) -> int:
2542
- """
2543
- Returns the size of the stdout log of this task.
2544
-
2545
- Similar to `stdout`, the size returned is the latest size of the log
2546
- (so for a running attempt, this value will increase as the task produces
2547
- more output).
2548
-
2549
- Returns
2550
- -------
2551
- int
2552
- Size of the stdout log content (in bytes)
2553
- """
2554
- ...
2555
- @property
2556
- def stderr(self) -> str:
2557
- """
2558
- Returns the full standard error of this task.
2559
-
2560
- If you specify a specific attempt for this task, it will return the
2561
- standard error for that attempt. If you do not specify an attempt,
2562
- this will return the current standard error for the latest *started*
2563
- attempt. In both cases, multiple calls to this
2564
- method will return the most up-to-date log (so if an attempt is not
2565
- done, each call will fetch the latest log).
2566
-
2567
- Returns
2568
- -------
2569
- str
2570
- Standard error of this task
2571
- """
2572
- ...
2573
- @property
2574
- def stderr_size(self) -> int:
2575
- """
2576
- Returns the size of the stderr log of this task.
2577
-
2578
- Similar to `stderr`, the size returned is the latest size of the log
2579
- (so for a running attempt, this value will increase as the task produces
2580
- more output).
2581
-
2582
- Returns
2583
- -------
2584
- int
2585
- Size of the stderr log content (in bytes)
2586
- """
2587
- ...
2588
- @property
2589
- def current_attempt(self) -> int:
2590
- """
2591
- Get the relevant attempt for this Task.
2592
-
2593
- Returns the specific attempt used when
2594
- initializing the instance, or the latest *started* attempt for the Task.
2595
-
2596
- Returns
2597
- -------
2598
- int
2599
- attempt id for this task object
2600
- """
2601
- ...
2602
- @property
2603
- def code(self) -> typing.Optional[metaflow.client.core.MetaflowCode]:
2604
- """
2605
- Returns the MetaflowCode object for this task, if present.
2606
-
2607
- Not all tasks save their code so this call may return None in those cases.
2608
-
2609
- Returns
2610
- -------
2611
- MetaflowCode
2612
- Code package for this task
2613
- """
2614
- ...
2615
- @property
2616
- def environment_info(self) -> typing.Dict[str, typing.Any]:
2617
- """
2618
- Returns information about the environment that was used to execute this task. As an
2619
- example, if the Conda environment is selected, this will return information about the
2620
- dependencies that were used in the environment.
2621
-
2622
- This environment information is only available for tasks that have a code package.
2623
-
2624
- Returns
2625
- -------
2626
- Dict
2627
- Dictionary describing the environment
2628
- """
2629
- ...
2630
- def loglines(self, stream: str, as_unicode: bool = True, meta_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Iterator[typing.Tuple[datetime.datetime, str]]:
2631
- """
2632
- Return an iterator over (utc_timestamp, logline) tuples.
2633
-
2634
- Parameters
2635
- ----------
2636
- stream : str
2637
- Either 'stdout' or 'stderr'.
2638
- as_unicode : bool, default: True
2639
- If as_unicode=False, each logline is returned as a byte object. Otherwise,
2640
- it is returned as a (unicode) string.
2641
-
2642
- Yields
2643
- ------
2644
- Tuple[datetime, str]
2645
- Tuple of timestamp, logline pairs.
2646
- """
2647
- ...
2648
- def __iter__(self) -> typing.Iterator[metaflow.client.core.DataArtifact]:
2649
- """
2650
- Iterate over all children DataArtifact of this Task
2651
-
2652
- Yields
2653
- ------
2654
- DataArtifact
2655
- A DataArtifact in this Step
2656
- """
2657
- ...
2658
- def __getitem__(self, name: str) -> metaflow.client.core.DataArtifact:
2659
- """
2660
- Returns the DataArtifact object with the artifact name 'name'
2661
-
2662
- Parameters
2663
- ----------
2664
- name : str
2665
- Data artifact name
2666
-
2667
- Returns
2668
- -------
2669
- DataArtifact
2670
- DataArtifact for this artifact name in this task
2671
-
2672
- Raises
2673
- ------
2674
- KeyError
2675
- If the name does not identify a valid DataArtifact object
2676
- """
2677
- ...
2678
- def __getstate__(self):
2679
- ...
2680
- def __setstate__(self, state):
2681
- ...
2682
- ...
2683
-
2684
- class DataArtifact(metaflow.client.core.MetaflowObject, metaclass=type):
2685
- """
2686
- A single data artifact and associated metadata. Note that this object does
2687
- not contain other objects as it is the leaf object in the hierarchy.
2688
-
2689
- Attributes
2690
- ----------
2691
- data : object
2692
- The data contained in this artifact, that is, the object produced during
2693
- execution of this run.
2694
- sha : string
2695
- A unique ID of this artifact.
2696
- finished_at : datetime
2697
- Corresponds roughly to the `Task.finished_at` time of the parent `Task`.
2698
- An alias for `DataArtifact.created_at`.
2699
- """
2700
- @property
2701
- def data(self) -> typing.Any:
2702
- """
2703
- Unpickled representation of the data contained in this artifact.
2704
-
2705
- Returns
2706
- -------
2707
- object
2708
- Object contained in this artifact
2709
- """
2710
- ...
2711
- @property
2712
- def size(self) -> int:
2713
- """
2714
- Returns the size (in bytes) of the pickled object representing this
2715
- DataArtifact
2716
-
2717
- Returns
2718
- -------
2719
- int
2720
- size of the pickled representation of data artifact (in bytes)
2721
- """
2722
- ...
2723
- @property
2724
- def sha(self) -> str:
2725
- """
2726
- Unique identifier for this artifact.
2727
-
2728
- This is a unique hash of the artifact (historically SHA1 hash)
2729
-
2730
- Returns
2731
- -------
2732
- str
2733
- Hash of this artifact
2734
- """
2735
- ...
2736
- @property
2737
- def finished_at(self) -> datetime.datetime:
2738
- """
2739
- Creation time for this artifact.
2740
-
2741
- Alias for created_at.
2742
-
2743
- Returns
2744
- -------
2745
- datetime
2746
- Creation time
2747
- """
2748
- ...
2749
- def __getstate__(self):
2750
- ...
2751
- def __setstate__(self, state):
2752
- ...
2753
- ...
2754
-
2755
- class Runner(object, metaclass=type):
2756
- """
2757
- Metaflow's Runner API that presents a programmatic interface
2758
- to run flows and perform other operations either synchronously or asynchronously.
2759
- The class expects a path to the flow file along with optional arguments
2760
- that match top-level options on the command-line.
2761
-
2762
- This class works as a context manager, calling `cleanup()` to remove
2763
- temporary files at exit.
2764
-
2765
- Example:
2766
- ```python
2767
- with Runner('slowflow.py', pylint=False) as runner:
2768
- result = runner.run(alpha=5, tags=["abc", "def"], max_workers=5)
2769
- print(result.run.finished)
782
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
2770
783
  ```
2771
-
2772
- Parameters
2773
- ----------
2774
- flow_file : str
2775
- Path to the flow file to run
2776
- show_output : bool, default True
2777
- Show the 'stdout' and 'stderr' to the console by default,
2778
- Only applicable for synchronous 'run' and 'resume' functions.
2779
- profile : Optional[str], default None
2780
- Metaflow profile to use to run this run. If not specified, the default
2781
- profile is used (or the one already set using `METAFLOW_PROFILE`)
2782
- env : Optional[Dict], default None
2783
- Additional environment variables to set for the Run. This overrides the
2784
- environment set for this process.
2785
- cwd : Optional[str], default None
2786
- The directory to run the subprocess in; if not specified, the current
2787
- directory is used.
2788
- file_read_timeout : int, default 3600
2789
- The timeout until which we try to read the runner attribute file.
2790
- **kwargs : Any
2791
- Additional arguments that you would pass to `python myflow.py` before
2792
- the `run` command.
2793
- """
2794
- def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
2795
- ...
2796
- def __enter__(self) -> metaflow.runner.metaflow_runner.Runner:
2797
- ...
2798
- def __aenter__(self) -> metaflow.runner.metaflow_runner.Runner:
2799
- ...
2800
- def _Runner__get_executing_run(self, tfp_runner_attribute, command_obj):
2801
- ...
2802
- def run(self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun:
2803
- """
2804
- Blocking execution of the run. This method will wait until
2805
- the run has completed execution.
2806
-
2807
- Parameters
2808
- ----------
2809
- **kwargs : Any
2810
- Additional arguments that you would pass to `python myflow.py` after
2811
- the `run` command, in particular, any parameters accepted by the flow.
2812
-
2813
- Returns
2814
- -------
2815
- ExecutingRun
2816
- ExecutingRun containing the results of the run.
2817
- """
2818
- ...
2819
- def resume(self, **kwargs):
2820
- """
2821
- Blocking resume execution of the run.
2822
- This method will wait until the resumed run has completed execution.
2823
-
2824
- Parameters
2825
- ----------
2826
- **kwargs : Any
2827
- Additional arguments that you would pass to `python ./myflow.py` after
2828
- the `resume` command.
2829
-
2830
- Returns
2831
- -------
2832
- ExecutingRun
2833
- ExecutingRun containing the results of the resumed run.
2834
- """
2835
- ...
2836
- def async_run(self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun:
2837
- """
2838
- Non-blocking execution of the run. This method will return as soon as the
2839
- run has launched.
2840
-
2841
- Note that this method is asynchronous and needs to be `await`ed.
2842
-
2843
- Parameters
2844
- ----------
2845
- **kwargs : Any
2846
- Additional arguments that you would pass to `python myflow.py` after
2847
- the `run` command, in particular, any parameters accepted by the flow.
2848
-
2849
- Returns
2850
- -------
2851
- ExecutingRun
2852
- ExecutingRun representing the run that was started.
2853
- """
2854
- ...
2855
- def async_resume(self, **kwargs):
2856
- """
2857
- Non-blocking resume execution of the run.
2858
- This method will return as soon as the resume has launched.
2859
-
2860
- Note that this method is asynchronous and needs to be `await`ed.
2861
-
2862
- Parameters
2863
- ----------
2864
- **kwargs : Any
2865
- Additional arguments that you would pass to `python myflow.py` after
2866
- the `resume` command.
2867
-
2868
- Returns
2869
- -------
2870
- ExecutingRun
2871
- ExecutingRun representing the resumed run that was started.
2872
- """
2873
- ...
2874
- def __exit__(self, exc_type, exc_value, traceback):
2875
- ...
2876
- def __aexit__(self, exc_type, exc_value, traceback):
2877
- ...
2878
- def cleanup(self):
2879
- """
2880
- Delete any temporary files created during execution.
2881
- """
2882
- ...
2883
- ...
2884
-
2885
- class NBRunner(object, metaclass=type):
2886
- """
2887
- A wrapper over `Runner` for executing flows defined in a Jupyter
2888
- notebook cell.
2889
-
2890
- Instantiate this class on the last line of a notebook cell where
2891
- a `flow` is defined. In contrast to `Runner`, this class is not
2892
- meant to be used in a context manager. Instead, use a blocking helper
2893
- function like `nbrun` (which calls `cleanup()` internally) or call
2894
- `cleanup()` explictly when using non-blocking APIs.
2895
-
2896
- ```python
2897
- run = NBRunner(FlowName).nbrun()
784
+ This is equivalent to:
2898
785
  ```
2899
-
2900
- Parameters
2901
- ----------
2902
- flow : FlowSpec
2903
- Flow defined in the same cell
2904
- show_output : bool, default True
2905
- Show the 'stdout' and 'stderr' to the console by default,
2906
- Only applicable for synchronous 'run' and 'resume' functions.
2907
- profile : Optional[str], default None
2908
- Metaflow profile to use to run this run. If not specified, the default
2909
- profile is used (or the one already set using `METAFLOW_PROFILE`)
2910
- env : Optional[Dict], default None
2911
- Additional environment variables to set for the Run. This overrides the
2912
- environment set for this process.
2913
- base_dir : Optional[str], default None
2914
- The directory to run the subprocess in; if not specified, the current
2915
- working directory is used.
2916
- file_read_timeout : int, default 3600
2917
- The timeout until which we try to read the runner attribute file.
2918
- **kwargs : Any
2919
- Additional arguments that you would pass to `python myflow.py` before
2920
- the `run` command.
2921
- """
2922
- def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
2923
- ...
2924
- def nbrun(self, **kwargs):
2925
- """
2926
- Blocking execution of the run. This method will wait until
2927
- the run has completed execution.
2928
-
2929
- Note that in contrast to `run`, this method returns a
2930
- `metaflow.Run` object directly and calls `cleanup()` internally
2931
- to support a common notebook pattern of executing a flow and
2932
- retrieving its results immediately.
2933
-
2934
- Parameters
2935
- ----------
2936
- **kwargs : Any
2937
- Additional arguments that you would pass to `python myflow.py` after
2938
- the `run` command, in particular, any parameters accepted by the flow.
2939
-
2940
- Returns
2941
- -------
2942
- Run
2943
- A `metaflow.Run` object representing the finished run.
2944
- """
2945
- ...
2946
- def nbresume(self, **kwargs):
2947
- """
2948
- Blocking resuming of a run. This method will wait until
2949
- the resumed run has completed execution.
2950
-
2951
- Note that in contrast to `resume`, this method returns a
2952
- `metaflow.Run` object directly and calls `cleanup()` internally
2953
- to support a common notebook pattern of executing a flow and
2954
- retrieving its results immediately.
2955
-
2956
- Parameters
2957
- ----------
2958
- **kwargs : Any
2959
- Additional arguments that you would pass to `python myflow.py` after
2960
- the `resume` command.
2961
-
2962
- Returns
2963
- -------
2964
- Run
2965
- A `metaflow.Run` object representing the resumed run.
2966
- """
2967
- ...
2968
- def run(self, **kwargs):
2969
- """
2970
- Runs the flow.
2971
- """
2972
- ...
2973
- def resume(self, **kwargs):
2974
- """
2975
- Resumes the flow.
2976
- """
2977
- ...
2978
- def async_run(self, **kwargs):
2979
- """
2980
- Non-blocking execution of the run. This method will return as soon as the
2981
- run has launched. This method is equivalent to `Runner.async_run`.
2982
-
2983
- Note that this method is asynchronous and needs to be `await`ed.
2984
-
2985
-
2986
- Parameters
2987
- ----------
2988
- **kwargs : Any
2989
- Additional arguments that you would pass to `python myflow.py` after
2990
- the `run` command, in particular, any parameters accepted by the flow.
2991
-
2992
- Returns
2993
- -------
2994
- ExecutingRun
2995
- ExecutingRun representing the run that was started.
2996
- """
2997
- ...
2998
- def async_resume(self, **kwargs):
2999
- """
3000
- Non-blocking execution of the run. This method will return as soon as the
3001
- run has launched. This method is equivalent to `Runner.async_resume`.
3002
-
3003
- Note that this method is asynchronous and needs to be `await`ed.
3004
-
3005
- Parameters
3006
- ----------
3007
- **kwargs : Any
3008
- Additional arguments that you would pass to `python myflow.py` after
3009
- the `run` command, in particular, any parameters accepted by the flow.
3010
-
3011
- Returns
3012
- -------
3013
- ExecutingRun
3014
- ExecutingRun representing the run that was started.
3015
- """
3016
- ...
3017
- def cleanup(self):
3018
- """
3019
- Delete any temporary files created during execution.
3020
-
3021
- Call this method after using `async_run` or `async_resume`. You don't
3022
- have to call this after `nbrun` or `nbresume`.
3023
- """
3024
- ...
3025
- ...
3026
-
3027
- class Deployer(object, metaclass=type):
3028
- """
3029
- Use the `Deployer` class to configure and access one of the production
3030
- orchestrators supported by Metaflow.
3031
-
3032
- Parameters
3033
- ----------
3034
- flow_file : str
3035
- Path to the flow file to deploy.
3036
- show_output : bool, default True
3037
- Show the 'stdout' and 'stderr' to the console by default.
3038
- profile : Optional[str], default None
3039
- Metaflow profile to use for the deployment. If not specified, the default
3040
- profile is used.
3041
- env : Optional[Dict[str, str]], default None
3042
- Additional environment variables to set for the deployment.
3043
- cwd : Optional[str], default None
3044
- The directory to run the subprocess in; if not specified, the current
3045
- directory is used.
3046
- file_read_timeout : int, default 3600
3047
- The timeout until which we try to read the deployer attribute file.
3048
- **kwargs : Any
3049
- Additional arguments that you would pass to `python myflow.py` before
3050
- the deployment command.
3051
- """
3052
- def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
3053
- ...
3054
- def _Deployer__make_function(self, deployer_class):
3055
- """
3056
- Create a function for the given deployer class.
3057
-
3058
- Parameters
3059
- ----------
3060
- deployer_class : Type[DeployerImpl]
3061
- Deployer implementation class.
3062
-
3063
- Returns
3064
- -------
3065
- Callable
3066
- Function that initializes and returns an instance of the deployer class.
3067
- """
3068
- ...
3069
- ...
3070
-
3071
- class NBDeployer(object, metaclass=type):
3072
- """
3073
- A wrapper over `Deployer` for deploying flows defined in a Jupyter
3074
- notebook cell.
3075
-
3076
- Instantiate this class on the last line of a notebook cell where
3077
- a `flow` is defined. In contrast to `Deployer`, this class is not
3078
- meant to be used in a context manager.
3079
-
3080
- ```python
3081
- deployer = NBDeployer(FlowName)
3082
- ar = deployer.argo_workflows(name="madhur")
3083
- ar_obj = ar.create()
3084
- result = ar_obj.trigger(alpha=300)
3085
- print(result.status)
3086
- print(result.run)
3087
- result.terminate()
786
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
3088
787
  ```
3089
-
3090
- Parameters
3091
- ----------
3092
- flow : FlowSpec
3093
- Flow defined in the same cell
3094
- show_output : bool, default True
3095
- Show the 'stdout' and 'stderr' to the console by default,
3096
- profile : Optional[str], default None
3097
- Metaflow profile to use to deploy this run. If not specified, the default
3098
- profile is used (or the one already set using `METAFLOW_PROFILE`)
3099
- env : Optional[Dict[str, str]], default None
3100
- Additional environment variables to set. This overrides the
3101
- environment set for this process.
3102
- base_dir : Optional[str], default None
3103
- The directory to run the subprocess in; if not specified, the current
3104
- working directory is used.
3105
- **kwargs : Any
3106
- Additional arguments that you would pass to `python myflow.py` i.e. options
3107
- listed in `python myflow.py --help`
3108
- """
3109
- def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
3110
- ...
3111
- def cleanup(self):
3112
- """
3113
- Delete any temporary files created during execution.
3114
- """
3115
- ...
788
+ """
3116
789
  ...
3117
790
 
3118
791
  pkg_name: str
3119
792
 
3120
- class Checkpoint(object, metaclass=type):
3121
- def __init__(self, temp_dir_root = None, init_dir = False):
3122
- ...
3123
- @property
3124
- def directory(self):
3125
- ...
3126
- def save(self, path = None, metadata = None, latest = True, name = "mfchckpt", storage_format = "files"):
3127
- """
3128
- Saves the checkpoint to the datastore
3129
-
3130
- Parameters
3131
- ----------
3132
- path : Optional[Union[str, os.PathLike]], default: None
3133
- The path to save the checkpoint. Accepts a file path or a directory path.
3134
- - If a directory path is provided, all the contents within that directory will be saved.
3135
- When a checkpoint is reloaded during task retries, `the current.checkpoint.directory` will
3136
- contain the contents of this directory.
3137
- - If a file path is provided, the file will be directly saved to the datastore (with the same filename).
3138
- When the checkpoint is reloaded during task retries, the file with the same name will be available in the
3139
- `current.checkpoint.directory`.
3140
- - If no path is provided then the `Checkpoint.directory` will be saved as the checkpoint.
3141
-
3142
- name : Optional[str], default: "mfchckpt"
3143
- The name of the checkpoint.
3144
-
3145
- metadata : Optional[Dict], default: {}
3146
- Any metadata that needs to be saved with the checkpoint.
3147
-
3148
- latest : bool, default: True
3149
- If True, the checkpoint will be marked as the latest checkpoint.
3150
- This helps determine if the checkpoint gets loaded when the task restarts.
3151
-
3152
- storage_format : str, default: files
3153
- If `tar`, the contents of the directory will be tarred before saving to the datastore.
3154
- If `files`, saves directory directly to the datastore.
3155
- """
3156
- ...
3157
- def __enter__(self):
3158
- ...
3159
- def __exit__(self, exc_type, exc_val, exc_tb):
3160
- ...
3161
- def list(self, name: typing.Optional[str] = None, task: typing.Union["metaflow.Task", str, None] = None, attempt: typing.Union[int, str, None] = None, as_dict: bool = True, within_task: bool = True) -> typing.Iterable[typing.Union[typing.Dict, metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]]:
3162
- """
3163
- lists the checkpoints in the datastore based on the Task.
3164
- It will always be task scoped.
3165
-
3166
- Usage:
3167
- ------
3168
-
3169
- ```python
3170
-
3171
- Checkpoint().list(name="best") # lists checkpoints in the current task with the name "best"
3172
- Checkpoint().list(task="anotherflow/somerunid/somestep/sometask", name="best") # Identical as the above one but
3173
- Checkpoint().list() # lists all the checkpoints in the current task
3174
-
3175
- ```
3176
-
3177
- Parameters
3178
- ----------
3179
-
3180
- - `name`:
3181
- - name of the checkpoint to filter for
3182
- - `task`:
3183
- - Task object outside the one that is currently set in the `Checkpoint` object; Can be a pathspec string.
3184
- - `attempt`:
3185
- - attempt number of the task (optional filter. If none, then lists all checkpoints from all attempts)
3186
- """
3187
- ...
3188
- def load(self, reference: typing.Union[str, typing.Dict, metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact], path: typing.Optional[str] = None):
3189
- """
3190
- loads a checkpoint reference from the datastore. (resembles a read op)
3191
-
3192
- Parameters
3193
- ----------
3194
-
3195
- `reference` :
3196
- - can be a string, dict or a CheckpointArtifact object:
3197
- - string: a string reference to the checkpoint (checkpoint key)
3198
- - dict: a dictionary reference to the checkpoint
3199
- - CheckpointArtifact: a CheckpointArtifact object reference to the checkpoint
3200
- """
3201
- ...
3202
- ...
3203
-
3204
- def load_model(reference: typing.Union[str, metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.MetaflowDataArtifactReference, dict], path: str):
3205
- ...
3206
-
3207
- def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
3208
- ...
3209
-