ob-metaflow-stubs 6.0.3.121__py2.py3-none-any.whl → 6.0.3.122__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +233 -233
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +4 -2
- metaflow-stubs/metaflow_current.pyi +34 -34
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/mflog/__init__.pyi +2 -2
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +11 -11
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +12 -12
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -3
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +5 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +5 -5
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +12 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +8 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +5 -5
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +8 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +3 -2
- metaflow-stubs/plugins/logs_cli.pyi +2 -2
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +3 -3
- metaflow-stubs/runner/deployer.pyi +30 -30
- metaflow-stubs/runner/deployer_impl.pyi +3 -2
- metaflow-stubs/runner/metaflow_runner.pyi +7 -3
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +7 -3
- metaflow-stubs/runner/utils.pyi +109 -10
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.121.dist-info → ob_metaflow_stubs-6.0.3.122.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.122.dist-info/RECORD +174 -0
- ob_metaflow_stubs-6.0.3.121.dist-info/RECORD +0 -174
- {ob_metaflow_stubs-6.0.3.121.dist-info → ob_metaflow_stubs-6.0.3.122.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.121.dist-info → ob_metaflow_stubs-6.0.3.122.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-12-
|
3
|
+
# MF version: 2.12.35.1+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-12-06T18:19:55.683902 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import datetime
|
12
11
|
import typing
|
12
|
+
import datetime
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
15
15
|
|
@@ -140,6 +140,72 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
140
140
|
"""
|
141
141
|
...
|
142
142
|
|
143
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
144
|
+
"""
|
145
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
146
|
+
"""
|
147
|
+
...
|
148
|
+
|
149
|
+
@typing.overload
|
150
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
151
|
+
"""
|
152
|
+
Specifies the number of times the task corresponding
|
153
|
+
to a step needs to be retried.
|
154
|
+
|
155
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
156
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
157
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
158
|
+
|
159
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
160
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
161
|
+
ensuring that the flow execution can continue.
|
162
|
+
"""
|
163
|
+
...
|
164
|
+
|
165
|
+
@typing.overload
|
166
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
167
|
+
...
|
168
|
+
|
169
|
+
@typing.overload
|
170
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
171
|
+
...
|
172
|
+
|
173
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
174
|
+
"""
|
175
|
+
Specifies the number of times the task corresponding
|
176
|
+
to a step needs to be retried.
|
177
|
+
|
178
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
179
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
180
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
181
|
+
|
182
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
183
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
184
|
+
ensuring that the flow execution can continue.
|
185
|
+
"""
|
186
|
+
...
|
187
|
+
|
188
|
+
@typing.overload
|
189
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
190
|
+
"""
|
191
|
+
Specifies environment variables to be set prior to the execution of a step.
|
192
|
+
"""
|
193
|
+
...
|
194
|
+
|
195
|
+
@typing.overload
|
196
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
197
|
+
...
|
198
|
+
|
199
|
+
@typing.overload
|
200
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
201
|
+
...
|
202
|
+
|
203
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
204
|
+
"""
|
205
|
+
Specifies environment variables to be set prior to the execution of a step.
|
206
|
+
"""
|
207
|
+
...
|
208
|
+
|
143
209
|
@typing.overload
|
144
210
|
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
145
211
|
"""
|
@@ -160,33 +226,33 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
160
226
|
...
|
161
227
|
|
162
228
|
@typing.overload
|
163
|
-
def
|
229
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
164
230
|
"""
|
165
|
-
Specifies the
|
231
|
+
Specifies the PyPI packages for the step.
|
166
232
|
|
167
233
|
Information in this decorator will augment any
|
168
|
-
attributes set in the `@
|
169
|
-
you can use `@
|
170
|
-
steps and use `@
|
234
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
235
|
+
you can use `@pypi_base` to set packages required by all
|
236
|
+
steps and use `@pypi` to specify step-specific overrides.
|
171
237
|
"""
|
172
238
|
...
|
173
239
|
|
174
240
|
@typing.overload
|
175
|
-
def
|
241
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
176
242
|
...
|
177
243
|
|
178
244
|
@typing.overload
|
179
|
-
def
|
245
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
180
246
|
...
|
181
247
|
|
182
|
-
def
|
248
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
183
249
|
"""
|
184
|
-
Specifies the
|
250
|
+
Specifies the PyPI packages for the step.
|
185
251
|
|
186
252
|
Information in this decorator will augment any
|
187
|
-
attributes set in the `@
|
188
|
-
you can use `@
|
189
|
-
steps and use `@
|
253
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
254
|
+
you can use `@pypi_base` to set packages required by all
|
255
|
+
steps and use `@pypi` to specify step-specific overrides.
|
190
256
|
"""
|
191
257
|
...
|
192
258
|
|
@@ -239,9 +305,24 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
239
305
|
"""
|
240
306
|
...
|
241
307
|
|
242
|
-
|
308
|
+
@typing.overload
|
309
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
243
310
|
"""
|
244
|
-
|
311
|
+
Enables checkpointing for a step.
|
312
|
+
"""
|
313
|
+
...
|
314
|
+
|
315
|
+
@typing.overload
|
316
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
317
|
+
...
|
318
|
+
|
319
|
+
@typing.overload
|
320
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
321
|
+
...
|
322
|
+
|
323
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
324
|
+
"""
|
325
|
+
Enables checkpointing for a step.
|
245
326
|
"""
|
246
327
|
...
|
247
328
|
|
@@ -277,201 +358,126 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
277
358
|
...
|
278
359
|
|
279
360
|
@typing.overload
|
280
|
-
def
|
361
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
281
362
|
"""
|
282
|
-
|
283
|
-
|
284
|
-
This decorator is useful if this step may hang indefinitely.
|
285
|
-
|
286
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
287
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
288
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
363
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
289
364
|
|
290
|
-
Note that
|
291
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
365
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
292
366
|
"""
|
293
367
|
...
|
294
368
|
|
295
369
|
@typing.overload
|
296
|
-
def
|
370
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
297
371
|
...
|
298
372
|
|
299
373
|
@typing.overload
|
300
|
-
def
|
374
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
301
375
|
...
|
302
376
|
|
303
|
-
def
|
377
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
304
378
|
"""
|
305
|
-
|
306
|
-
|
307
|
-
This decorator is useful if this step may hang indefinitely.
|
308
|
-
|
309
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
310
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
311
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
379
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
312
380
|
|
313
|
-
Note that
|
314
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
315
|
-
"""
|
316
|
-
...
|
317
|
-
|
318
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
319
|
-
"""
|
320
|
-
Specifies that this step should execute on Kubernetes.
|
381
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
321
382
|
"""
|
322
383
|
...
|
323
384
|
|
324
385
|
@typing.overload
|
325
|
-
def
|
386
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
326
387
|
"""
|
327
|
-
|
388
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
389
|
+
the execution of a step.
|
328
390
|
"""
|
329
391
|
...
|
330
392
|
|
331
393
|
@typing.overload
|
332
|
-
def
|
394
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
333
395
|
...
|
334
396
|
|
335
397
|
@typing.overload
|
336
|
-
def
|
398
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
337
399
|
...
|
338
400
|
|
339
|
-
def
|
401
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
340
402
|
"""
|
341
|
-
|
403
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
404
|
+
the execution of a step.
|
342
405
|
"""
|
343
406
|
...
|
344
407
|
|
345
408
|
@typing.overload
|
346
|
-
def
|
409
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
347
410
|
"""
|
348
|
-
Specifies the
|
411
|
+
Specifies the Conda environment for the step.
|
349
412
|
|
350
413
|
Information in this decorator will augment any
|
351
|
-
attributes set in the `@
|
352
|
-
you can use `@
|
353
|
-
steps and use `@
|
414
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
415
|
+
you can use `@conda_base` to set packages required by all
|
416
|
+
steps and use `@conda` to specify step-specific overrides.
|
354
417
|
"""
|
355
418
|
...
|
356
419
|
|
357
420
|
@typing.overload
|
358
|
-
def
|
421
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
359
422
|
...
|
360
423
|
|
361
424
|
@typing.overload
|
362
|
-
def
|
425
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
363
426
|
...
|
364
427
|
|
365
|
-
def
|
428
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
366
429
|
"""
|
367
|
-
Specifies the
|
430
|
+
Specifies the Conda environment for the step.
|
368
431
|
|
369
432
|
Information in this decorator will augment any
|
370
|
-
attributes set in the `@
|
371
|
-
you can use `@
|
372
|
-
steps and use `@
|
373
|
-
"""
|
374
|
-
...
|
375
|
-
|
376
|
-
@typing.overload
|
377
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
378
|
-
"""
|
379
|
-
Specifies environment variables to be set prior to the execution of a step.
|
380
|
-
"""
|
381
|
-
...
|
382
|
-
|
383
|
-
@typing.overload
|
384
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
385
|
-
...
|
386
|
-
|
387
|
-
@typing.overload
|
388
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
389
|
-
...
|
390
|
-
|
391
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
392
|
-
"""
|
393
|
-
Specifies environment variables to be set prior to the execution of a step.
|
433
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
434
|
+
you can use `@conda_base` to set packages required by all
|
435
|
+
steps and use `@conda` to specify step-specific overrides.
|
394
436
|
"""
|
395
437
|
...
|
396
438
|
|
397
439
|
@typing.overload
|
398
|
-
def
|
440
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
399
441
|
"""
|
400
|
-
Specifies
|
401
|
-
to a step needs to be retried.
|
442
|
+
Specifies a timeout for your step.
|
402
443
|
|
403
|
-
This decorator is useful
|
404
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
405
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
444
|
+
This decorator is useful if this step may hang indefinitely.
|
406
445
|
|
407
|
-
This can be used in conjunction with the `@
|
408
|
-
|
409
|
-
|
446
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
447
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
448
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
449
|
+
|
450
|
+
Note that all the values specified in parameters are added together so if you specify
|
451
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
410
452
|
"""
|
411
453
|
...
|
412
454
|
|
413
455
|
@typing.overload
|
414
|
-
def
|
456
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
415
457
|
...
|
416
458
|
|
417
459
|
@typing.overload
|
418
|
-
def
|
460
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
419
461
|
...
|
420
462
|
|
421
|
-
def
|
463
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
422
464
|
"""
|
423
|
-
Specifies
|
424
|
-
to a step needs to be retried.
|
425
|
-
|
426
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
427
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
428
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
465
|
+
Specifies a timeout for your step.
|
429
466
|
|
430
|
-
This
|
431
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
432
|
-
ensuring that the flow execution can continue.
|
433
|
-
"""
|
434
|
-
...
|
435
|
-
|
436
|
-
@typing.overload
|
437
|
-
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
438
|
-
"""
|
439
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
467
|
+
This decorator is useful if this step may hang indefinitely.
|
440
468
|
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
@typing.overload
|
446
|
-
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
447
|
-
...
|
448
|
-
|
449
|
-
@typing.overload
|
450
|
-
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
451
|
-
...
|
452
|
-
|
453
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
454
|
-
"""
|
455
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
469
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
470
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
471
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
456
472
|
|
457
|
-
Note that
|
458
|
-
|
459
|
-
...
|
460
|
-
|
461
|
-
@typing.overload
|
462
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
463
|
-
"""
|
464
|
-
Internal decorator to support Fast bakery
|
473
|
+
Note that all the values specified in parameters are added together so if you specify
|
474
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
465
475
|
"""
|
466
476
|
...
|
467
477
|
|
468
|
-
|
469
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
470
|
-
...
|
471
|
-
|
472
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
478
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
473
479
|
"""
|
474
|
-
|
480
|
+
Specifies that this step should execute on Kubernetes.
|
475
481
|
"""
|
476
482
|
...
|
477
483
|
|
@@ -497,91 +503,19 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
497
503
|
...
|
498
504
|
|
499
505
|
@typing.overload
|
500
|
-
def
|
501
|
-
"""
|
502
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
503
|
-
the execution of a step.
|
504
|
-
"""
|
505
|
-
...
|
506
|
-
|
507
|
-
@typing.overload
|
508
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
509
|
-
...
|
510
|
-
|
511
|
-
@typing.overload
|
512
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
513
|
-
...
|
514
|
-
|
515
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
516
|
-
"""
|
517
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
518
|
-
the execution of a step.
|
519
|
-
"""
|
520
|
-
...
|
521
|
-
|
522
|
-
@typing.overload
|
523
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
524
|
-
"""
|
525
|
-
Specifies the PyPI packages for all steps of the flow.
|
526
|
-
|
527
|
-
Use `@pypi_base` to set common packages required by all
|
528
|
-
steps and use `@pypi` to specify step-specific overrides.
|
529
|
-
"""
|
530
|
-
...
|
531
|
-
|
532
|
-
@typing.overload
|
533
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
534
|
-
...
|
535
|
-
|
536
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
537
|
-
"""
|
538
|
-
Specifies the PyPI packages for all steps of the flow.
|
539
|
-
|
540
|
-
Use `@pypi_base` to set common packages required by all
|
541
|
-
steps and use `@pypi` to specify step-specific overrides.
|
542
|
-
"""
|
543
|
-
...
|
544
|
-
|
545
|
-
@typing.overload
|
546
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
506
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
547
507
|
"""
|
548
|
-
|
549
|
-
production scheduler.
|
508
|
+
Internal decorator to support Fast bakery
|
550
509
|
"""
|
551
510
|
...
|
552
511
|
|
553
512
|
@typing.overload
|
554
|
-
def
|
555
|
-
...
|
556
|
-
|
557
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
558
|
-
"""
|
559
|
-
Specifies the times when the flow should be run when running on a
|
560
|
-
production scheduler.
|
561
|
-
"""
|
513
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
562
514
|
...
|
563
515
|
|
564
|
-
def
|
516
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
565
517
|
"""
|
566
|
-
|
567
|
-
|
568
|
-
User code call
|
569
|
-
-----------
|
570
|
-
@nim(
|
571
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
572
|
-
backend='managed'
|
573
|
-
)
|
574
|
-
|
575
|
-
Valid backend options
|
576
|
-
---------------------
|
577
|
-
- 'managed': Outerbounds selects a compute provider based on the model.
|
578
|
-
- 🚧 'dataplane': Run in your account.
|
579
|
-
|
580
|
-
Valid model options
|
581
|
-
----------------
|
582
|
-
- 'meta/llama3-8b-instruct': 8B parameter model
|
583
|
-
- 'meta/llama3-70b-instruct': 70B parameter model
|
584
|
-
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
518
|
+
Internal decorator to support Fast bakery
|
585
519
|
"""
|
586
520
|
...
|
587
521
|
|
@@ -666,13 +600,59 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
666
600
|
"""
|
667
601
|
...
|
668
602
|
|
669
|
-
|
603
|
+
@typing.overload
|
604
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
670
605
|
"""
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
|
606
|
+
Specifies the PyPI packages for all steps of the flow.
|
607
|
+
|
608
|
+
Use `@pypi_base` to set common packages required by all
|
609
|
+
steps and use `@pypi` to specify step-specific overrides.
|
610
|
+
"""
|
611
|
+
...
|
612
|
+
|
613
|
+
@typing.overload
|
614
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
615
|
+
...
|
616
|
+
|
617
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
618
|
+
"""
|
619
|
+
Specifies the PyPI packages for all steps of the flow.
|
620
|
+
|
621
|
+
Use `@pypi_base` to set common packages required by all
|
622
|
+
steps and use `@pypi` to specify step-specific overrides.
|
623
|
+
"""
|
624
|
+
...
|
625
|
+
|
626
|
+
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
627
|
+
"""
|
628
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
629
|
+
|
630
|
+
User code call
|
631
|
+
-----------
|
632
|
+
@nim(
|
633
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
634
|
+
backend='managed'
|
635
|
+
)
|
636
|
+
|
637
|
+
Valid backend options
|
638
|
+
---------------------
|
639
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
640
|
+
- 🚧 'dataplane': Run in your account.
|
641
|
+
|
642
|
+
Valid model options
|
643
|
+
----------------
|
644
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
645
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
646
|
+
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
647
|
+
"""
|
648
|
+
...
|
649
|
+
|
650
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
651
|
+
"""
|
652
|
+
Specifies what flows belong to the same project.
|
653
|
+
|
654
|
+
A project-specific namespace is created for all flows that
|
655
|
+
use the same `@project(name)`.
|
676
656
|
"""
|
677
657
|
...
|
678
658
|
|
@@ -749,6 +729,35 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
749
729
|
"""
|
750
730
|
...
|
751
731
|
|
732
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
733
|
+
"""
|
734
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
735
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
736
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
737
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
738
|
+
starts only after all sensors finish.
|
739
|
+
"""
|
740
|
+
...
|
741
|
+
|
742
|
+
@typing.overload
|
743
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
744
|
+
"""
|
745
|
+
Specifies the times when the flow should be run when running on a
|
746
|
+
production scheduler.
|
747
|
+
"""
|
748
|
+
...
|
749
|
+
|
750
|
+
@typing.overload
|
751
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
752
|
+
...
|
753
|
+
|
754
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
755
|
+
"""
|
756
|
+
Specifies the times when the flow should be run when running on a
|
757
|
+
production scheduler.
|
758
|
+
"""
|
759
|
+
...
|
760
|
+
|
752
761
|
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
753
762
|
"""
|
754
763
|
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
@@ -779,14 +788,5 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
779
788
|
"""
|
780
789
|
...
|
781
790
|
|
782
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
783
|
-
"""
|
784
|
-
Specifies what flows belong to the same project.
|
785
|
-
|
786
|
-
A project-specific namespace is created for all flows that
|
787
|
-
use the same `@project(name)`.
|
788
|
-
"""
|
789
|
-
...
|
790
|
-
|
791
791
|
pkg_name: str
|
792
792
|
|