ob-metaflow-stubs 6.0.3.175rc0__py2.py3-none-any.whl → 6.0.3.175rc1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +593 -593
- metaflow-stubs/cards.pyi +1 -1
- metaflow-stubs/cli.pyi +1 -1
- metaflow-stubs/cli_components/__init__.pyi +1 -1
- metaflow-stubs/cli_components/utils.pyi +1 -1
- metaflow-stubs/client/__init__.pyi +1 -1
- metaflow-stubs/client/core.pyi +4 -4
- metaflow-stubs/client/filecache.pyi +1 -1
- metaflow-stubs/events.pyi +1 -1
- metaflow-stubs/exception.pyi +1 -1
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/info_file.pyi +1 -1
- metaflow-stubs/metadata_provider/__init__.pyi +1 -1
- metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +1 -1
- metaflow-stubs/metaflow_config.pyi +1 -1
- metaflow-stubs/metaflow_current.pyi +81 -81
- metaflow-stubs/metaflow_git.pyi +1 -1
- metaflow-stubs/mf_extensions/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
- metaflow-stubs/multicore_utils.pyi +1 -1
- metaflow-stubs/ob_internal.pyi +1 -1
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +8 -8
- metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
- metaflow-stubs/plugins/airflow/exception.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
- metaflow-stubs/plugins/argo/__init__.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +1 -1
- metaflow-stubs/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
- metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/cards/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_client.pyi +1 -1
- metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
- metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
- metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
- metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
- metaflow-stubs/plugins/cards/exception.pyi +1 -1
- metaflow-stubs/plugins/catch_decorator.pyi +1 -1
- metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/local.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
- metaflow-stubs/plugins/debug_logger.pyi +1 -1
- metaflow-stubs/plugins/debug_monitor.pyi +1 -1
- metaflow-stubs/plugins/environment_decorator.pyi +1 -1
- metaflow-stubs/plugins/events_decorator.pyi +1 -1
- metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
- metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
- metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
- metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
- metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
- metaflow-stubs/plugins/perimeters.pyi +1 -1
- metaflow-stubs/plugins/project_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
- metaflow-stubs/plugins/pypi/utils.pyi +1 -1
- metaflow-stubs/plugins/resources_decorator.pyi +1 -1
- metaflow-stubs/plugins/retry_decorator.pyi +1 -1
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
- metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/plugins/storage_executor.pyi +1 -1
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
- metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
- metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +1 -1
- metaflow-stubs/pylint_wrapper.pyi +1 -1
- metaflow-stubs/runner/__init__.pyi +1 -1
- metaflow-stubs/runner/deployer.pyi +6 -6
- metaflow-stubs/runner/deployer_impl.pyi +1 -1
- metaflow-stubs/runner/metaflow_runner.pyi +2 -2
- metaflow-stubs/runner/nbdeploy.pyi +1 -1
- metaflow-stubs/runner/nbrun.pyi +1 -1
- metaflow-stubs/runner/subprocess_manager.pyi +1 -1
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +1 -1
- metaflow-stubs/system/system_logger.pyi +1 -1
- metaflow-stubs/system/system_monitor.pyi +1 -1
- metaflow-stubs/tagging_util.pyi +1 -1
- metaflow-stubs/tuple_util.pyi +1 -1
- metaflow-stubs/user_configs/__init__.pyi +1 -1
- metaflow-stubs/user_configs/config_decorators.pyi +3 -3
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +3 -3
- {ob_metaflow_stubs-6.0.3.175rc0.dist-info → ob_metaflow_stubs-6.0.3.175rc1.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.175rc1.dist-info/RECORD +216 -0
- ob_metaflow_stubs-6.0.3.175rc0.dist-info/RECORD +0 -216
- {ob_metaflow_stubs-6.0.3.175rc0.dist-info → ob_metaflow_stubs-6.0.3.175rc1.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.175rc0.dist-info → ob_metaflow_stubs-6.0.3.175rc1.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
3
|
# MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
|
4
|
-
# Generated on 2025-05-
|
4
|
+
# Generated on 2025-05-31T01:09:57.775269 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -35,10 +35,10 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
|
|
35
35
|
from .user_configs.config_parameters import config_expr as config_expr
|
36
36
|
from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
|
37
37
|
from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
|
38
|
-
from . import cards as cards
|
39
|
-
from . import metaflow_git as metaflow_git
|
40
38
|
from . import tuple_util as tuple_util
|
39
|
+
from . import cards as cards
|
41
40
|
from . import events as events
|
41
|
+
from . import metaflow_git as metaflow_git
|
42
42
|
from . import runner as runner
|
43
43
|
from . import plugins as plugins
|
44
44
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
@@ -155,53 +155,81 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
155
155
|
...
|
156
156
|
|
157
157
|
@typing.overload
|
158
|
-
def
|
158
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
159
159
|
"""
|
160
|
-
Specifies the
|
160
|
+
Specifies the resources needed when executing this step.
|
161
161
|
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
162
|
+
Use `@resources` to specify the resource requirements
|
163
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
164
|
+
|
165
|
+
You can choose the compute layer on the command line by executing e.g.
|
166
|
+
```
|
167
|
+
python myflow.py run --with batch
|
168
|
+
```
|
169
|
+
or
|
170
|
+
```
|
171
|
+
python myflow.py run --with kubernetes
|
172
|
+
```
|
173
|
+
which executes the flow on the desired system using the
|
174
|
+
requirements specified in `@resources`.
|
166
175
|
|
167
176
|
|
168
177
|
Parameters
|
169
178
|
----------
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
179
|
+
cpu : int, default 1
|
180
|
+
Number of CPUs required for this step.
|
181
|
+
gpu : int, optional, default None
|
182
|
+
Number of GPUs required for this step.
|
183
|
+
disk : int, optional, default None
|
184
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
185
|
+
memory : int, default 4096
|
186
|
+
Memory size (in MB) required for this step.
|
187
|
+
shared_memory : int, optional, default None
|
188
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
189
|
+
This parameter maps to the `--shm-size` option in Docker.
|
176
190
|
"""
|
177
191
|
...
|
178
192
|
|
179
193
|
@typing.overload
|
180
|
-
def
|
194
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
181
195
|
...
|
182
196
|
|
183
197
|
@typing.overload
|
184
|
-
def
|
198
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
185
199
|
...
|
186
200
|
|
187
|
-
def
|
201
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
188
202
|
"""
|
189
|
-
Specifies the
|
203
|
+
Specifies the resources needed when executing this step.
|
190
204
|
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
205
|
+
Use `@resources` to specify the resource requirements
|
206
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
207
|
+
|
208
|
+
You can choose the compute layer on the command line by executing e.g.
|
209
|
+
```
|
210
|
+
python myflow.py run --with batch
|
211
|
+
```
|
212
|
+
or
|
213
|
+
```
|
214
|
+
python myflow.py run --with kubernetes
|
215
|
+
```
|
216
|
+
which executes the flow on the desired system using the
|
217
|
+
requirements specified in `@resources`.
|
195
218
|
|
196
219
|
|
197
220
|
Parameters
|
198
221
|
----------
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
222
|
+
cpu : int, default 1
|
223
|
+
Number of CPUs required for this step.
|
224
|
+
gpu : int, optional, default None
|
225
|
+
Number of GPUs required for this step.
|
226
|
+
disk : int, optional, default None
|
227
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
228
|
+
memory : int, default 4096
|
229
|
+
Memory size (in MB) required for this step.
|
230
|
+
shared_memory : int, optional, default None
|
231
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
232
|
+
This parameter maps to the `--shm-size` option in Docker.
|
205
233
|
"""
|
206
234
|
...
|
207
235
|
|
@@ -256,6 +284,63 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
256
284
|
"""
|
257
285
|
...
|
258
286
|
|
287
|
+
@typing.overload
|
288
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
289
|
+
"""
|
290
|
+
Enables loading / saving of models within a step.
|
291
|
+
|
292
|
+
|
293
|
+
|
294
|
+
Parameters
|
295
|
+
----------
|
296
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
297
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
298
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
299
|
+
- `current.checkpoint`
|
300
|
+
- `current.model`
|
301
|
+
- `current.huggingface_hub`
|
302
|
+
|
303
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
304
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
305
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
306
|
+
|
307
|
+
temp_dir_root : str, default: None
|
308
|
+
The root directory under which `current.model.loaded` will store loaded models
|
309
|
+
"""
|
310
|
+
...
|
311
|
+
|
312
|
+
@typing.overload
|
313
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
314
|
+
...
|
315
|
+
|
316
|
+
@typing.overload
|
317
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
318
|
+
...
|
319
|
+
|
320
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
321
|
+
"""
|
322
|
+
Enables loading / saving of models within a step.
|
323
|
+
|
324
|
+
|
325
|
+
|
326
|
+
Parameters
|
327
|
+
----------
|
328
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
329
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
330
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
331
|
+
- `current.checkpoint`
|
332
|
+
- `current.model`
|
333
|
+
- `current.huggingface_hub`
|
334
|
+
|
335
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
336
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
337
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
338
|
+
|
339
|
+
temp_dir_root : str, default: None
|
340
|
+
The root directory under which `current.model.loaded` will store loaded models
|
341
|
+
"""
|
342
|
+
...
|
343
|
+
|
259
344
|
def ollama(*, models: list, backend: str, force_pull: bool, skip_push_check: bool, debug: bool) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
260
345
|
"""
|
261
346
|
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
@@ -293,6 +378,71 @@ def ollama(*, models: list, backend: str, force_pull: bool, skip_push_check: boo
|
|
293
378
|
"""
|
294
379
|
...
|
295
380
|
|
381
|
+
def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
382
|
+
"""
|
383
|
+
Specifies that this step is used to deploy an instance of the app.
|
384
|
+
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
385
|
+
|
386
|
+
|
387
|
+
Parameters
|
388
|
+
----------
|
389
|
+
app_port : int
|
390
|
+
Number of GPUs to use.
|
391
|
+
app_name : str
|
392
|
+
Name of the app to deploy.
|
393
|
+
"""
|
394
|
+
...
|
395
|
+
|
396
|
+
@typing.overload
|
397
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
398
|
+
"""
|
399
|
+
Decorator prototype for all step decorators. This function gets specialized
|
400
|
+
and imported for all decorators types by _import_plugin_decorators().
|
401
|
+
"""
|
402
|
+
...
|
403
|
+
|
404
|
+
@typing.overload
|
405
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
406
|
+
...
|
407
|
+
|
408
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
409
|
+
"""
|
410
|
+
Decorator prototype for all step decorators. This function gets specialized
|
411
|
+
and imported for all decorators types by _import_plugin_decorators().
|
412
|
+
"""
|
413
|
+
...
|
414
|
+
|
415
|
+
@typing.overload
|
416
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
417
|
+
"""
|
418
|
+
Internal decorator to support Fast bakery
|
419
|
+
"""
|
420
|
+
...
|
421
|
+
|
422
|
+
@typing.overload
|
423
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
424
|
+
...
|
425
|
+
|
426
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
427
|
+
"""
|
428
|
+
Internal decorator to support Fast bakery
|
429
|
+
"""
|
430
|
+
...
|
431
|
+
|
432
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
433
|
+
"""
|
434
|
+
Specifies that this step should execute on DGX cloud.
|
435
|
+
|
436
|
+
|
437
|
+
Parameters
|
438
|
+
----------
|
439
|
+
gpu : int
|
440
|
+
Number of GPUs to use.
|
441
|
+
gpu_type : str
|
442
|
+
Type of Nvidia GPU to use.
|
443
|
+
"""
|
444
|
+
...
|
445
|
+
|
296
446
|
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
297
447
|
"""
|
298
448
|
Specifies that this step should execute on Kubernetes.
|
@@ -378,75 +528,77 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
378
528
|
"""
|
379
529
|
...
|
380
530
|
|
381
|
-
|
382
|
-
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
531
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
383
532
|
"""
|
384
|
-
|
385
|
-
|
533
|
+
Specifies that this step should execute on DGX cloud.
|
386
534
|
|
387
535
|
|
388
536
|
Parameters
|
389
537
|
----------
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
538
|
+
gpu : int
|
539
|
+
Number of GPUs to use.
|
540
|
+
gpu_type : str
|
541
|
+
Type of Nvidia GPU to use.
|
542
|
+
queue_timeout : int
|
543
|
+
Time to keep the job in NVCF's queue.
|
544
|
+
"""
|
545
|
+
...
|
546
|
+
|
547
|
+
@typing.overload
|
548
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
549
|
+
"""
|
550
|
+
Specifies environment variables to be set prior to the execution of a step.
|
396
551
|
|
397
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
398
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
399
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
400
552
|
|
401
|
-
|
402
|
-
|
553
|
+
Parameters
|
554
|
+
----------
|
555
|
+
vars : Dict[str, str], default {}
|
556
|
+
Dictionary of environment variables to set.
|
403
557
|
"""
|
404
558
|
...
|
405
559
|
|
406
560
|
@typing.overload
|
407
|
-
def
|
561
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
408
562
|
...
|
409
563
|
|
410
564
|
@typing.overload
|
411
|
-
def
|
565
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
412
566
|
...
|
413
567
|
|
414
|
-
def
|
568
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
415
569
|
"""
|
416
|
-
|
417
|
-
|
570
|
+
Specifies environment variables to be set prior to the execution of a step.
|
418
571
|
|
419
572
|
|
420
573
|
Parameters
|
421
574
|
----------
|
422
|
-
|
423
|
-
|
424
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
425
|
-
- `current.checkpoint`
|
426
|
-
- `current.model`
|
427
|
-
- `current.huggingface_hub`
|
428
|
-
|
429
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
430
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
431
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
432
|
-
|
433
|
-
temp_dir_root : str, default: None
|
434
|
-
The root directory under which `current.model.loaded` will store loaded models
|
575
|
+
vars : Dict[str, str], default {}
|
576
|
+
Dictionary of environment variables to set.
|
435
577
|
"""
|
436
578
|
...
|
437
579
|
|
438
|
-
def
|
580
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
439
581
|
"""
|
440
|
-
|
441
|
-
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
582
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
442
583
|
|
443
584
|
|
444
585
|
Parameters
|
445
586
|
----------
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
587
|
+
temp_dir_root : str, optional
|
588
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
589
|
+
|
590
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
591
|
+
The list of repos (models/datasets) to load.
|
592
|
+
|
593
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
594
|
+
|
595
|
+
- If repo (model/dataset) is not found in the datastore:
|
596
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
597
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
598
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
599
|
+
|
600
|
+
- If repo is found in the datastore:
|
601
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
450
602
|
"""
|
451
603
|
...
|
452
604
|
|
@@ -505,184 +657,6 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
505
657
|
"""
|
506
658
|
...
|
507
659
|
|
508
|
-
@typing.overload
|
509
|
-
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
510
|
-
"""
|
511
|
-
Enables checkpointing for a step.
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
Parameters
|
516
|
-
----------
|
517
|
-
load_policy : str, default: "fresh"
|
518
|
-
The policy for loading the checkpoint. The following policies are supported:
|
519
|
-
- "eager": Loads the the latest available checkpoint within the namespace.
|
520
|
-
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
521
|
-
will be loaded at the start of the task.
|
522
|
-
- "none": Do not load any checkpoint
|
523
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
524
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
525
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
526
|
-
created within the task will be loaded when the task is retries execution on failure.
|
527
|
-
|
528
|
-
temp_dir_root : str, default: None
|
529
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
530
|
-
"""
|
531
|
-
...
|
532
|
-
|
533
|
-
@typing.overload
|
534
|
-
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
535
|
-
...
|
536
|
-
|
537
|
-
@typing.overload
|
538
|
-
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
539
|
-
...
|
540
|
-
|
541
|
-
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
542
|
-
"""
|
543
|
-
Enables checkpointing for a step.
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
Parameters
|
548
|
-
----------
|
549
|
-
load_policy : str, default: "fresh"
|
550
|
-
The policy for loading the checkpoint. The following policies are supported:
|
551
|
-
- "eager": Loads the the latest available checkpoint within the namespace.
|
552
|
-
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
553
|
-
will be loaded at the start of the task.
|
554
|
-
- "none": Do not load any checkpoint
|
555
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
556
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
557
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
558
|
-
created within the task will be loaded when the task is retries execution on failure.
|
559
|
-
|
560
|
-
temp_dir_root : str, default: None
|
561
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
562
|
-
"""
|
563
|
-
...
|
564
|
-
|
565
|
-
@typing.overload
|
566
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
567
|
-
"""
|
568
|
-
Internal decorator to support Fast bakery
|
569
|
-
"""
|
570
|
-
...
|
571
|
-
|
572
|
-
@typing.overload
|
573
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
574
|
-
...
|
575
|
-
|
576
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
577
|
-
"""
|
578
|
-
Internal decorator to support Fast bakery
|
579
|
-
"""
|
580
|
-
...
|
581
|
-
|
582
|
-
@typing.overload
|
583
|
-
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
584
|
-
"""
|
585
|
-
Specifies the resources needed when executing this step.
|
586
|
-
|
587
|
-
Use `@resources` to specify the resource requirements
|
588
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
589
|
-
|
590
|
-
You can choose the compute layer on the command line by executing e.g.
|
591
|
-
```
|
592
|
-
python myflow.py run --with batch
|
593
|
-
```
|
594
|
-
or
|
595
|
-
```
|
596
|
-
python myflow.py run --with kubernetes
|
597
|
-
```
|
598
|
-
which executes the flow on the desired system using the
|
599
|
-
requirements specified in `@resources`.
|
600
|
-
|
601
|
-
|
602
|
-
Parameters
|
603
|
-
----------
|
604
|
-
cpu : int, default 1
|
605
|
-
Number of CPUs required for this step.
|
606
|
-
gpu : int, optional, default None
|
607
|
-
Number of GPUs required for this step.
|
608
|
-
disk : int, optional, default None
|
609
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
610
|
-
memory : int, default 4096
|
611
|
-
Memory size (in MB) required for this step.
|
612
|
-
shared_memory : int, optional, default None
|
613
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
614
|
-
This parameter maps to the `--shm-size` option in Docker.
|
615
|
-
"""
|
616
|
-
...
|
617
|
-
|
618
|
-
@typing.overload
|
619
|
-
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
620
|
-
...
|
621
|
-
|
622
|
-
@typing.overload
|
623
|
-
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
624
|
-
...
|
625
|
-
|
626
|
-
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
627
|
-
"""
|
628
|
-
Specifies the resources needed when executing this step.
|
629
|
-
|
630
|
-
Use `@resources` to specify the resource requirements
|
631
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
632
|
-
|
633
|
-
You can choose the compute layer on the command line by executing e.g.
|
634
|
-
```
|
635
|
-
python myflow.py run --with batch
|
636
|
-
```
|
637
|
-
or
|
638
|
-
```
|
639
|
-
python myflow.py run --with kubernetes
|
640
|
-
```
|
641
|
-
which executes the flow on the desired system using the
|
642
|
-
requirements specified in `@resources`.
|
643
|
-
|
644
|
-
|
645
|
-
Parameters
|
646
|
-
----------
|
647
|
-
cpu : int, default 1
|
648
|
-
Number of CPUs required for this step.
|
649
|
-
gpu : int, optional, default None
|
650
|
-
Number of GPUs required for this step.
|
651
|
-
disk : int, optional, default None
|
652
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
653
|
-
memory : int, default 4096
|
654
|
-
Memory size (in MB) required for this step.
|
655
|
-
shared_memory : int, optional, default None
|
656
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
657
|
-
This parameter maps to the `--shm-size` option in Docker.
|
658
|
-
"""
|
659
|
-
...
|
660
|
-
|
661
|
-
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
662
|
-
"""
|
663
|
-
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
664
|
-
|
665
|
-
|
666
|
-
Parameters
|
667
|
-
----------
|
668
|
-
temp_dir_root : str, optional
|
669
|
-
The root directory that will hold the temporary directory where objects will be downloaded.
|
670
|
-
|
671
|
-
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
672
|
-
The list of repos (models/datasets) to load.
|
673
|
-
|
674
|
-
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
675
|
-
|
676
|
-
- If repo (model/dataset) is not found in the datastore:
|
677
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
678
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
679
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
680
|
-
|
681
|
-
- If repo is found in the datastore:
|
682
|
-
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
683
|
-
"""
|
684
|
-
...
|
685
|
-
|
686
660
|
@typing.overload
|
687
661
|
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
688
662
|
"""
|
@@ -792,65 +766,118 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
792
766
|
...
|
793
767
|
|
794
768
|
@typing.overload
|
795
|
-
def
|
769
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
796
770
|
"""
|
797
|
-
|
771
|
+
Enables checkpointing for a step.
|
772
|
+
|
798
773
|
|
799
774
|
|
800
775
|
Parameters
|
801
776
|
----------
|
802
|
-
|
803
|
-
|
777
|
+
load_policy : str, default: "fresh"
|
778
|
+
The policy for loading the checkpoint. The following policies are supported:
|
779
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
780
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
781
|
+
will be loaded at the start of the task.
|
782
|
+
- "none": Do not load any checkpoint
|
783
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
784
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
785
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
786
|
+
created within the task will be loaded when the task is retries execution on failure.
|
787
|
+
|
788
|
+
temp_dir_root : str, default: None
|
789
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
804
790
|
"""
|
805
791
|
...
|
806
792
|
|
807
793
|
@typing.overload
|
808
|
-
def
|
794
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
809
795
|
...
|
810
796
|
|
811
797
|
@typing.overload
|
812
|
-
def
|
798
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
813
799
|
...
|
814
800
|
|
815
|
-
def
|
801
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
816
802
|
"""
|
817
|
-
|
803
|
+
Enables checkpointing for a step.
|
804
|
+
|
818
805
|
|
819
806
|
|
820
807
|
Parameters
|
821
808
|
----------
|
822
|
-
|
823
|
-
|
809
|
+
load_policy : str, default: "fresh"
|
810
|
+
The policy for loading the checkpoint. The following policies are supported:
|
811
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
812
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
813
|
+
will be loaded at the start of the task.
|
814
|
+
- "none": Do not load any checkpoint
|
815
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
816
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
817
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
818
|
+
created within the task will be loaded when the task is retries execution on failure.
|
819
|
+
|
820
|
+
temp_dir_root : str, default: None
|
821
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
824
822
|
"""
|
825
823
|
...
|
826
824
|
|
827
|
-
|
828
|
-
|
829
|
-
|
825
|
+
@typing.overload
|
826
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
827
|
+
"""
|
828
|
+
Specifies the Conda environment for the step.
|
829
|
+
|
830
|
+
Information in this decorator will augment any
|
831
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
832
|
+
you can use `@conda_base` to set packages required by all
|
833
|
+
steps and use `@conda` to specify step-specific overrides.
|
830
834
|
|
831
835
|
|
832
836
|
Parameters
|
833
837
|
----------
|
834
|
-
|
835
|
-
|
836
|
-
|
837
|
-
|
838
|
-
|
839
|
-
|
838
|
+
packages : Dict[str, str], default {}
|
839
|
+
Packages to use for this step. The key is the name of the package
|
840
|
+
and the value is the version to use.
|
841
|
+
libraries : Dict[str, str], default {}
|
842
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
843
|
+
python : str, optional, default None
|
844
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
845
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
846
|
+
disabled : bool, default False
|
847
|
+
If set to True, disables @conda.
|
840
848
|
"""
|
841
849
|
...
|
842
850
|
|
843
|
-
|
851
|
+
@typing.overload
|
852
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
853
|
+
...
|
854
|
+
|
855
|
+
@typing.overload
|
856
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
857
|
+
...
|
858
|
+
|
859
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
844
860
|
"""
|
845
|
-
Specifies
|
861
|
+
Specifies the Conda environment for the step.
|
862
|
+
|
863
|
+
Information in this decorator will augment any
|
864
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
865
|
+
you can use `@conda_base` to set packages required by all
|
866
|
+
steps and use `@conda` to specify step-specific overrides.
|
846
867
|
|
847
868
|
|
848
869
|
Parameters
|
849
870
|
----------
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
871
|
+
packages : Dict[str, str], default {}
|
872
|
+
Packages to use for this step. The key is the name of the package
|
873
|
+
and the value is the version to use.
|
874
|
+
libraries : Dict[str, str], default {}
|
875
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
876
|
+
python : str, optional, default None
|
877
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
878
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
879
|
+
disabled : bool, default False
|
880
|
+
If set to True, disables @conda.
|
854
881
|
"""
|
855
882
|
...
|
856
883
|
|
@@ -890,80 +917,146 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
890
917
|
...
|
891
918
|
|
892
919
|
@typing.overload
|
893
|
-
def
|
920
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
894
921
|
"""
|
895
|
-
Specifies the
|
922
|
+
Specifies the PyPI packages for the step.
|
896
923
|
|
897
924
|
Information in this decorator will augment any
|
898
|
-
attributes set in the `@
|
899
|
-
you can use `@
|
900
|
-
steps and use `@
|
925
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
926
|
+
you can use `@pypi_base` to set packages required by all
|
927
|
+
steps and use `@pypi` to specify step-specific overrides.
|
901
928
|
|
902
929
|
|
903
930
|
Parameters
|
904
931
|
----------
|
905
|
-
packages : Dict[str, str], default {}
|
932
|
+
packages : Dict[str, str], default: {}
|
906
933
|
Packages to use for this step. The key is the name of the package
|
907
934
|
and the value is the version to use.
|
908
|
-
|
909
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
910
|
-
python : str, optional, default None
|
935
|
+
python : str, optional, default: None
|
911
936
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
912
937
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
913
|
-
disabled : bool, default False
|
914
|
-
If set to True, disables @conda.
|
915
938
|
"""
|
916
939
|
...
|
917
940
|
|
918
941
|
@typing.overload
|
919
|
-
def
|
942
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
920
943
|
...
|
921
944
|
|
922
945
|
@typing.overload
|
923
|
-
def
|
946
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
924
947
|
...
|
925
948
|
|
926
|
-
def
|
949
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
927
950
|
"""
|
928
|
-
Specifies the
|
951
|
+
Specifies the PyPI packages for the step.
|
929
952
|
|
930
953
|
Information in this decorator will augment any
|
931
|
-
attributes set in the `@
|
932
|
-
you can use `@
|
933
|
-
steps and use `@
|
954
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
955
|
+
you can use `@pypi_base` to set packages required by all
|
956
|
+
steps and use `@pypi` to specify step-specific overrides.
|
934
957
|
|
935
958
|
|
936
959
|
Parameters
|
937
960
|
----------
|
938
|
-
packages : Dict[str, str], default {}
|
961
|
+
packages : Dict[str, str], default: {}
|
939
962
|
Packages to use for this step. The key is the name of the package
|
940
963
|
and the value is the version to use.
|
941
|
-
|
942
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
943
|
-
python : str, optional, default None
|
964
|
+
python : str, optional, default: None
|
944
965
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
945
966
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
946
|
-
disabled : bool, default False
|
947
|
-
If set to True, disables @conda.
|
948
967
|
"""
|
949
968
|
...
|
950
969
|
|
951
970
|
@typing.overload
|
952
|
-
def
|
971
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
953
972
|
"""
|
954
|
-
|
955
|
-
|
973
|
+
Specifies the event(s) that this flow depends on.
|
974
|
+
|
975
|
+
```
|
976
|
+
@trigger(event='foo')
|
977
|
+
```
|
978
|
+
or
|
979
|
+
```
|
980
|
+
@trigger(events=['foo', 'bar'])
|
981
|
+
```
|
982
|
+
|
983
|
+
Additionally, you can specify the parameter mappings
|
984
|
+
to map event payload to Metaflow parameters for the flow.
|
985
|
+
```
|
986
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
987
|
+
```
|
988
|
+
or
|
989
|
+
```
|
990
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
991
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
992
|
+
```
|
993
|
+
|
994
|
+
'parameters' can also be a list of strings and tuples like so:
|
995
|
+
```
|
996
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
997
|
+
```
|
998
|
+
This is equivalent to:
|
999
|
+
```
|
1000
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1001
|
+
```
|
1002
|
+
|
1003
|
+
|
1004
|
+
Parameters
|
1005
|
+
----------
|
1006
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1007
|
+
Event dependency for this flow.
|
1008
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1009
|
+
Events dependency for this flow.
|
1010
|
+
options : Dict[str, Any], default {}
|
1011
|
+
Backend-specific configuration for tuning eventing behavior.
|
956
1012
|
"""
|
957
1013
|
...
|
958
1014
|
|
959
1015
|
@typing.overload
|
960
|
-
def
|
1016
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
961
1017
|
...
|
962
1018
|
|
963
|
-
def
|
1019
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
964
1020
|
"""
|
965
|
-
|
966
|
-
|
1021
|
+
Specifies the event(s) that this flow depends on.
|
1022
|
+
|
1023
|
+
```
|
1024
|
+
@trigger(event='foo')
|
1025
|
+
```
|
1026
|
+
or
|
1027
|
+
```
|
1028
|
+
@trigger(events=['foo', 'bar'])
|
1029
|
+
```
|
1030
|
+
|
1031
|
+
Additionally, you can specify the parameter mappings
|
1032
|
+
to map event payload to Metaflow parameters for the flow.
|
1033
|
+
```
|
1034
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1035
|
+
```
|
1036
|
+
or
|
1037
|
+
```
|
1038
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1039
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1040
|
+
```
|
1041
|
+
|
1042
|
+
'parameters' can also be a list of strings and tuples like so:
|
1043
|
+
```
|
1044
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1045
|
+
```
|
1046
|
+
This is equivalent to:
|
1047
|
+
```
|
1048
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1049
|
+
```
|
1050
|
+
|
1051
|
+
|
1052
|
+
Parameters
|
1053
|
+
----------
|
1054
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1055
|
+
Event dependency for this flow.
|
1056
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1057
|
+
Events dependency for this flow.
|
1058
|
+
options : Dict[str, Any], default {}
|
1059
|
+
Backend-specific configuration for tuning eventing behavior.
|
967
1060
|
"""
|
968
1061
|
...
|
969
1062
|
|
@@ -1069,95 +1162,223 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1069
1162
|
...
|
1070
1163
|
|
1071
1164
|
@typing.overload
|
1072
|
-
def
|
1165
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1073
1166
|
"""
|
1074
|
-
Specifies the
|
1075
|
-
|
1076
|
-
```
|
1077
|
-
@trigger(event='foo')
|
1078
|
-
```
|
1079
|
-
or
|
1080
|
-
```
|
1081
|
-
@trigger(events=['foo', 'bar'])
|
1082
|
-
```
|
1083
|
-
|
1084
|
-
Additionally, you can specify the parameter mappings
|
1085
|
-
to map event payload to Metaflow parameters for the flow.
|
1086
|
-
```
|
1087
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1088
|
-
```
|
1089
|
-
or
|
1090
|
-
```
|
1091
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1092
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1093
|
-
```
|
1167
|
+
Specifies the Conda environment for all steps of the flow.
|
1094
1168
|
|
1095
|
-
|
1096
|
-
|
1097
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1098
|
-
```
|
1099
|
-
This is equivalent to:
|
1100
|
-
```
|
1101
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1102
|
-
```
|
1169
|
+
Use `@conda_base` to set common libraries required by all
|
1170
|
+
steps and use `@conda` to specify step-specific additions.
|
1103
1171
|
|
1104
1172
|
|
1105
1173
|
Parameters
|
1106
1174
|
----------
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1175
|
+
packages : Dict[str, str], default {}
|
1176
|
+
Packages to use for this flow. The key is the name of the package
|
1177
|
+
and the value is the version to use.
|
1178
|
+
libraries : Dict[str, str], default {}
|
1179
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1180
|
+
python : str, optional, default None
|
1181
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1182
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1183
|
+
disabled : bool, default False
|
1184
|
+
If set to True, disables Conda.
|
1113
1185
|
"""
|
1114
1186
|
...
|
1115
1187
|
|
1116
1188
|
@typing.overload
|
1117
|
-
def
|
1189
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1118
1190
|
...
|
1119
1191
|
|
1120
|
-
def
|
1192
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1121
1193
|
"""
|
1122
|
-
Specifies the
|
1194
|
+
Specifies the Conda environment for all steps of the flow.
|
1123
1195
|
|
1124
|
-
|
1125
|
-
|
1126
|
-
```
|
1127
|
-
or
|
1128
|
-
```
|
1129
|
-
@trigger(events=['foo', 'bar'])
|
1130
|
-
```
|
1196
|
+
Use `@conda_base` to set common libraries required by all
|
1197
|
+
steps and use `@conda` to specify step-specific additions.
|
1131
1198
|
|
1132
|
-
Additionally, you can specify the parameter mappings
|
1133
|
-
to map event payload to Metaflow parameters for the flow.
|
1134
|
-
```
|
1135
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1136
|
-
```
|
1137
|
-
or
|
1138
|
-
```
|
1139
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1140
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1141
|
-
```
|
1142
1199
|
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1200
|
+
Parameters
|
1201
|
+
----------
|
1202
|
+
packages : Dict[str, str], default {}
|
1203
|
+
Packages to use for this flow. The key is the name of the package
|
1204
|
+
and the value is the version to use.
|
1205
|
+
libraries : Dict[str, str], default {}
|
1206
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1207
|
+
python : str, optional, default None
|
1208
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1209
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1210
|
+
disabled : bool, default False
|
1211
|
+
If set to True, disables Conda.
|
1212
|
+
"""
|
1213
|
+
...
|
1214
|
+
|
1215
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1216
|
+
"""
|
1217
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1218
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1219
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1220
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1221
|
+
starts only after all sensors finish.
|
1151
1222
|
|
1152
1223
|
|
1153
1224
|
Parameters
|
1154
1225
|
----------
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1226
|
+
timeout : int
|
1227
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1228
|
+
poke_interval : int
|
1229
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1230
|
+
mode : str
|
1231
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1232
|
+
exponential_backoff : bool
|
1233
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1234
|
+
pool : str
|
1235
|
+
the slot pool this task should run in,
|
1236
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1237
|
+
soft_fail : bool
|
1238
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1239
|
+
name : str
|
1240
|
+
Name of the sensor on Airflow
|
1241
|
+
description : str
|
1242
|
+
Description of sensor in the Airflow UI
|
1243
|
+
bucket_key : Union[str, List[str]]
|
1244
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1245
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1246
|
+
bucket_name : str
|
1247
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1248
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1249
|
+
wildcard_match : bool
|
1250
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1251
|
+
aws_conn_id : str
|
1252
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1253
|
+
verify : bool
|
1254
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1255
|
+
"""
|
1256
|
+
...
|
1257
|
+
|
1258
|
+
@typing.overload
|
1259
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1260
|
+
"""
|
1261
|
+
Specifies the times when the flow should be run when running on a
|
1262
|
+
production scheduler.
|
1263
|
+
|
1264
|
+
|
1265
|
+
Parameters
|
1266
|
+
----------
|
1267
|
+
hourly : bool, default False
|
1268
|
+
Run the workflow hourly.
|
1269
|
+
daily : bool, default True
|
1270
|
+
Run the workflow daily.
|
1271
|
+
weekly : bool, default False
|
1272
|
+
Run the workflow weekly.
|
1273
|
+
cron : str, optional, default None
|
1274
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1275
|
+
specified by this expression.
|
1276
|
+
timezone : str, optional, default None
|
1277
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1278
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1279
|
+
"""
|
1280
|
+
...
|
1281
|
+
|
1282
|
+
@typing.overload
|
1283
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1284
|
+
...
|
1285
|
+
|
1286
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1287
|
+
"""
|
1288
|
+
Specifies the times when the flow should be run when running on a
|
1289
|
+
production scheduler.
|
1290
|
+
|
1291
|
+
|
1292
|
+
Parameters
|
1293
|
+
----------
|
1294
|
+
hourly : bool, default False
|
1295
|
+
Run the workflow hourly.
|
1296
|
+
daily : bool, default True
|
1297
|
+
Run the workflow daily.
|
1298
|
+
weekly : bool, default False
|
1299
|
+
Run the workflow weekly.
|
1300
|
+
cron : str, optional, default None
|
1301
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1302
|
+
specified by this expression.
|
1303
|
+
timezone : str, optional, default None
|
1304
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1305
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1306
|
+
"""
|
1307
|
+
...
|
1308
|
+
|
1309
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1310
|
+
"""
|
1311
|
+
Specifies what flows belong to the same project.
|
1312
|
+
|
1313
|
+
A project-specific namespace is created for all flows that
|
1314
|
+
use the same `@project(name)`.
|
1315
|
+
|
1316
|
+
|
1317
|
+
Parameters
|
1318
|
+
----------
|
1319
|
+
name : str
|
1320
|
+
Project name. Make sure that the name is unique amongst all
|
1321
|
+
projects that use the same production scheduler. The name may
|
1322
|
+
contain only lowercase alphanumeric characters and underscores.
|
1323
|
+
|
1324
|
+
branch : Optional[str], default None
|
1325
|
+
The branch to use. If not specified, the branch is set to
|
1326
|
+
`user.<username>` unless `production` is set to `True`. This can
|
1327
|
+
also be set on the command line using `--branch` as a top-level option.
|
1328
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
1329
|
+
|
1330
|
+
production : bool, default False
|
1331
|
+
Whether or not the branch is the production branch. This can also be set on the
|
1332
|
+
command line using `--production` as a top-level option. It is an error to specify
|
1333
|
+
`production` in the decorator and on the command line.
|
1334
|
+
The project branch name will be:
|
1335
|
+
- if `branch` is specified:
|
1336
|
+
- if `production` is True: `prod.<branch>`
|
1337
|
+
- if `production` is False: `test.<branch>`
|
1338
|
+
- if `branch` is not specified:
|
1339
|
+
- if `production` is True: `prod`
|
1340
|
+
- if `production` is False: `user.<username>`
|
1341
|
+
"""
|
1342
|
+
...
|
1343
|
+
|
1344
|
+
@typing.overload
|
1345
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1346
|
+
"""
|
1347
|
+
Specifies the PyPI packages for all steps of the flow.
|
1348
|
+
|
1349
|
+
Use `@pypi_base` to set common packages required by all
|
1350
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1351
|
+
|
1352
|
+
Parameters
|
1353
|
+
----------
|
1354
|
+
packages : Dict[str, str], default: {}
|
1355
|
+
Packages to use for this flow. The key is the name of the package
|
1356
|
+
and the value is the version to use.
|
1357
|
+
python : str, optional, default: None
|
1358
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1359
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1360
|
+
"""
|
1361
|
+
...
|
1362
|
+
|
1363
|
+
@typing.overload
|
1364
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1365
|
+
...
|
1366
|
+
|
1367
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1368
|
+
"""
|
1369
|
+
Specifies the PyPI packages for all steps of the flow.
|
1370
|
+
|
1371
|
+
Use `@pypi_base` to set common packages required by all
|
1372
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1373
|
+
|
1374
|
+
Parameters
|
1375
|
+
----------
|
1376
|
+
packages : Dict[str, str], default: {}
|
1377
|
+
Packages to use for this flow. The key is the name of the package
|
1378
|
+
and the value is the version to use.
|
1379
|
+
python : str, optional, default: None
|
1380
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1381
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1161
1382
|
"""
|
1162
1383
|
...
|
1163
1384
|
|
@@ -1275,151 +1496,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
1275
1496
|
"""
|
1276
1497
|
...
|
1277
1498
|
|
1278
|
-
@typing.overload
|
1279
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1280
|
-
"""
|
1281
|
-
Specifies the Conda environment for all steps of the flow.
|
1282
|
-
|
1283
|
-
Use `@conda_base` to set common libraries required by all
|
1284
|
-
steps and use `@conda` to specify step-specific additions.
|
1285
|
-
|
1286
|
-
|
1287
|
-
Parameters
|
1288
|
-
----------
|
1289
|
-
packages : Dict[str, str], default {}
|
1290
|
-
Packages to use for this flow. The key is the name of the package
|
1291
|
-
and the value is the version to use.
|
1292
|
-
libraries : Dict[str, str], default {}
|
1293
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1294
|
-
python : str, optional, default None
|
1295
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1296
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1297
|
-
disabled : bool, default False
|
1298
|
-
If set to True, disables Conda.
|
1299
|
-
"""
|
1300
|
-
...
|
1301
|
-
|
1302
|
-
@typing.overload
|
1303
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1304
|
-
...
|
1305
|
-
|
1306
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1307
|
-
"""
|
1308
|
-
Specifies the Conda environment for all steps of the flow.
|
1309
|
-
|
1310
|
-
Use `@conda_base` to set common libraries required by all
|
1311
|
-
steps and use `@conda` to specify step-specific additions.
|
1312
|
-
|
1313
|
-
|
1314
|
-
Parameters
|
1315
|
-
----------
|
1316
|
-
packages : Dict[str, str], default {}
|
1317
|
-
Packages to use for this flow. The key is the name of the package
|
1318
|
-
and the value is the version to use.
|
1319
|
-
libraries : Dict[str, str], default {}
|
1320
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1321
|
-
python : str, optional, default None
|
1322
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1323
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1324
|
-
disabled : bool, default False
|
1325
|
-
If set to True, disables Conda.
|
1326
|
-
"""
|
1327
|
-
...
|
1328
|
-
|
1329
|
-
@typing.overload
|
1330
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1331
|
-
"""
|
1332
|
-
Specifies the times when the flow should be run when running on a
|
1333
|
-
production scheduler.
|
1334
|
-
|
1335
|
-
|
1336
|
-
Parameters
|
1337
|
-
----------
|
1338
|
-
hourly : bool, default False
|
1339
|
-
Run the workflow hourly.
|
1340
|
-
daily : bool, default True
|
1341
|
-
Run the workflow daily.
|
1342
|
-
weekly : bool, default False
|
1343
|
-
Run the workflow weekly.
|
1344
|
-
cron : str, optional, default None
|
1345
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1346
|
-
specified by this expression.
|
1347
|
-
timezone : str, optional, default None
|
1348
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1349
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1350
|
-
"""
|
1351
|
-
...
|
1352
|
-
|
1353
|
-
@typing.overload
|
1354
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1355
|
-
...
|
1356
|
-
|
1357
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1358
|
-
"""
|
1359
|
-
Specifies the times when the flow should be run when running on a
|
1360
|
-
production scheduler.
|
1361
|
-
|
1362
|
-
|
1363
|
-
Parameters
|
1364
|
-
----------
|
1365
|
-
hourly : bool, default False
|
1366
|
-
Run the workflow hourly.
|
1367
|
-
daily : bool, default True
|
1368
|
-
Run the workflow daily.
|
1369
|
-
weekly : bool, default False
|
1370
|
-
Run the workflow weekly.
|
1371
|
-
cron : str, optional, default None
|
1372
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1373
|
-
specified by this expression.
|
1374
|
-
timezone : str, optional, default None
|
1375
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1376
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1377
|
-
"""
|
1378
|
-
...
|
1379
|
-
|
1380
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1381
|
-
"""
|
1382
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1383
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1384
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1385
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1386
|
-
starts only after all sensors finish.
|
1387
|
-
|
1388
|
-
|
1389
|
-
Parameters
|
1390
|
-
----------
|
1391
|
-
timeout : int
|
1392
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1393
|
-
poke_interval : int
|
1394
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1395
|
-
mode : str
|
1396
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1397
|
-
exponential_backoff : bool
|
1398
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1399
|
-
pool : str
|
1400
|
-
the slot pool this task should run in,
|
1401
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1402
|
-
soft_fail : bool
|
1403
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1404
|
-
name : str
|
1405
|
-
Name of the sensor on Airflow
|
1406
|
-
description : str
|
1407
|
-
Description of sensor in the Airflow UI
|
1408
|
-
bucket_key : Union[str, List[str]]
|
1409
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1410
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1411
|
-
bucket_name : str
|
1412
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1413
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1414
|
-
wildcard_match : bool
|
1415
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1416
|
-
aws_conn_id : str
|
1417
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1418
|
-
verify : bool
|
1419
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1420
|
-
"""
|
1421
|
-
...
|
1422
|
-
|
1423
1499
|
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1424
1500
|
"""
|
1425
1501
|
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
@@ -1463,81 +1539,5 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1463
1539
|
"""
|
1464
1540
|
...
|
1465
1541
|
|
1466
|
-
@typing.overload
|
1467
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1468
|
-
"""
|
1469
|
-
Specifies the PyPI packages for all steps of the flow.
|
1470
|
-
|
1471
|
-
Use `@pypi_base` to set common packages required by all
|
1472
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1473
|
-
|
1474
|
-
Parameters
|
1475
|
-
----------
|
1476
|
-
packages : Dict[str, str], default: {}
|
1477
|
-
Packages to use for this flow. The key is the name of the package
|
1478
|
-
and the value is the version to use.
|
1479
|
-
python : str, optional, default: None
|
1480
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1481
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1482
|
-
"""
|
1483
|
-
...
|
1484
|
-
|
1485
|
-
@typing.overload
|
1486
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1487
|
-
...
|
1488
|
-
|
1489
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1490
|
-
"""
|
1491
|
-
Specifies the PyPI packages for all steps of the flow.
|
1492
|
-
|
1493
|
-
Use `@pypi_base` to set common packages required by all
|
1494
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1495
|
-
|
1496
|
-
Parameters
|
1497
|
-
----------
|
1498
|
-
packages : Dict[str, str], default: {}
|
1499
|
-
Packages to use for this flow. The key is the name of the package
|
1500
|
-
and the value is the version to use.
|
1501
|
-
python : str, optional, default: None
|
1502
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1503
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1504
|
-
"""
|
1505
|
-
...
|
1506
|
-
|
1507
|
-
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1508
|
-
"""
|
1509
|
-
Specifies what flows belong to the same project.
|
1510
|
-
|
1511
|
-
A project-specific namespace is created for all flows that
|
1512
|
-
use the same `@project(name)`.
|
1513
|
-
|
1514
|
-
|
1515
|
-
Parameters
|
1516
|
-
----------
|
1517
|
-
name : str
|
1518
|
-
Project name. Make sure that the name is unique amongst all
|
1519
|
-
projects that use the same production scheduler. The name may
|
1520
|
-
contain only lowercase alphanumeric characters and underscores.
|
1521
|
-
|
1522
|
-
branch : Optional[str], default None
|
1523
|
-
The branch to use. If not specified, the branch is set to
|
1524
|
-
`user.<username>` unless `production` is set to `True`. This can
|
1525
|
-
also be set on the command line using `--branch` as a top-level option.
|
1526
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
1527
|
-
|
1528
|
-
production : bool, default False
|
1529
|
-
Whether or not the branch is the production branch. This can also be set on the
|
1530
|
-
command line using `--production` as a top-level option. It is an error to specify
|
1531
|
-
`production` in the decorator and on the command line.
|
1532
|
-
The project branch name will be:
|
1533
|
-
- if `branch` is specified:
|
1534
|
-
- if `production` is True: `prod.<branch>`
|
1535
|
-
- if `production` is False: `test.<branch>`
|
1536
|
-
- if `branch` is not specified:
|
1537
|
-
- if `production` is True: `prod`
|
1538
|
-
- if `production` is False: `user.<username>`
|
1539
|
-
"""
|
1540
|
-
...
|
1541
|
-
|
1542
1542
|
pkg_name: str
|
1543
1543
|
|