ob-metaflow-stubs 6.0.3.128__py2.py3-none-any.whl → 6.0.3.130__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (191) hide show
  1. metaflow-stubs/__init__.pyi +315 -314
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +3 -3
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +5 -5
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +6 -2
  20. metaflow-stubs/metaflow_current.pyi +122 -122
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  61. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  62. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  70. metaflow-stubs/multicore_utils.pyi +2 -2
  71. metaflow-stubs/parameters.pyi +5 -5
  72. metaflow-stubs/plugins/__init__.pyi +10 -10
  73. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  74. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  75. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  76. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  78. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  79. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  80. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  82. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  83. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -8
  84. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -4
  85. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  86. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  87. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  89. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  90. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  92. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  93. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +9 -10
  94. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +14 -12
  96. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  98. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  99. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  100. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  101. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  102. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  103. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  105. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  106. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  107. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  108. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  109. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  110. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  111. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  112. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  113. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  114. metaflow-stubs/plugins/cards/card_decorator.pyi +4 -2
  115. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  116. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  117. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  118. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  119. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  123. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  124. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  125. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  126. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  128. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  130. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  131. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  132. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  133. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  134. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  135. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  138. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  139. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  140. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  141. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  142. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  143. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  144. metaflow-stubs/plugins/kubernetes/__init__.pyi +3 -2
  145. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +25 -2
  146. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +5 -22
  147. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  148. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +12 -3
  149. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +10 -4
  150. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  151. metaflow-stubs/plugins/perimeters.pyi +2 -2
  152. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  154. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  156. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  157. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  158. metaflow-stubs/plugins/pypi/utils.pyi +6 -2
  159. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  160. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  162. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  163. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  164. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  165. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  166. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  167. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  168. metaflow-stubs/profilers/__init__.pyi +2 -2
  169. metaflow-stubs/pylint_wrapper.pyi +2 -2
  170. metaflow-stubs/runner/__init__.pyi +2 -2
  171. metaflow-stubs/runner/deployer.pyi +29 -29
  172. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  173. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  174. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  175. metaflow-stubs/runner/nbrun.pyi +2 -2
  176. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  177. metaflow-stubs/runner/utils.pyi +3 -3
  178. metaflow-stubs/system/__init__.pyi +2 -2
  179. metaflow-stubs/system/system_logger.pyi +2 -2
  180. metaflow-stubs/system/system_monitor.pyi +2 -2
  181. metaflow-stubs/tagging_util.pyi +2 -2
  182. metaflow-stubs/tuple_util.pyi +2 -2
  183. metaflow-stubs/user_configs/__init__.pyi +2 -2
  184. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  185. metaflow-stubs/user_configs/config_options.pyi +9 -4
  186. metaflow-stubs/user_configs/config_parameters.pyi +8 -8
  187. {ob_metaflow_stubs-6.0.3.128.dist-info → ob_metaflow_stubs-6.0.3.130.dist-info}/METADATA +1 -1
  188. ob_metaflow_stubs-6.0.3.130.dist-info/RECORD +191 -0
  189. ob_metaflow_stubs-6.0.3.128.dist-info/RECORD +0 -191
  190. {ob_metaflow_stubs-6.0.3.128.dist-info → ob_metaflow_stubs-6.0.3.130.dist-info}/WHEEL +0 -0
  191. {ob_metaflow_stubs-6.0.3.128.dist-info → ob_metaflow_stubs-6.0.3.130.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.39.1+obcheckpoint(0.1.6);ob(v1) #
4
- # Generated on 2024-12-18T07:29:34.132398 #
3
+ # MF version: 2.13.1.1+obcheckpoint(0.1.6);ob(v1) #
4
+ # Generated on 2025-01-06T16:50:32.764131 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -31,12 +31,13 @@ from .parameters import Parameter as Parameter
31
31
  from .parameters import JSONTypeClass as JSONTypeClass
32
32
  from .parameters import JSONType as JSONType
33
33
  from .user_configs.config_parameters import Config as Config
34
+ from .user_configs.config_parameters import ConfigValue as ConfigValue
34
35
  from .user_configs.config_parameters import config_expr as config_expr
35
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
36
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
37
- from . import events as events
38
- from . import cards as cards
39
38
  from . import tuple_util as tuple_util
39
+ from . import cards as cards
40
+ from . import events as events
40
41
  from . import runner as runner
41
42
  from . import plugins as plugins
42
43
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -145,137 +146,93 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
145
146
  """
146
147
  ...
147
148
 
148
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
149
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
149
150
  """
150
151
  Specifies that this step should execute on Kubernetes.
151
152
  """
152
153
  ...
153
154
 
154
- @typing.overload
155
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
156
- """
157
- Specifies the Conda environment for the step.
158
-
159
- Information in this decorator will augment any
160
- attributes set in the `@conda_base` flow-level decorator. Hence,
161
- you can use `@conda_base` to set packages required by all
162
- steps and use `@conda` to specify step-specific overrides.
163
- """
164
- ...
165
-
166
- @typing.overload
167
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
168
- ...
169
-
170
- @typing.overload
171
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
172
- ...
173
-
174
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
155
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
175
156
  """
176
- Specifies the Conda environment for the step.
177
-
178
- Information in this decorator will augment any
179
- attributes set in the `@conda_base` flow-level decorator. Hence,
180
- you can use `@conda_base` to set packages required by all
181
- steps and use `@conda` to specify step-specific overrides.
157
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
182
158
  """
183
159
  ...
184
160
 
185
161
  @typing.overload
186
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
162
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
187
163
  """
188
- Specifies the resources needed when executing this step.
164
+ Specifies the number of times the task corresponding
165
+ to a step needs to be retried.
189
166
 
190
- Use `@resources` to specify the resource requirements
191
- independently of the specific compute layer (`@batch`, `@kubernetes`).
167
+ This decorator is useful for handling transient errors, such as networking issues.
168
+ If your task contains operations that can't be retried safely, e.g. database updates,
169
+ it is advisable to annotate it with `@retry(times=0)`.
192
170
 
193
- You can choose the compute layer on the command line by executing e.g.
194
- ```
195
- python myflow.py run --with batch
196
- ```
197
- or
198
- ```
199
- python myflow.py run --with kubernetes
200
- ```
201
- which executes the flow on the desired system using the
202
- requirements specified in `@resources`.
171
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
172
+ decorator will execute a no-op task after all retries have been exhausted,
173
+ ensuring that the flow execution can continue.
203
174
  """
204
175
  ...
205
176
 
206
177
  @typing.overload
207
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
178
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
208
179
  ...
209
180
 
210
181
  @typing.overload
211
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
182
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
212
183
  ...
213
184
 
214
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
185
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
215
186
  """
216
- Specifies the resources needed when executing this step.
187
+ Specifies the number of times the task corresponding
188
+ to a step needs to be retried.
217
189
 
218
- Use `@resources` to specify the resource requirements
219
- independently of the specific compute layer (`@batch`, `@kubernetes`).
190
+ This decorator is useful for handling transient errors, such as networking issues.
191
+ If your task contains operations that can't be retried safely, e.g. database updates,
192
+ it is advisable to annotate it with `@retry(times=0)`.
220
193
 
221
- You can choose the compute layer on the command line by executing e.g.
222
- ```
223
- python myflow.py run --with batch
224
- ```
225
- or
226
- ```
227
- python myflow.py run --with kubernetes
228
- ```
229
- which executes the flow on the desired system using the
230
- requirements specified in `@resources`.
194
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
195
+ decorator will execute a no-op task after all retries have been exhausted,
196
+ ensuring that the flow execution can continue.
231
197
  """
232
198
  ...
233
199
 
234
200
  @typing.overload
235
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
201
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
236
202
  """
237
- Specifies the PyPI packages for the step.
203
+ Specifies a timeout for your step.
238
204
 
239
- Information in this decorator will augment any
240
- attributes set in the `@pyi_base` flow-level decorator. Hence,
241
- you can use `@pypi_base` to set packages required by all
242
- steps and use `@pypi` to specify step-specific overrides.
243
- """
244
- ...
245
-
246
- @typing.overload
247
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
248
- ...
249
-
250
- @typing.overload
251
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
252
- ...
253
-
254
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
255
- """
256
- Specifies the PyPI packages for the step.
205
+ This decorator is useful if this step may hang indefinitely.
257
206
 
258
- Information in this decorator will augment any
259
- attributes set in the `@pyi_base` flow-level decorator. Hence,
260
- you can use `@pypi_base` to set packages required by all
261
- steps and use `@pypi` to specify step-specific overrides.
207
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
208
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
209
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
210
+
211
+ Note that all the values specified in parameters are added together so if you specify
212
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
262
213
  """
263
214
  ...
264
215
 
265
216
  @typing.overload
266
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
267
- """
268
- Internal decorator to support Fast bakery
269
- """
217
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
270
218
  ...
271
219
 
272
220
  @typing.overload
273
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
221
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
274
222
  ...
275
223
 
276
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
224
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
277
225
  """
278
- Internal decorator to support Fast bakery
226
+ Specifies a timeout for your step.
227
+
228
+ This decorator is useful if this step may hang indefinitely.
229
+
230
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
231
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
232
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
233
+
234
+ Note that all the values specified in parameters are added together so if you specify
235
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
279
236
  """
280
237
  ...
281
238
 
@@ -310,27 +267,6 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
310
267
  """
311
268
  ...
312
269
 
313
- @typing.overload
314
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
315
- """
316
- Enables loading / saving of models within a step.
317
- """
318
- ...
319
-
320
- @typing.overload
321
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
322
- ...
323
-
324
- @typing.overload
325
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
326
- ...
327
-
328
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
329
- """
330
- Enables loading / saving of models within a step.
331
- """
332
- ...
333
-
334
270
  @typing.overload
335
271
  def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
336
272
  """
@@ -351,157 +287,222 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
351
287
  ...
352
288
 
353
289
  @typing.overload
354
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
290
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
355
291
  """
356
- Specifies secrets to be retrieved and injected as environment variables prior to
357
- the execution of a step.
292
+ Enables checkpointing for a step.
358
293
  """
359
294
  ...
360
295
 
361
296
  @typing.overload
362
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
297
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
363
298
  ...
364
299
 
365
300
  @typing.overload
366
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
301
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
367
302
  ...
368
303
 
369
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
304
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
370
305
  """
371
- Specifies secrets to be retrieved and injected as environment variables prior to
372
- the execution of a step.
306
+ Enables checkpointing for a step.
373
307
  """
374
308
  ...
375
309
 
376
310
  @typing.overload
377
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
311
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
378
312
  """
379
- Specifies the number of times the task corresponding
380
- to a step needs to be retried.
381
-
382
- This decorator is useful for handling transient errors, such as networking issues.
383
- If your task contains operations that can't be retried safely, e.g. database updates,
384
- it is advisable to annotate it with `@retry(times=0)`.
385
-
386
- This can be used in conjunction with the `@catch` decorator. The `@catch`
387
- decorator will execute a no-op task after all retries have been exhausted,
388
- ensuring that the flow execution can continue.
313
+ Specifies environment variables to be set prior to the execution of a step.
389
314
  """
390
315
  ...
391
316
 
392
317
  @typing.overload
393
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
318
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
394
319
  ...
395
320
 
396
321
  @typing.overload
397
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
322
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
398
323
  ...
399
324
 
400
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
325
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
401
326
  """
402
- Specifies the number of times the task corresponding
403
- to a step needs to be retried.
404
-
405
- This decorator is useful for handling transient errors, such as networking issues.
406
- If your task contains operations that can't be retried safely, e.g. database updates,
407
- it is advisable to annotate it with `@retry(times=0)`.
408
-
409
- This can be used in conjunction with the `@catch` decorator. The `@catch`
410
- decorator will execute a no-op task after all retries have been exhausted,
411
- ensuring that the flow execution can continue.
327
+ Specifies environment variables to be set prior to the execution of a step.
412
328
  """
413
329
  ...
414
330
 
415
331
  @typing.overload
416
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
332
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
417
333
  """
418
- Enables checkpointing for a step.
334
+ Internal decorator to support Fast bakery
419
335
  """
420
336
  ...
421
337
 
422
338
  @typing.overload
423
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
424
- ...
425
-
426
- @typing.overload
427
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
428
- ...
429
-
430
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
431
- """
432
- Enables checkpointing for a step.
433
- """
339
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
434
340
  ...
435
341
 
436
- def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
342
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
437
343
  """
438
- Specifies that this step should execute on DGX cloud.
344
+ Internal decorator to support Fast bakery
439
345
  """
440
346
  ...
441
347
 
442
348
  @typing.overload
443
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
349
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
444
350
  """
445
- Specifies environment variables to be set prior to the execution of a step.
351
+ Specifies the resources needed when executing this step.
352
+
353
+ Use `@resources` to specify the resource requirements
354
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
355
+
356
+ You can choose the compute layer on the command line by executing e.g.
357
+ ```
358
+ python myflow.py run --with batch
359
+ ```
360
+ or
361
+ ```
362
+ python myflow.py run --with kubernetes
363
+ ```
364
+ which executes the flow on the desired system using the
365
+ requirements specified in `@resources`.
446
366
  """
447
367
  ...
448
368
 
449
369
  @typing.overload
450
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
370
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
451
371
  ...
452
372
 
453
373
  @typing.overload
454
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
455
- ...
456
-
457
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
458
- """
459
- Specifies environment variables to be set prior to the execution of a step.
460
- """
374
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
461
375
  ...
462
376
 
463
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
464
- """
465
- Decorator that helps cache, version and store models/datasets from huggingface hub.
377
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
466
378
  """
467
- ...
468
-
469
- @typing.overload
470
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
471
- """
472
- Specifies a timeout for your step.
379
+ Specifies the resources needed when executing this step.
473
380
 
474
- This decorator is useful if this step may hang indefinitely.
381
+ Use `@resources` to specify the resource requirements
382
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
475
383
 
476
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
477
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
478
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
384
+ You can choose the compute layer on the command line by executing e.g.
385
+ ```
386
+ python myflow.py run --with batch
387
+ ```
388
+ or
389
+ ```
390
+ python myflow.py run --with kubernetes
391
+ ```
392
+ which executes the flow on the desired system using the
393
+ requirements specified in `@resources`.
394
+ """
395
+ ...
396
+
397
+ @typing.overload
398
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
399
+ """
400
+ Specifies the Conda environment for the step.
479
401
 
480
- Note that all the values specified in parameters are added together so if you specify
481
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
402
+ Information in this decorator will augment any
403
+ attributes set in the `@conda_base` flow-level decorator. Hence,
404
+ you can use `@conda_base` to set packages required by all
405
+ steps and use `@conda` to specify step-specific overrides.
482
406
  """
483
407
  ...
484
408
 
485
409
  @typing.overload
486
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
410
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
487
411
  ...
488
412
 
489
413
  @typing.overload
490
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
414
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
491
415
  ...
492
416
 
493
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
417
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
494
418
  """
495
- Specifies a timeout for your step.
419
+ Specifies the Conda environment for the step.
496
420
 
497
- This decorator is useful if this step may hang indefinitely.
421
+ Information in this decorator will augment any
422
+ attributes set in the `@conda_base` flow-level decorator. Hence,
423
+ you can use `@conda_base` to set packages required by all
424
+ steps and use `@conda` to specify step-specific overrides.
425
+ """
426
+ ...
427
+
428
+ def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
429
+ """
430
+ Specifies that this step should execute on DGX cloud.
431
+ """
432
+ ...
433
+
434
+ @typing.overload
435
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
436
+ """
437
+ Specifies secrets to be retrieved and injected as environment variables prior to
438
+ the execution of a step.
439
+ """
440
+ ...
441
+
442
+ @typing.overload
443
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
444
+ ...
445
+
446
+ @typing.overload
447
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
448
+ ...
449
+
450
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
451
+ """
452
+ Specifies secrets to be retrieved and injected as environment variables prior to
453
+ the execution of a step.
454
+ """
455
+ ...
456
+
457
+ @typing.overload
458
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
459
+ """
460
+ Enables loading / saving of models within a step.
461
+ """
462
+ ...
463
+
464
+ @typing.overload
465
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
466
+ ...
467
+
468
+ @typing.overload
469
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
470
+ ...
471
+
472
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
473
+ """
474
+ Enables loading / saving of models within a step.
475
+ """
476
+ ...
477
+
478
+ @typing.overload
479
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
480
+ """
481
+ Specifies the PyPI packages for the step.
498
482
 
499
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
500
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
501
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
483
+ Information in this decorator will augment any
484
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
485
+ you can use `@pypi_base` to set packages required by all
486
+ steps and use `@pypi` to specify step-specific overrides.
487
+ """
488
+ ...
489
+
490
+ @typing.overload
491
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
492
+ ...
493
+
494
+ @typing.overload
495
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
496
+ ...
497
+
498
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
499
+ """
500
+ Specifies the PyPI packages for the step.
502
501
 
503
- Note that all the values specified in parameters are added together so if you specify
504
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
502
+ Information in this decorator will augment any
503
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
504
+ you can use `@pypi_base` to set packages required by all
505
+ steps and use `@pypi` to specify step-specific overrides.
505
506
  """
506
507
  ...
507
508
 
@@ -530,22 +531,22 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
530
531
  """
531
532
  ...
532
533
 
533
- @typing.overload
534
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
534
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
535
535
  """
536
- Specifies the times when the flow should be run when running on a
537
- production scheduler.
536
+ Specifies what flows belong to the same project.
537
+
538
+ A project-specific namespace is created for all flows that
539
+ use the same `@project(name)`.
538
540
  """
539
541
  ...
540
542
 
541
- @typing.overload
542
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
543
- ...
544
-
545
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
543
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
546
544
  """
547
- Specifies the times when the flow should be run when running on a
548
- production scheduler.
545
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
546
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
547
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
548
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
549
+ starts only after all sensors finish.
549
550
  """
550
551
  ...
551
552
 
@@ -572,125 +573,103 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
572
573
  """
573
574
  ...
574
575
 
575
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
576
- """
577
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
578
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
579
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
580
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
581
- starts only after all sensors finish.
582
- """
583
- ...
584
-
585
576
  @typing.overload
586
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
577
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
587
578
  """
588
- Specifies the event(s) that this flow depends on.
579
+ Specifies the flow(s) that this flow depends on.
589
580
 
590
581
  ```
591
- @trigger(event='foo')
582
+ @trigger_on_finish(flow='FooFlow')
592
583
  ```
593
584
  or
594
585
  ```
595
- @trigger(events=['foo', 'bar'])
586
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
596
587
  ```
588
+ This decorator respects the @project decorator and triggers the flow
589
+ when upstream runs within the same namespace complete successfully
597
590
 
598
- Additionally, you can specify the parameter mappings
599
- to map event payload to Metaflow parameters for the flow.
591
+ Additionally, you can specify project aware upstream flow dependencies
592
+ by specifying the fully qualified project_flow_name.
600
593
  ```
601
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
594
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
602
595
  ```
603
596
  or
604
597
  ```
605
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
606
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
598
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
607
599
  ```
608
600
 
609
- 'parameters' can also be a list of strings and tuples like so:
610
- ```
611
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
612
- ```
613
- This is equivalent to:
601
+ You can also specify just the project or project branch (other values will be
602
+ inferred from the current project or project branch):
614
603
  ```
615
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
604
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
616
605
  ```
606
+
607
+ Note that `branch` is typically one of:
608
+ - `prod`
609
+ - `user.bob`
610
+ - `test.my_experiment`
611
+ - `prod.staging`
617
612
  """
618
613
  ...
619
614
 
620
615
  @typing.overload
621
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
616
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
622
617
  ...
623
618
 
624
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
619
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
625
620
  """
626
- Specifies the event(s) that this flow depends on.
621
+ Specifies the flow(s) that this flow depends on.
627
622
 
628
623
  ```
629
- @trigger(event='foo')
624
+ @trigger_on_finish(flow='FooFlow')
630
625
  ```
631
626
  or
632
627
  ```
633
- @trigger(events=['foo', 'bar'])
628
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
634
629
  ```
630
+ This decorator respects the @project decorator and triggers the flow
631
+ when upstream runs within the same namespace complete successfully
635
632
 
636
- Additionally, you can specify the parameter mappings
637
- to map event payload to Metaflow parameters for the flow.
633
+ Additionally, you can specify project aware upstream flow dependencies
634
+ by specifying the fully qualified project_flow_name.
638
635
  ```
639
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
636
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
640
637
  ```
641
638
  or
642
639
  ```
643
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
644
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
640
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
645
641
  ```
646
642
 
647
- 'parameters' can also be a list of strings and tuples like so:
648
- ```
649
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
650
- ```
651
- This is equivalent to:
643
+ You can also specify just the project or project branch (other values will be
644
+ inferred from the current project or project branch):
652
645
  ```
653
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
646
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
654
647
  ```
655
- """
656
- ...
657
-
658
- @typing.overload
659
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
660
- """
661
- Specifies the PyPI packages for all steps of the flow.
662
648
 
663
- Use `@pypi_base` to set common packages required by all
664
- steps and use `@pypi` to specify step-specific overrides.
649
+ Note that `branch` is typically one of:
650
+ - `prod`
651
+ - `user.bob`
652
+ - `test.my_experiment`
653
+ - `prod.staging`
665
654
  """
666
655
  ...
667
656
 
668
657
  @typing.overload
669
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
670
- ...
671
-
672
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
658
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
673
659
  """
674
- Specifies the PyPI packages for all steps of the flow.
675
-
676
- Use `@pypi_base` to set common packages required by all
677
- steps and use `@pypi` to specify step-specific overrides.
660
+ Specifies the times when the flow should be run when running on a
661
+ production scheduler.
678
662
  """
679
663
  ...
680
664
 
681
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
682
- """
683
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
684
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
685
- """
665
+ @typing.overload
666
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
686
667
  ...
687
668
 
688
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
669
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
689
670
  """
690
- Specifies what flows belong to the same project.
691
-
692
- A project-specific namespace is created for all flows that
693
- use the same `@project(name)`.
671
+ Specifies the times when the flow should be run when running on a
672
+ production scheduler.
694
673
  """
695
674
  ...
696
675
 
@@ -719,83 +698,105 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
719
698
  ...
720
699
 
721
700
  @typing.overload
722
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
701
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
723
702
  """
724
- Specifies the flow(s) that this flow depends on.
703
+ Specifies the PyPI packages for all steps of the flow.
704
+
705
+ Use `@pypi_base` to set common packages required by all
706
+ steps and use `@pypi` to specify step-specific overrides.
707
+ """
708
+ ...
709
+
710
+ @typing.overload
711
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
712
+ ...
713
+
714
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
715
+ """
716
+ Specifies the PyPI packages for all steps of the flow.
717
+
718
+ Use `@pypi_base` to set common packages required by all
719
+ steps and use `@pypi` to specify step-specific overrides.
720
+ """
721
+ ...
722
+
723
+ @typing.overload
724
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
725
+ """
726
+ Specifies the event(s) that this flow depends on.
725
727
 
726
728
  ```
727
- @trigger_on_finish(flow='FooFlow')
729
+ @trigger(event='foo')
728
730
  ```
729
731
  or
730
732
  ```
731
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
733
+ @trigger(events=['foo', 'bar'])
732
734
  ```
733
- This decorator respects the @project decorator and triggers the flow
734
- when upstream runs within the same namespace complete successfully
735
735
 
736
- Additionally, you can specify project aware upstream flow dependencies
737
- by specifying the fully qualified project_flow_name.
736
+ Additionally, you can specify the parameter mappings
737
+ to map event payload to Metaflow parameters for the flow.
738
738
  ```
739
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
739
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
740
740
  ```
741
741
  or
742
742
  ```
743
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
743
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
744
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
744
745
  ```
745
746
 
746
- You can also specify just the project or project branch (other values will be
747
- inferred from the current project or project branch):
747
+ 'parameters' can also be a list of strings and tuples like so:
748
748
  ```
749
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
749
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
750
+ ```
751
+ This is equivalent to:
752
+ ```
753
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
750
754
  ```
751
-
752
- Note that `branch` is typically one of:
753
- - `prod`
754
- - `user.bob`
755
- - `test.my_experiment`
756
- - `prod.staging`
757
755
  """
758
756
  ...
759
757
 
760
758
  @typing.overload
761
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
759
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
762
760
  ...
763
761
 
764
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
762
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
765
763
  """
766
- Specifies the flow(s) that this flow depends on.
764
+ Specifies the event(s) that this flow depends on.
767
765
 
768
766
  ```
769
- @trigger_on_finish(flow='FooFlow')
767
+ @trigger(event='foo')
770
768
  ```
771
769
  or
772
770
  ```
773
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
771
+ @trigger(events=['foo', 'bar'])
774
772
  ```
775
- This decorator respects the @project decorator and triggers the flow
776
- when upstream runs within the same namespace complete successfully
777
773
 
778
- Additionally, you can specify project aware upstream flow dependencies
779
- by specifying the fully qualified project_flow_name.
774
+ Additionally, you can specify the parameter mappings
775
+ to map event payload to Metaflow parameters for the flow.
780
776
  ```
781
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
777
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
782
778
  ```
783
779
  or
784
780
  ```
785
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
781
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
782
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
786
783
  ```
787
784
 
788
- You can also specify just the project or project branch (other values will be
789
- inferred from the current project or project branch):
785
+ 'parameters' can also be a list of strings and tuples like so:
790
786
  ```
791
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
787
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
792
788
  ```
793
-
794
- Note that `branch` is typically one of:
795
- - `prod`
796
- - `user.bob`
797
- - `test.my_experiment`
798
- - `prod.staging`
789
+ This is equivalent to:
790
+ ```
791
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
792
+ ```
793
+ """
794
+ ...
795
+
796
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
797
+ """
798
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
799
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
799
800
  """
800
801
  ...
801
802