ob-metaflow-stubs 6.0.3.130__py2.py3-none-any.whl → 6.0.3.131__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (191) hide show
  1. metaflow-stubs/__init__.pyi +283 -283
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +102 -102
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +5 -5
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +5 -5
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  61. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  62. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  70. metaflow-stubs/multicore_utils.pyi +2 -2
  71. metaflow-stubs/parameters.pyi +4 -4
  72. metaflow-stubs/plugins/__init__.pyi +20 -12
  73. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  74. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  75. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  76. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  78. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  79. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  80. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  82. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  83. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  84. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  86. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  87. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  89. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  90. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  92. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  93. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  96. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  98. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  99. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  100. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  101. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  102. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +4 -4
  103. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  105. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  106. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  107. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  108. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  109. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  110. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  111. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  112. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  113. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  114. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  116. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  117. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  118. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  119. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  123. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  124. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  125. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  126. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  128. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  130. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  131. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  132. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  133. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  134. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  135. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  138. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  139. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  140. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  141. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  142. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  143. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  144. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  146. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  147. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  148. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  149. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  150. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  151. metaflow-stubs/plugins/perimeters.pyi +2 -2
  152. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  154. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  156. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  157. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  158. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  159. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  160. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  162. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  163. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  164. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  165. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  166. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  167. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  168. metaflow-stubs/profilers/__init__.pyi +2 -2
  169. metaflow-stubs/pylint_wrapper.pyi +2 -2
  170. metaflow-stubs/runner/__init__.pyi +2 -2
  171. metaflow-stubs/runner/deployer.pyi +7 -7
  172. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  173. metaflow-stubs/runner/metaflow_runner.pyi +14 -6
  174. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  175. metaflow-stubs/runner/nbrun.pyi +2 -2
  176. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  177. metaflow-stubs/runner/utils.pyi +2 -2
  178. metaflow-stubs/system/__init__.pyi +2 -2
  179. metaflow-stubs/system/system_logger.pyi +3 -3
  180. metaflow-stubs/system/system_monitor.pyi +2 -2
  181. metaflow-stubs/tagging_util.pyi +2 -2
  182. metaflow-stubs/tuple_util.pyi +2 -2
  183. metaflow-stubs/user_configs/__init__.pyi +2 -2
  184. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  185. metaflow-stubs/user_configs/config_options.pyi +3 -3
  186. metaflow-stubs/user_configs/config_parameters.pyi +7 -7
  187. {ob_metaflow_stubs-6.0.3.130.dist-info → ob_metaflow_stubs-6.0.3.131.dist-info}/METADATA +1 -1
  188. ob_metaflow_stubs-6.0.3.131.dist-info/RECORD +191 -0
  189. ob_metaflow_stubs-6.0.3.130.dist-info/RECORD +0 -191
  190. {ob_metaflow_stubs-6.0.3.130.dist-info → ob_metaflow_stubs-6.0.3.131.dist-info}/WHEEL +0 -0
  191. {ob_metaflow_stubs-6.0.3.130.dist-info → ob_metaflow_stubs-6.0.3.131.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.13.1.1+obcheckpoint(0.1.6);ob(v1) #
4
- # Generated on 2025-01-06T16:50:32.764131 #
3
+ # MF version: 2.13.2.1+obcheckpoint(0.1.6);ob(v1) #
4
+ # Generated on 2025-01-08T23:52:28.392925 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -146,202 +146,179 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
146
146
  """
147
147
  ...
148
148
 
149
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
150
- """
151
- Specifies that this step should execute on Kubernetes.
152
- """
153
- ...
154
-
155
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
156
- """
157
- Decorator that helps cache, version and store models/datasets from huggingface hub.
158
- """
159
- ...
160
-
161
149
  @typing.overload
162
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
150
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
163
151
  """
164
- Specifies the number of times the task corresponding
165
- to a step needs to be retried.
166
-
167
- This decorator is useful for handling transient errors, such as networking issues.
168
- If your task contains operations that can't be retried safely, e.g. database updates,
169
- it is advisable to annotate it with `@retry(times=0)`.
170
-
171
- This can be used in conjunction with the `@catch` decorator. The `@catch`
172
- decorator will execute a no-op task after all retries have been exhausted,
173
- ensuring that the flow execution can continue.
152
+ Decorator prototype for all step decorators. This function gets specialized
153
+ and imported for all decorators types by _import_plugin_decorators().
174
154
  """
175
155
  ...
176
156
 
177
157
  @typing.overload
178
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
179
- ...
180
-
181
- @typing.overload
182
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
158
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
183
159
  ...
184
160
 
185
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
161
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
186
162
  """
187
- Specifies the number of times the task corresponding
188
- to a step needs to be retried.
189
-
190
- This decorator is useful for handling transient errors, such as networking issues.
191
- If your task contains operations that can't be retried safely, e.g. database updates,
192
- it is advisable to annotate it with `@retry(times=0)`.
193
-
194
- This can be used in conjunction with the `@catch` decorator. The `@catch`
195
- decorator will execute a no-op task after all retries have been exhausted,
196
- ensuring that the flow execution can continue.
163
+ Decorator prototype for all step decorators. This function gets specialized
164
+ and imported for all decorators types by _import_plugin_decorators().
197
165
  """
198
166
  ...
199
167
 
200
168
  @typing.overload
201
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
169
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
202
170
  """
203
- Specifies a timeout for your step.
204
-
205
- This decorator is useful if this step may hang indefinitely.
206
-
207
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
208
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
209
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
210
-
211
- Note that all the values specified in parameters are added together so if you specify
212
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
171
+ Specifies environment variables to be set prior to the execution of a step.
213
172
  """
214
173
  ...
215
174
 
216
175
  @typing.overload
217
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
176
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
218
177
  ...
219
178
 
220
179
  @typing.overload
221
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
180
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
222
181
  ...
223
182
 
224
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
183
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
225
184
  """
226
- Specifies a timeout for your step.
227
-
228
- This decorator is useful if this step may hang indefinitely.
229
-
230
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
231
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
232
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
233
-
234
- Note that all the values specified in parameters are added together so if you specify
235
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
185
+ Specifies environment variables to be set prior to the execution of a step.
186
+ """
187
+ ...
188
+
189
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
190
+ """
191
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
236
192
  """
237
193
  ...
238
194
 
239
195
  @typing.overload
240
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
196
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
241
197
  """
242
- Specifies that the step will success under all circumstances.
198
+ Specifies the Conda environment for the step.
243
199
 
244
- The decorator will create an optional artifact, specified by `var`, which
245
- contains the exception raised. You can use it to detect the presence
246
- of errors, indicating that all happy-path artifacts produced by the step
247
- are missing.
200
+ Information in this decorator will augment any
201
+ attributes set in the `@conda_base` flow-level decorator. Hence,
202
+ you can use `@conda_base` to set packages required by all
203
+ steps and use `@conda` to specify step-specific overrides.
248
204
  """
249
205
  ...
250
206
 
251
207
  @typing.overload
252
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
208
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
253
209
  ...
254
210
 
255
211
  @typing.overload
256
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
212
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
257
213
  ...
258
214
 
259
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
215
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
260
216
  """
261
- Specifies that the step will success under all circumstances.
217
+ Specifies the Conda environment for the step.
262
218
 
263
- The decorator will create an optional artifact, specified by `var`, which
264
- contains the exception raised. You can use it to detect the presence
265
- of errors, indicating that all happy-path artifacts produced by the step
266
- are missing.
267
- """
268
- ...
269
-
270
- @typing.overload
271
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
272
- """
273
- Decorator prototype for all step decorators. This function gets specialized
274
- and imported for all decorators types by _import_plugin_decorators().
219
+ Information in this decorator will augment any
220
+ attributes set in the `@conda_base` flow-level decorator. Hence,
221
+ you can use `@conda_base` to set packages required by all
222
+ steps and use `@conda` to specify step-specific overrides.
275
223
  """
276
224
  ...
277
225
 
278
- @typing.overload
279
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
280
- ...
281
-
282
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
226
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
283
227
  """
284
- Decorator prototype for all step decorators. This function gets specialized
285
- and imported for all decorators types by _import_plugin_decorators().
228
+ Specifies that this step should execute on Kubernetes.
286
229
  """
287
230
  ...
288
231
 
289
232
  @typing.overload
290
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
233
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
291
234
  """
292
- Enables checkpointing for a step.
235
+ Specifies secrets to be retrieved and injected as environment variables prior to
236
+ the execution of a step.
293
237
  """
294
238
  ...
295
239
 
296
240
  @typing.overload
297
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
241
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
298
242
  ...
299
243
 
300
244
  @typing.overload
301
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
245
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
302
246
  ...
303
247
 
304
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
248
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
305
249
  """
306
- Enables checkpointing for a step.
250
+ Specifies secrets to be retrieved and injected as environment variables prior to
251
+ the execution of a step.
307
252
  """
308
253
  ...
309
254
 
310
255
  @typing.overload
311
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
256
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
312
257
  """
313
- Specifies environment variables to be set prior to the execution of a step.
258
+ Specifies that the step will success under all circumstances.
259
+
260
+ The decorator will create an optional artifact, specified by `var`, which
261
+ contains the exception raised. You can use it to detect the presence
262
+ of errors, indicating that all happy-path artifacts produced by the step
263
+ are missing.
314
264
  """
315
265
  ...
316
266
 
317
267
  @typing.overload
318
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
268
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
319
269
  ...
320
270
 
321
271
  @typing.overload
322
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
272
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
323
273
  ...
324
274
 
325
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
275
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
326
276
  """
327
- Specifies environment variables to be set prior to the execution of a step.
277
+ Specifies that the step will success under all circumstances.
278
+
279
+ The decorator will create an optional artifact, specified by `var`, which
280
+ contains the exception raised. You can use it to detect the presence
281
+ of errors, indicating that all happy-path artifacts produced by the step
282
+ are missing.
328
283
  """
329
284
  ...
330
285
 
331
286
  @typing.overload
332
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
287
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
333
288
  """
334
- Internal decorator to support Fast bakery
289
+ Specifies the number of times the task corresponding
290
+ to a step needs to be retried.
291
+
292
+ This decorator is useful for handling transient errors, such as networking issues.
293
+ If your task contains operations that can't be retried safely, e.g. database updates,
294
+ it is advisable to annotate it with `@retry(times=0)`.
295
+
296
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
297
+ decorator will execute a no-op task after all retries have been exhausted,
298
+ ensuring that the flow execution can continue.
335
299
  """
336
300
  ...
337
301
 
338
302
  @typing.overload
339
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
303
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
340
304
  ...
341
305
 
342
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
306
+ @typing.overload
307
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
308
+ ...
309
+
310
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
343
311
  """
344
- Internal decorator to support Fast bakery
312
+ Specifies the number of times the task corresponding
313
+ to a step needs to be retried.
314
+
315
+ This decorator is useful for handling transient errors, such as networking issues.
316
+ If your task contains operations that can't be retried safely, e.g. database updates,
317
+ it is advisable to annotate it with `@retry(times=0)`.
318
+
319
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
320
+ decorator will execute a no-op task after all retries have been exhausted,
321
+ ensuring that the flow execution can continue.
345
322
  """
346
323
  ...
347
324
 
@@ -395,262 +372,244 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
395
372
  ...
396
373
 
397
374
  @typing.overload
398
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
375
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
399
376
  """
400
- Specifies the Conda environment for the step.
401
-
402
- Information in this decorator will augment any
403
- attributes set in the `@conda_base` flow-level decorator. Hence,
404
- you can use `@conda_base` to set packages required by all
405
- steps and use `@conda` to specify step-specific overrides.
377
+ Enables loading / saving of models within a step.
406
378
  """
407
379
  ...
408
380
 
409
381
  @typing.overload
410
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
382
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
411
383
  ...
412
384
 
413
385
  @typing.overload
414
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
415
- ...
416
-
417
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
418
- """
419
- Specifies the Conda environment for the step.
420
-
421
- Information in this decorator will augment any
422
- attributes set in the `@conda_base` flow-level decorator. Hence,
423
- you can use `@conda_base` to set packages required by all
424
- steps and use `@conda` to specify step-specific overrides.
425
- """
386
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
426
387
  ...
427
388
 
428
- def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
389
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
429
390
  """
430
- Specifies that this step should execute on DGX cloud.
391
+ Enables loading / saving of models within a step.
431
392
  """
432
393
  ...
433
394
 
434
395
  @typing.overload
435
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
396
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
436
397
  """
437
- Specifies secrets to be retrieved and injected as environment variables prior to
438
- the execution of a step.
398
+ Specifies the PyPI packages for the step.
399
+
400
+ Information in this decorator will augment any
401
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
402
+ you can use `@pypi_base` to set packages required by all
403
+ steps and use `@pypi` to specify step-specific overrides.
439
404
  """
440
405
  ...
441
406
 
442
407
  @typing.overload
443
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
408
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
444
409
  ...
445
410
 
446
411
  @typing.overload
447
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
412
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
448
413
  ...
449
414
 
450
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
415
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
451
416
  """
452
- Specifies secrets to be retrieved and injected as environment variables prior to
453
- the execution of a step.
417
+ Specifies the PyPI packages for the step.
418
+
419
+ Information in this decorator will augment any
420
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
421
+ you can use `@pypi_base` to set packages required by all
422
+ steps and use `@pypi` to specify step-specific overrides.
454
423
  """
455
424
  ...
456
425
 
457
426
  @typing.overload
458
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
459
- """
460
- Enables loading / saving of models within a step.
427
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
428
+ """
429
+ Specifies a timeout for your step.
430
+
431
+ This decorator is useful if this step may hang indefinitely.
432
+
433
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
434
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
435
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
436
+
437
+ Note that all the values specified in parameters are added together so if you specify
438
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
461
439
  """
462
440
  ...
463
441
 
464
442
  @typing.overload
465
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
466
- ...
467
-
468
- @typing.overload
469
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
443
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
470
444
  ...
471
445
 
472
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
446
+ @typing.overload
447
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
448
+ ...
449
+
450
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
473
451
  """
474
- Enables loading / saving of models within a step.
452
+ Specifies a timeout for your step.
453
+
454
+ This decorator is useful if this step may hang indefinitely.
455
+
456
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
457
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
458
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
459
+
460
+ Note that all the values specified in parameters are added together so if you specify
461
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
475
462
  """
476
463
  ...
477
464
 
478
465
  @typing.overload
479
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
466
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
480
467
  """
481
- Specifies the PyPI packages for the step.
482
-
483
- Information in this decorator will augment any
484
- attributes set in the `@pyi_base` flow-level decorator. Hence,
485
- you can use `@pypi_base` to set packages required by all
486
- steps and use `@pypi` to specify step-specific overrides.
468
+ Enables checkpointing for a step.
487
469
  """
488
470
  ...
489
471
 
490
472
  @typing.overload
491
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
473
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
492
474
  ...
493
475
 
494
476
  @typing.overload
495
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
477
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
496
478
  ...
497
479
 
498
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
480
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
499
481
  """
500
- Specifies the PyPI packages for the step.
501
-
502
- Information in this decorator will augment any
503
- attributes set in the `@pyi_base` flow-level decorator. Hence,
504
- you can use `@pypi_base` to set packages required by all
505
- steps and use `@pypi` to specify step-specific overrides.
482
+ Enables checkpointing for a step.
506
483
  """
507
484
  ...
508
485
 
509
- @typing.overload
510
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
486
+ def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
511
487
  """
512
- Creates a human-readable report, a Metaflow Card, after this step completes.
513
-
514
- Note that you may add multiple `@card` decorators in a step with different parameters.
488
+ Specifies that this step should execute on DGX cloud.
515
489
  """
516
490
  ...
517
491
 
518
492
  @typing.overload
519
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
493
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
494
+ """
495
+ Internal decorator to support Fast bakery
496
+ """
520
497
  ...
521
498
 
522
499
  @typing.overload
523
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
500
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
524
501
  ...
525
502
 
526
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
503
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
527
504
  """
528
- Creates a human-readable report, a Metaflow Card, after this step completes.
529
-
530
- Note that you may add multiple `@card` decorators in a step with different parameters.
505
+ Internal decorator to support Fast bakery
531
506
  """
532
507
  ...
533
508
 
534
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
509
+ @typing.overload
510
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
535
511
  """
536
- Specifies what flows belong to the same project.
512
+ Creates a human-readable report, a Metaflow Card, after this step completes.
537
513
 
538
- A project-specific namespace is created for all flows that
539
- use the same `@project(name)`.
540
- """
541
- ...
542
-
543
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
544
- """
545
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
546
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
547
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
548
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
549
- starts only after all sensors finish.
514
+ Note that you may add multiple `@card` decorators in a step with different parameters.
550
515
  """
551
516
  ...
552
517
 
553
518
  @typing.overload
554
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
555
- """
556
- Specifies the Conda environment for all steps of the flow.
557
-
558
- Use `@conda_base` to set common libraries required by all
559
- steps and use `@conda` to specify step-specific additions.
560
- """
519
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
561
520
  ...
562
521
 
563
522
  @typing.overload
564
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
523
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
565
524
  ...
566
525
 
567
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
526
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
568
527
  """
569
- Specifies the Conda environment for all steps of the flow.
528
+ Creates a human-readable report, a Metaflow Card, after this step completes.
570
529
 
571
- Use `@conda_base` to set common libraries required by all
572
- steps and use `@conda` to specify step-specific additions.
530
+ Note that you may add multiple `@card` decorators in a step with different parameters.
573
531
  """
574
532
  ...
575
533
 
576
534
  @typing.overload
577
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
535
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
578
536
  """
579
- Specifies the flow(s) that this flow depends on.
537
+ Specifies the event(s) that this flow depends on.
580
538
 
581
539
  ```
582
- @trigger_on_finish(flow='FooFlow')
540
+ @trigger(event='foo')
583
541
  ```
584
542
  or
585
543
  ```
586
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
544
+ @trigger(events=['foo', 'bar'])
587
545
  ```
588
- This decorator respects the @project decorator and triggers the flow
589
- when upstream runs within the same namespace complete successfully
590
546
 
591
- Additionally, you can specify project aware upstream flow dependencies
592
- by specifying the fully qualified project_flow_name.
547
+ Additionally, you can specify the parameter mappings
548
+ to map event payload to Metaflow parameters for the flow.
593
549
  ```
594
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
550
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
595
551
  ```
596
552
  or
597
553
  ```
598
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
554
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
555
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
599
556
  ```
600
557
 
601
- You can also specify just the project or project branch (other values will be
602
- inferred from the current project or project branch):
558
+ 'parameters' can also be a list of strings and tuples like so:
603
559
  ```
604
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
560
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
561
+ ```
562
+ This is equivalent to:
563
+ ```
564
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
605
565
  ```
606
-
607
- Note that `branch` is typically one of:
608
- - `prod`
609
- - `user.bob`
610
- - `test.my_experiment`
611
- - `prod.staging`
612
566
  """
613
567
  ...
614
568
 
615
569
  @typing.overload
616
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
570
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
617
571
  ...
618
572
 
619
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
573
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
620
574
  """
621
- Specifies the flow(s) that this flow depends on.
575
+ Specifies the event(s) that this flow depends on.
622
576
 
623
577
  ```
624
- @trigger_on_finish(flow='FooFlow')
578
+ @trigger(event='foo')
625
579
  ```
626
580
  or
627
581
  ```
628
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
582
+ @trigger(events=['foo', 'bar'])
629
583
  ```
630
- This decorator respects the @project decorator and triggers the flow
631
- when upstream runs within the same namespace complete successfully
632
584
 
633
- Additionally, you can specify project aware upstream flow dependencies
634
- by specifying the fully qualified project_flow_name.
585
+ Additionally, you can specify the parameter mappings
586
+ to map event payload to Metaflow parameters for the flow.
635
587
  ```
636
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
588
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
637
589
  ```
638
590
  or
639
591
  ```
640
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
592
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
593
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
641
594
  ```
642
595
 
643
- You can also specify just the project or project branch (other values will be
644
- inferred from the current project or project branch):
596
+ 'parameters' can also be a list of strings and tuples like so:
645
597
  ```
646
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
598
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
647
599
  ```
600
+ This is equivalent to:
601
+ ```
602
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
603
+ ```
604
+ """
605
+ ...
606
+
607
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
608
+ """
609
+ Specifies what flows belong to the same project.
648
610
 
649
- Note that `branch` is typically one of:
650
- - `prod`
651
- - `user.bob`
652
- - `test.my_experiment`
653
- - `prod.staging`
611
+ A project-specific namespace is created for all flows that
612
+ use the same `@project(name)`.
654
613
  """
655
614
  ...
656
615
 
@@ -698,98 +657,116 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
698
657
  ...
699
658
 
700
659
  @typing.overload
701
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
660
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
702
661
  """
703
- Specifies the PyPI packages for all steps of the flow.
662
+ Specifies the Conda environment for all steps of the flow.
704
663
 
705
- Use `@pypi_base` to set common packages required by all
706
- steps and use `@pypi` to specify step-specific overrides.
664
+ Use `@conda_base` to set common libraries required by all
665
+ steps and use `@conda` to specify step-specific additions.
707
666
  """
708
667
  ...
709
668
 
710
669
  @typing.overload
711
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
670
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
712
671
  ...
713
672
 
714
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
673
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
715
674
  """
716
- Specifies the PyPI packages for all steps of the flow.
675
+ Specifies the Conda environment for all steps of the flow.
717
676
 
718
- Use `@pypi_base` to set common packages required by all
719
- steps and use `@pypi` to specify step-specific overrides.
677
+ Use `@conda_base` to set common libraries required by all
678
+ steps and use `@conda` to specify step-specific additions.
679
+ """
680
+ ...
681
+
682
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
683
+ """
684
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
685
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
686
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
687
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
688
+ starts only after all sensors finish.
720
689
  """
721
690
  ...
722
691
 
723
692
  @typing.overload
724
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
693
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
725
694
  """
726
- Specifies the event(s) that this flow depends on.
695
+ Specifies the flow(s) that this flow depends on.
727
696
 
728
697
  ```
729
- @trigger(event='foo')
698
+ @trigger_on_finish(flow='FooFlow')
730
699
  ```
731
700
  or
732
701
  ```
733
- @trigger(events=['foo', 'bar'])
702
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
734
703
  ```
704
+ This decorator respects the @project decorator and triggers the flow
705
+ when upstream runs within the same namespace complete successfully
735
706
 
736
- Additionally, you can specify the parameter mappings
737
- to map event payload to Metaflow parameters for the flow.
707
+ Additionally, you can specify project aware upstream flow dependencies
708
+ by specifying the fully qualified project_flow_name.
738
709
  ```
739
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
710
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
740
711
  ```
741
712
  or
742
713
  ```
743
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
744
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
714
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
745
715
  ```
746
716
 
747
- 'parameters' can also be a list of strings and tuples like so:
748
- ```
749
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
750
- ```
751
- This is equivalent to:
717
+ You can also specify just the project or project branch (other values will be
718
+ inferred from the current project or project branch):
752
719
  ```
753
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
720
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
754
721
  ```
722
+
723
+ Note that `branch` is typically one of:
724
+ - `prod`
725
+ - `user.bob`
726
+ - `test.my_experiment`
727
+ - `prod.staging`
755
728
  """
756
729
  ...
757
730
 
758
731
  @typing.overload
759
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
732
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
760
733
  ...
761
734
 
762
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
735
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
763
736
  """
764
- Specifies the event(s) that this flow depends on.
737
+ Specifies the flow(s) that this flow depends on.
765
738
 
766
739
  ```
767
- @trigger(event='foo')
740
+ @trigger_on_finish(flow='FooFlow')
768
741
  ```
769
742
  or
770
743
  ```
771
- @trigger(events=['foo', 'bar'])
744
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
772
745
  ```
746
+ This decorator respects the @project decorator and triggers the flow
747
+ when upstream runs within the same namespace complete successfully
773
748
 
774
- Additionally, you can specify the parameter mappings
775
- to map event payload to Metaflow parameters for the flow.
749
+ Additionally, you can specify project aware upstream flow dependencies
750
+ by specifying the fully qualified project_flow_name.
776
751
  ```
777
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
752
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
778
753
  ```
779
754
  or
780
755
  ```
781
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
782
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
756
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
783
757
  ```
784
758
 
785
- 'parameters' can also be a list of strings and tuples like so:
786
- ```
787
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
788
- ```
789
- This is equivalent to:
759
+ You can also specify just the project or project branch (other values will be
760
+ inferred from the current project or project branch):
790
761
  ```
791
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
762
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
792
763
  ```
764
+
765
+ Note that `branch` is typically one of:
766
+ - `prod`
767
+ - `user.bob`
768
+ - `test.my_experiment`
769
+ - `prod.staging`
793
770
  """
794
771
  ...
795
772
 
@@ -800,5 +777,28 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
800
777
  """
801
778
  ...
802
779
 
780
+ @typing.overload
781
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
782
+ """
783
+ Specifies the PyPI packages for all steps of the flow.
784
+
785
+ Use `@pypi_base` to set common packages required by all
786
+ steps and use `@pypi` to specify step-specific overrides.
787
+ """
788
+ ...
789
+
790
+ @typing.overload
791
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
792
+ ...
793
+
794
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
795
+ """
796
+ Specifies the PyPI packages for all steps of the flow.
797
+
798
+ Use `@pypi_base` to set common packages required by all
799
+ steps and use `@pypi` to specify step-specific overrides.
800
+ """
801
+ ...
802
+
803
803
  pkg_name: str
804
804