ob-metaflow-stubs 6.0.3.175rc0__py2.py3-none-any.whl → 6.0.3.175rc1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. metaflow-stubs/__init__.pyi +593 -593
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +1 -1
  9. metaflow-stubs/events.pyi +1 -1
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +81 -81
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +1 -1
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +1 -1
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +1 -1
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  83. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  84. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  85. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  86. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  88. metaflow-stubs/multicore_utils.pyi +1 -1
  89. metaflow-stubs/ob_internal.pyi +1 -1
  90. metaflow-stubs/parameters.pyi +3 -3
  91. metaflow-stubs/plugins/__init__.pyi +8 -8
  92. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  93. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  94. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  95. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  96. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  97. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  98. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  99. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  100. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  101. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  102. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  103. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  104. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  105. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +1 -1
  106. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  107. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  108. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  109. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  110. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  111. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  112. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  113. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  114. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  115. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  116. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  117. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  118. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  119. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  120. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  121. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  122. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  123. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  124. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  125. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  126. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  127. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  128. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  129. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  130. metaflow-stubs/plugins/cards/card_client.pyi +1 -1
  131. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  132. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  133. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  134. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  135. metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
  136. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  137. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  138. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  139. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  140. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  141. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  142. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  143. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  144. metaflow-stubs/plugins/catch_decorator.pyi +1 -1
  145. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  146. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  147. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  148. metaflow-stubs/plugins/datatools/s3/s3.pyi +2 -2
  149. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  150. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  151. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  152. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  153. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  154. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  155. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  156. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  157. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  158. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  159. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  160. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  161. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  162. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  163. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  164. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  165. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  166. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  167. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  168. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  169. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  170. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  171. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  172. metaflow-stubs/plugins/perimeters.pyi +1 -1
  173. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  174. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  175. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  176. metaflow-stubs/plugins/pypi/conda_environment.pyi +2 -2
  177. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  178. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  179. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  180. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  181. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  182. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  183. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  184. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  185. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  186. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  187. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  188. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  189. metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
  190. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  192. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  193. metaflow-stubs/profilers/__init__.pyi +1 -1
  194. metaflow-stubs/pylint_wrapper.pyi +1 -1
  195. metaflow-stubs/runner/__init__.pyi +1 -1
  196. metaflow-stubs/runner/deployer.pyi +6 -6
  197. metaflow-stubs/runner/deployer_impl.pyi +1 -1
  198. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  199. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  200. metaflow-stubs/runner/nbrun.pyi +1 -1
  201. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  202. metaflow-stubs/runner/utils.pyi +2 -2
  203. metaflow-stubs/system/__init__.pyi +1 -1
  204. metaflow-stubs/system/system_logger.pyi +1 -1
  205. metaflow-stubs/system/system_monitor.pyi +1 -1
  206. metaflow-stubs/tagging_util.pyi +1 -1
  207. metaflow-stubs/tuple_util.pyi +1 -1
  208. metaflow-stubs/user_configs/__init__.pyi +1 -1
  209. metaflow-stubs/user_configs/config_decorators.pyi +3 -3
  210. metaflow-stubs/user_configs/config_options.pyi +3 -3
  211. metaflow-stubs/user_configs/config_parameters.pyi +3 -3
  212. {ob_metaflow_stubs-6.0.3.175rc0.dist-info → ob_metaflow_stubs-6.0.3.175rc1.dist-info}/METADATA +1 -1
  213. ob_metaflow_stubs-6.0.3.175rc1.dist-info/RECORD +216 -0
  214. ob_metaflow_stubs-6.0.3.175rc0.dist-info/RECORD +0 -216
  215. {ob_metaflow_stubs-6.0.3.175rc0.dist-info → ob_metaflow_stubs-6.0.3.175rc1.dist-info}/WHEEL +0 -0
  216. {ob_metaflow_stubs-6.0.3.175rc0.dist-info → ob_metaflow_stubs-6.0.3.175rc1.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-05-30T00:54:07.424022 #
4
+ # Generated on 2025-05-31T01:09:57.775269 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -35,10 +35,10 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import cards as cards
39
- from . import metaflow_git as metaflow_git
40
38
  from . import tuple_util as tuple_util
39
+ from . import cards as cards
41
40
  from . import events as events
41
+ from . import metaflow_git as metaflow_git
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -155,53 +155,81 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
155
155
  ...
156
156
 
157
157
  @typing.overload
158
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
158
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
159
159
  """
160
- Specifies the PyPI packages for the step.
160
+ Specifies the resources needed when executing this step.
161
161
 
162
- Information in this decorator will augment any
163
- attributes set in the `@pyi_base` flow-level decorator. Hence,
164
- you can use `@pypi_base` to set packages required by all
165
- steps and use `@pypi` to specify step-specific overrides.
162
+ Use `@resources` to specify the resource requirements
163
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
164
+
165
+ You can choose the compute layer on the command line by executing e.g.
166
+ ```
167
+ python myflow.py run --with batch
168
+ ```
169
+ or
170
+ ```
171
+ python myflow.py run --with kubernetes
172
+ ```
173
+ which executes the flow on the desired system using the
174
+ requirements specified in `@resources`.
166
175
 
167
176
 
168
177
  Parameters
169
178
  ----------
170
- packages : Dict[str, str], default: {}
171
- Packages to use for this step. The key is the name of the package
172
- and the value is the version to use.
173
- python : str, optional, default: None
174
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
175
- that the version used will correspond to the version of the Python interpreter used to start the run.
179
+ cpu : int, default 1
180
+ Number of CPUs required for this step.
181
+ gpu : int, optional, default None
182
+ Number of GPUs required for this step.
183
+ disk : int, optional, default None
184
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
185
+ memory : int, default 4096
186
+ Memory size (in MB) required for this step.
187
+ shared_memory : int, optional, default None
188
+ The value for the size (in MiB) of the /dev/shm volume for this step.
189
+ This parameter maps to the `--shm-size` option in Docker.
176
190
  """
177
191
  ...
178
192
 
179
193
  @typing.overload
180
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
194
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
181
195
  ...
182
196
 
183
197
  @typing.overload
184
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
198
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
185
199
  ...
186
200
 
187
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
201
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
188
202
  """
189
- Specifies the PyPI packages for the step.
203
+ Specifies the resources needed when executing this step.
190
204
 
191
- Information in this decorator will augment any
192
- attributes set in the `@pyi_base` flow-level decorator. Hence,
193
- you can use `@pypi_base` to set packages required by all
194
- steps and use `@pypi` to specify step-specific overrides.
205
+ Use `@resources` to specify the resource requirements
206
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
207
+
208
+ You can choose the compute layer on the command line by executing e.g.
209
+ ```
210
+ python myflow.py run --with batch
211
+ ```
212
+ or
213
+ ```
214
+ python myflow.py run --with kubernetes
215
+ ```
216
+ which executes the flow on the desired system using the
217
+ requirements specified in `@resources`.
195
218
 
196
219
 
197
220
  Parameters
198
221
  ----------
199
- packages : Dict[str, str], default: {}
200
- Packages to use for this step. The key is the name of the package
201
- and the value is the version to use.
202
- python : str, optional, default: None
203
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
204
- that the version used will correspond to the version of the Python interpreter used to start the run.
222
+ cpu : int, default 1
223
+ Number of CPUs required for this step.
224
+ gpu : int, optional, default None
225
+ Number of GPUs required for this step.
226
+ disk : int, optional, default None
227
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
228
+ memory : int, default 4096
229
+ Memory size (in MB) required for this step.
230
+ shared_memory : int, optional, default None
231
+ The value for the size (in MiB) of the /dev/shm volume for this step.
232
+ This parameter maps to the `--shm-size` option in Docker.
205
233
  """
206
234
  ...
207
235
 
@@ -256,6 +284,63 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
256
284
  """
257
285
  ...
258
286
 
287
+ @typing.overload
288
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
289
+ """
290
+ Enables loading / saving of models within a step.
291
+
292
+
293
+
294
+ Parameters
295
+ ----------
296
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
297
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
298
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
299
+ - `current.checkpoint`
300
+ - `current.model`
301
+ - `current.huggingface_hub`
302
+
303
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
304
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
305
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
306
+
307
+ temp_dir_root : str, default: None
308
+ The root directory under which `current.model.loaded` will store loaded models
309
+ """
310
+ ...
311
+
312
+ @typing.overload
313
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
314
+ ...
315
+
316
+ @typing.overload
317
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
318
+ ...
319
+
320
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
321
+ """
322
+ Enables loading / saving of models within a step.
323
+
324
+
325
+
326
+ Parameters
327
+ ----------
328
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
329
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
330
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
331
+ - `current.checkpoint`
332
+ - `current.model`
333
+ - `current.huggingface_hub`
334
+
335
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
336
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
337
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
338
+
339
+ temp_dir_root : str, default: None
340
+ The root directory under which `current.model.loaded` will store loaded models
341
+ """
342
+ ...
343
+
259
344
  def ollama(*, models: list, backend: str, force_pull: bool, skip_push_check: bool, debug: bool) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
260
345
  """
261
346
  This decorator is used to run Ollama APIs as Metaflow task sidecars.
@@ -293,6 +378,71 @@ def ollama(*, models: list, backend: str, force_pull: bool, skip_push_check: boo
293
378
  """
294
379
  ...
295
380
 
381
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
382
+ """
383
+ Specifies that this step is used to deploy an instance of the app.
384
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
385
+
386
+
387
+ Parameters
388
+ ----------
389
+ app_port : int
390
+ Number of GPUs to use.
391
+ app_name : str
392
+ Name of the app to deploy.
393
+ """
394
+ ...
395
+
396
+ @typing.overload
397
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
398
+ """
399
+ Decorator prototype for all step decorators. This function gets specialized
400
+ and imported for all decorators types by _import_plugin_decorators().
401
+ """
402
+ ...
403
+
404
+ @typing.overload
405
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
406
+ ...
407
+
408
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
409
+ """
410
+ Decorator prototype for all step decorators. This function gets specialized
411
+ and imported for all decorators types by _import_plugin_decorators().
412
+ """
413
+ ...
414
+
415
+ @typing.overload
416
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
417
+ """
418
+ Internal decorator to support Fast bakery
419
+ """
420
+ ...
421
+
422
+ @typing.overload
423
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
424
+ ...
425
+
426
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
427
+ """
428
+ Internal decorator to support Fast bakery
429
+ """
430
+ ...
431
+
432
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
433
+ """
434
+ Specifies that this step should execute on DGX cloud.
435
+
436
+
437
+ Parameters
438
+ ----------
439
+ gpu : int
440
+ Number of GPUs to use.
441
+ gpu_type : str
442
+ Type of Nvidia GPU to use.
443
+ """
444
+ ...
445
+
296
446
  def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
297
447
  """
298
448
  Specifies that this step should execute on Kubernetes.
@@ -378,75 +528,77 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
378
528
  """
379
529
  ...
380
530
 
381
- @typing.overload
382
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
531
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
383
532
  """
384
- Enables loading / saving of models within a step.
385
-
533
+ Specifies that this step should execute on DGX cloud.
386
534
 
387
535
 
388
536
  Parameters
389
537
  ----------
390
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
391
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
392
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
393
- - `current.checkpoint`
394
- - `current.model`
395
- - `current.huggingface_hub`
538
+ gpu : int
539
+ Number of GPUs to use.
540
+ gpu_type : str
541
+ Type of Nvidia GPU to use.
542
+ queue_timeout : int
543
+ Time to keep the job in NVCF's queue.
544
+ """
545
+ ...
546
+
547
+ @typing.overload
548
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
549
+ """
550
+ Specifies environment variables to be set prior to the execution of a step.
396
551
 
397
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
398
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
399
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
400
552
 
401
- temp_dir_root : str, default: None
402
- The root directory under which `current.model.loaded` will store loaded models
553
+ Parameters
554
+ ----------
555
+ vars : Dict[str, str], default {}
556
+ Dictionary of environment variables to set.
403
557
  """
404
558
  ...
405
559
 
406
560
  @typing.overload
407
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
561
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
408
562
  ...
409
563
 
410
564
  @typing.overload
411
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
565
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
412
566
  ...
413
567
 
414
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
568
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
415
569
  """
416
- Enables loading / saving of models within a step.
417
-
570
+ Specifies environment variables to be set prior to the execution of a step.
418
571
 
419
572
 
420
573
  Parameters
421
574
  ----------
422
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
423
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
424
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
425
- - `current.checkpoint`
426
- - `current.model`
427
- - `current.huggingface_hub`
428
-
429
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
430
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
431
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
432
-
433
- temp_dir_root : str, default: None
434
- The root directory under which `current.model.loaded` will store loaded models
575
+ vars : Dict[str, str], default {}
576
+ Dictionary of environment variables to set.
435
577
  """
436
578
  ...
437
579
 
438
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
580
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
439
581
  """
440
- Specifies that this step is used to deploy an instance of the app.
441
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
582
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
442
583
 
443
584
 
444
585
  Parameters
445
586
  ----------
446
- app_port : int
447
- Number of GPUs to use.
448
- app_name : str
449
- Name of the app to deploy.
587
+ temp_dir_root : str, optional
588
+ The root directory that will hold the temporary directory where objects will be downloaded.
589
+
590
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
591
+ The list of repos (models/datasets) to load.
592
+
593
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
594
+
595
+ - If repo (model/dataset) is not found in the datastore:
596
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
597
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
598
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
599
+
600
+ - If repo is found in the datastore:
601
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
450
602
  """
451
603
  ...
452
604
 
@@ -505,184 +657,6 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
505
657
  """
506
658
  ...
507
659
 
508
- @typing.overload
509
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
510
- """
511
- Enables checkpointing for a step.
512
-
513
-
514
-
515
- Parameters
516
- ----------
517
- load_policy : str, default: "fresh"
518
- The policy for loading the checkpoint. The following policies are supported:
519
- - "eager": Loads the the latest available checkpoint within the namespace.
520
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
521
- will be loaded at the start of the task.
522
- - "none": Do not load any checkpoint
523
- - "fresh": Loads the lastest checkpoint created within the running Task.
524
- This mode helps loading checkpoints across various retry attempts of the same task.
525
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
526
- created within the task will be loaded when the task is retries execution on failure.
527
-
528
- temp_dir_root : str, default: None
529
- The root directory under which `current.checkpoint.directory` will be created.
530
- """
531
- ...
532
-
533
- @typing.overload
534
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
535
- ...
536
-
537
- @typing.overload
538
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
539
- ...
540
-
541
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
542
- """
543
- Enables checkpointing for a step.
544
-
545
-
546
-
547
- Parameters
548
- ----------
549
- load_policy : str, default: "fresh"
550
- The policy for loading the checkpoint. The following policies are supported:
551
- - "eager": Loads the the latest available checkpoint within the namespace.
552
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
553
- will be loaded at the start of the task.
554
- - "none": Do not load any checkpoint
555
- - "fresh": Loads the lastest checkpoint created within the running Task.
556
- This mode helps loading checkpoints across various retry attempts of the same task.
557
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
558
- created within the task will be loaded when the task is retries execution on failure.
559
-
560
- temp_dir_root : str, default: None
561
- The root directory under which `current.checkpoint.directory` will be created.
562
- """
563
- ...
564
-
565
- @typing.overload
566
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
567
- """
568
- Internal decorator to support Fast bakery
569
- """
570
- ...
571
-
572
- @typing.overload
573
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
574
- ...
575
-
576
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
577
- """
578
- Internal decorator to support Fast bakery
579
- """
580
- ...
581
-
582
- @typing.overload
583
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
584
- """
585
- Specifies the resources needed when executing this step.
586
-
587
- Use `@resources` to specify the resource requirements
588
- independently of the specific compute layer (`@batch`, `@kubernetes`).
589
-
590
- You can choose the compute layer on the command line by executing e.g.
591
- ```
592
- python myflow.py run --with batch
593
- ```
594
- or
595
- ```
596
- python myflow.py run --with kubernetes
597
- ```
598
- which executes the flow on the desired system using the
599
- requirements specified in `@resources`.
600
-
601
-
602
- Parameters
603
- ----------
604
- cpu : int, default 1
605
- Number of CPUs required for this step.
606
- gpu : int, optional, default None
607
- Number of GPUs required for this step.
608
- disk : int, optional, default None
609
- Disk size (in MB) required for this step. Only applies on Kubernetes.
610
- memory : int, default 4096
611
- Memory size (in MB) required for this step.
612
- shared_memory : int, optional, default None
613
- The value for the size (in MiB) of the /dev/shm volume for this step.
614
- This parameter maps to the `--shm-size` option in Docker.
615
- """
616
- ...
617
-
618
- @typing.overload
619
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
620
- ...
621
-
622
- @typing.overload
623
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
624
- ...
625
-
626
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
627
- """
628
- Specifies the resources needed when executing this step.
629
-
630
- Use `@resources` to specify the resource requirements
631
- independently of the specific compute layer (`@batch`, `@kubernetes`).
632
-
633
- You can choose the compute layer on the command line by executing e.g.
634
- ```
635
- python myflow.py run --with batch
636
- ```
637
- or
638
- ```
639
- python myflow.py run --with kubernetes
640
- ```
641
- which executes the flow on the desired system using the
642
- requirements specified in `@resources`.
643
-
644
-
645
- Parameters
646
- ----------
647
- cpu : int, default 1
648
- Number of CPUs required for this step.
649
- gpu : int, optional, default None
650
- Number of GPUs required for this step.
651
- disk : int, optional, default None
652
- Disk size (in MB) required for this step. Only applies on Kubernetes.
653
- memory : int, default 4096
654
- Memory size (in MB) required for this step.
655
- shared_memory : int, optional, default None
656
- The value for the size (in MiB) of the /dev/shm volume for this step.
657
- This parameter maps to the `--shm-size` option in Docker.
658
- """
659
- ...
660
-
661
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
662
- """
663
- Decorator that helps cache, version and store models/datasets from huggingface hub.
664
-
665
-
666
- Parameters
667
- ----------
668
- temp_dir_root : str, optional
669
- The root directory that will hold the temporary directory where objects will be downloaded.
670
-
671
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
672
- The list of repos (models/datasets) to load.
673
-
674
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
675
-
676
- - If repo (model/dataset) is not found in the datastore:
677
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
678
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
679
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
680
-
681
- - If repo is found in the datastore:
682
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
683
- """
684
- ...
685
-
686
660
  @typing.overload
687
661
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
688
662
  """
@@ -792,65 +766,118 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
792
766
  ...
793
767
 
794
768
  @typing.overload
795
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
769
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
796
770
  """
797
- Specifies environment variables to be set prior to the execution of a step.
771
+ Enables checkpointing for a step.
772
+
798
773
 
799
774
 
800
775
  Parameters
801
776
  ----------
802
- vars : Dict[str, str], default {}
803
- Dictionary of environment variables to set.
777
+ load_policy : str, default: "fresh"
778
+ The policy for loading the checkpoint. The following policies are supported:
779
+ - "eager": Loads the the latest available checkpoint within the namespace.
780
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
781
+ will be loaded at the start of the task.
782
+ - "none": Do not load any checkpoint
783
+ - "fresh": Loads the lastest checkpoint created within the running Task.
784
+ This mode helps loading checkpoints across various retry attempts of the same task.
785
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
786
+ created within the task will be loaded when the task is retries execution on failure.
787
+
788
+ temp_dir_root : str, default: None
789
+ The root directory under which `current.checkpoint.directory` will be created.
804
790
  """
805
791
  ...
806
792
 
807
793
  @typing.overload
808
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
794
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
809
795
  ...
810
796
 
811
797
  @typing.overload
812
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
798
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
813
799
  ...
814
800
 
815
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
801
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
816
802
  """
817
- Specifies environment variables to be set prior to the execution of a step.
803
+ Enables checkpointing for a step.
804
+
818
805
 
819
806
 
820
807
  Parameters
821
808
  ----------
822
- vars : Dict[str, str], default {}
823
- Dictionary of environment variables to set.
809
+ load_policy : str, default: "fresh"
810
+ The policy for loading the checkpoint. The following policies are supported:
811
+ - "eager": Loads the the latest available checkpoint within the namespace.
812
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
813
+ will be loaded at the start of the task.
814
+ - "none": Do not load any checkpoint
815
+ - "fresh": Loads the lastest checkpoint created within the running Task.
816
+ This mode helps loading checkpoints across various retry attempts of the same task.
817
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
818
+ created within the task will be loaded when the task is retries execution on failure.
819
+
820
+ temp_dir_root : str, default: None
821
+ The root directory under which `current.checkpoint.directory` will be created.
824
822
  """
825
823
  ...
826
824
 
827
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
828
- """
829
- Specifies that this step should execute on DGX cloud.
825
+ @typing.overload
826
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
827
+ """
828
+ Specifies the Conda environment for the step.
829
+
830
+ Information in this decorator will augment any
831
+ attributes set in the `@conda_base` flow-level decorator. Hence,
832
+ you can use `@conda_base` to set packages required by all
833
+ steps and use `@conda` to specify step-specific overrides.
830
834
 
831
835
 
832
836
  Parameters
833
837
  ----------
834
- gpu : int
835
- Number of GPUs to use.
836
- gpu_type : str
837
- Type of Nvidia GPU to use.
838
- queue_timeout : int
839
- Time to keep the job in NVCF's queue.
838
+ packages : Dict[str, str], default {}
839
+ Packages to use for this step. The key is the name of the package
840
+ and the value is the version to use.
841
+ libraries : Dict[str, str], default {}
842
+ Supported for backward compatibility. When used with packages, packages will take precedence.
843
+ python : str, optional, default None
844
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
845
+ that the version used will correspond to the version of the Python interpreter used to start the run.
846
+ disabled : bool, default False
847
+ If set to True, disables @conda.
840
848
  """
841
849
  ...
842
850
 
843
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
851
+ @typing.overload
852
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
853
+ ...
854
+
855
+ @typing.overload
856
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
857
+ ...
858
+
859
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
844
860
  """
845
- Specifies that this step should execute on DGX cloud.
861
+ Specifies the Conda environment for the step.
862
+
863
+ Information in this decorator will augment any
864
+ attributes set in the `@conda_base` flow-level decorator. Hence,
865
+ you can use `@conda_base` to set packages required by all
866
+ steps and use `@conda` to specify step-specific overrides.
846
867
 
847
868
 
848
869
  Parameters
849
870
  ----------
850
- gpu : int
851
- Number of GPUs to use.
852
- gpu_type : str
853
- Type of Nvidia GPU to use.
871
+ packages : Dict[str, str], default {}
872
+ Packages to use for this step. The key is the name of the package
873
+ and the value is the version to use.
874
+ libraries : Dict[str, str], default {}
875
+ Supported for backward compatibility. When used with packages, packages will take precedence.
876
+ python : str, optional, default None
877
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
878
+ that the version used will correspond to the version of the Python interpreter used to start the run.
879
+ disabled : bool, default False
880
+ If set to True, disables @conda.
854
881
  """
855
882
  ...
856
883
 
@@ -890,80 +917,146 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
890
917
  ...
891
918
 
892
919
  @typing.overload
893
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
920
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
894
921
  """
895
- Specifies the Conda environment for the step.
922
+ Specifies the PyPI packages for the step.
896
923
 
897
924
  Information in this decorator will augment any
898
- attributes set in the `@conda_base` flow-level decorator. Hence,
899
- you can use `@conda_base` to set packages required by all
900
- steps and use `@conda` to specify step-specific overrides.
925
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
926
+ you can use `@pypi_base` to set packages required by all
927
+ steps and use `@pypi` to specify step-specific overrides.
901
928
 
902
929
 
903
930
  Parameters
904
931
  ----------
905
- packages : Dict[str, str], default {}
932
+ packages : Dict[str, str], default: {}
906
933
  Packages to use for this step. The key is the name of the package
907
934
  and the value is the version to use.
908
- libraries : Dict[str, str], default {}
909
- Supported for backward compatibility. When used with packages, packages will take precedence.
910
- python : str, optional, default None
935
+ python : str, optional, default: None
911
936
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
912
937
  that the version used will correspond to the version of the Python interpreter used to start the run.
913
- disabled : bool, default False
914
- If set to True, disables @conda.
915
938
  """
916
939
  ...
917
940
 
918
941
  @typing.overload
919
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
942
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
920
943
  ...
921
944
 
922
945
  @typing.overload
923
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
946
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
924
947
  ...
925
948
 
926
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
949
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
927
950
  """
928
- Specifies the Conda environment for the step.
951
+ Specifies the PyPI packages for the step.
929
952
 
930
953
  Information in this decorator will augment any
931
- attributes set in the `@conda_base` flow-level decorator. Hence,
932
- you can use `@conda_base` to set packages required by all
933
- steps and use `@conda` to specify step-specific overrides.
954
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
955
+ you can use `@pypi_base` to set packages required by all
956
+ steps and use `@pypi` to specify step-specific overrides.
934
957
 
935
958
 
936
959
  Parameters
937
960
  ----------
938
- packages : Dict[str, str], default {}
961
+ packages : Dict[str, str], default: {}
939
962
  Packages to use for this step. The key is the name of the package
940
963
  and the value is the version to use.
941
- libraries : Dict[str, str], default {}
942
- Supported for backward compatibility. When used with packages, packages will take precedence.
943
- python : str, optional, default None
964
+ python : str, optional, default: None
944
965
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
945
966
  that the version used will correspond to the version of the Python interpreter used to start the run.
946
- disabled : bool, default False
947
- If set to True, disables @conda.
948
967
  """
949
968
  ...
950
969
 
951
970
  @typing.overload
952
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
971
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
953
972
  """
954
- Decorator prototype for all step decorators. This function gets specialized
955
- and imported for all decorators types by _import_plugin_decorators().
973
+ Specifies the event(s) that this flow depends on.
974
+
975
+ ```
976
+ @trigger(event='foo')
977
+ ```
978
+ or
979
+ ```
980
+ @trigger(events=['foo', 'bar'])
981
+ ```
982
+
983
+ Additionally, you can specify the parameter mappings
984
+ to map event payload to Metaflow parameters for the flow.
985
+ ```
986
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
987
+ ```
988
+ or
989
+ ```
990
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
991
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
992
+ ```
993
+
994
+ 'parameters' can also be a list of strings and tuples like so:
995
+ ```
996
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
997
+ ```
998
+ This is equivalent to:
999
+ ```
1000
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1001
+ ```
1002
+
1003
+
1004
+ Parameters
1005
+ ----------
1006
+ event : Union[str, Dict[str, Any]], optional, default None
1007
+ Event dependency for this flow.
1008
+ events : List[Union[str, Dict[str, Any]]], default []
1009
+ Events dependency for this flow.
1010
+ options : Dict[str, Any], default {}
1011
+ Backend-specific configuration for tuning eventing behavior.
956
1012
  """
957
1013
  ...
958
1014
 
959
1015
  @typing.overload
960
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1016
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
961
1017
  ...
962
1018
 
963
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1019
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
964
1020
  """
965
- Decorator prototype for all step decorators. This function gets specialized
966
- and imported for all decorators types by _import_plugin_decorators().
1021
+ Specifies the event(s) that this flow depends on.
1022
+
1023
+ ```
1024
+ @trigger(event='foo')
1025
+ ```
1026
+ or
1027
+ ```
1028
+ @trigger(events=['foo', 'bar'])
1029
+ ```
1030
+
1031
+ Additionally, you can specify the parameter mappings
1032
+ to map event payload to Metaflow parameters for the flow.
1033
+ ```
1034
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1035
+ ```
1036
+ or
1037
+ ```
1038
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1039
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1040
+ ```
1041
+
1042
+ 'parameters' can also be a list of strings and tuples like so:
1043
+ ```
1044
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1045
+ ```
1046
+ This is equivalent to:
1047
+ ```
1048
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1049
+ ```
1050
+
1051
+
1052
+ Parameters
1053
+ ----------
1054
+ event : Union[str, Dict[str, Any]], optional, default None
1055
+ Event dependency for this flow.
1056
+ events : List[Union[str, Dict[str, Any]]], default []
1057
+ Events dependency for this flow.
1058
+ options : Dict[str, Any], default {}
1059
+ Backend-specific configuration for tuning eventing behavior.
967
1060
  """
968
1061
  ...
969
1062
 
@@ -1069,95 +1162,223 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1069
1162
  ...
1070
1163
 
1071
1164
  @typing.overload
1072
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1165
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1073
1166
  """
1074
- Specifies the event(s) that this flow depends on.
1075
-
1076
- ```
1077
- @trigger(event='foo')
1078
- ```
1079
- or
1080
- ```
1081
- @trigger(events=['foo', 'bar'])
1082
- ```
1083
-
1084
- Additionally, you can specify the parameter mappings
1085
- to map event payload to Metaflow parameters for the flow.
1086
- ```
1087
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1088
- ```
1089
- or
1090
- ```
1091
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1092
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1093
- ```
1167
+ Specifies the Conda environment for all steps of the flow.
1094
1168
 
1095
- 'parameters' can also be a list of strings and tuples like so:
1096
- ```
1097
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1098
- ```
1099
- This is equivalent to:
1100
- ```
1101
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1102
- ```
1169
+ Use `@conda_base` to set common libraries required by all
1170
+ steps and use `@conda` to specify step-specific additions.
1103
1171
 
1104
1172
 
1105
1173
  Parameters
1106
1174
  ----------
1107
- event : Union[str, Dict[str, Any]], optional, default None
1108
- Event dependency for this flow.
1109
- events : List[Union[str, Dict[str, Any]]], default []
1110
- Events dependency for this flow.
1111
- options : Dict[str, Any], default {}
1112
- Backend-specific configuration for tuning eventing behavior.
1175
+ packages : Dict[str, str], default {}
1176
+ Packages to use for this flow. The key is the name of the package
1177
+ and the value is the version to use.
1178
+ libraries : Dict[str, str], default {}
1179
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1180
+ python : str, optional, default None
1181
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1182
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1183
+ disabled : bool, default False
1184
+ If set to True, disables Conda.
1113
1185
  """
1114
1186
  ...
1115
1187
 
1116
1188
  @typing.overload
1117
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1189
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1118
1190
  ...
1119
1191
 
1120
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1192
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1121
1193
  """
1122
- Specifies the event(s) that this flow depends on.
1194
+ Specifies the Conda environment for all steps of the flow.
1123
1195
 
1124
- ```
1125
- @trigger(event='foo')
1126
- ```
1127
- or
1128
- ```
1129
- @trigger(events=['foo', 'bar'])
1130
- ```
1196
+ Use `@conda_base` to set common libraries required by all
1197
+ steps and use `@conda` to specify step-specific additions.
1131
1198
 
1132
- Additionally, you can specify the parameter mappings
1133
- to map event payload to Metaflow parameters for the flow.
1134
- ```
1135
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1136
- ```
1137
- or
1138
- ```
1139
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1140
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1141
- ```
1142
1199
 
1143
- 'parameters' can also be a list of strings and tuples like so:
1144
- ```
1145
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1146
- ```
1147
- This is equivalent to:
1148
- ```
1149
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1150
- ```
1200
+ Parameters
1201
+ ----------
1202
+ packages : Dict[str, str], default {}
1203
+ Packages to use for this flow. The key is the name of the package
1204
+ and the value is the version to use.
1205
+ libraries : Dict[str, str], default {}
1206
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1207
+ python : str, optional, default None
1208
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1209
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1210
+ disabled : bool, default False
1211
+ If set to True, disables Conda.
1212
+ """
1213
+ ...
1214
+
1215
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1216
+ """
1217
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1218
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1219
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1220
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1221
+ starts only after all sensors finish.
1151
1222
 
1152
1223
 
1153
1224
  Parameters
1154
1225
  ----------
1155
- event : Union[str, Dict[str, Any]], optional, default None
1156
- Event dependency for this flow.
1157
- events : List[Union[str, Dict[str, Any]]], default []
1158
- Events dependency for this flow.
1159
- options : Dict[str, Any], default {}
1160
- Backend-specific configuration for tuning eventing behavior.
1226
+ timeout : int
1227
+ Time, in seconds before the task times out and fails. (Default: 3600)
1228
+ poke_interval : int
1229
+ Time in seconds that the job should wait in between each try. (Default: 60)
1230
+ mode : str
1231
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1232
+ exponential_backoff : bool
1233
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1234
+ pool : str
1235
+ the slot pool this task should run in,
1236
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1237
+ soft_fail : bool
1238
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1239
+ name : str
1240
+ Name of the sensor on Airflow
1241
+ description : str
1242
+ Description of sensor in the Airflow UI
1243
+ bucket_key : Union[str, List[str]]
1244
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1245
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1246
+ bucket_name : str
1247
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1248
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1249
+ wildcard_match : bool
1250
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1251
+ aws_conn_id : str
1252
+ a reference to the s3 connection on Airflow. (Default: None)
1253
+ verify : bool
1254
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1255
+ """
1256
+ ...
1257
+
1258
+ @typing.overload
1259
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1260
+ """
1261
+ Specifies the times when the flow should be run when running on a
1262
+ production scheduler.
1263
+
1264
+
1265
+ Parameters
1266
+ ----------
1267
+ hourly : bool, default False
1268
+ Run the workflow hourly.
1269
+ daily : bool, default True
1270
+ Run the workflow daily.
1271
+ weekly : bool, default False
1272
+ Run the workflow weekly.
1273
+ cron : str, optional, default None
1274
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1275
+ specified by this expression.
1276
+ timezone : str, optional, default None
1277
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1278
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1279
+ """
1280
+ ...
1281
+
1282
+ @typing.overload
1283
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1284
+ ...
1285
+
1286
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1287
+ """
1288
+ Specifies the times when the flow should be run when running on a
1289
+ production scheduler.
1290
+
1291
+
1292
+ Parameters
1293
+ ----------
1294
+ hourly : bool, default False
1295
+ Run the workflow hourly.
1296
+ daily : bool, default True
1297
+ Run the workflow daily.
1298
+ weekly : bool, default False
1299
+ Run the workflow weekly.
1300
+ cron : str, optional, default None
1301
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1302
+ specified by this expression.
1303
+ timezone : str, optional, default None
1304
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1305
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1306
+ """
1307
+ ...
1308
+
1309
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1310
+ """
1311
+ Specifies what flows belong to the same project.
1312
+
1313
+ A project-specific namespace is created for all flows that
1314
+ use the same `@project(name)`.
1315
+
1316
+
1317
+ Parameters
1318
+ ----------
1319
+ name : str
1320
+ Project name. Make sure that the name is unique amongst all
1321
+ projects that use the same production scheduler. The name may
1322
+ contain only lowercase alphanumeric characters and underscores.
1323
+
1324
+ branch : Optional[str], default None
1325
+ The branch to use. If not specified, the branch is set to
1326
+ `user.<username>` unless `production` is set to `True`. This can
1327
+ also be set on the command line using `--branch` as a top-level option.
1328
+ It is an error to specify `branch` in the decorator and on the command line.
1329
+
1330
+ production : bool, default False
1331
+ Whether or not the branch is the production branch. This can also be set on the
1332
+ command line using `--production` as a top-level option. It is an error to specify
1333
+ `production` in the decorator and on the command line.
1334
+ The project branch name will be:
1335
+ - if `branch` is specified:
1336
+ - if `production` is True: `prod.<branch>`
1337
+ - if `production` is False: `test.<branch>`
1338
+ - if `branch` is not specified:
1339
+ - if `production` is True: `prod`
1340
+ - if `production` is False: `user.<username>`
1341
+ """
1342
+ ...
1343
+
1344
+ @typing.overload
1345
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1346
+ """
1347
+ Specifies the PyPI packages for all steps of the flow.
1348
+
1349
+ Use `@pypi_base` to set common packages required by all
1350
+ steps and use `@pypi` to specify step-specific overrides.
1351
+
1352
+ Parameters
1353
+ ----------
1354
+ packages : Dict[str, str], default: {}
1355
+ Packages to use for this flow. The key is the name of the package
1356
+ and the value is the version to use.
1357
+ python : str, optional, default: None
1358
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1359
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1360
+ """
1361
+ ...
1362
+
1363
+ @typing.overload
1364
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1365
+ ...
1366
+
1367
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1368
+ """
1369
+ Specifies the PyPI packages for all steps of the flow.
1370
+
1371
+ Use `@pypi_base` to set common packages required by all
1372
+ steps and use `@pypi` to specify step-specific overrides.
1373
+
1374
+ Parameters
1375
+ ----------
1376
+ packages : Dict[str, str], default: {}
1377
+ Packages to use for this flow. The key is the name of the package
1378
+ and the value is the version to use.
1379
+ python : str, optional, default: None
1380
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1381
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1161
1382
  """
1162
1383
  ...
1163
1384
 
@@ -1275,151 +1496,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1275
1496
  """
1276
1497
  ...
1277
1498
 
1278
- @typing.overload
1279
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1280
- """
1281
- Specifies the Conda environment for all steps of the flow.
1282
-
1283
- Use `@conda_base` to set common libraries required by all
1284
- steps and use `@conda` to specify step-specific additions.
1285
-
1286
-
1287
- Parameters
1288
- ----------
1289
- packages : Dict[str, str], default {}
1290
- Packages to use for this flow. The key is the name of the package
1291
- and the value is the version to use.
1292
- libraries : Dict[str, str], default {}
1293
- Supported for backward compatibility. When used with packages, packages will take precedence.
1294
- python : str, optional, default None
1295
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1296
- that the version used will correspond to the version of the Python interpreter used to start the run.
1297
- disabled : bool, default False
1298
- If set to True, disables Conda.
1299
- """
1300
- ...
1301
-
1302
- @typing.overload
1303
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1304
- ...
1305
-
1306
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1307
- """
1308
- Specifies the Conda environment for all steps of the flow.
1309
-
1310
- Use `@conda_base` to set common libraries required by all
1311
- steps and use `@conda` to specify step-specific additions.
1312
-
1313
-
1314
- Parameters
1315
- ----------
1316
- packages : Dict[str, str], default {}
1317
- Packages to use for this flow. The key is the name of the package
1318
- and the value is the version to use.
1319
- libraries : Dict[str, str], default {}
1320
- Supported for backward compatibility. When used with packages, packages will take precedence.
1321
- python : str, optional, default None
1322
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1323
- that the version used will correspond to the version of the Python interpreter used to start the run.
1324
- disabled : bool, default False
1325
- If set to True, disables Conda.
1326
- """
1327
- ...
1328
-
1329
- @typing.overload
1330
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1331
- """
1332
- Specifies the times when the flow should be run when running on a
1333
- production scheduler.
1334
-
1335
-
1336
- Parameters
1337
- ----------
1338
- hourly : bool, default False
1339
- Run the workflow hourly.
1340
- daily : bool, default True
1341
- Run the workflow daily.
1342
- weekly : bool, default False
1343
- Run the workflow weekly.
1344
- cron : str, optional, default None
1345
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1346
- specified by this expression.
1347
- timezone : str, optional, default None
1348
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1349
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1350
- """
1351
- ...
1352
-
1353
- @typing.overload
1354
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1355
- ...
1356
-
1357
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1358
- """
1359
- Specifies the times when the flow should be run when running on a
1360
- production scheduler.
1361
-
1362
-
1363
- Parameters
1364
- ----------
1365
- hourly : bool, default False
1366
- Run the workflow hourly.
1367
- daily : bool, default True
1368
- Run the workflow daily.
1369
- weekly : bool, default False
1370
- Run the workflow weekly.
1371
- cron : str, optional, default None
1372
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1373
- specified by this expression.
1374
- timezone : str, optional, default None
1375
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1376
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1377
- """
1378
- ...
1379
-
1380
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1381
- """
1382
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1383
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1384
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1385
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1386
- starts only after all sensors finish.
1387
-
1388
-
1389
- Parameters
1390
- ----------
1391
- timeout : int
1392
- Time, in seconds before the task times out and fails. (Default: 3600)
1393
- poke_interval : int
1394
- Time in seconds that the job should wait in between each try. (Default: 60)
1395
- mode : str
1396
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1397
- exponential_backoff : bool
1398
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1399
- pool : str
1400
- the slot pool this task should run in,
1401
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1402
- soft_fail : bool
1403
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1404
- name : str
1405
- Name of the sensor on Airflow
1406
- description : str
1407
- Description of sensor in the Airflow UI
1408
- bucket_key : Union[str, List[str]]
1409
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1410
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1411
- bucket_name : str
1412
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1413
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1414
- wildcard_match : bool
1415
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1416
- aws_conn_id : str
1417
- a reference to the s3 connection on Airflow. (Default: None)
1418
- verify : bool
1419
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1420
- """
1421
- ...
1422
-
1423
1499
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1424
1500
  """
1425
1501
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1463,81 +1539,5 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1463
1539
  """
1464
1540
  ...
1465
1541
 
1466
- @typing.overload
1467
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1468
- """
1469
- Specifies the PyPI packages for all steps of the flow.
1470
-
1471
- Use `@pypi_base` to set common packages required by all
1472
- steps and use `@pypi` to specify step-specific overrides.
1473
-
1474
- Parameters
1475
- ----------
1476
- packages : Dict[str, str], default: {}
1477
- Packages to use for this flow. The key is the name of the package
1478
- and the value is the version to use.
1479
- python : str, optional, default: None
1480
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1481
- that the version used will correspond to the version of the Python interpreter used to start the run.
1482
- """
1483
- ...
1484
-
1485
- @typing.overload
1486
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1487
- ...
1488
-
1489
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1490
- """
1491
- Specifies the PyPI packages for all steps of the flow.
1492
-
1493
- Use `@pypi_base` to set common packages required by all
1494
- steps and use `@pypi` to specify step-specific overrides.
1495
-
1496
- Parameters
1497
- ----------
1498
- packages : Dict[str, str], default: {}
1499
- Packages to use for this flow. The key is the name of the package
1500
- and the value is the version to use.
1501
- python : str, optional, default: None
1502
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1503
- that the version used will correspond to the version of the Python interpreter used to start the run.
1504
- """
1505
- ...
1506
-
1507
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1508
- """
1509
- Specifies what flows belong to the same project.
1510
-
1511
- A project-specific namespace is created for all flows that
1512
- use the same `@project(name)`.
1513
-
1514
-
1515
- Parameters
1516
- ----------
1517
- name : str
1518
- Project name. Make sure that the name is unique amongst all
1519
- projects that use the same production scheduler. The name may
1520
- contain only lowercase alphanumeric characters and underscores.
1521
-
1522
- branch : Optional[str], default None
1523
- The branch to use. If not specified, the branch is set to
1524
- `user.<username>` unless `production` is set to `True`. This can
1525
- also be set on the command line using `--branch` as a top-level option.
1526
- It is an error to specify `branch` in the decorator and on the command line.
1527
-
1528
- production : bool, default False
1529
- Whether or not the branch is the production branch. This can also be set on the
1530
- command line using `--production` as a top-level option. It is an error to specify
1531
- `production` in the decorator and on the command line.
1532
- The project branch name will be:
1533
- - if `branch` is specified:
1534
- - if `production` is True: `prod.<branch>`
1535
- - if `production` is False: `test.<branch>`
1536
- - if `branch` is not specified:
1537
- - if `production` is True: `prod`
1538
- - if `production` is False: `user.<username>`
1539
- """
1540
- ...
1541
-
1542
1542
  pkg_name: str
1543
1543