ob-metaflow-stubs 6.0.3.123__py2.py3-none-any.whl → 6.0.3.126__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (181) hide show
  1. metaflow-stubs/__init__.pyi +253 -248
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +9 -21
  4. metaflow-stubs/{plugins/package_cli.pyi → cli_components/__init__.pyi} +3 -3
  5. metaflow-stubs/cli_components/utils.pyi +35 -0
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +10 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +18 -11
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +4 -2
  20. metaflow-stubs/metaflow_current.pyi +58 -58
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  43. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  44. metaflow-stubs/{mflog → mf_extensions/outerbounds/plugins}/__init__.pyi +2 -2
  45. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +6 -0
  46. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +92 -0
  47. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  49. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  50. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +3 -2
  52. metaflow-stubs/multicore_utils.pyi +2 -2
  53. metaflow-stubs/parameters.pyi +33 -16
  54. metaflow-stubs/plugins/__init__.pyi +16 -15
  55. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  56. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  57. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  58. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  60. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  61. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  62. metaflow-stubs/plugins/argo/__init__.pyi +2 -5
  63. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  64. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  65. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -3
  66. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  68. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +31 -3
  69. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  71. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  72. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  74. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  75. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  76. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
  78. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -3
  79. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +3 -2
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  85. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  92. metaflow-stubs/plugins/cards/__init__.pyi +2 -3
  93. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  94. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  95. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  96. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  99. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  100. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  101. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  102. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  103. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  104. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  105. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  106. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  107. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  110. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  111. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  112. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  113. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  114. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  115. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  116. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  119. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  120. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  121. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
  122. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  123. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  124. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  125. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  126. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -5
  127. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  128. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  129. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  130. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  131. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  132. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  133. metaflow-stubs/plugins/perimeters.pyi +2 -2
  134. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  135. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  136. metaflow-stubs/plugins/pypi/conda_decorator.pyi +6 -2
  137. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  138. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  139. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  140. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  141. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  142. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  143. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  144. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  145. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/snowflake/__init__.pyi +14 -0
  147. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  148. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  149. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  150. metaflow-stubs/profilers/__init__.pyi +2 -2
  151. metaflow-stubs/pylint_wrapper.pyi +2 -2
  152. metaflow-stubs/runner/__init__.pyi +2 -2
  153. metaflow-stubs/runner/deployer.pyi +30 -30
  154. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  155. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  156. metaflow-stubs/runner/nbdeploy.pyi +4 -2
  157. metaflow-stubs/runner/nbrun.pyi +3 -3
  158. metaflow-stubs/runner/subprocess_manager.pyi +5 -2
  159. metaflow-stubs/runner/utils.pyi +8 -8
  160. metaflow-stubs/system/__init__.pyi +2 -2
  161. metaflow-stubs/system/system_logger.pyi +2 -2
  162. metaflow-stubs/system/system_monitor.pyi +2 -2
  163. metaflow-stubs/tagging_util.pyi +2 -2
  164. metaflow-stubs/tuple_util.pyi +2 -2
  165. metaflow-stubs/{plugins/aws/step_functions/production_token.pyi → user_configs/__init__.pyi} +5 -11
  166. metaflow-stubs/user_configs/config_decorators.pyi +253 -0
  167. metaflow-stubs/user_configs/config_options.pyi +82 -0
  168. metaflow-stubs/user_configs/config_parameters.pyi +217 -0
  169. {ob_metaflow_stubs-6.0.3.123.dist-info → ob_metaflow_stubs-6.0.3.126.dist-info}/METADATA +1 -1
  170. ob_metaflow_stubs-6.0.3.126.dist-info/RECORD +173 -0
  171. metaflow-stubs/clone_util.pyi +0 -14
  172. metaflow-stubs/mflog/mflog.pyi +0 -69
  173. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +0 -103
  174. metaflow-stubs/plugins/cards/card_cli.pyi +0 -146
  175. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +0 -33
  176. metaflow-stubs/plugins/logs_cli.pyi +0 -46
  177. metaflow-stubs/plugins/tag_cli.pyi +0 -19
  178. metaflow-stubs/procpoll.pyi +0 -53
  179. ob_metaflow_stubs-6.0.3.123.dist-info/RECORD +0 -174
  180. {ob_metaflow_stubs-6.0.3.123.dist-info → ob_metaflow_stubs-6.0.3.126.dist-info}/WHEEL +0 -0
  181. {ob_metaflow_stubs-6.0.3.123.dist-info → ob_metaflow_stubs-6.0.3.126.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.36.1+obcheckpoint(0.1.4);ob(v1) #
4
- # Generated on 2024-12-09T17:36:48.672361 #
3
+ # MF version: 2.12.39.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-12-17T01:10:55.898283 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -22,6 +22,7 @@ from .multicore_utils import parallel_map as parallel_map
22
22
  from . import metaflow_current as metaflow_current
23
23
  from .metaflow_current import current as current
24
24
  from . import parameters as parameters
25
+ from . import user_configs as user_configs
25
26
  from . import tagging_util as tagging_util
26
27
  from . import metadata_provider as metadata_provider
27
28
  from . import flowspec as flowspec
@@ -29,9 +30,13 @@ from .flowspec import FlowSpec as FlowSpec
29
30
  from .parameters import Parameter as Parameter
30
31
  from .parameters import JSONTypeClass as JSONTypeClass
31
32
  from .parameters import JSONType as JSONType
33
+ from .user_configs.config_parameters import Config as Config
34
+ from .user_configs.config_parameters import config_expr as config_expr
35
+ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
36
+ from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
37
+ from . import events as events
32
38
  from . import tuple_util as tuple_util
33
39
  from . import cards as cards
34
- from . import events as events
35
40
  from . import runner as runner
36
41
  from . import plugins as plugins
37
42
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -58,10 +63,10 @@ from .runner.nbdeploy import NBDeployer as NBDeployer
58
63
  from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.final_api import Checkpoint as Checkpoint
59
64
  from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures import load_model as load_model
60
65
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_aws_client as get_aws_client
66
+ from .mf_extensions.outerbounds.plugins.snowflake.snowflake import Snowflake as Snowflake
67
+ from . import cli_components as cli_components
61
68
  from . import system as system
62
69
  from . import pylint_wrapper as pylint_wrapper
63
- from . import procpoll as procpoll
64
- from . import clone_util as clone_util
65
70
  from . import cli as cli
66
71
  from . import profilers as profilers
67
72
 
@@ -140,33 +145,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
140
145
  """
141
146
  ...
142
147
 
143
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
144
- """
145
- Specifies that this step should execute on Kubernetes.
146
- """
147
- ...
148
-
149
- @typing.overload
150
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
- """
152
- Enables checkpointing for a step.
153
- """
154
- ...
155
-
156
- @typing.overload
157
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
158
- ...
159
-
160
- @typing.overload
161
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
162
- ...
163
-
164
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
165
- """
166
- Enables checkpointing for a step.
167
- """
168
- ...
169
-
170
148
  @typing.overload
171
149
  def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
172
150
  """
@@ -199,85 +177,135 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
199
177
  ...
200
178
 
201
179
  @typing.overload
202
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
180
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
203
181
  """
204
- Specifies environment variables to be set prior to the execution of a step.
182
+ Decorator prototype for all step decorators. This function gets specialized
183
+ and imported for all decorators types by _import_plugin_decorators().
205
184
  """
206
185
  ...
207
186
 
208
187
  @typing.overload
209
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
188
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
189
+ ...
190
+
191
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
192
+ """
193
+ Decorator prototype for all step decorators. This function gets specialized
194
+ and imported for all decorators types by _import_plugin_decorators().
195
+ """
210
196
  ...
211
197
 
212
198
  @typing.overload
213
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
199
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
200
+ """
201
+ Specifies secrets to be retrieved and injected as environment variables prior to
202
+ the execution of a step.
203
+ """
214
204
  ...
215
205
 
216
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
206
+ @typing.overload
207
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
208
+ ...
209
+
210
+ @typing.overload
211
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
212
+ ...
213
+
214
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
217
215
  """
218
- Specifies environment variables to be set prior to the execution of a step.
216
+ Specifies secrets to be retrieved and injected as environment variables prior to
217
+ the execution of a step.
219
218
  """
220
219
  ...
221
220
 
222
221
  @typing.overload
223
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
222
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
224
223
  """
225
- Specifies the PyPI packages for the step.
224
+ Specifies a timeout for your step.
226
225
 
227
- Information in this decorator will augment any
228
- attributes set in the `@pyi_base` flow-level decorator. Hence,
229
- you can use `@pypi_base` to set packages required by all
230
- steps and use `@pypi` to specify step-specific overrides.
226
+ This decorator is useful if this step may hang indefinitely.
227
+
228
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
229
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
230
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
231
+
232
+ Note that all the values specified in parameters are added together so if you specify
233
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
231
234
  """
232
235
  ...
233
236
 
234
237
  @typing.overload
235
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
238
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
236
239
  ...
237
240
 
238
241
  @typing.overload
239
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
242
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
240
243
  ...
241
244
 
242
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
245
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
243
246
  """
244
- Specifies the PyPI packages for the step.
247
+ Specifies a timeout for your step.
245
248
 
246
- Information in this decorator will augment any
247
- attributes set in the `@pyi_base` flow-level decorator. Hence,
248
- you can use `@pypi_base` to set packages required by all
249
- steps and use `@pypi` to specify step-specific overrides.
249
+ This decorator is useful if this step may hang indefinitely.
250
+
251
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
252
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
253
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
254
+
255
+ Note that all the values specified in parameters are added together so if you specify
256
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
250
257
  """
251
258
  ...
252
259
 
253
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
260
+ @typing.overload
261
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
254
262
  """
255
- Decorator that helps cache, version and store models/datasets from huggingface hub.
263
+ Enables checkpointing for a step.
256
264
  """
257
265
  ...
258
266
 
259
267
  @typing.overload
260
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
268
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
269
+ ...
270
+
271
+ @typing.overload
272
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
273
+ ...
274
+
275
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
261
276
  """
262
- Creates a human-readable report, a Metaflow Card, after this step completes.
277
+ Enables checkpointing for a step.
278
+ """
279
+ ...
280
+
281
+ @typing.overload
282
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
283
+ """
284
+ Specifies that the step will success under all circumstances.
263
285
 
264
- Note that you may add multiple `@card` decorators in a step with different parameters.
286
+ The decorator will create an optional artifact, specified by `var`, which
287
+ contains the exception raised. You can use it to detect the presence
288
+ of errors, indicating that all happy-path artifacts produced by the step
289
+ are missing.
265
290
  """
266
291
  ...
267
292
 
268
293
  @typing.overload
269
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
294
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
270
295
  ...
271
296
 
272
297
  @typing.overload
273
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
298
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
274
299
  ...
275
300
 
276
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
301
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
277
302
  """
278
- Creates a human-readable report, a Metaflow Card, after this step completes.
303
+ Specifies that the step will success under all circumstances.
279
304
 
280
- Note that you may add multiple `@card` decorators in a step with different parameters.
305
+ The decorator will create an optional artifact, specified by `var`, which
306
+ contains the exception raised. You can use it to detect the presence
307
+ of errors, indicating that all happy-path artifacts produced by the step
308
+ are missing.
281
309
  """
282
310
  ...
283
311
 
@@ -330,115 +358,105 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
330
358
  """
331
359
  ...
332
360
 
361
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
362
+ """
363
+ Specifies that this step should execute on Kubernetes.
364
+ """
365
+ ...
366
+
333
367
  @typing.overload
334
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
368
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
335
369
  """
336
- Decorator prototype for all step decorators. This function gets specialized
337
- and imported for all decorators types by _import_plugin_decorators().
370
+ Specifies the PyPI packages for the step.
371
+
372
+ Information in this decorator will augment any
373
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
374
+ you can use `@pypi_base` to set packages required by all
375
+ steps and use `@pypi` to specify step-specific overrides.
338
376
  """
339
377
  ...
340
378
 
341
379
  @typing.overload
342
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
380
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
343
381
  ...
344
382
 
345
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
383
+ @typing.overload
384
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
385
+ ...
386
+
387
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
346
388
  """
347
- Decorator prototype for all step decorators. This function gets specialized
348
- and imported for all decorators types by _import_plugin_decorators().
389
+ Specifies the PyPI packages for the step.
390
+
391
+ Information in this decorator will augment any
392
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
393
+ you can use `@pypi_base` to set packages required by all
394
+ steps and use `@pypi` to specify step-specific overrides.
395
+ """
396
+ ...
397
+
398
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
399
+ """
400
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
349
401
  """
350
402
  ...
351
403
 
352
404
  @typing.overload
353
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
405
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
354
406
  """
355
- Specifies that the step will success under all circumstances.
356
-
357
- The decorator will create an optional artifact, specified by `var`, which
358
- contains the exception raised. You can use it to detect the presence
359
- of errors, indicating that all happy-path artifacts produced by the step
360
- are missing.
407
+ Enables loading / saving of models within a step.
361
408
  """
362
409
  ...
363
410
 
364
411
  @typing.overload
365
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
412
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
366
413
  ...
367
414
 
368
415
  @typing.overload
369
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
416
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
370
417
  ...
371
418
 
372
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
419
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
373
420
  """
374
- Specifies that the step will success under all circumstances.
375
-
376
- The decorator will create an optional artifact, specified by `var`, which
377
- contains the exception raised. You can use it to detect the presence
378
- of errors, indicating that all happy-path artifacts produced by the step
379
- are missing.
421
+ Enables loading / saving of models within a step.
380
422
  """
381
423
  ...
382
424
 
383
425
  @typing.overload
384
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
426
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
385
427
  """
386
- Specifies a timeout for your step.
387
-
388
- This decorator is useful if this step may hang indefinitely.
389
-
390
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
391
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
392
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
393
-
394
- Note that all the values specified in parameters are added together so if you specify
395
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
428
+ Specifies environment variables to be set prior to the execution of a step.
396
429
  """
397
430
  ...
398
431
 
399
432
  @typing.overload
400
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
433
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
401
434
  ...
402
435
 
403
436
  @typing.overload
404
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
437
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
405
438
  ...
406
439
 
407
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
440
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
408
441
  """
409
- Specifies a timeout for your step.
410
-
411
- This decorator is useful if this step may hang indefinitely.
412
-
413
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
414
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
415
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
416
-
417
- Note that all the values specified in parameters are added together so if you specify
418
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
442
+ Specifies environment variables to be set prior to the execution of a step.
419
443
  """
420
444
  ...
421
445
 
422
446
  @typing.overload
423
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
447
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
424
448
  """
425
- Specifies secrets to be retrieved and injected as environment variables prior to
426
- the execution of a step.
449
+ Internal decorator to support Fast bakery
427
450
  """
428
451
  ...
429
452
 
430
453
  @typing.overload
431
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
432
- ...
433
-
434
- @typing.overload
435
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
454
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
436
455
  ...
437
456
 
438
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
457
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
439
458
  """
440
- Specifies secrets to be retrieved and injected as environment variables prior to
441
- the execution of a step.
459
+ Internal decorator to support Fast bakery
442
460
  """
443
461
  ...
444
462
 
@@ -482,40 +500,60 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
482
500
  ...
483
501
 
484
502
  @typing.overload
485
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
503
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
486
504
  """
487
- Internal decorator to support Fast bakery
505
+ Creates a human-readable report, a Metaflow Card, after this step completes.
506
+
507
+ Note that you may add multiple `@card` decorators in a step with different parameters.
488
508
  """
489
509
  ...
490
510
 
491
511
  @typing.overload
492
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
512
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
493
513
  ...
494
514
 
495
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
515
+ @typing.overload
516
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
517
+ ...
518
+
519
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
496
520
  """
497
- Internal decorator to support Fast bakery
521
+ Creates a human-readable report, a Metaflow Card, after this step completes.
522
+
523
+ Note that you may add multiple `@card` decorators in a step with different parameters.
498
524
  """
499
525
  ...
500
526
 
501
- @typing.overload
502
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
527
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
503
528
  """
504
- Enables loading / saving of models within a step.
529
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
530
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
531
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
532
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
533
+ starts only after all sensors finish.
505
534
  """
506
535
  ...
507
536
 
508
537
  @typing.overload
509
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
538
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
539
+ """
540
+ Specifies the PyPI packages for all steps of the flow.
541
+
542
+ Use `@pypi_base` to set common packages required by all
543
+ steps and use `@pypi` to specify step-specific overrides.
544
+ """
510
545
  ...
511
546
 
512
547
  @typing.overload
513
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
548
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
514
549
  ...
515
550
 
516
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
551
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
517
552
  """
518
- Enables loading / saving of models within a step.
553
+ Specifies the PyPI packages for all steps of the flow.
554
+
555
+ Use `@pypi_base` to set common packages required by all
556
+ steps and use `@pypi` to specify step-specific overrides.
519
557
  """
520
558
  ...
521
559
 
@@ -544,115 +582,53 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
544
582
  ...
545
583
 
546
584
  @typing.overload
547
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
585
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
548
586
  """
549
- Specifies the event(s) that this flow depends on.
550
-
551
- ```
552
- @trigger(event='foo')
553
- ```
554
- or
555
- ```
556
- @trigger(events=['foo', 'bar'])
557
- ```
558
-
559
- Additionally, you can specify the parameter mappings
560
- to map event payload to Metaflow parameters for the flow.
561
- ```
562
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
563
- ```
564
- or
565
- ```
566
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
567
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
568
- ```
569
-
570
- 'parameters' can also be a list of strings and tuples like so:
571
- ```
572
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
573
- ```
574
- This is equivalent to:
575
- ```
576
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
577
- ```
587
+ Specifies the times when the flow should be run when running on a
588
+ production scheduler.
578
589
  """
579
590
  ...
580
591
 
581
592
  @typing.overload
582
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
583
- ...
584
-
585
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
586
- """
587
- Specifies the event(s) that this flow depends on.
588
-
589
- ```
590
- @trigger(event='foo')
591
- ```
592
- or
593
- ```
594
- @trigger(events=['foo', 'bar'])
595
- ```
596
-
597
- Additionally, you can specify the parameter mappings
598
- to map event payload to Metaflow parameters for the flow.
599
- ```
600
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
601
- ```
602
- or
603
- ```
604
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
605
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
606
- ```
607
-
608
- 'parameters' can also be a list of strings and tuples like so:
609
- ```
610
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
611
- ```
612
- This is equivalent to:
613
- ```
614
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
615
- ```
616
- """
593
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
617
594
  ...
618
595
 
619
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
596
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
620
597
  """
621
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
622
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
623
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
624
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
625
- starts only after all sensors finish.
598
+ Specifies the times when the flow should be run when running on a
599
+ production scheduler.
626
600
  """
627
601
  ...
628
602
 
629
603
  @typing.overload
630
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
604
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
631
605
  """
632
- Specifies the PyPI packages for all steps of the flow.
606
+ Specifies the Conda environment for all steps of the flow.
633
607
 
634
- Use `@pypi_base` to set common packages required by all
635
- steps and use `@pypi` to specify step-specific overrides.
608
+ Use `@conda_base` to set common libraries required by all
609
+ steps and use `@conda` to specify step-specific additions.
636
610
  """
637
611
  ...
638
612
 
639
613
  @typing.overload
640
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
614
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
641
615
  ...
642
616
 
643
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
617
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
644
618
  """
645
- Specifies the PyPI packages for all steps of the flow.
619
+ Specifies the Conda environment for all steps of the flow.
646
620
 
647
- Use `@pypi_base` to set common packages required by all
648
- steps and use `@pypi` to specify step-specific overrides.
621
+ Use `@conda_base` to set common libraries required by all
622
+ steps and use `@conda` to specify step-specific additions.
649
623
  """
650
624
  ...
651
625
 
652
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
626
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
653
627
  """
654
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
655
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
628
+ Specifies what flows belong to the same project.
629
+
630
+ A project-specific namespace is created for all flows that
631
+ use the same `@project(name)`.
656
632
  """
657
633
  ...
658
634
 
@@ -737,54 +713,83 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
737
713
  """
738
714
  ...
739
715
 
740
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
741
- """
742
- Specifies what flows belong to the same project.
743
-
744
- A project-specific namespace is created for all flows that
745
- use the same `@project(name)`.
746
- """
747
- ...
748
-
749
716
  @typing.overload
750
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
717
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
751
718
  """
752
- Specifies the Conda environment for all steps of the flow.
719
+ Specifies the event(s) that this flow depends on.
753
720
 
754
- Use `@conda_base` to set common libraries required by all
755
- steps and use `@conda` to specify step-specific additions.
721
+ ```
722
+ @trigger(event='foo')
723
+ ```
724
+ or
725
+ ```
726
+ @trigger(events=['foo', 'bar'])
727
+ ```
728
+
729
+ Additionally, you can specify the parameter mappings
730
+ to map event payload to Metaflow parameters for the flow.
731
+ ```
732
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
733
+ ```
734
+ or
735
+ ```
736
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
737
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
738
+ ```
739
+
740
+ 'parameters' can also be a list of strings and tuples like so:
741
+ ```
742
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
743
+ ```
744
+ This is equivalent to:
745
+ ```
746
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
747
+ ```
756
748
  """
757
749
  ...
758
750
 
759
751
  @typing.overload
760
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
752
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
761
753
  ...
762
754
 
763
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
755
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
764
756
  """
765
- Specifies the Conda environment for all steps of the flow.
757
+ Specifies the event(s) that this flow depends on.
766
758
 
767
- Use `@conda_base` to set common libraries required by all
768
- steps and use `@conda` to specify step-specific additions.
769
- """
770
- ...
771
-
772
- @typing.overload
773
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
774
- """
775
- Specifies the times when the flow should be run when running on a
776
- production scheduler.
759
+ ```
760
+ @trigger(event='foo')
761
+ ```
762
+ or
763
+ ```
764
+ @trigger(events=['foo', 'bar'])
765
+ ```
766
+
767
+ Additionally, you can specify the parameter mappings
768
+ to map event payload to Metaflow parameters for the flow.
769
+ ```
770
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
771
+ ```
772
+ or
773
+ ```
774
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
775
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
776
+ ```
777
+
778
+ 'parameters' can also be a list of strings and tuples like so:
779
+ ```
780
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
781
+ ```
782
+ This is equivalent to:
783
+ ```
784
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
785
+ ```
777
786
  """
778
787
  ...
779
788
 
780
- @typing.overload
781
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
782
- ...
783
-
784
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
789
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
785
790
  """
786
- Specifies the times when the flow should be run when running on a
787
- production scheduler.
791
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
792
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
788
793
  """
789
794
  ...
790
795