ob-metaflow-stubs 4.7__py2.py3-none-any.whl → 4.9__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. metaflow-stubs/__init__.pyi +668 -619
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +3 -3
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +35 -12
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +6 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +4 -4
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +38 -5
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +269 -4
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +229 -4
  37. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +292 -0
  38. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  40. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  46. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +23 -3
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +260 -0
  58. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  61. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  62. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  63. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  64. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  65. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
  67. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  68. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  73. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  80. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  81. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  84. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  85. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  86. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/__init__.pyi +4 -4
  88. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/s3/__init__.pyi +4 -4
  90. metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
  91. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  92. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  93. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  94. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  95. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  96. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -2
  99. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  104. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  106. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +18 -10
  107. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +12 -12
  108. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +14 -31
  110. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +82 -28
  111. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  112. metaflow-stubs/plugins/package_cli.pyi +2 -2
  113. metaflow-stubs/plugins/parallel_decorator.pyi +47 -2
  114. metaflow-stubs/plugins/perimeters.pyi +2 -2
  115. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  117. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  119. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  121. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  122. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  124. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  125. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  126. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +3 -3
  127. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  128. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  129. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +26 -3
  130. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  131. metaflow-stubs/procpoll.pyi +2 -2
  132. metaflow-stubs/profilers/__init__.pyi +2 -2
  133. metaflow-stubs/pylint_wrapper.pyi +2 -2
  134. metaflow-stubs/runner/__init__.pyi +2 -2
  135. metaflow-stubs/runner/deployer.pyi +354 -0
  136. metaflow-stubs/runner/metaflow_runner.pyi +11 -11
  137. metaflow-stubs/runner/nbdeploy.pyi +54 -0
  138. metaflow-stubs/runner/nbrun.pyi +7 -7
  139. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  140. metaflow-stubs/runner/utils.pyi +25 -0
  141. metaflow-stubs/system/__init__.pyi +3 -3
  142. metaflow-stubs/system/system_logger.pyi +2 -2
  143. metaflow-stubs/system/system_monitor.pyi +2 -2
  144. metaflow-stubs/tagging_util.pyi +2 -2
  145. metaflow-stubs/tuple_util.pyi +2 -2
  146. {ob_metaflow_stubs-4.7.dist-info → ob_metaflow_stubs-4.9.dist-info}/METADATA +1 -1
  147. ob_metaflow_stubs-4.9.dist-info/RECORD +150 -0
  148. ob_metaflow_stubs-4.7.dist-info/RECORD +0 -145
  149. {ob_metaflow_stubs-4.7.dist-info → ob_metaflow_stubs-4.9.dist-info}/WHEEL +0 -0
  150. {ob_metaflow_stubs-4.7.dist-info → ob_metaflow_stubs-4.9.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.7.2+ob(v1) #
4
- # Generated on 2024-07-16T17:10:32.274678 #
3
+ # MF version: 2.12.10.1+ob(v1) #
4
+ # Generated on 2024-07-31T05:43:20.962506 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
- import datetime
11
+ import metaflow.events
12
+ import metaflow.flowspec
13
+ import metaflow.metaflow_current
14
+ import metaflow._vendor.click.types
13
15
  import metaflow.plugins.datatools.s3.s3
14
- import metaflow.parameters
16
+ import metaflow.client.core
15
17
  import metaflow.datastore.inputs
18
+ import datetime
16
19
  import metaflow.runner.metaflow_runner
20
+ import typing
21
+ import metaflow.parameters
17
22
  import io
18
- import metaflow.metaflow_current
19
- import metaflow.flowspec
20
- import metaflow._vendor.click.types
21
- import metaflow.client.core
22
- import metaflow.events
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -728,279 +728,141 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
728
728
  ...
729
729
 
730
730
  @typing.overload
731
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
731
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
732
  """
733
- Specifies the number of times the task corresponding
734
- to a step needs to be retried.
735
-
736
- This decorator is useful for handling transient errors, such as networking issues.
737
- If your task contains operations that can't be retried safely, e.g. database updates,
738
- it is advisable to annotate it with `@retry(times=0)`.
739
-
740
- This can be used in conjunction with the `@catch` decorator. The `@catch`
741
- decorator will execute a no-op task after all retries have been exhausted,
742
- ensuring that the flow execution can continue.
733
+ Specifies environment variables to be set prior to the execution of a step.
743
734
 
744
735
  Parameters
745
736
  ----------
746
- times : int, default 3
747
- Number of times to retry this task.
748
- minutes_between_retries : int, default 2
749
- Number of minutes between retries.
737
+ vars : Dict[str, str], default {}
738
+ Dictionary of environment variables to set.
750
739
  """
751
740
  ...
752
741
 
753
742
  @typing.overload
754
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
743
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
755
744
  ...
756
745
 
757
746
  @typing.overload
758
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
747
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
759
748
  ...
760
749
 
761
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
750
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
762
751
  """
763
- Specifies the number of times the task corresponding
764
- to a step needs to be retried.
765
-
766
- This decorator is useful for handling transient errors, such as networking issues.
767
- If your task contains operations that can't be retried safely, e.g. database updates,
768
- it is advisable to annotate it with `@retry(times=0)`.
769
-
770
- This can be used in conjunction with the `@catch` decorator. The `@catch`
771
- decorator will execute a no-op task after all retries have been exhausted,
772
- ensuring that the flow execution can continue.
752
+ Specifies environment variables to be set prior to the execution of a step.
773
753
 
774
754
  Parameters
775
755
  ----------
776
- times : int, default 3
777
- Number of times to retry this task.
778
- minutes_between_retries : int, default 2
779
- Number of minutes between retries.
756
+ vars : Dict[str, str], default {}
757
+ Dictionary of environment variables to set.
780
758
  """
781
759
  ...
782
760
 
783
761
  @typing.overload
784
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
762
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
785
763
  """
786
- Specifies the resources needed when executing this step.
787
-
788
- Use `@resources` to specify the resource requirements
789
- independently of the specific compute layer (`@batch`, `@kubernetes`).
764
+ Specifies the Conda environment for the step.
790
765
 
791
- You can choose the compute layer on the command line by executing e.g.
792
- ```
793
- python myflow.py run --with batch
794
- ```
795
- or
796
- ```
797
- python myflow.py run --with kubernetes
798
- ```
799
- which executes the flow on the desired system using the
800
- requirements specified in `@resources`.
766
+ Information in this decorator will augment any
767
+ attributes set in the `@conda_base` flow-level decorator. Hence,
768
+ you can use `@conda_base` to set packages required by all
769
+ steps and use `@conda` to specify step-specific overrides.
801
770
 
802
771
  Parameters
803
772
  ----------
804
- cpu : int, default 1
805
- Number of CPUs required for this step.
806
- gpu : int, default 0
807
- Number of GPUs required for this step.
808
- disk : int, optional, default None
809
- Disk size (in MB) required for this step. Only applies on Kubernetes.
810
- memory : int, default 4096
811
- Memory size (in MB) required for this step.
812
- shared_memory : int, optional, default None
813
- The value for the size (in MiB) of the /dev/shm volume for this step.
814
- This parameter maps to the `--shm-size` option in Docker.
773
+ packages : Dict[str, str], default {}
774
+ Packages to use for this step. The key is the name of the package
775
+ and the value is the version to use.
776
+ libraries : Dict[str, str], default {}
777
+ Supported for backward compatibility. When used with packages, packages will take precedence.
778
+ python : str, optional, default None
779
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
780
+ that the version used will correspond to the version of the Python interpreter used to start the run.
781
+ disabled : bool, default False
782
+ If set to True, disables @conda.
815
783
  """
816
784
  ...
817
785
 
818
786
  @typing.overload
819
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
787
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
820
788
  ...
821
789
 
822
790
  @typing.overload
823
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
791
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
824
792
  ...
825
793
 
826
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
794
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
827
795
  """
828
- Specifies the resources needed when executing this step.
829
-
830
- Use `@resources` to specify the resource requirements
831
- independently of the specific compute layer (`@batch`, `@kubernetes`).
796
+ Specifies the Conda environment for the step.
832
797
 
833
- You can choose the compute layer on the command line by executing e.g.
834
- ```
835
- python myflow.py run --with batch
836
- ```
837
- or
838
- ```
839
- python myflow.py run --with kubernetes
840
- ```
841
- which executes the flow on the desired system using the
842
- requirements specified in `@resources`.
798
+ Information in this decorator will augment any
799
+ attributes set in the `@conda_base` flow-level decorator. Hence,
800
+ you can use `@conda_base` to set packages required by all
801
+ steps and use `@conda` to specify step-specific overrides.
843
802
 
844
803
  Parameters
845
804
  ----------
846
- cpu : int, default 1
847
- Number of CPUs required for this step.
848
- gpu : int, default 0
849
- Number of GPUs required for this step.
850
- disk : int, optional, default None
851
- Disk size (in MB) required for this step. Only applies on Kubernetes.
852
- memory : int, default 4096
853
- Memory size (in MB) required for this step.
854
- shared_memory : int, optional, default None
855
- The value for the size (in MiB) of the /dev/shm volume for this step.
856
- This parameter maps to the `--shm-size` option in Docker.
805
+ packages : Dict[str, str], default {}
806
+ Packages to use for this step. The key is the name of the package
807
+ and the value is the version to use.
808
+ libraries : Dict[str, str], default {}
809
+ Supported for backward compatibility. When used with packages, packages will take precedence.
810
+ python : str, optional, default None
811
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
812
+ that the version used will correspond to the version of the Python interpreter used to start the run.
813
+ disabled : bool, default False
814
+ If set to True, disables @conda.
857
815
  """
858
816
  ...
859
817
 
860
818
  @typing.overload
861
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
819
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
862
820
  """
863
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
821
+ Creates a human-readable report, a Metaflow Card, after this step completes.
822
+
823
+ Note that you may add multiple `@card` decorators in a step with different parameters.
864
824
 
865
825
  Parameters
866
826
  ----------
867
- cpu : int, default 1
868
- Number of CPUs required for this step. If `@resources` is
869
- also present, the maximum value from all decorators is used.
870
- gpu : int, default 0
871
- Number of GPUs required for this step. If `@resources` is
872
- also present, the maximum value from all decorators is used.
873
- memory : int, default 4096
874
- Memory size (in MB) required for this step. If
875
- `@resources` is also present, the maximum value from all decorators is
876
- used.
877
- image : str, optional, default None
878
- Docker image to use when launching on AWS Batch. If not specified, and
879
- METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
880
- not, a default Docker image mapping to the current version of Python is used.
881
- queue : str, default METAFLOW_BATCH_JOB_QUEUE
882
- AWS Batch Job Queue to submit the job to.
883
- iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
884
- AWS IAM role that AWS Batch container uses to access AWS cloud resources.
885
- execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
886
- AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
887
- (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
888
- shared_memory : int, optional, default None
889
- The value for the size (in MiB) of the /dev/shm volume for this step.
890
- This parameter maps to the `--shm-size` option in Docker.
891
- max_swap : int, optional, default None
892
- The total amount of swap memory (in MiB) a container can use for this
893
- step. This parameter is translated to the `--memory-swap` option in
894
- Docker where the value is the sum of the container memory plus the
895
- `max_swap` value.
896
- swappiness : int, optional, default None
897
- This allows you to tune memory swappiness behavior for this step.
898
- A swappiness value of 0 causes swapping not to happen unless absolutely
899
- necessary. A swappiness value of 100 causes pages to be swapped very
900
- aggressively. Accepted values are whole numbers between 0 and 100.
901
- use_tmpfs : bool, default False
902
- This enables an explicit tmpfs mount for this step. Note that tmpfs is
903
- not available on Fargate compute environments
904
- tmpfs_tempdir : bool, default True
905
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
906
- tmpfs_size : int, optional, default None
907
- The value for the size (in MiB) of the tmpfs mount for this step.
908
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
909
- memory allocated for this step.
910
- tmpfs_path : str, optional, default None
911
- Path to tmpfs mount for this step. Defaults to /metaflow_temp.
912
- inferentia : int, default 0
913
- Number of Inferentia chips required for this step.
914
- trainium : int, default None
915
- Alias for inferentia. Use only one of the two.
916
- efa : int, default 0
917
- Number of elastic fabric adapter network devices to attach to container
918
- ephemeral_storage : int, default None
919
- The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
920
- This is only relevant for Fargate compute environments
921
- log_driver: str, optional, default None
922
- The log driver to use for the Amazon ECS container.
923
- log_options: List[str], optional, default None
924
- List of strings containing options for the chosen log driver. The configurable values
925
- depend on the `log driver` chosen. Validation of these options is not supported yet.
926
- Example: [`awslogs-group:aws/batch/job`]
827
+ type : str, default 'default'
828
+ Card type.
829
+ id : str, optional, default None
830
+ If multiple cards are present, use this id to identify this card.
831
+ options : Dict[str, Any], default {}
832
+ Options passed to the card. The contents depend on the card type.
833
+ timeout : int, default 45
834
+ Interrupt reporting if it takes more than this many seconds.
835
+
836
+
927
837
  """
928
838
  ...
929
839
 
930
840
  @typing.overload
931
- def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
841
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
932
842
  ...
933
843
 
934
844
  @typing.overload
935
- def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
845
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
936
846
  ...
937
847
 
938
- def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
848
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
939
849
  """
940
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
850
+ Creates a human-readable report, a Metaflow Card, after this step completes.
851
+
852
+ Note that you may add multiple `@card` decorators in a step with different parameters.
941
853
 
942
854
  Parameters
943
855
  ----------
944
- cpu : int, default 1
945
- Number of CPUs required for this step. If `@resources` is
946
- also present, the maximum value from all decorators is used.
947
- gpu : int, default 0
948
- Number of GPUs required for this step. If `@resources` is
949
- also present, the maximum value from all decorators is used.
950
- memory : int, default 4096
951
- Memory size (in MB) required for this step. If
952
- `@resources` is also present, the maximum value from all decorators is
953
- used.
954
- image : str, optional, default None
955
- Docker image to use when launching on AWS Batch. If not specified, and
956
- METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
957
- not, a default Docker image mapping to the current version of Python is used.
958
- queue : str, default METAFLOW_BATCH_JOB_QUEUE
959
- AWS Batch Job Queue to submit the job to.
960
- iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
961
- AWS IAM role that AWS Batch container uses to access AWS cloud resources.
962
- execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
963
- AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
964
- (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
965
- shared_memory : int, optional, default None
966
- The value for the size (in MiB) of the /dev/shm volume for this step.
967
- This parameter maps to the `--shm-size` option in Docker.
968
- max_swap : int, optional, default None
969
- The total amount of swap memory (in MiB) a container can use for this
970
- step. This parameter is translated to the `--memory-swap` option in
971
- Docker where the value is the sum of the container memory plus the
972
- `max_swap` value.
973
- swappiness : int, optional, default None
974
- This allows you to tune memory swappiness behavior for this step.
975
- A swappiness value of 0 causes swapping not to happen unless absolutely
976
- necessary. A swappiness value of 100 causes pages to be swapped very
977
- aggressively. Accepted values are whole numbers between 0 and 100.
978
- use_tmpfs : bool, default False
979
- This enables an explicit tmpfs mount for this step. Note that tmpfs is
980
- not available on Fargate compute environments
981
- tmpfs_tempdir : bool, default True
982
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
983
- tmpfs_size : int, optional, default None
984
- The value for the size (in MiB) of the tmpfs mount for this step.
985
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
986
- memory allocated for this step.
987
- tmpfs_path : str, optional, default None
988
- Path to tmpfs mount for this step. Defaults to /metaflow_temp.
989
- inferentia : int, default 0
990
- Number of Inferentia chips required for this step.
991
- trainium : int, default None
992
- Alias for inferentia. Use only one of the two.
993
- efa : int, default 0
994
- Number of elastic fabric adapter network devices to attach to container
995
- ephemeral_storage : int, default None
996
- The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
997
- This is only relevant for Fargate compute environments
998
- log_driver: str, optional, default None
999
- The log driver to use for the Amazon ECS container.
1000
- log_options: List[str], optional, default None
1001
- List of strings containing options for the chosen log driver. The configurable values
1002
- depend on the `log driver` chosen. Validation of these options is not supported yet.
1003
- Example: [`awslogs-group:aws/batch/job`]
856
+ type : str, default 'default'
857
+ Card type.
858
+ id : str, optional, default None
859
+ If multiple cards are present, use this id to identify this card.
860
+ options : Dict[str, Any], default {}
861
+ Options passed to the card. The contents depend on the card type.
862
+ timeout : int, default 45
863
+ Interrupt reporting if it takes more than this many seconds.
864
+
865
+
1004
866
  """
1005
867
  ...
1006
868
 
@@ -1061,6 +923,55 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1061
923
  """
1062
924
  ...
1063
925
 
926
+ @typing.overload
927
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
928
+ """
929
+ Specifies that the step will success under all circumstances.
930
+
931
+ The decorator will create an optional artifact, specified by `var`, which
932
+ contains the exception raised. You can use it to detect the presence
933
+ of errors, indicating that all happy-path artifacts produced by the step
934
+ are missing.
935
+
936
+ Parameters
937
+ ----------
938
+ var : str, optional, default None
939
+ Name of the artifact in which to store the caught exception.
940
+ If not specified, the exception is not stored.
941
+ print_exception : bool, default True
942
+ Determines whether or not the exception is printed to
943
+ stdout when caught.
944
+ """
945
+ ...
946
+
947
+ @typing.overload
948
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
949
+ ...
950
+
951
+ @typing.overload
952
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
953
+ ...
954
+
955
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
956
+ """
957
+ Specifies that the step will success under all circumstances.
958
+
959
+ The decorator will create an optional artifact, specified by `var`, which
960
+ contains the exception raised. You can use it to detect the presence
961
+ of errors, indicating that all happy-path artifacts produced by the step
962
+ are missing.
963
+
964
+ Parameters
965
+ ----------
966
+ var : str, optional, default None
967
+ Name of the artifact in which to store the caught exception.
968
+ If not specified, the exception is not stored.
969
+ print_exception : bool, default True
970
+ Determines whether or not the exception is printed to
971
+ stdout when caught.
972
+ """
973
+ ...
974
+
1064
975
  @typing.overload
1065
976
  def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1066
977
  """
@@ -1094,6 +1005,78 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1094
1005
  """
1095
1006
  ...
1096
1007
 
1008
+ @typing.overload
1009
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1010
+ """
1011
+ Decorator prototype for all step decorators. This function gets specialized
1012
+ and imported for all decorators types by _import_plugin_decorators().
1013
+ """
1014
+ ...
1015
+
1016
+ @typing.overload
1017
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1018
+ ...
1019
+
1020
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1021
+ """
1022
+ Decorator prototype for all step decorators. This function gets specialized
1023
+ and imported for all decorators types by _import_plugin_decorators().
1024
+ """
1025
+ ...
1026
+
1027
+ @typing.overload
1028
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1029
+ """
1030
+ Specifies the number of times the task corresponding
1031
+ to a step needs to be retried.
1032
+
1033
+ This decorator is useful for handling transient errors, such as networking issues.
1034
+ If your task contains operations that can't be retried safely, e.g. database updates,
1035
+ it is advisable to annotate it with `@retry(times=0)`.
1036
+
1037
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1038
+ decorator will execute a no-op task after all retries have been exhausted,
1039
+ ensuring that the flow execution can continue.
1040
+
1041
+ Parameters
1042
+ ----------
1043
+ times : int, default 3
1044
+ Number of times to retry this task.
1045
+ minutes_between_retries : int, default 2
1046
+ Number of minutes between retries.
1047
+ """
1048
+ ...
1049
+
1050
+ @typing.overload
1051
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1052
+ ...
1053
+
1054
+ @typing.overload
1055
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1056
+ ...
1057
+
1058
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1059
+ """
1060
+ Specifies the number of times the task corresponding
1061
+ to a step needs to be retried.
1062
+
1063
+ This decorator is useful for handling transient errors, such as networking issues.
1064
+ If your task contains operations that can't be retried safely, e.g. database updates,
1065
+ it is advisable to annotate it with `@retry(times=0)`.
1066
+
1067
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1068
+ decorator will execute a no-op task after all retries have been exhausted,
1069
+ ensuring that the flow execution can continue.
1070
+
1071
+ Parameters
1072
+ ----------
1073
+ times : int, default 3
1074
+ Number of times to retry this task.
1075
+ minutes_between_retries : int, default 2
1076
+ Number of minutes between retries.
1077
+ """
1078
+ ...
1079
+
1097
1080
  def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1098
1081
  """
1099
1082
  Specifies that this step should execute on Kubernetes.
@@ -1154,108 +1137,149 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1154
1137
  ...
1155
1138
 
1156
1139
  @typing.overload
1157
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1140
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1158
1141
  """
1159
- Specifies the Conda environment for the step.
1160
-
1161
- Information in this decorator will augment any
1162
- attributes set in the `@conda_base` flow-level decorator. Hence,
1163
- you can use `@conda_base` to set packages required by all
1164
- steps and use `@conda` to specify step-specific overrides.
1142
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
1165
1143
 
1166
1144
  Parameters
1167
1145
  ----------
1168
- packages : Dict[str, str], default {}
1169
- Packages to use for this step. The key is the name of the package
1170
- and the value is the version to use.
1171
- libraries : Dict[str, str], default {}
1172
- Supported for backward compatibility. When used with packages, packages will take precedence.
1173
- python : str, optional, default None
1174
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1175
- that the version used will correspond to the version of the Python interpreter used to start the run.
1176
- disabled : bool, default False
1177
- If set to True, disables @conda.
1146
+ cpu : int, default 1
1147
+ Number of CPUs required for this step. If `@resources` is
1148
+ also present, the maximum value from all decorators is used.
1149
+ gpu : int, default 0
1150
+ Number of GPUs required for this step. If `@resources` is
1151
+ also present, the maximum value from all decorators is used.
1152
+ memory : int, default 4096
1153
+ Memory size (in MB) required for this step. If
1154
+ `@resources` is also present, the maximum value from all decorators is
1155
+ used.
1156
+ image : str, optional, default None
1157
+ Docker image to use when launching on AWS Batch. If not specified, and
1158
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
1159
+ not, a default Docker image mapping to the current version of Python is used.
1160
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
1161
+ AWS Batch Job Queue to submit the job to.
1162
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
1163
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
1164
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
1165
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
1166
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
1167
+ shared_memory : int, optional, default None
1168
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1169
+ This parameter maps to the `--shm-size` option in Docker.
1170
+ max_swap : int, optional, default None
1171
+ The total amount of swap memory (in MiB) a container can use for this
1172
+ step. This parameter is translated to the `--memory-swap` option in
1173
+ Docker where the value is the sum of the container memory plus the
1174
+ `max_swap` value.
1175
+ swappiness : int, optional, default None
1176
+ This allows you to tune memory swappiness behavior for this step.
1177
+ A swappiness value of 0 causes swapping not to happen unless absolutely
1178
+ necessary. A swappiness value of 100 causes pages to be swapped very
1179
+ aggressively. Accepted values are whole numbers between 0 and 100.
1180
+ use_tmpfs : bool, default False
1181
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
1182
+ not available on Fargate compute environments
1183
+ tmpfs_tempdir : bool, default True
1184
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1185
+ tmpfs_size : int, optional, default None
1186
+ The value for the size (in MiB) of the tmpfs mount for this step.
1187
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1188
+ memory allocated for this step.
1189
+ tmpfs_path : str, optional, default None
1190
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
1191
+ inferentia : int, default 0
1192
+ Number of Inferentia chips required for this step.
1193
+ trainium : int, default None
1194
+ Alias for inferentia. Use only one of the two.
1195
+ efa : int, default 0
1196
+ Number of elastic fabric adapter network devices to attach to container
1197
+ ephemeral_storage : int, default None
1198
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
1199
+ This is only relevant for Fargate compute environments
1200
+ log_driver: str, optional, default None
1201
+ The log driver to use for the Amazon ECS container.
1202
+ log_options: List[str], optional, default None
1203
+ List of strings containing options for the chosen log driver. The configurable values
1204
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
1205
+ Example: [`awslogs-group:aws/batch/job`]
1178
1206
  """
1179
1207
  ...
1180
1208
 
1181
1209
  @typing.overload
1182
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1210
+ def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1183
1211
  ...
1184
1212
 
1185
1213
  @typing.overload
1186
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1214
+ def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1187
1215
  ...
1188
1216
 
1189
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1217
+ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
1190
1218
  """
1191
- Specifies the Conda environment for the step.
1192
-
1193
- Information in this decorator will augment any
1194
- attributes set in the `@conda_base` flow-level decorator. Hence,
1195
- you can use `@conda_base` to set packages required by all
1196
- steps and use `@conda` to specify step-specific overrides.
1219
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
1197
1220
 
1198
1221
  Parameters
1199
1222
  ----------
1200
- packages : Dict[str, str], default {}
1201
- Packages to use for this step. The key is the name of the package
1202
- and the value is the version to use.
1203
- libraries : Dict[str, str], default {}
1204
- Supported for backward compatibility. When used with packages, packages will take precedence.
1205
- python : str, optional, default None
1206
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1207
- that the version used will correspond to the version of the Python interpreter used to start the run.
1208
- disabled : bool, default False
1209
- If set to True, disables @conda.
1210
- """
1211
- ...
1212
-
1213
- @typing.overload
1214
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1215
- """
1216
- Specifies that the step will success under all circumstances.
1217
-
1218
- The decorator will create an optional artifact, specified by `var`, which
1219
- contains the exception raised. You can use it to detect the presence
1220
- of errors, indicating that all happy-path artifacts produced by the step
1221
- are missing.
1222
-
1223
- Parameters
1224
- ----------
1225
- var : str, optional, default None
1226
- Name of the artifact in which to store the caught exception.
1227
- If not specified, the exception is not stored.
1228
- print_exception : bool, default True
1229
- Determines whether or not the exception is printed to
1230
- stdout when caught.
1231
- """
1232
- ...
1233
-
1234
- @typing.overload
1235
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1236
- ...
1237
-
1238
- @typing.overload
1239
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1240
- ...
1241
-
1242
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1243
- """
1244
- Specifies that the step will success under all circumstances.
1245
-
1246
- The decorator will create an optional artifact, specified by `var`, which
1247
- contains the exception raised. You can use it to detect the presence
1248
- of errors, indicating that all happy-path artifacts produced by the step
1249
- are missing.
1250
-
1251
- Parameters
1252
- ----------
1253
- var : str, optional, default None
1254
- Name of the artifact in which to store the caught exception.
1255
- If not specified, the exception is not stored.
1256
- print_exception : bool, default True
1257
- Determines whether or not the exception is printed to
1258
- stdout when caught.
1223
+ cpu : int, default 1
1224
+ Number of CPUs required for this step. If `@resources` is
1225
+ also present, the maximum value from all decorators is used.
1226
+ gpu : int, default 0
1227
+ Number of GPUs required for this step. If `@resources` is
1228
+ also present, the maximum value from all decorators is used.
1229
+ memory : int, default 4096
1230
+ Memory size (in MB) required for this step. If
1231
+ `@resources` is also present, the maximum value from all decorators is
1232
+ used.
1233
+ image : str, optional, default None
1234
+ Docker image to use when launching on AWS Batch. If not specified, and
1235
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
1236
+ not, a default Docker image mapping to the current version of Python is used.
1237
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
1238
+ AWS Batch Job Queue to submit the job to.
1239
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
1240
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
1241
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
1242
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
1243
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
1244
+ shared_memory : int, optional, default None
1245
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1246
+ This parameter maps to the `--shm-size` option in Docker.
1247
+ max_swap : int, optional, default None
1248
+ The total amount of swap memory (in MiB) a container can use for this
1249
+ step. This parameter is translated to the `--memory-swap` option in
1250
+ Docker where the value is the sum of the container memory plus the
1251
+ `max_swap` value.
1252
+ swappiness : int, optional, default None
1253
+ This allows you to tune memory swappiness behavior for this step.
1254
+ A swappiness value of 0 causes swapping not to happen unless absolutely
1255
+ necessary. A swappiness value of 100 causes pages to be swapped very
1256
+ aggressively. Accepted values are whole numbers between 0 and 100.
1257
+ use_tmpfs : bool, default False
1258
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
1259
+ not available on Fargate compute environments
1260
+ tmpfs_tempdir : bool, default True
1261
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1262
+ tmpfs_size : int, optional, default None
1263
+ The value for the size (in MiB) of the tmpfs mount for this step.
1264
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1265
+ memory allocated for this step.
1266
+ tmpfs_path : str, optional, default None
1267
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
1268
+ inferentia : int, default 0
1269
+ Number of Inferentia chips required for this step.
1270
+ trainium : int, default None
1271
+ Alias for inferentia. Use only one of the two.
1272
+ efa : int, default 0
1273
+ Number of elastic fabric adapter network devices to attach to container
1274
+ ephemeral_storage : int, default None
1275
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
1276
+ This is only relevant for Fargate compute environments
1277
+ log_driver: str, optional, default None
1278
+ The log driver to use for the Amazon ECS container.
1279
+ log_options: List[str], optional, default None
1280
+ List of strings containing options for the chosen log driver. The configurable values
1281
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
1282
+ Example: [`awslogs-group:aws/batch/job`]
1259
1283
  """
1260
1284
  ...
1261
1285
 
@@ -1309,187 +1333,167 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1309
1333
  ...
1310
1334
 
1311
1335
  @typing.overload
1312
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1336
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1313
1337
  """
1314
- Creates a human-readable report, a Metaflow Card, after this step completes.
1338
+ Specifies the resources needed when executing this step.
1315
1339
 
1316
- Note that you may add multiple `@card` decorators in a step with different parameters.
1340
+ Use `@resources` to specify the resource requirements
1341
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1342
+
1343
+ You can choose the compute layer on the command line by executing e.g.
1344
+ ```
1345
+ python myflow.py run --with batch
1346
+ ```
1347
+ or
1348
+ ```
1349
+ python myflow.py run --with kubernetes
1350
+ ```
1351
+ which executes the flow on the desired system using the
1352
+ requirements specified in `@resources`.
1317
1353
 
1318
1354
  Parameters
1319
1355
  ----------
1320
- type : str, default 'default'
1321
- Card type.
1322
- id : str, optional, default None
1323
- If multiple cards are present, use this id to identify this card.
1324
- options : Dict[str, Any], default {}
1325
- Options passed to the card. The contents depend on the card type.
1326
- timeout : int, default 45
1327
- Interrupt reporting if it takes more than this many seconds.
1328
-
1329
-
1356
+ cpu : int, default 1
1357
+ Number of CPUs required for this step.
1358
+ gpu : int, default 0
1359
+ Number of GPUs required for this step.
1360
+ disk : int, optional, default None
1361
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1362
+ memory : int, default 4096
1363
+ Memory size (in MB) required for this step.
1364
+ shared_memory : int, optional, default None
1365
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1366
+ This parameter maps to the `--shm-size` option in Docker.
1330
1367
  """
1331
1368
  ...
1332
1369
 
1333
1370
  @typing.overload
1334
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1371
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1335
1372
  ...
1336
1373
 
1337
1374
  @typing.overload
1338
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1375
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1339
1376
  ...
1340
1377
 
1341
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1378
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1342
1379
  """
1343
- Creates a human-readable report, a Metaflow Card, after this step completes.
1380
+ Specifies the resources needed when executing this step.
1344
1381
 
1345
- Note that you may add multiple `@card` decorators in a step with different parameters.
1382
+ Use `@resources` to specify the resource requirements
1383
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1384
+
1385
+ You can choose the compute layer on the command line by executing e.g.
1386
+ ```
1387
+ python myflow.py run --with batch
1388
+ ```
1389
+ or
1390
+ ```
1391
+ python myflow.py run --with kubernetes
1392
+ ```
1393
+ which executes the flow on the desired system using the
1394
+ requirements specified in `@resources`.
1346
1395
 
1347
1396
  Parameters
1348
1397
  ----------
1349
- type : str, default 'default'
1350
- Card type.
1351
- id : str, optional, default None
1352
- If multiple cards are present, use this id to identify this card.
1353
- options : Dict[str, Any], default {}
1354
- Options passed to the card. The contents depend on the card type.
1355
- timeout : int, default 45
1356
- Interrupt reporting if it takes more than this many seconds.
1357
-
1358
-
1398
+ cpu : int, default 1
1399
+ Number of CPUs required for this step.
1400
+ gpu : int, default 0
1401
+ Number of GPUs required for this step.
1402
+ disk : int, optional, default None
1403
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1404
+ memory : int, default 4096
1405
+ Memory size (in MB) required for this step.
1406
+ shared_memory : int, optional, default None
1407
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1408
+ This parameter maps to the `--shm-size` option in Docker.
1359
1409
  """
1360
1410
  ...
1361
1411
 
1362
- @typing.overload
1363
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1412
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1364
1413
  """
1365
- Specifies environment variables to be set prior to the execution of a step.
1414
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1415
+
1416
+ User code call
1417
+ -----------
1418
+ @nim(
1419
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1420
+ backend='managed'
1421
+ )
1422
+
1423
+ Valid backend options
1424
+ ---------------------
1425
+ - 'managed': Outerbounds selects a compute provider based on the model.
1426
+ - 🚧 'dataplane': Run in your account.
1427
+
1428
+ Valid model options
1429
+ ----------------
1430
+ - 'meta/llama3-8b-instruct': 8B parameter model
1431
+ - 'meta/llama3-70b-instruct': 70B parameter model
1432
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1366
1433
 
1367
1434
  Parameters
1368
1435
  ----------
1369
- vars : Dict[str, str], default {}
1370
- Dictionary of environment variables to set.
1436
+ models: list[NIM]
1437
+ List of NIM containers running models in sidecars.
1438
+ backend: str
1439
+ Compute provider to run the NIM container.
1371
1440
  """
1372
1441
  ...
1373
1442
 
1374
- @typing.overload
1375
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1376
- ...
1377
-
1378
- @typing.overload
1379
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1380
- ...
1381
-
1382
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1443
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1383
1444
  """
1384
- Specifies environment variables to be set prior to the execution of a step.
1445
+ Specifies what flows belong to the same project.
1446
+
1447
+ A project-specific namespace is created for all flows that
1448
+ use the same `@project(name)`.
1385
1449
 
1386
1450
  Parameters
1387
1451
  ----------
1388
- vars : Dict[str, str], default {}
1389
- Dictionary of environment variables to set.
1452
+ name : str
1453
+ Project name. Make sure that the name is unique amongst all
1454
+ projects that use the same production scheduler. The name may
1455
+ contain only lowercase alphanumeric characters and underscores.
1456
+
1457
+
1390
1458
  """
1391
1459
  ...
1392
1460
 
1393
1461
  @typing.overload
1394
- def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1462
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1395
1463
  """
1396
- Specifies the flow(s) that this flow depends on.
1397
-
1398
- ```
1399
- @trigger_on_finish(flow='FooFlow')
1400
- ```
1401
- or
1402
- ```
1403
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1404
- ```
1405
- This decorator respects the @project decorator and triggers the flow
1406
- when upstream runs within the same namespace complete successfully
1407
-
1408
- Additionally, you can specify project aware upstream flow dependencies
1409
- by specifying the fully qualified project_flow_name.
1410
- ```
1411
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1412
- ```
1413
- or
1414
- ```
1415
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1416
- ```
1417
-
1418
- You can also specify just the project or project branch (other values will be
1419
- inferred from the current project or project branch):
1420
- ```
1421
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1422
- ```
1423
-
1424
- Note that `branch` is typically one of:
1425
- - `prod`
1426
- - `user.bob`
1427
- - `test.my_experiment`
1428
- - `prod.staging`
1464
+ Specifies the PyPI packages for all steps of the flow.
1429
1465
 
1466
+ Use `@pypi_base` to set common packages required by all
1467
+ steps and use `@pypi` to specify step-specific overrides.
1430
1468
  Parameters
1431
1469
  ----------
1432
- flow : Union[str, Dict[str, str]], optional, default None
1433
- Upstream flow dependency for this flow.
1434
- flows : List[Union[str, Dict[str, str]]], default []
1435
- Upstream flow dependencies for this flow.
1436
- options : Dict[str, Any], default {}
1437
- Backend-specific configuration for tuning eventing behavior.
1438
-
1439
-
1470
+ packages : Dict[str, str], default: {}
1471
+ Packages to use for this flow. The key is the name of the package
1472
+ and the value is the version to use.
1473
+ python : str, optional, default: None
1474
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1475
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1440
1476
  """
1441
1477
  ...
1442
1478
 
1443
1479
  @typing.overload
1444
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1480
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1445
1481
  ...
1446
1482
 
1447
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1483
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1448
1484
  """
1449
- Specifies the flow(s) that this flow depends on.
1450
-
1451
- ```
1452
- @trigger_on_finish(flow='FooFlow')
1453
- ```
1454
- or
1455
- ```
1456
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1457
- ```
1458
- This decorator respects the @project decorator and triggers the flow
1459
- when upstream runs within the same namespace complete successfully
1460
-
1461
- Additionally, you can specify project aware upstream flow dependencies
1462
- by specifying the fully qualified project_flow_name.
1463
- ```
1464
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1465
- ```
1466
- or
1467
- ```
1468
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1469
- ```
1470
-
1471
- You can also specify just the project or project branch (other values will be
1472
- inferred from the current project or project branch):
1473
- ```
1474
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1475
- ```
1476
-
1477
- Note that `branch` is typically one of:
1478
- - `prod`
1479
- - `user.bob`
1480
- - `test.my_experiment`
1481
- - `prod.staging`
1485
+ Specifies the PyPI packages for all steps of the flow.
1482
1486
 
1487
+ Use `@pypi_base` to set common packages required by all
1488
+ steps and use `@pypi` to specify step-specific overrides.
1483
1489
  Parameters
1484
1490
  ----------
1485
- flow : Union[str, Dict[str, str]], optional, default None
1486
- Upstream flow dependency for this flow.
1487
- flows : List[Union[str, Dict[str, str]]], default []
1488
- Upstream flow dependencies for this flow.
1489
- options : Dict[str, Any], default {}
1490
- Backend-specific configuration for tuning eventing behavior.
1491
-
1492
-
1491
+ packages : Dict[str, str], default: {}
1492
+ Packages to use for this flow. The key is the name of the package
1493
+ and the value is the version to use.
1494
+ python : str, optional, default: None
1495
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1496
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1493
1497
  """
1494
1498
  ...
1495
1499
 
@@ -1536,90 +1540,188 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1536
1540
  ...
1537
1541
 
1538
1542
  @typing.overload
1539
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1543
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1540
1544
  """
1541
- Specifies the PyPI packages for all steps of the flow.
1545
+ Specifies the times when the flow should be run when running on a
1546
+ production scheduler.
1542
1547
 
1543
- Use `@pypi_base` to set common packages required by all
1544
- steps and use `@pypi` to specify step-specific overrides.
1545
1548
  Parameters
1546
1549
  ----------
1547
- packages : Dict[str, str], default: {}
1548
- Packages to use for this flow. The key is the name of the package
1549
- and the value is the version to use.
1550
- python : str, optional, default: None
1551
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1552
- that the version used will correspond to the version of the Python interpreter used to start the run.
1550
+ hourly : bool, default False
1551
+ Run the workflow hourly.
1552
+ daily : bool, default True
1553
+ Run the workflow daily.
1554
+ weekly : bool, default False
1555
+ Run the workflow weekly.
1556
+ cron : str, optional, default None
1557
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1558
+ specified by this expression.
1559
+ timezone : str, optional, default None
1560
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1561
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1553
1562
  """
1554
1563
  ...
1555
1564
 
1556
1565
  @typing.overload
1557
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1566
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1558
1567
  ...
1559
1568
 
1560
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1569
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1561
1570
  """
1562
- Specifies the PyPI packages for all steps of the flow.
1571
+ Specifies the times when the flow should be run when running on a
1572
+ production scheduler.
1563
1573
 
1564
- Use `@pypi_base` to set common packages required by all
1565
- steps and use `@pypi` to specify step-specific overrides.
1566
1574
  Parameters
1567
1575
  ----------
1568
- packages : Dict[str, str], default: {}
1569
- Packages to use for this flow. The key is the name of the package
1570
- and the value is the version to use.
1571
- python : str, optional, default: None
1572
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1573
- that the version used will correspond to the version of the Python interpreter used to start the run.
1576
+ hourly : bool, default False
1577
+ Run the workflow hourly.
1578
+ daily : bool, default True
1579
+ Run the workflow daily.
1580
+ weekly : bool, default False
1581
+ Run the workflow weekly.
1582
+ cron : str, optional, default None
1583
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1584
+ specified by this expression.
1585
+ timezone : str, optional, default None
1586
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1587
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1574
1588
  """
1575
1589
  ...
1576
1590
 
1577
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1591
+ @typing.overload
1592
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1578
1593
  """
1579
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1594
+ Specifies the event(s) that this flow depends on.
1580
1595
 
1581
- User code call
1582
- -----------
1583
- @nim(
1584
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1585
- backend='managed'
1586
- )
1596
+ ```
1597
+ @trigger(event='foo')
1598
+ ```
1599
+ or
1600
+ ```
1601
+ @trigger(events=['foo', 'bar'])
1602
+ ```
1587
1603
 
1588
- Valid backend options
1589
- ---------------------
1590
- - 'managed': Outerbounds selects a compute provider based on the model.
1591
- - 🚧 'dataplane': Run in your account.
1604
+ Additionally, you can specify the parameter mappings
1605
+ to map event payload to Metaflow parameters for the flow.
1606
+ ```
1607
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1608
+ ```
1609
+ or
1610
+ ```
1611
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1612
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1613
+ ```
1592
1614
 
1593
- Valid model options
1594
- ----------------
1595
- - 'meta/llama3-8b-instruct': 8B parameter model
1596
- - 'meta/llama3-70b-instruct': 70B parameter model
1597
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1615
+ 'parameters' can also be a list of strings and tuples like so:
1616
+ ```
1617
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1618
+ ```
1619
+ This is equivalent to:
1620
+ ```
1621
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1622
+ ```
1623
+
1624
+ Parameters
1625
+ ----------
1626
+ event : Union[str, Dict[str, Any]], optional, default None
1627
+ Event dependency for this flow.
1628
+ events : List[Union[str, Dict[str, Any]]], default []
1629
+ Events dependency for this flow.
1630
+ options : Dict[str, Any], default {}
1631
+ Backend-specific configuration for tuning eventing behavior.
1632
+
1633
+
1634
+ """
1635
+ ...
1636
+
1637
+ @typing.overload
1638
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1639
+ ...
1640
+
1641
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1642
+ """
1643
+ Specifies the event(s) that this flow depends on.
1644
+
1645
+ ```
1646
+ @trigger(event='foo')
1647
+ ```
1648
+ or
1649
+ ```
1650
+ @trigger(events=['foo', 'bar'])
1651
+ ```
1652
+
1653
+ Additionally, you can specify the parameter mappings
1654
+ to map event payload to Metaflow parameters for the flow.
1655
+ ```
1656
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1657
+ ```
1658
+ or
1659
+ ```
1660
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1661
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1662
+ ```
1663
+
1664
+ 'parameters' can also be a list of strings and tuples like so:
1665
+ ```
1666
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1667
+ ```
1668
+ This is equivalent to:
1669
+ ```
1670
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1671
+ ```
1598
1672
 
1599
1673
  Parameters
1600
1674
  ----------
1601
- models: list[NIM]
1602
- List of NIM containers running models in sidecars.
1603
- backend: str
1604
- Compute provider to run the NIM container.
1675
+ event : Union[str, Dict[str, Any]], optional, default None
1676
+ Event dependency for this flow.
1677
+ events : List[Union[str, Dict[str, Any]]], default []
1678
+ Events dependency for this flow.
1679
+ options : Dict[str, Any], default {}
1680
+ Backend-specific configuration for tuning eventing behavior.
1681
+
1682
+
1605
1683
  """
1606
1684
  ...
1607
1685
 
1608
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1686
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1609
1687
  """
1610
- Specifies what flows belong to the same project.
1611
-
1612
- A project-specific namespace is created for all flows that
1613
- use the same `@project(name)`.
1688
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1689
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1614
1690
 
1615
1691
  Parameters
1616
1692
  ----------
1693
+ timeout : int
1694
+ Time, in seconds before the task times out and fails. (Default: 3600)
1695
+ poke_interval : int
1696
+ Time in seconds that the job should wait in between each try. (Default: 60)
1697
+ mode : str
1698
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1699
+ exponential_backoff : bool
1700
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1701
+ pool : str
1702
+ the slot pool this task should run in,
1703
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1704
+ soft_fail : bool
1705
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1617
1706
  name : str
1618
- Project name. Make sure that the name is unique amongst all
1619
- projects that use the same production scheduler. The name may
1620
- contain only lowercase alphanumeric characters and underscores.
1621
-
1622
-
1707
+ Name of the sensor on Airflow
1708
+ description : str
1709
+ Description of sensor in the Airflow UI
1710
+ external_dag_id : str
1711
+ The dag_id that contains the task you want to wait for.
1712
+ external_task_ids : List[str]
1713
+ The list of task_ids that you want to wait for.
1714
+ If None (default value) the sensor waits for the DAG. (Default: None)
1715
+ allowed_states : List[str]
1716
+ Iterable of allowed states, (Default: ['success'])
1717
+ failed_states : List[str]
1718
+ Iterable of failed or dis-allowed states. (Default: None)
1719
+ execution_delta : datetime.timedelta
1720
+ time difference with the previous execution to look at,
1721
+ the default is the same logical date as the current task or DAG. (Default: None)
1722
+ check_existence: bool
1723
+ Set to True to check if the external task exists or check if
1724
+ the DAG to wait for exists. (Default: True)
1623
1725
  """
1624
1726
  ...
1625
1727
 
@@ -1672,87 +1774,49 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1672
1774
  """
1673
1775
  ...
1674
1776
 
1675
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1676
- """
1677
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1678
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1679
-
1680
- Parameters
1681
- ----------
1682
- timeout : int
1683
- Time, in seconds before the task times out and fails. (Default: 3600)
1684
- poke_interval : int
1685
- Time in seconds that the job should wait in between each try. (Default: 60)
1686
- mode : str
1687
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1688
- exponential_backoff : bool
1689
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1690
- pool : str
1691
- the slot pool this task should run in,
1692
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1693
- soft_fail : bool
1694
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1695
- name : str
1696
- Name of the sensor on Airflow
1697
- description : str
1698
- Description of sensor in the Airflow UI
1699
- external_dag_id : str
1700
- The dag_id that contains the task you want to wait for.
1701
- external_task_ids : List[str]
1702
- The list of task_ids that you want to wait for.
1703
- If None (default value) the sensor waits for the DAG. (Default: None)
1704
- allowed_states : List[str]
1705
- Iterable of allowed states, (Default: ['success'])
1706
- failed_states : List[str]
1707
- Iterable of failed or dis-allowed states. (Default: None)
1708
- execution_delta : datetime.timedelta
1709
- time difference with the previous execution to look at,
1710
- the default is the same logical date as the current task or DAG. (Default: None)
1711
- check_existence: bool
1712
- Set to True to check if the external task exists or check if
1713
- the DAG to wait for exists. (Default: True)
1714
- """
1715
- ...
1716
-
1717
1777
  @typing.overload
1718
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1778
+ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1719
1779
  """
1720
- Specifies the event(s) that this flow depends on.
1780
+ Specifies the flow(s) that this flow depends on.
1721
1781
 
1722
1782
  ```
1723
- @trigger(event='foo')
1783
+ @trigger_on_finish(flow='FooFlow')
1724
1784
  ```
1725
1785
  or
1726
1786
  ```
1727
- @trigger(events=['foo', 'bar'])
1787
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1728
1788
  ```
1789
+ This decorator respects the @project decorator and triggers the flow
1790
+ when upstream runs within the same namespace complete successfully
1729
1791
 
1730
- Additionally, you can specify the parameter mappings
1731
- to map event payload to Metaflow parameters for the flow.
1792
+ Additionally, you can specify project aware upstream flow dependencies
1793
+ by specifying the fully qualified project_flow_name.
1732
1794
  ```
1733
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1795
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1734
1796
  ```
1735
1797
  or
1736
1798
  ```
1737
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1738
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1799
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1739
1800
  ```
1740
1801
 
1741
- 'parameters' can also be a list of strings and tuples like so:
1742
- ```
1743
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1744
- ```
1745
- This is equivalent to:
1802
+ You can also specify just the project or project branch (other values will be
1803
+ inferred from the current project or project branch):
1746
1804
  ```
1747
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1805
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1748
1806
  ```
1749
1807
 
1808
+ Note that `branch` is typically one of:
1809
+ - `prod`
1810
+ - `user.bob`
1811
+ - `test.my_experiment`
1812
+ - `prod.staging`
1813
+
1750
1814
  Parameters
1751
1815
  ----------
1752
- event : Union[str, Dict[str, Any]], optional, default None
1753
- Event dependency for this flow.
1754
- events : List[Union[str, Dict[str, Any]]], default []
1755
- Events dependency for this flow.
1816
+ flow : Union[str, Dict[str, str]], optional, default None
1817
+ Upstream flow dependency for this flow.
1818
+ flows : List[Union[str, Dict[str, str]]], default []
1819
+ Upstream flow dependencies for this flow.
1756
1820
  options : Dict[str, Any], default {}
1757
1821
  Backend-specific configuration for tuning eventing behavior.
1758
1822
 
@@ -1761,47 +1825,51 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1761
1825
  ...
1762
1826
 
1763
1827
  @typing.overload
1764
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1828
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1765
1829
  ...
1766
1830
 
1767
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1831
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1768
1832
  """
1769
- Specifies the event(s) that this flow depends on.
1833
+ Specifies the flow(s) that this flow depends on.
1770
1834
 
1771
1835
  ```
1772
- @trigger(event='foo')
1836
+ @trigger_on_finish(flow='FooFlow')
1773
1837
  ```
1774
1838
  or
1775
1839
  ```
1776
- @trigger(events=['foo', 'bar'])
1840
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1777
1841
  ```
1842
+ This decorator respects the @project decorator and triggers the flow
1843
+ when upstream runs within the same namespace complete successfully
1778
1844
 
1779
- Additionally, you can specify the parameter mappings
1780
- to map event payload to Metaflow parameters for the flow.
1845
+ Additionally, you can specify project aware upstream flow dependencies
1846
+ by specifying the fully qualified project_flow_name.
1781
1847
  ```
1782
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1848
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1783
1849
  ```
1784
1850
  or
1785
1851
  ```
1786
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1787
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1852
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1788
1853
  ```
1789
1854
 
1790
- 'parameters' can also be a list of strings and tuples like so:
1791
- ```
1792
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1793
- ```
1794
- This is equivalent to:
1855
+ You can also specify just the project or project branch (other values will be
1856
+ inferred from the current project or project branch):
1795
1857
  ```
1796
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1858
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1797
1859
  ```
1798
1860
 
1861
+ Note that `branch` is typically one of:
1862
+ - `prod`
1863
+ - `user.bob`
1864
+ - `test.my_experiment`
1865
+ - `prod.staging`
1866
+
1799
1867
  Parameters
1800
1868
  ----------
1801
- event : Union[str, Dict[str, Any]], optional, default None
1802
- Event dependency for this flow.
1803
- events : List[Union[str, Dict[str, Any]]], default []
1804
- Events dependency for this flow.
1869
+ flow : Union[str, Dict[str, str]], optional, default None
1870
+ Upstream flow dependency for this flow.
1871
+ flows : List[Union[str, Dict[str, str]]], default []
1872
+ Upstream flow dependencies for this flow.
1805
1873
  options : Dict[str, Any], default {}
1806
1874
  Backend-specific configuration for tuning eventing behavior.
1807
1875
 
@@ -1809,55 +1877,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1809
1877
  """
1810
1878
  ...
1811
1879
 
1812
- @typing.overload
1813
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1814
- """
1815
- Specifies the times when the flow should be run when running on a
1816
- production scheduler.
1817
-
1818
- Parameters
1819
- ----------
1820
- hourly : bool, default False
1821
- Run the workflow hourly.
1822
- daily : bool, default True
1823
- Run the workflow daily.
1824
- weekly : bool, default False
1825
- Run the workflow weekly.
1826
- cron : str, optional, default None
1827
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1828
- specified by this expression.
1829
- timezone : str, optional, default None
1830
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1831
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1832
- """
1833
- ...
1834
-
1835
- @typing.overload
1836
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1837
- ...
1838
-
1839
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1840
- """
1841
- Specifies the times when the flow should be run when running on a
1842
- production scheduler.
1843
-
1844
- Parameters
1845
- ----------
1846
- hourly : bool, default False
1847
- Run the workflow hourly.
1848
- daily : bool, default True
1849
- Run the workflow daily.
1850
- weekly : bool, default False
1851
- Run the workflow weekly.
1852
- cron : str, optional, default None
1853
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1854
- specified by this expression.
1855
- timezone : str, optional, default None
1856
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1857
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1858
- """
1859
- ...
1860
-
1861
1880
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1862
1881
  """
1863
1882
  Switch namespace to the one provided.
@@ -3006,6 +3025,36 @@ class NBRunner(object, metaclass=type):
3006
3025
  ...
3007
3026
  ...
3008
3027
 
3028
+ class Deployer(object, metaclass=type):
3029
+ def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, **kwargs):
3030
+ ...
3031
+ def _Deployer__make_function(self, deployer_class):
3032
+ """
3033
+ Create a function for the given deployer class.
3034
+
3035
+ Parameters
3036
+ ----------
3037
+ deployer_class : Type[DeployerImpl]
3038
+ Deployer implementation class.
3039
+
3040
+ Returns
3041
+ -------
3042
+ Callable
3043
+ Function that initializes and returns an instance of the deployer class.
3044
+ """
3045
+ ...
3046
+ ...
3047
+
3048
+ class NBDeployer(object, metaclass=type):
3049
+ def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", **kwargs):
3050
+ ...
3051
+ def cleanup(self):
3052
+ """
3053
+ Delete any temporary files created during execution.
3054
+ """
3055
+ ...
3056
+ ...
3057
+
3009
3058
  def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
3010
3059
  ...
3011
3060