metaflow-stubs 2.12.5__py2.py3-none-any.whl → 2.12.6__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. metaflow-stubs/__init__.pyi +620 -620
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +3 -3
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +5 -5
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -2
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +3 -3
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  37. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  45. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  56. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  61. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  62. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  63. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  65. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  78. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  81. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  82. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  83. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  84. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
  89. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  91. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  92. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  93. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  95. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -4
  97. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  99. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  106. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  107. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  108. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  110. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  111. metaflow-stubs/plugins/package_cli.pyi +2 -2
  112. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  114. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  117. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  119. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  120. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  123. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  124. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  126. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  127. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  128. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  129. metaflow-stubs/procpoll.pyi +2 -2
  130. metaflow-stubs/pylint_wrapper.pyi +2 -2
  131. metaflow-stubs/runner/__init__.pyi +2 -2
  132. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  133. metaflow-stubs/runner/nbrun.pyi +2 -2
  134. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  135. metaflow-stubs/tagging_util.pyi +2 -2
  136. metaflow-stubs/tuple_util.pyi +2 -2
  137. metaflow-stubs/version.pyi +2 -2
  138. {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.6.dist-info}/METADATA +2 -2
  139. metaflow_stubs-2.12.6.dist-info/RECORD +142 -0
  140. {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.6.dist-info}/WHEEL +1 -1
  141. metaflow_stubs-2.12.5.dist-info/RECORD +0 -142
  142. {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.6.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.5 #
4
- # Generated on 2024-06-20T19:51:33.982232 #
3
+ # MF version: 2.12.6 #
4
+ # Generated on 2024-07-01T22:37:35.546266 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.client.core
12
- import metaflow.runner.metaflow_runner
13
- import metaflow._vendor.click.types
14
- import typing
15
11
  import metaflow.parameters
16
12
  import metaflow.flowspec
17
- import metaflow.plugins.datatools.s3.s3
18
13
  import datetime
19
- import io
20
- import metaflow.metaflow_current
21
14
  import metaflow.datastore.inputs
15
+ import metaflow.plugins.datatools.s3.s3
16
+ import metaflow.client.core
22
17
  import metaflow.events
18
+ import metaflow.runner.metaflow_runner
19
+ import metaflow._vendor.click.types
20
+ import typing
21
+ import metaflow.metaflow_current
22
+ import io
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -728,182 +728,145 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
728
728
  ...
729
729
 
730
730
  @typing.overload
731
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
731
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
732
  """
733
- Specifies secrets to be retrieved and injected as environment variables prior to
734
- the execution of a step.
733
+ Specifies the number of times the task corresponding
734
+ to a step needs to be retried.
735
+
736
+ This decorator is useful for handling transient errors, such as networking issues.
737
+ If your task contains operations that can't be retried safely, e.g. database updates,
738
+ it is advisable to annotate it with `@retry(times=0)`.
739
+
740
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
741
+ decorator will execute a no-op task after all retries have been exhausted,
742
+ ensuring that the flow execution can continue.
735
743
 
736
744
  Parameters
737
745
  ----------
738
- sources : List[Union[str, Dict[str, Any]]], default: []
739
- List of secret specs, defining how the secrets are to be retrieved
746
+ times : int, default 3
747
+ Number of times to retry this task.
748
+ minutes_between_retries : int, default 2
749
+ Number of minutes between retries.
740
750
  """
741
751
  ...
742
752
 
743
753
  @typing.overload
744
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
754
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
745
755
  ...
746
756
 
747
757
  @typing.overload
748
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
758
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
749
759
  ...
750
760
 
751
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
761
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
752
762
  """
753
- Specifies secrets to be retrieved and injected as environment variables prior to
754
- the execution of a step.
763
+ Specifies the number of times the task corresponding
764
+ to a step needs to be retried.
765
+
766
+ This decorator is useful for handling transient errors, such as networking issues.
767
+ If your task contains operations that can't be retried safely, e.g. database updates,
768
+ it is advisable to annotate it with `@retry(times=0)`.
769
+
770
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
771
+ decorator will execute a no-op task after all retries have been exhausted,
772
+ ensuring that the flow execution can continue.
755
773
 
756
774
  Parameters
757
775
  ----------
758
- sources : List[Union[str, Dict[str, Any]]], default: []
759
- List of secret specs, defining how the secrets are to be retrieved
776
+ times : int, default 3
777
+ Number of times to retry this task.
778
+ minutes_between_retries : int, default 2
779
+ Number of minutes between retries.
760
780
  """
761
781
  ...
762
782
 
763
783
  @typing.overload
764
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
784
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
765
785
  """
766
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
786
+ Specifies the Conda environment for the step.
787
+
788
+ Information in this decorator will augment any
789
+ attributes set in the `@conda_base` flow-level decorator. Hence,
790
+ you can use `@conda_base` to set packages required by all
791
+ steps and use `@conda` to specify step-specific overrides.
767
792
 
768
793
  Parameters
769
794
  ----------
770
- cpu : int, default 1
771
- Number of CPUs required for this step. If `@resources` is
772
- also present, the maximum value from all decorators is used.
773
- gpu : int, default 0
774
- Number of GPUs required for this step. If `@resources` is
775
- also present, the maximum value from all decorators is used.
776
- memory : int, default 4096
777
- Memory size (in MB) required for this step. If
778
- `@resources` is also present, the maximum value from all decorators is
779
- used.
780
- image : str, optional, default None
781
- Docker image to use when launching on AWS Batch. If not specified, and
782
- METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
783
- not, a default Docker image mapping to the current version of Python is used.
784
- queue : str, default METAFLOW_BATCH_JOB_QUEUE
785
- AWS Batch Job Queue to submit the job to.
786
- iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
787
- AWS IAM role that AWS Batch container uses to access AWS cloud resources.
788
- execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
789
- AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
790
- (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
791
- shared_memory : int, optional, default None
792
- The value for the size (in MiB) of the /dev/shm volume for this step.
793
- This parameter maps to the `--shm-size` option in Docker.
794
- max_swap : int, optional, default None
795
- The total amount of swap memory (in MiB) a container can use for this
796
- step. This parameter is translated to the `--memory-swap` option in
797
- Docker where the value is the sum of the container memory plus the
798
- `max_swap` value.
799
- swappiness : int, optional, default None
800
- This allows you to tune memory swappiness behavior for this step.
801
- A swappiness value of 0 causes swapping not to happen unless absolutely
802
- necessary. A swappiness value of 100 causes pages to be swapped very
803
- aggressively. Accepted values are whole numbers between 0 and 100.
804
- use_tmpfs : bool, default False
805
- This enables an explicit tmpfs mount for this step. Note that tmpfs is
806
- not available on Fargate compute environments
807
- tmpfs_tempdir : bool, default True
808
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
809
- tmpfs_size : int, optional, default None
810
- The value for the size (in MiB) of the tmpfs mount for this step.
811
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
812
- memory allocated for this step.
813
- tmpfs_path : str, optional, default None
814
- Path to tmpfs mount for this step. Defaults to /metaflow_temp.
815
- inferentia : int, default 0
816
- Number of Inferentia chips required for this step.
817
- trainium : int, default None
818
- Alias for inferentia. Use only one of the two.
819
- efa : int, default 0
820
- Number of elastic fabric adapter network devices to attach to container
821
- ephemeral_storage : int, default None
822
- The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
823
- This is only relevant for Fargate compute environments
824
- log_driver: str, optional, default None
825
- The log driver to use for the Amazon ECS container.
826
- log_options: List[str], optional, default None
827
- List of strings containing options for the chosen log driver. The configurable values
828
- depend on the `log driver` chosen. Validation of these options is not supported yet.
829
- Example: [`awslogs-group:aws/batch/job`]
795
+ packages : Dict[str, str], default {}
796
+ Packages to use for this step. The key is the name of the package
797
+ and the value is the version to use.
798
+ libraries : Dict[str, str], default {}
799
+ Supported for backward compatibility. When used with packages, packages will take precedence.
800
+ python : str, optional, default None
801
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
802
+ that the version used will correspond to the version of the Python interpreter used to start the run.
803
+ disabled : bool, default False
804
+ If set to True, disables @conda.
830
805
  """
831
806
  ...
832
807
 
833
808
  @typing.overload
834
- def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
809
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
835
810
  ...
836
811
 
837
812
  @typing.overload
838
- def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
813
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
839
814
  ...
840
815
 
841
- def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
816
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
842
817
  """
843
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
818
+ Specifies the Conda environment for the step.
819
+
820
+ Information in this decorator will augment any
821
+ attributes set in the `@conda_base` flow-level decorator. Hence,
822
+ you can use `@conda_base` to set packages required by all
823
+ steps and use `@conda` to specify step-specific overrides.
844
824
 
845
825
  Parameters
846
826
  ----------
847
- cpu : int, default 1
848
- Number of CPUs required for this step. If `@resources` is
849
- also present, the maximum value from all decorators is used.
850
- gpu : int, default 0
851
- Number of GPUs required for this step. If `@resources` is
852
- also present, the maximum value from all decorators is used.
853
- memory : int, default 4096
854
- Memory size (in MB) required for this step. If
855
- `@resources` is also present, the maximum value from all decorators is
856
- used.
857
- image : str, optional, default None
858
- Docker image to use when launching on AWS Batch. If not specified, and
859
- METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
860
- not, a default Docker image mapping to the current version of Python is used.
861
- queue : str, default METAFLOW_BATCH_JOB_QUEUE
862
- AWS Batch Job Queue to submit the job to.
863
- iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
864
- AWS IAM role that AWS Batch container uses to access AWS cloud resources.
865
- execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
866
- AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
867
- (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
868
- shared_memory : int, optional, default None
869
- The value for the size (in MiB) of the /dev/shm volume for this step.
870
- This parameter maps to the `--shm-size` option in Docker.
871
- max_swap : int, optional, default None
872
- The total amount of swap memory (in MiB) a container can use for this
873
- step. This parameter is translated to the `--memory-swap` option in
874
- Docker where the value is the sum of the container memory plus the
875
- `max_swap` value.
876
- swappiness : int, optional, default None
877
- This allows you to tune memory swappiness behavior for this step.
878
- A swappiness value of 0 causes swapping not to happen unless absolutely
879
- necessary. A swappiness value of 100 causes pages to be swapped very
880
- aggressively. Accepted values are whole numbers between 0 and 100.
881
- use_tmpfs : bool, default False
882
- This enables an explicit tmpfs mount for this step. Note that tmpfs is
883
- not available on Fargate compute environments
884
- tmpfs_tempdir : bool, default True
885
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
886
- tmpfs_size : int, optional, default None
887
- The value for the size (in MiB) of the tmpfs mount for this step.
888
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
889
- memory allocated for this step.
890
- tmpfs_path : str, optional, default None
891
- Path to tmpfs mount for this step. Defaults to /metaflow_temp.
892
- inferentia : int, default 0
893
- Number of Inferentia chips required for this step.
894
- trainium : int, default None
895
- Alias for inferentia. Use only one of the two.
896
- efa : int, default 0
897
- Number of elastic fabric adapter network devices to attach to container
898
- ephemeral_storage : int, default None
899
- The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
900
- This is only relevant for Fargate compute environments
901
- log_driver: str, optional, default None
902
- The log driver to use for the Amazon ECS container.
903
- log_options: List[str], optional, default None
904
- List of strings containing options for the chosen log driver. The configurable values
905
- depend on the `log driver` chosen. Validation of these options is not supported yet.
906
- Example: [`awslogs-group:aws/batch/job`]
827
+ packages : Dict[str, str], default {}
828
+ Packages to use for this step. The key is the name of the package
829
+ and the value is the version to use.
830
+ libraries : Dict[str, str], default {}
831
+ Supported for backward compatibility. When used with packages, packages will take precedence.
832
+ python : str, optional, default None
833
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
834
+ that the version used will correspond to the version of the Python interpreter used to start the run.
835
+ disabled : bool, default False
836
+ If set to True, disables @conda.
837
+ """
838
+ ...
839
+
840
+ @typing.overload
841
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
842
+ """
843
+ Specifies secrets to be retrieved and injected as environment variables prior to
844
+ the execution of a step.
845
+
846
+ Parameters
847
+ ----------
848
+ sources : List[Union[str, Dict[str, Any]]], default: []
849
+ List of secret specs, defining how the secrets are to be retrieved
850
+ """
851
+ ...
852
+
853
+ @typing.overload
854
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
855
+ ...
856
+
857
+ @typing.overload
858
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
859
+ ...
860
+
861
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
862
+ """
863
+ Specifies secrets to be retrieved and injected as environment variables prior to
864
+ the execution of a step.
865
+
866
+ Parameters
867
+ ----------
868
+ sources : List[Union[str, Dict[str, Any]]], default: []
869
+ List of secret specs, defining how the secrets are to be retrieved
907
870
  """
908
871
  ...
909
872
 
@@ -967,243 +930,59 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
967
930
  ...
968
931
 
969
932
  @typing.overload
970
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
933
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
971
934
  """
972
- Specifies the Conda environment for the step.
973
-
974
- Information in this decorator will augment any
975
- attributes set in the `@conda_base` flow-level decorator. Hence,
976
- you can use `@conda_base` to set packages required by all
977
- steps and use `@conda` to specify step-specific overrides.
978
-
979
- Parameters
980
- ----------
981
- packages : Dict[str, str], default {}
982
- Packages to use for this step. The key is the name of the package
983
- and the value is the version to use.
984
- libraries : Dict[str, str], default {}
985
- Supported for backward compatibility. When used with packages, packages will take precedence.
986
- python : str, optional, default None
987
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
988
- that the version used will correspond to the version of the Python interpreter used to start the run.
989
- disabled : bool, default False
990
- If set to True, disables @conda.
991
- """
992
- ...
993
-
994
- @typing.overload
995
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
996
- ...
997
-
998
- @typing.overload
999
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1000
- ...
1001
-
1002
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1003
- """
1004
- Specifies the Conda environment for the step.
1005
-
1006
- Information in this decorator will augment any
1007
- attributes set in the `@conda_base` flow-level decorator. Hence,
1008
- you can use `@conda_base` to set packages required by all
1009
- steps and use `@conda` to specify step-specific overrides.
1010
-
1011
- Parameters
1012
- ----------
1013
- packages : Dict[str, str], default {}
1014
- Packages to use for this step. The key is the name of the package
1015
- and the value is the version to use.
1016
- libraries : Dict[str, str], default {}
1017
- Supported for backward compatibility. When used with packages, packages will take precedence.
1018
- python : str, optional, default None
1019
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1020
- that the version used will correspond to the version of the Python interpreter used to start the run.
1021
- disabled : bool, default False
1022
- If set to True, disables @conda.
1023
- """
1024
- ...
1025
-
1026
- @typing.overload
1027
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1028
- """
1029
- Creates a human-readable report, a Metaflow Card, after this step completes.
1030
-
1031
- Note that you may add multiple `@card` decorators in a step with different parameters.
1032
-
1033
- Parameters
1034
- ----------
1035
- type : str, default 'default'
1036
- Card type.
1037
- id : str, optional, default None
1038
- If multiple cards are present, use this id to identify this card.
1039
- options : Dict[str, Any], default {}
1040
- Options passed to the card. The contents depend on the card type.
1041
- timeout : int, default 45
1042
- Interrupt reporting if it takes more than this many seconds.
1043
-
1044
-
1045
- """
1046
- ...
1047
-
1048
- @typing.overload
1049
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1050
- ...
1051
-
1052
- @typing.overload
1053
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1054
- ...
1055
-
1056
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1057
- """
1058
- Creates a human-readable report, a Metaflow Card, after this step completes.
1059
-
1060
- Note that you may add multiple `@card` decorators in a step with different parameters.
1061
-
1062
- Parameters
1063
- ----------
1064
- type : str, default 'default'
1065
- Card type.
1066
- id : str, optional, default None
1067
- If multiple cards are present, use this id to identify this card.
1068
- options : Dict[str, Any], default {}
1069
- Options passed to the card. The contents depend on the card type.
1070
- timeout : int, default 45
1071
- Interrupt reporting if it takes more than this many seconds.
1072
-
1073
-
1074
- """
1075
- ...
1076
-
1077
- @typing.overload
1078
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1079
- """
1080
- Specifies environment variables to be set prior to the execution of a step.
1081
-
1082
- Parameters
1083
- ----------
1084
- vars : Dict[str, str], default {}
1085
- Dictionary of environment variables to set.
1086
- """
1087
- ...
1088
-
1089
- @typing.overload
1090
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1091
- ...
1092
-
1093
- @typing.overload
1094
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1095
- ...
1096
-
1097
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1098
- """
1099
- Specifies environment variables to be set prior to the execution of a step.
935
+ Specifies a timeout for your step.
1100
936
 
1101
- Parameters
1102
- ----------
1103
- vars : Dict[str, str], default {}
1104
- Dictionary of environment variables to set.
1105
- """
1106
- ...
1107
-
1108
- @typing.overload
1109
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1110
- """
1111
- Specifies the number of times the task corresponding
1112
- to a step needs to be retried.
937
+ This decorator is useful if this step may hang indefinitely.
1113
938
 
1114
- This decorator is useful for handling transient errors, such as networking issues.
1115
- If your task contains operations that can't be retried safely, e.g. database updates,
1116
- it is advisable to annotate it with `@retry(times=0)`.
939
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
940
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
941
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1117
942
 
1118
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1119
- decorator will execute a no-op task after all retries have been exhausted,
1120
- ensuring that the flow execution can continue.
943
+ Note that all the values specified in parameters are added together so if you specify
944
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1121
945
 
1122
946
  Parameters
1123
947
  ----------
1124
- times : int, default 3
1125
- Number of times to retry this task.
1126
- minutes_between_retries : int, default 2
1127
- Number of minutes between retries.
948
+ seconds : int, default 0
949
+ Number of seconds to wait prior to timing out.
950
+ minutes : int, default 0
951
+ Number of minutes to wait prior to timing out.
952
+ hours : int, default 0
953
+ Number of hours to wait prior to timing out.
1128
954
  """
1129
955
  ...
1130
956
 
1131
957
  @typing.overload
1132
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
958
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1133
959
  ...
1134
960
 
1135
961
  @typing.overload
1136
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1137
- ...
1138
-
1139
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1140
- """
1141
- Specifies the number of times the task corresponding
1142
- to a step needs to be retried.
1143
-
1144
- This decorator is useful for handling transient errors, such as networking issues.
1145
- If your task contains operations that can't be retried safely, e.g. database updates,
1146
- it is advisable to annotate it with `@retry(times=0)`.
1147
-
1148
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1149
- decorator will execute a no-op task after all retries have been exhausted,
1150
- ensuring that the flow execution can continue.
1151
-
1152
- Parameters
1153
- ----------
1154
- times : int, default 3
1155
- Number of times to retry this task.
1156
- minutes_between_retries : int, default 2
1157
- Number of minutes between retries.
1158
- """
962
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1159
963
  ...
1160
964
 
1161
- @typing.overload
1162
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
965
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1163
966
  """
1164
- Specifies that the step will success under all circumstances.
967
+ Specifies a timeout for your step.
1165
968
 
1166
- The decorator will create an optional artifact, specified by `var`, which
1167
- contains the exception raised. You can use it to detect the presence
1168
- of errors, indicating that all happy-path artifacts produced by the step
1169
- are missing.
969
+ This decorator is useful if this step may hang indefinitely.
1170
970
 
1171
- Parameters
1172
- ----------
1173
- var : str, optional, default None
1174
- Name of the artifact in which to store the caught exception.
1175
- If not specified, the exception is not stored.
1176
- print_exception : bool, default True
1177
- Determines whether or not the exception is printed to
1178
- stdout when caught.
1179
- """
1180
- ...
1181
-
1182
- @typing.overload
1183
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1184
- ...
1185
-
1186
- @typing.overload
1187
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1188
- ...
1189
-
1190
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1191
- """
1192
- Specifies that the step will success under all circumstances.
971
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
972
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
973
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1193
974
 
1194
- The decorator will create an optional artifact, specified by `var`, which
1195
- contains the exception raised. You can use it to detect the presence
1196
- of errors, indicating that all happy-path artifacts produced by the step
1197
- are missing.
975
+ Note that all the values specified in parameters are added together so if you specify
976
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1198
977
 
1199
978
  Parameters
1200
979
  ----------
1201
- var : str, optional, default None
1202
- Name of the artifact in which to store the caught exception.
1203
- If not specified, the exception is not stored.
1204
- print_exception : bool, default True
1205
- Determines whether or not the exception is printed to
1206
- stdout when caught.
980
+ seconds : int, default 0
981
+ Number of seconds to wait prior to timing out.
982
+ minutes : int, default 0
983
+ Number of minutes to wait prior to timing out.
984
+ hours : int, default 0
985
+ Number of hours to wait prior to timing out.
1207
986
  """
1208
987
  ...
1209
988
 
@@ -1285,108 +1064,450 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1285
1064
  ...
1286
1065
 
1287
1066
  @typing.overload
1288
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1067
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1289
1068
  """
1290
- Specifies a timeout for your step.
1291
-
1292
- This decorator is useful if this step may hang indefinitely.
1069
+ Specifies the PyPI packages for the step.
1293
1070
 
1294
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1295
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1296
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1071
+ Information in this decorator will augment any
1072
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1073
+ you can use `@pypi_base` to set packages required by all
1074
+ steps and use `@pypi` to specify step-specific overrides.
1297
1075
 
1298
- Note that all the values specified in parameters are added together so if you specify
1299
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1076
+ Parameters
1077
+ ----------
1078
+ packages : Dict[str, str], default: {}
1079
+ Packages to use for this step. The key is the name of the package
1080
+ and the value is the version to use.
1081
+ python : str, optional, default: None
1082
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1083
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1084
+ """
1085
+ ...
1086
+
1087
+ @typing.overload
1088
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1089
+ ...
1090
+
1091
+ @typing.overload
1092
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1093
+ ...
1094
+
1095
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1096
+ """
1097
+ Specifies the PyPI packages for the step.
1098
+
1099
+ Information in this decorator will augment any
1100
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1101
+ you can use `@pypi_base` to set packages required by all
1102
+ steps and use `@pypi` to specify step-specific overrides.
1300
1103
 
1301
1104
  Parameters
1302
1105
  ----------
1303
- seconds : int, default 0
1304
- Number of seconds to wait prior to timing out.
1305
- minutes : int, default 0
1306
- Number of minutes to wait prior to timing out.
1307
- hours : int, default 0
1308
- Number of hours to wait prior to timing out.
1106
+ packages : Dict[str, str], default: {}
1107
+ Packages to use for this step. The key is the name of the package
1108
+ and the value is the version to use.
1109
+ python : str, optional, default: None
1110
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1111
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1309
1112
  """
1310
1113
  ...
1311
1114
 
1312
1115
  @typing.overload
1313
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1116
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1117
+ """
1118
+ Specifies environment variables to be set prior to the execution of a step.
1119
+
1120
+ Parameters
1121
+ ----------
1122
+ vars : Dict[str, str], default {}
1123
+ Dictionary of environment variables to set.
1124
+ """
1314
1125
  ...
1315
1126
 
1316
1127
  @typing.overload
1317
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1128
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1318
1129
  ...
1319
1130
 
1320
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1131
+ @typing.overload
1132
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1133
+ ...
1134
+
1135
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1321
1136
  """
1322
- Specifies a timeout for your step.
1137
+ Specifies environment variables to be set prior to the execution of a step.
1323
1138
 
1324
- This decorator is useful if this step may hang indefinitely.
1139
+ Parameters
1140
+ ----------
1141
+ vars : Dict[str, str], default {}
1142
+ Dictionary of environment variables to set.
1143
+ """
1144
+ ...
1145
+
1146
+ @typing.overload
1147
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1148
+ """
1149
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1325
1150
 
1326
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1327
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1328
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1151
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1152
+
1153
+ Parameters
1154
+ ----------
1155
+ type : str, default 'default'
1156
+ Card type.
1157
+ id : str, optional, default None
1158
+ If multiple cards are present, use this id to identify this card.
1159
+ options : Dict[str, Any], default {}
1160
+ Options passed to the card. The contents depend on the card type.
1161
+ timeout : int, default 45
1162
+ Interrupt reporting if it takes more than this many seconds.
1329
1163
 
1330
- Note that all the values specified in parameters are added together so if you specify
1331
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1164
+
1165
+ """
1166
+ ...
1167
+
1168
+ @typing.overload
1169
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1170
+ ...
1171
+
1172
+ @typing.overload
1173
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1174
+ ...
1175
+
1176
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1177
+ """
1178
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1179
+
1180
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1332
1181
 
1333
1182
  Parameters
1334
1183
  ----------
1335
- seconds : int, default 0
1336
- Number of seconds to wait prior to timing out.
1337
- minutes : int, default 0
1338
- Number of minutes to wait prior to timing out.
1339
- hours : int, default 0
1340
- Number of hours to wait prior to timing out.
1184
+ type : str, default 'default'
1185
+ Card type.
1186
+ id : str, optional, default None
1187
+ If multiple cards are present, use this id to identify this card.
1188
+ options : Dict[str, Any], default {}
1189
+ Options passed to the card. The contents depend on the card type.
1190
+ timeout : int, default 45
1191
+ Interrupt reporting if it takes more than this many seconds.
1192
+
1193
+
1341
1194
  """
1342
1195
  ...
1343
1196
 
1344
1197
  @typing.overload
1345
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1198
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1346
1199
  """
1347
- Specifies the PyPI packages for the step.
1200
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
1348
1201
 
1349
- Information in this decorator will augment any
1350
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1351
- you can use `@pypi_base` to set packages required by all
1352
- steps and use `@pypi` to specify step-specific overrides.
1202
+ Parameters
1203
+ ----------
1204
+ cpu : int, default 1
1205
+ Number of CPUs required for this step. If `@resources` is
1206
+ also present, the maximum value from all decorators is used.
1207
+ gpu : int, default 0
1208
+ Number of GPUs required for this step. If `@resources` is
1209
+ also present, the maximum value from all decorators is used.
1210
+ memory : int, default 4096
1211
+ Memory size (in MB) required for this step. If
1212
+ `@resources` is also present, the maximum value from all decorators is
1213
+ used.
1214
+ image : str, optional, default None
1215
+ Docker image to use when launching on AWS Batch. If not specified, and
1216
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
1217
+ not, a default Docker image mapping to the current version of Python is used.
1218
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
1219
+ AWS Batch Job Queue to submit the job to.
1220
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
1221
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
1222
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
1223
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
1224
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
1225
+ shared_memory : int, optional, default None
1226
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1227
+ This parameter maps to the `--shm-size` option in Docker.
1228
+ max_swap : int, optional, default None
1229
+ The total amount of swap memory (in MiB) a container can use for this
1230
+ step. This parameter is translated to the `--memory-swap` option in
1231
+ Docker where the value is the sum of the container memory plus the
1232
+ `max_swap` value.
1233
+ swappiness : int, optional, default None
1234
+ This allows you to tune memory swappiness behavior for this step.
1235
+ A swappiness value of 0 causes swapping not to happen unless absolutely
1236
+ necessary. A swappiness value of 100 causes pages to be swapped very
1237
+ aggressively. Accepted values are whole numbers between 0 and 100.
1238
+ use_tmpfs : bool, default False
1239
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
1240
+ not available on Fargate compute environments
1241
+ tmpfs_tempdir : bool, default True
1242
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1243
+ tmpfs_size : int, optional, default None
1244
+ The value for the size (in MiB) of the tmpfs mount for this step.
1245
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1246
+ memory allocated for this step.
1247
+ tmpfs_path : str, optional, default None
1248
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
1249
+ inferentia : int, default 0
1250
+ Number of Inferentia chips required for this step.
1251
+ trainium : int, default None
1252
+ Alias for inferentia. Use only one of the two.
1253
+ efa : int, default 0
1254
+ Number of elastic fabric adapter network devices to attach to container
1255
+ ephemeral_storage : int, default None
1256
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
1257
+ This is only relevant for Fargate compute environments
1258
+ log_driver: str, optional, default None
1259
+ The log driver to use for the Amazon ECS container.
1260
+ log_options: List[str], optional, default None
1261
+ List of strings containing options for the chosen log driver. The configurable values
1262
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
1263
+ Example: [`awslogs-group:aws/batch/job`]
1264
+ """
1265
+ ...
1266
+
1267
+ @typing.overload
1268
+ def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1269
+ ...
1270
+
1271
+ @typing.overload
1272
+ def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1273
+ ...
1274
+
1275
+ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
1276
+ """
1277
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
1353
1278
 
1354
1279
  Parameters
1355
1280
  ----------
1356
- packages : Dict[str, str], default: {}
1357
- Packages to use for this step. The key is the name of the package
1358
- and the value is the version to use.
1359
- python : str, optional, default: None
1360
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1361
- that the version used will correspond to the version of the Python interpreter used to start the run.
1281
+ cpu : int, default 1
1282
+ Number of CPUs required for this step. If `@resources` is
1283
+ also present, the maximum value from all decorators is used.
1284
+ gpu : int, default 0
1285
+ Number of GPUs required for this step. If `@resources` is
1286
+ also present, the maximum value from all decorators is used.
1287
+ memory : int, default 4096
1288
+ Memory size (in MB) required for this step. If
1289
+ `@resources` is also present, the maximum value from all decorators is
1290
+ used.
1291
+ image : str, optional, default None
1292
+ Docker image to use when launching on AWS Batch. If not specified, and
1293
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
1294
+ not, a default Docker image mapping to the current version of Python is used.
1295
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
1296
+ AWS Batch Job Queue to submit the job to.
1297
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
1298
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
1299
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
1300
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
1301
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
1302
+ shared_memory : int, optional, default None
1303
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1304
+ This parameter maps to the `--shm-size` option in Docker.
1305
+ max_swap : int, optional, default None
1306
+ The total amount of swap memory (in MiB) a container can use for this
1307
+ step. This parameter is translated to the `--memory-swap` option in
1308
+ Docker where the value is the sum of the container memory plus the
1309
+ `max_swap` value.
1310
+ swappiness : int, optional, default None
1311
+ This allows you to tune memory swappiness behavior for this step.
1312
+ A swappiness value of 0 causes swapping not to happen unless absolutely
1313
+ necessary. A swappiness value of 100 causes pages to be swapped very
1314
+ aggressively. Accepted values are whole numbers between 0 and 100.
1315
+ use_tmpfs : bool, default False
1316
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
1317
+ not available on Fargate compute environments
1318
+ tmpfs_tempdir : bool, default True
1319
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1320
+ tmpfs_size : int, optional, default None
1321
+ The value for the size (in MiB) of the tmpfs mount for this step.
1322
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1323
+ memory allocated for this step.
1324
+ tmpfs_path : str, optional, default None
1325
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
1326
+ inferentia : int, default 0
1327
+ Number of Inferentia chips required for this step.
1328
+ trainium : int, default None
1329
+ Alias for inferentia. Use only one of the two.
1330
+ efa : int, default 0
1331
+ Number of elastic fabric adapter network devices to attach to container
1332
+ ephemeral_storage : int, default None
1333
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
1334
+ This is only relevant for Fargate compute environments
1335
+ log_driver: str, optional, default None
1336
+ The log driver to use for the Amazon ECS container.
1337
+ log_options: List[str], optional, default None
1338
+ List of strings containing options for the chosen log driver. The configurable values
1339
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
1340
+ Example: [`awslogs-group:aws/batch/job`]
1362
1341
  """
1363
1342
  ...
1364
1343
 
1365
1344
  @typing.overload
1366
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1345
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1346
+ """
1347
+ Specifies that the step will success under all circumstances.
1348
+
1349
+ The decorator will create an optional artifact, specified by `var`, which
1350
+ contains the exception raised. You can use it to detect the presence
1351
+ of errors, indicating that all happy-path artifacts produced by the step
1352
+ are missing.
1353
+
1354
+ Parameters
1355
+ ----------
1356
+ var : str, optional, default None
1357
+ Name of the artifact in which to store the caught exception.
1358
+ If not specified, the exception is not stored.
1359
+ print_exception : bool, default True
1360
+ Determines whether or not the exception is printed to
1361
+ stdout when caught.
1362
+ """
1367
1363
  ...
1368
1364
 
1369
1365
  @typing.overload
1370
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1366
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1371
1367
  ...
1372
1368
 
1373
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1369
+ @typing.overload
1370
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1371
+ ...
1372
+
1373
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1374
1374
  """
1375
- Specifies the PyPI packages for the step.
1375
+ Specifies that the step will success under all circumstances.
1376
1376
 
1377
- Information in this decorator will augment any
1378
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1379
- you can use `@pypi_base` to set packages required by all
1380
- steps and use `@pypi` to specify step-specific overrides.
1377
+ The decorator will create an optional artifact, specified by `var`, which
1378
+ contains the exception raised. You can use it to detect the presence
1379
+ of errors, indicating that all happy-path artifacts produced by the step
1380
+ are missing.
1381
1381
 
1382
1382
  Parameters
1383
1383
  ----------
1384
- packages : Dict[str, str], default: {}
1385
- Packages to use for this step. The key is the name of the package
1386
- and the value is the version to use.
1387
- python : str, optional, default: None
1388
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1389
- that the version used will correspond to the version of the Python interpreter used to start the run.
1384
+ var : str, optional, default None
1385
+ Name of the artifact in which to store the caught exception.
1386
+ If not specified, the exception is not stored.
1387
+ print_exception : bool, default True
1388
+ Determines whether or not the exception is printed to
1389
+ stdout when caught.
1390
+ """
1391
+ ...
1392
+
1393
+ @typing.overload
1394
+ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1395
+ """
1396
+ Specifies the flow(s) that this flow depends on.
1397
+
1398
+ ```
1399
+ @trigger_on_finish(flow='FooFlow')
1400
+ ```
1401
+ or
1402
+ ```
1403
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1404
+ ```
1405
+ This decorator respects the @project decorator and triggers the flow
1406
+ when upstream runs within the same namespace complete successfully
1407
+
1408
+ Additionally, you can specify project aware upstream flow dependencies
1409
+ by specifying the fully qualified project_flow_name.
1410
+ ```
1411
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1412
+ ```
1413
+ or
1414
+ ```
1415
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1416
+ ```
1417
+
1418
+ You can also specify just the project or project branch (other values will be
1419
+ inferred from the current project or project branch):
1420
+ ```
1421
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1422
+ ```
1423
+
1424
+ Note that `branch` is typically one of:
1425
+ - `prod`
1426
+ - `user.bob`
1427
+ - `test.my_experiment`
1428
+ - `prod.staging`
1429
+
1430
+ Parameters
1431
+ ----------
1432
+ flow : Union[str, Dict[str, str]], optional, default None
1433
+ Upstream flow dependency for this flow.
1434
+ flows : List[Union[str, Dict[str, str]]], default []
1435
+ Upstream flow dependencies for this flow.
1436
+ options : Dict[str, Any], default {}
1437
+ Backend-specific configuration for tuning eventing behavior.
1438
+
1439
+
1440
+ """
1441
+ ...
1442
+
1443
+ @typing.overload
1444
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1445
+ ...
1446
+
1447
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1448
+ """
1449
+ Specifies the flow(s) that this flow depends on.
1450
+
1451
+ ```
1452
+ @trigger_on_finish(flow='FooFlow')
1453
+ ```
1454
+ or
1455
+ ```
1456
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1457
+ ```
1458
+ This decorator respects the @project decorator and triggers the flow
1459
+ when upstream runs within the same namespace complete successfully
1460
+
1461
+ Additionally, you can specify project aware upstream flow dependencies
1462
+ by specifying the fully qualified project_flow_name.
1463
+ ```
1464
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1465
+ ```
1466
+ or
1467
+ ```
1468
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1469
+ ```
1470
+
1471
+ You can also specify just the project or project branch (other values will be
1472
+ inferred from the current project or project branch):
1473
+ ```
1474
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1475
+ ```
1476
+
1477
+ Note that `branch` is typically one of:
1478
+ - `prod`
1479
+ - `user.bob`
1480
+ - `test.my_experiment`
1481
+ - `prod.staging`
1482
+
1483
+ Parameters
1484
+ ----------
1485
+ flow : Union[str, Dict[str, str]], optional, default None
1486
+ Upstream flow dependency for this flow.
1487
+ flows : List[Union[str, Dict[str, str]]], default []
1488
+ Upstream flow dependencies for this flow.
1489
+ options : Dict[str, Any], default {}
1490
+ Backend-specific configuration for tuning eventing behavior.
1491
+
1492
+
1493
+ """
1494
+ ...
1495
+
1496
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1497
+ """
1498
+ Specifies what flows belong to the same project.
1499
+
1500
+ A project-specific namespace is created for all flows that
1501
+ use the same `@project(name)`.
1502
+
1503
+ Parameters
1504
+ ----------
1505
+ name : str
1506
+ Project name. Make sure that the name is unique amongst all
1507
+ projects that use the same production scheduler. The name may
1508
+ contain only lowercase alphanumeric characters and underscores.
1509
+
1510
+
1390
1511
  """
1391
1512
  ...
1392
1513
 
@@ -1521,93 +1642,62 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1521
1642
  @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1522
1643
  ```
1523
1644
 
1524
- Parameters
1525
- ----------
1526
- event : Union[str, Dict[str, Any]], optional, default None
1527
- Event dependency for this flow.
1528
- events : List[Union[str, Dict[str, Any]]], default []
1529
- Events dependency for this flow.
1530
- options : Dict[str, Any], default {}
1531
- Backend-specific configuration for tuning eventing behavior.
1532
-
1533
-
1534
- """
1535
- ...
1536
-
1537
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1538
- """
1539
- Specifies what flows belong to the same project.
1540
-
1541
- A project-specific namespace is created for all flows that
1542
- use the same `@project(name)`.
1543
-
1544
- Parameters
1545
- ----------
1546
- name : str
1547
- Project name. Make sure that the name is unique amongst all
1548
- projects that use the same production scheduler. The name may
1549
- contain only lowercase alphanumeric characters and underscores.
1550
-
1645
+ Parameters
1646
+ ----------
1647
+ event : Union[str, Dict[str, Any]], optional, default None
1648
+ Event dependency for this flow.
1649
+ events : List[Union[str, Dict[str, Any]]], default []
1650
+ Events dependency for this flow.
1651
+ options : Dict[str, Any], default {}
1652
+ Backend-specific configuration for tuning eventing behavior.
1653
+
1551
1654
 
1552
1655
  """
1553
1656
  ...
1554
1657
 
1555
1658
  @typing.overload
1556
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1659
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1557
1660
  """
1558
- Specifies the Conda environment for all steps of the flow.
1559
-
1560
- Use `@conda_base` to set common libraries required by all
1561
- steps and use `@conda` to specify step-specific additions.
1661
+ Specifies the PyPI packages for all steps of the flow.
1562
1662
 
1663
+ Use `@pypi_base` to set common packages required by all
1664
+ steps and use `@pypi` to specify step-specific overrides.
1563
1665
  Parameters
1564
1666
  ----------
1565
- packages : Dict[str, str], default {}
1667
+ packages : Dict[str, str], default: {}
1566
1668
  Packages to use for this flow. The key is the name of the package
1567
1669
  and the value is the version to use.
1568
- libraries : Dict[str, str], default {}
1569
- Supported for backward compatibility. When used with packages, packages will take precedence.
1570
- python : str, optional, default None
1670
+ python : str, optional, default: None
1571
1671
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1572
1672
  that the version used will correspond to the version of the Python interpreter used to start the run.
1573
- disabled : bool, default False
1574
- If set to True, disables Conda.
1575
1673
  """
1576
1674
  ...
1577
1675
 
1578
1676
  @typing.overload
1579
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1677
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1580
1678
  ...
1581
1679
 
1582
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1680
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1583
1681
  """
1584
- Specifies the Conda environment for all steps of the flow.
1585
-
1586
- Use `@conda_base` to set common libraries required by all
1587
- steps and use `@conda` to specify step-specific additions.
1682
+ Specifies the PyPI packages for all steps of the flow.
1588
1683
 
1684
+ Use `@pypi_base` to set common packages required by all
1685
+ steps and use `@pypi` to specify step-specific overrides.
1589
1686
  Parameters
1590
1687
  ----------
1591
- packages : Dict[str, str], default {}
1688
+ packages : Dict[str, str], default: {}
1592
1689
  Packages to use for this flow. The key is the name of the package
1593
1690
  and the value is the version to use.
1594
- libraries : Dict[str, str], default {}
1595
- Supported for backward compatibility. When used with packages, packages will take precedence.
1596
- python : str, optional, default None
1691
+ python : str, optional, default: None
1597
1692
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1598
1693
  that the version used will correspond to the version of the Python interpreter used to start the run.
1599
- disabled : bool, default False
1600
- If set to True, disables Conda.
1601
1694
  """
1602
1695
  ...
1603
1696
 
1604
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1697
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1605
1698
  """
1606
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1607
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1608
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1609
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1610
- starts only after all sensors finish.
1699
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1700
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1611
1701
 
1612
1702
  Parameters
1613
1703
  ----------
@@ -1628,25 +1718,31 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1628
1718
  Name of the sensor on Airflow
1629
1719
  description : str
1630
1720
  Description of sensor in the Airflow UI
1631
- bucket_key : Union[str, List[str]]
1632
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1633
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1634
- bucket_name : str
1635
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1636
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1637
- wildcard_match : bool
1638
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1639
- aws_conn_id : str
1640
- a reference to the s3 connection on Airflow. (Default: None)
1641
- verify : bool
1642
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1721
+ external_dag_id : str
1722
+ The dag_id that contains the task you want to wait for.
1723
+ external_task_ids : List[str]
1724
+ The list of task_ids that you want to wait for.
1725
+ If None (default value) the sensor waits for the DAG. (Default: None)
1726
+ allowed_states : List[str]
1727
+ Iterable of allowed states, (Default: ['success'])
1728
+ failed_states : List[str]
1729
+ Iterable of failed or dis-allowed states. (Default: None)
1730
+ execution_delta : datetime.timedelta
1731
+ time difference with the previous execution to look at,
1732
+ the default is the same logical date as the current task or DAG. (Default: None)
1733
+ check_existence: bool
1734
+ Set to True to check if the external task exists or check if
1735
+ the DAG to wait for exists. (Default: True)
1643
1736
  """
1644
1737
  ...
1645
1738
 
1646
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1739
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1647
1740
  """
1648
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1649
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1741
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1742
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1743
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1744
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1745
+ starts only after all sensors finish.
1650
1746
 
1651
1747
  Parameters
1652
1748
  ----------
@@ -1667,163 +1763,67 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1667
1763
  Name of the sensor on Airflow
1668
1764
  description : str
1669
1765
  Description of sensor in the Airflow UI
1670
- external_dag_id : str
1671
- The dag_id that contains the task you want to wait for.
1672
- external_task_ids : List[str]
1673
- The list of task_ids that you want to wait for.
1674
- If None (default value) the sensor waits for the DAG. (Default: None)
1675
- allowed_states : List[str]
1676
- Iterable of allowed states, (Default: ['success'])
1677
- failed_states : List[str]
1678
- Iterable of failed or dis-allowed states. (Default: None)
1679
- execution_delta : datetime.timedelta
1680
- time difference with the previous execution to look at,
1681
- the default is the same logical date as the current task or DAG. (Default: None)
1682
- check_existence: bool
1683
- Set to True to check if the external task exists or check if
1684
- the DAG to wait for exists. (Default: True)
1766
+ bucket_key : Union[str, List[str]]
1767
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1768
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1769
+ bucket_name : str
1770
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1771
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1772
+ wildcard_match : bool
1773
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1774
+ aws_conn_id : str
1775
+ a reference to the s3 connection on Airflow. (Default: None)
1776
+ verify : bool
1777
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1685
1778
  """
1686
1779
  ...
1687
1780
 
1688
1781
  @typing.overload
1689
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1782
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1690
1783
  """
1691
- Specifies the PyPI packages for all steps of the flow.
1784
+ Specifies the Conda environment for all steps of the flow.
1785
+
1786
+ Use `@conda_base` to set common libraries required by all
1787
+ steps and use `@conda` to specify step-specific additions.
1692
1788
 
1693
- Use `@pypi_base` to set common packages required by all
1694
- steps and use `@pypi` to specify step-specific overrides.
1695
1789
  Parameters
1696
1790
  ----------
1697
- packages : Dict[str, str], default: {}
1791
+ packages : Dict[str, str], default {}
1698
1792
  Packages to use for this flow. The key is the name of the package
1699
1793
  and the value is the version to use.
1700
- python : str, optional, default: None
1794
+ libraries : Dict[str, str], default {}
1795
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1796
+ python : str, optional, default None
1701
1797
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1702
1798
  that the version used will correspond to the version of the Python interpreter used to start the run.
1799
+ disabled : bool, default False
1800
+ If set to True, disables Conda.
1703
1801
  """
1704
1802
  ...
1705
1803
 
1706
1804
  @typing.overload
1707
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1805
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1708
1806
  ...
1709
1807
 
1710
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1808
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1711
1809
  """
1712
- Specifies the PyPI packages for all steps of the flow.
1810
+ Specifies the Conda environment for all steps of the flow.
1811
+
1812
+ Use `@conda_base` to set common libraries required by all
1813
+ steps and use `@conda` to specify step-specific additions.
1713
1814
 
1714
- Use `@pypi_base` to set common packages required by all
1715
- steps and use `@pypi` to specify step-specific overrides.
1716
1815
  Parameters
1717
1816
  ----------
1718
- packages : Dict[str, str], default: {}
1817
+ packages : Dict[str, str], default {}
1719
1818
  Packages to use for this flow. The key is the name of the package
1720
1819
  and the value is the version to use.
1721
- python : str, optional, default: None
1820
+ libraries : Dict[str, str], default {}
1821
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1822
+ python : str, optional, default None
1722
1823
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1723
1824
  that the version used will correspond to the version of the Python interpreter used to start the run.
1724
- """
1725
- ...
1726
-
1727
- @typing.overload
1728
- def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1729
- """
1730
- Specifies the flow(s) that this flow depends on.
1731
-
1732
- ```
1733
- @trigger_on_finish(flow='FooFlow')
1734
- ```
1735
- or
1736
- ```
1737
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1738
- ```
1739
- This decorator respects the @project decorator and triggers the flow
1740
- when upstream runs within the same namespace complete successfully
1741
-
1742
- Additionally, you can specify project aware upstream flow dependencies
1743
- by specifying the fully qualified project_flow_name.
1744
- ```
1745
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1746
- ```
1747
- or
1748
- ```
1749
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1750
- ```
1751
-
1752
- You can also specify just the project or project branch (other values will be
1753
- inferred from the current project or project branch):
1754
- ```
1755
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1756
- ```
1757
-
1758
- Note that `branch` is typically one of:
1759
- - `prod`
1760
- - `user.bob`
1761
- - `test.my_experiment`
1762
- - `prod.staging`
1763
-
1764
- Parameters
1765
- ----------
1766
- flow : Union[str, Dict[str, str]], optional, default None
1767
- Upstream flow dependency for this flow.
1768
- flows : List[Union[str, Dict[str, str]]], default []
1769
- Upstream flow dependencies for this flow.
1770
- options : Dict[str, Any], default {}
1771
- Backend-specific configuration for tuning eventing behavior.
1772
-
1773
-
1774
- """
1775
- ...
1776
-
1777
- @typing.overload
1778
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1779
- ...
1780
-
1781
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1782
- """
1783
- Specifies the flow(s) that this flow depends on.
1784
-
1785
- ```
1786
- @trigger_on_finish(flow='FooFlow')
1787
- ```
1788
- or
1789
- ```
1790
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1791
- ```
1792
- This decorator respects the @project decorator and triggers the flow
1793
- when upstream runs within the same namespace complete successfully
1794
-
1795
- Additionally, you can specify project aware upstream flow dependencies
1796
- by specifying the fully qualified project_flow_name.
1797
- ```
1798
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1799
- ```
1800
- or
1801
- ```
1802
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1803
- ```
1804
-
1805
- You can also specify just the project or project branch (other values will be
1806
- inferred from the current project or project branch):
1807
- ```
1808
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1809
- ```
1810
-
1811
- Note that `branch` is typically one of:
1812
- - `prod`
1813
- - `user.bob`
1814
- - `test.my_experiment`
1815
- - `prod.staging`
1816
-
1817
- Parameters
1818
- ----------
1819
- flow : Union[str, Dict[str, str]], optional, default None
1820
- Upstream flow dependency for this flow.
1821
- flows : List[Union[str, Dict[str, str]]], default []
1822
- Upstream flow dependencies for this flow.
1823
- options : Dict[str, Any], default {}
1824
- Backend-specific configuration for tuning eventing behavior.
1825
-
1826
-
1825
+ disabled : bool, default False
1826
+ If set to True, disables Conda.
1827
1827
  """
1828
1828
  ...
1829
1829