ob-metaflow-stubs 4.0__py2.py3-none-any.whl → 4.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. metaflow-stubs/__init__.pyi +442 -442
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +3 -3
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +4 -2
  16. metaflow-stubs/metaflow_current.pyi +4 -4
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +3 -3
  20. metaflow-stubs/plugins/__init__.pyi +3 -3
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +7 -3
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +3 -3
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  37. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  45. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  47. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  56. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  60. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  61. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  62. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  63. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  65. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  78. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  81. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  82. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  83. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  84. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  89. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  91. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  92. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  93. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  95. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  97. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  99. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  106. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  107. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  108. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  109. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  110. metaflow-stubs/plugins/package_cli.pyi +2 -2
  111. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/perimeters.pyi +2 -2
  113. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  114. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  117. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/pypi_environment.pyi +4 -4
  119. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  120. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  123. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  124. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  126. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  127. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  128. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  129. metaflow-stubs/procpoll.pyi +2 -2
  130. metaflow-stubs/profilers/__init__.pyi +2 -2
  131. metaflow-stubs/pylint_wrapper.pyi +2 -2
  132. metaflow-stubs/runner/__init__.pyi +2 -2
  133. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  134. metaflow-stubs/runner/nbrun.pyi +2 -2
  135. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  136. metaflow-stubs/tagging_util.pyi +2 -2
  137. metaflow-stubs/tuple_util.pyi +2 -2
  138. {ob_metaflow_stubs-4.0.dist-info → ob_metaflow_stubs-4.1.dist-info}/METADATA +1 -1
  139. ob_metaflow_stubs-4.1.dist-info/RECORD +142 -0
  140. ob_metaflow_stubs-4.0.dist-info/RECORD +0 -142
  141. {ob_metaflow_stubs-4.0.dist-info → ob_metaflow_stubs-4.1.dist-info}/WHEEL +0 -0
  142. {ob_metaflow_stubs-4.0.dist-info → ob_metaflow_stubs-4.1.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.5.2+ob(v1) #
4
- # Generated on 2024-06-24T23:02:43.033000 #
3
+ # MF version: 2.12.6.1+ob(v1) #
4
+ # Generated on 2024-07-03T18:20:45.077392 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import io
12
- import metaflow.client.core
13
- import metaflow.parameters
14
- import metaflow.metaflow_current
15
11
  import metaflow.flowspec
16
- import metaflow.runner.metaflow_runner
17
- import typing
18
- import datetime
19
- import metaflow.plugins.datatools.s3.s3
20
12
  import metaflow.datastore.inputs
21
13
  import metaflow.events
22
14
  import metaflow._vendor.click.types
15
+ import metaflow.metaflow_current
16
+ import metaflow.parameters
17
+ import typing
18
+ import metaflow.runner.metaflow_runner
19
+ import io
20
+ import metaflow.client.core
21
+ import metaflow.plugins.datatools.s3.s3
22
+ import datetime
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -727,6 +727,114 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
727
727
  """
728
728
  ...
729
729
 
730
+ @typing.overload
731
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
+ """
733
+ Specifies that the step will success under all circumstances.
734
+
735
+ The decorator will create an optional artifact, specified by `var`, which
736
+ contains the exception raised. You can use it to detect the presence
737
+ of errors, indicating that all happy-path artifacts produced by the step
738
+ are missing.
739
+
740
+ Parameters
741
+ ----------
742
+ var : str, optional, default None
743
+ Name of the artifact in which to store the caught exception.
744
+ If not specified, the exception is not stored.
745
+ print_exception : bool, default True
746
+ Determines whether or not the exception is printed to
747
+ stdout when caught.
748
+ """
749
+ ...
750
+
751
+ @typing.overload
752
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
753
+ ...
754
+
755
+ @typing.overload
756
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
757
+ ...
758
+
759
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
760
+ """
761
+ Specifies that the step will success under all circumstances.
762
+
763
+ The decorator will create an optional artifact, specified by `var`, which
764
+ contains the exception raised. You can use it to detect the presence
765
+ of errors, indicating that all happy-path artifacts produced by the step
766
+ are missing.
767
+
768
+ Parameters
769
+ ----------
770
+ var : str, optional, default None
771
+ Name of the artifact in which to store the caught exception.
772
+ If not specified, the exception is not stored.
773
+ print_exception : bool, default True
774
+ Determines whether or not the exception is printed to
775
+ stdout when caught.
776
+ """
777
+ ...
778
+
779
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
780
+ """
781
+ Specifies that this step should execute on Kubernetes.
782
+
783
+ Parameters
784
+ ----------
785
+ cpu : int, default 1
786
+ Number of CPUs required for this step. If `@resources` is
787
+ also present, the maximum value from all decorators is used.
788
+ memory : int, default 4096
789
+ Memory size (in MB) required for this step. If
790
+ `@resources` is also present, the maximum value from all decorators is
791
+ used.
792
+ disk : int, default 10240
793
+ Disk size (in MB) required for this step. If
794
+ `@resources` is also present, the maximum value from all decorators is
795
+ used.
796
+ image : str, optional, default None
797
+ Docker image to use when launching on Kubernetes. If not specified, and
798
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
799
+ not, a default Docker image mapping to the current version of Python is used.
800
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
801
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
802
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
803
+ Kubernetes service account to use when launching pod in Kubernetes.
804
+ secrets : List[str], optional, default None
805
+ Kubernetes secrets to use when launching pod in Kubernetes. These
806
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
807
+ in Metaflow configuration.
808
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
809
+ Kubernetes namespace to use when launching pod in Kubernetes.
810
+ gpu : int, optional, default None
811
+ Number of GPUs required for this step. A value of zero implies that
812
+ the scheduled node should not have GPUs.
813
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
814
+ The vendor of the GPUs to be used for this step.
815
+ tolerations : List[str], default []
816
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
817
+ Kubernetes tolerations to use when launching pod in Kubernetes.
818
+ use_tmpfs : bool, default False
819
+ This enables an explicit tmpfs mount for this step.
820
+ tmpfs_tempdir : bool, default True
821
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
822
+ tmpfs_size : int, optional, default: None
823
+ The value for the size (in MiB) of the tmpfs mount for this step.
824
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
825
+ memory allocated for this step.
826
+ tmpfs_path : str, optional, default /metaflow_temp
827
+ Path to tmpfs mount for this step.
828
+ persistent_volume_claims : Dict[str, str], optional, default None
829
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
830
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
831
+ shared_memory: int, optional
832
+ Shared memory size (in MiB) required for this step
833
+ port: int, optional
834
+ Port number to specify in the Kubernetes job object
835
+ """
836
+ ...
837
+
730
838
  @typing.overload
731
839
  def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
840
  """
@@ -778,55 +886,6 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
778
886
  """
779
887
  ...
780
888
 
781
- @typing.overload
782
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
783
- """
784
- Specifies the PyPI packages for the step.
785
-
786
- Information in this decorator will augment any
787
- attributes set in the `@pyi_base` flow-level decorator. Hence,
788
- you can use `@pypi_base` to set packages required by all
789
- steps and use `@pypi` to specify step-specific overrides.
790
-
791
- Parameters
792
- ----------
793
- packages : Dict[str, str], default: {}
794
- Packages to use for this step. The key is the name of the package
795
- and the value is the version to use.
796
- python : str, optional, default: None
797
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
798
- that the version used will correspond to the version of the Python interpreter used to start the run.
799
- """
800
- ...
801
-
802
- @typing.overload
803
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
804
- ...
805
-
806
- @typing.overload
807
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
808
- ...
809
-
810
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
811
- """
812
- Specifies the PyPI packages for the step.
813
-
814
- Information in this decorator will augment any
815
- attributes set in the `@pyi_base` flow-level decorator. Hence,
816
- you can use `@pypi_base` to set packages required by all
817
- steps and use `@pypi` to specify step-specific overrides.
818
-
819
- Parameters
820
- ----------
821
- packages : Dict[str, str], default: {}
822
- Packages to use for this step. The key is the name of the package
823
- and the value is the version to use.
824
- python : str, optional, default: None
825
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
826
- that the version used will correspond to the version of the Python interpreter used to start the run.
827
- """
828
- ...
829
-
830
889
  @typing.overload
831
890
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
832
891
  """
@@ -884,55 +943,6 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
884
943
  """
885
944
  ...
886
945
 
887
- @typing.overload
888
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
889
- """
890
- Specifies that the step will success under all circumstances.
891
-
892
- The decorator will create an optional artifact, specified by `var`, which
893
- contains the exception raised. You can use it to detect the presence
894
- of errors, indicating that all happy-path artifacts produced by the step
895
- are missing.
896
-
897
- Parameters
898
- ----------
899
- var : str, optional, default None
900
- Name of the artifact in which to store the caught exception.
901
- If not specified, the exception is not stored.
902
- print_exception : bool, default True
903
- Determines whether or not the exception is printed to
904
- stdout when caught.
905
- """
906
- ...
907
-
908
- @typing.overload
909
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
910
- ...
911
-
912
- @typing.overload
913
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
914
- ...
915
-
916
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
917
- """
918
- Specifies that the step will success under all circumstances.
919
-
920
- The decorator will create an optional artifact, specified by `var`, which
921
- contains the exception raised. You can use it to detect the presence
922
- of errors, indicating that all happy-path artifacts produced by the step
923
- are missing.
924
-
925
- Parameters
926
- ----------
927
- var : str, optional, default None
928
- Name of the artifact in which to store the caught exception.
929
- If not specified, the exception is not stored.
930
- print_exception : bool, default True
931
- Determines whether or not the exception is printed to
932
- stdout when caught.
933
- """
934
- ...
935
-
936
946
  @typing.overload
937
947
  def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
938
948
  """
@@ -1080,37 +1090,6 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1080
1090
  """
1081
1091
  ...
1082
1092
 
1083
- @typing.overload
1084
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1085
- """
1086
- Specifies environment variables to be set prior to the execution of a step.
1087
-
1088
- Parameters
1089
- ----------
1090
- vars : Dict[str, str], default {}
1091
- Dictionary of environment variables to set.
1092
- """
1093
- ...
1094
-
1095
- @typing.overload
1096
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1097
- ...
1098
-
1099
- @typing.overload
1100
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1101
- ...
1102
-
1103
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1104
- """
1105
- Specifies environment variables to be set prior to the execution of a step.
1106
-
1107
- Parameters
1108
- ----------
1109
- vars : Dict[str, str], default {}
1110
- Dictionary of environment variables to set.
1111
- """
1112
- ...
1113
-
1114
1093
  @typing.overload
1115
1094
  def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1116
1095
  """
@@ -1164,124 +1143,8 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1164
1143
  """
1165
1144
  ...
1166
1145
 
1167
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1168
- """
1169
- Specifies that this step should execute on Kubernetes.
1170
-
1171
- Parameters
1172
- ----------
1173
- cpu : int, default 1
1174
- Number of CPUs required for this step. If `@resources` is
1175
- also present, the maximum value from all decorators is used.
1176
- memory : int, default 4096
1177
- Memory size (in MB) required for this step. If
1178
- `@resources` is also present, the maximum value from all decorators is
1179
- used.
1180
- disk : int, default 10240
1181
- Disk size (in MB) required for this step. If
1182
- `@resources` is also present, the maximum value from all decorators is
1183
- used.
1184
- image : str, optional, default None
1185
- Docker image to use when launching on Kubernetes. If not specified, and
1186
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1187
- not, a default Docker image mapping to the current version of Python is used.
1188
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1189
- If given, the imagePullPolicy to be applied to the Docker image of the step.
1190
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1191
- Kubernetes service account to use when launching pod in Kubernetes.
1192
- secrets : List[str], optional, default None
1193
- Kubernetes secrets to use when launching pod in Kubernetes. These
1194
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1195
- in Metaflow configuration.
1196
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1197
- Kubernetes namespace to use when launching pod in Kubernetes.
1198
- gpu : int, optional, default None
1199
- Number of GPUs required for this step. A value of zero implies that
1200
- the scheduled node should not have GPUs.
1201
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1202
- The vendor of the GPUs to be used for this step.
1203
- tolerations : List[str], default []
1204
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1205
- Kubernetes tolerations to use when launching pod in Kubernetes.
1206
- use_tmpfs : bool, default False
1207
- This enables an explicit tmpfs mount for this step.
1208
- tmpfs_tempdir : bool, default True
1209
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1210
- tmpfs_size : int, optional, default: None
1211
- The value for the size (in MiB) of the tmpfs mount for this step.
1212
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1213
- memory allocated for this step.
1214
- tmpfs_path : str, optional, default /metaflow_temp
1215
- Path to tmpfs mount for this step.
1216
- persistent_volume_claims : Dict[str, str], optional, default None
1217
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1218
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1219
- shared_memory: int, optional
1220
- Shared memory size (in MiB) required for this step
1221
- port: int, optional
1222
- Port number to specify in the Kubernetes job object
1223
- """
1224
- ...
1225
-
1226
- @typing.overload
1227
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1228
- """
1229
- Specifies the Conda environment for the step.
1230
-
1231
- Information in this decorator will augment any
1232
- attributes set in the `@conda_base` flow-level decorator. Hence,
1233
- you can use `@conda_base` to set packages required by all
1234
- steps and use `@conda` to specify step-specific overrides.
1235
-
1236
- Parameters
1237
- ----------
1238
- packages : Dict[str, str], default {}
1239
- Packages to use for this step. The key is the name of the package
1240
- and the value is the version to use.
1241
- libraries : Dict[str, str], default {}
1242
- Supported for backward compatibility. When used with packages, packages will take precedence.
1243
- python : str, optional, default None
1244
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1245
- that the version used will correspond to the version of the Python interpreter used to start the run.
1246
- disabled : bool, default False
1247
- If set to True, disables @conda.
1248
- """
1249
- ...
1250
-
1251
- @typing.overload
1252
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1253
- ...
1254
-
1255
- @typing.overload
1256
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1257
- ...
1258
-
1259
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1260
- """
1261
- Specifies the Conda environment for the step.
1262
-
1263
- Information in this decorator will augment any
1264
- attributes set in the `@conda_base` flow-level decorator. Hence,
1265
- you can use `@conda_base` to set packages required by all
1266
- steps and use `@conda` to specify step-specific overrides.
1267
-
1268
- Parameters
1269
- ----------
1270
- packages : Dict[str, str], default {}
1271
- Packages to use for this step. The key is the name of the package
1272
- and the value is the version to use.
1273
- libraries : Dict[str, str], default {}
1274
- Supported for backward compatibility. When used with packages, packages will take precedence.
1275
- python : str, optional, default None
1276
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1277
- that the version used will correspond to the version of the Python interpreter used to start the run.
1278
- disabled : bool, default False
1279
- If set to True, disables @conda.
1280
- """
1281
- ...
1282
-
1283
- @typing.overload
1284
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1146
+ @typing.overload
1147
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1285
1148
  """
1286
1149
  Specifies secrets to be retrieved and injected as environment variables prior to
1287
1150
  the execution of a step.
@@ -1391,139 +1254,146 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1391
1254
  ...
1392
1255
 
1393
1256
  @typing.overload
1394
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1257
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1395
1258
  """
1396
- Specifies the PyPI packages for all steps of the flow.
1259
+ Specifies the Conda environment for the step.
1260
+
1261
+ Information in this decorator will augment any
1262
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1263
+ you can use `@conda_base` to set packages required by all
1264
+ steps and use `@conda` to specify step-specific overrides.
1397
1265
 
1398
- Use `@pypi_base` to set common packages required by all
1399
- steps and use `@pypi` to specify step-specific overrides.
1400
1266
  Parameters
1401
1267
  ----------
1402
- packages : Dict[str, str], default: {}
1403
- Packages to use for this flow. The key is the name of the package
1268
+ packages : Dict[str, str], default {}
1269
+ Packages to use for this step. The key is the name of the package
1404
1270
  and the value is the version to use.
1405
- python : str, optional, default: None
1271
+ libraries : Dict[str, str], default {}
1272
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1273
+ python : str, optional, default None
1406
1274
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1407
1275
  that the version used will correspond to the version of the Python interpreter used to start the run.
1276
+ disabled : bool, default False
1277
+ If set to True, disables @conda.
1408
1278
  """
1409
1279
  ...
1410
1280
 
1411
1281
  @typing.overload
1412
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1282
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1413
1283
  ...
1414
1284
 
1415
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1285
+ @typing.overload
1286
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1287
+ ...
1288
+
1289
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1416
1290
  """
1417
- Specifies the PyPI packages for all steps of the flow.
1291
+ Specifies the Conda environment for the step.
1292
+
1293
+ Information in this decorator will augment any
1294
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1295
+ you can use `@conda_base` to set packages required by all
1296
+ steps and use `@conda` to specify step-specific overrides.
1418
1297
 
1419
- Use `@pypi_base` to set common packages required by all
1420
- steps and use `@pypi` to specify step-specific overrides.
1421
1298
  Parameters
1422
1299
  ----------
1423
- packages : Dict[str, str], default: {}
1424
- Packages to use for this flow. The key is the name of the package
1300
+ packages : Dict[str, str], default {}
1301
+ Packages to use for this step. The key is the name of the package
1425
1302
  and the value is the version to use.
1426
- python : str, optional, default: None
1303
+ libraries : Dict[str, str], default {}
1304
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1305
+ python : str, optional, default None
1427
1306
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1428
1307
  that the version used will correspond to the version of the Python interpreter used to start the run.
1308
+ disabled : bool, default False
1309
+ If set to True, disables @conda.
1429
1310
  """
1430
1311
  ...
1431
1312
 
1432
1313
  @typing.overload
1433
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1314
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1434
1315
  """
1435
- Specifies the Conda environment for all steps of the flow.
1316
+ Specifies environment variables to be set prior to the execution of a step.
1436
1317
 
1437
- Use `@conda_base` to set common libraries required by all
1438
- steps and use `@conda` to specify step-specific additions.
1318
+ Parameters
1319
+ ----------
1320
+ vars : Dict[str, str], default {}
1321
+ Dictionary of environment variables to set.
1322
+ """
1323
+ ...
1324
+
1325
+ @typing.overload
1326
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1327
+ ...
1328
+
1329
+ @typing.overload
1330
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1331
+ ...
1332
+
1333
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1334
+ """
1335
+ Specifies environment variables to be set prior to the execution of a step.
1439
1336
 
1440
1337
  Parameters
1441
1338
  ----------
1442
- packages : Dict[str, str], default {}
1443
- Packages to use for this flow. The key is the name of the package
1339
+ vars : Dict[str, str], default {}
1340
+ Dictionary of environment variables to set.
1341
+ """
1342
+ ...
1343
+
1344
+ @typing.overload
1345
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1346
+ """
1347
+ Specifies the PyPI packages for the step.
1348
+
1349
+ Information in this decorator will augment any
1350
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1351
+ you can use `@pypi_base` to set packages required by all
1352
+ steps and use `@pypi` to specify step-specific overrides.
1353
+
1354
+ Parameters
1355
+ ----------
1356
+ packages : Dict[str, str], default: {}
1357
+ Packages to use for this step. The key is the name of the package
1444
1358
  and the value is the version to use.
1445
- libraries : Dict[str, str], default {}
1446
- Supported for backward compatibility. When used with packages, packages will take precedence.
1447
- python : str, optional, default None
1359
+ python : str, optional, default: None
1448
1360
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1449
1361
  that the version used will correspond to the version of the Python interpreter used to start the run.
1450
- disabled : bool, default False
1451
- If set to True, disables Conda.
1452
1362
  """
1453
1363
  ...
1454
1364
 
1455
1365
  @typing.overload
1456
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1366
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1457
1367
  ...
1458
1368
 
1459
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1369
+ @typing.overload
1370
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1371
+ ...
1372
+
1373
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1460
1374
  """
1461
- Specifies the Conda environment for all steps of the flow.
1375
+ Specifies the PyPI packages for the step.
1462
1376
 
1463
- Use `@conda_base` to set common libraries required by all
1464
- steps and use `@conda` to specify step-specific additions.
1377
+ Information in this decorator will augment any
1378
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1379
+ you can use `@pypi_base` to set packages required by all
1380
+ steps and use `@pypi` to specify step-specific overrides.
1465
1381
 
1466
1382
  Parameters
1467
1383
  ----------
1468
- packages : Dict[str, str], default {}
1469
- Packages to use for this flow. The key is the name of the package
1384
+ packages : Dict[str, str], default: {}
1385
+ Packages to use for this step. The key is the name of the package
1470
1386
  and the value is the version to use.
1471
- libraries : Dict[str, str], default {}
1472
- Supported for backward compatibility. When used with packages, packages will take precedence.
1473
- python : str, optional, default None
1387
+ python : str, optional, default: None
1474
1388
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1475
1389
  that the version used will correspond to the version of the Python interpreter used to start the run.
1476
- disabled : bool, default False
1477
- If set to True, disables Conda.
1478
1390
  """
1479
1391
  ...
1480
1392
 
1481
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1393
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1482
1394
  """
1483
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1484
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1485
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1486
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1487
- starts only after all sensors finish.
1488
-
1489
- Parameters
1490
- ----------
1491
- timeout : int
1492
- Time, in seconds before the task times out and fails. (Default: 3600)
1493
- poke_interval : int
1494
- Time in seconds that the job should wait in between each try. (Default: 60)
1495
- mode : str
1496
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1497
- exponential_backoff : bool
1498
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1499
- pool : str
1500
- the slot pool this task should run in,
1501
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1502
- soft_fail : bool
1503
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1504
- name : str
1505
- Name of the sensor on Airflow
1506
- description : str
1507
- Description of sensor in the Airflow UI
1508
- bucket_key : Union[str, List[str]]
1509
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1510
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1511
- bucket_name : str
1512
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1513
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1514
- wildcard_match : bool
1515
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1516
- aws_conn_id : str
1517
- a reference to the s3 connection on Airflow. (Default: None)
1518
- verify : bool
1519
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1520
- """
1521
- ...
1522
-
1523
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1524
- """
1525
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1526
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1395
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1396
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1527
1397
 
1528
1398
  Parameters
1529
1399
  ----------
@@ -1563,97 +1433,139 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1563
1433
  ...
1564
1434
 
1565
1435
  @typing.overload
1566
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1436
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1567
1437
  """
1568
- Specifies the event(s) that this flow depends on.
1569
-
1570
- ```
1571
- @trigger(event='foo')
1572
- ```
1573
- or
1574
- ```
1575
- @trigger(events=['foo', 'bar'])
1576
- ```
1577
-
1578
- Additionally, you can specify the parameter mappings
1579
- to map event payload to Metaflow parameters for the flow.
1580
- ```
1581
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1582
- ```
1583
- or
1584
- ```
1585
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1586
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1587
- ```
1438
+ Specifies the PyPI packages for all steps of the flow.
1588
1439
 
1589
- 'parameters' can also be a list of strings and tuples like so:
1590
- ```
1591
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1592
- ```
1593
- This is equivalent to:
1594
- ```
1595
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1596
- ```
1440
+ Use `@pypi_base` to set common packages required by all
1441
+ steps and use `@pypi` to specify step-specific overrides.
1442
+ Parameters
1443
+ ----------
1444
+ packages : Dict[str, str], default: {}
1445
+ Packages to use for this flow. The key is the name of the package
1446
+ and the value is the version to use.
1447
+ python : str, optional, default: None
1448
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1449
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1450
+ """
1451
+ ...
1452
+
1453
+ @typing.overload
1454
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1455
+ ...
1456
+
1457
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1458
+ """
1459
+ Specifies the PyPI packages for all steps of the flow.
1597
1460
 
1461
+ Use `@pypi_base` to set common packages required by all
1462
+ steps and use `@pypi` to specify step-specific overrides.
1598
1463
  Parameters
1599
1464
  ----------
1600
- event : Union[str, Dict[str, Any]], optional, default None
1601
- Event dependency for this flow.
1602
- events : List[Union[str, Dict[str, Any]]], default []
1603
- Events dependency for this flow.
1604
- options : Dict[str, Any], default {}
1605
- Backend-specific configuration for tuning eventing behavior.
1465
+ packages : Dict[str, str], default: {}
1466
+ Packages to use for this flow. The key is the name of the package
1467
+ and the value is the version to use.
1468
+ python : str, optional, default: None
1469
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1470
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1471
+ """
1472
+ ...
1473
+
1474
+ @typing.overload
1475
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1476
+ """
1477
+ Specifies the Conda environment for all steps of the flow.
1606
1478
 
1479
+ Use `@conda_base` to set common libraries required by all
1480
+ steps and use `@conda` to specify step-specific additions.
1607
1481
 
1482
+ Parameters
1483
+ ----------
1484
+ packages : Dict[str, str], default {}
1485
+ Packages to use for this flow. The key is the name of the package
1486
+ and the value is the version to use.
1487
+ libraries : Dict[str, str], default {}
1488
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1489
+ python : str, optional, default None
1490
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1491
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1492
+ disabled : bool, default False
1493
+ If set to True, disables Conda.
1608
1494
  """
1609
1495
  ...
1610
1496
 
1611
1497
  @typing.overload
1612
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1498
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1613
1499
  ...
1614
1500
 
1615
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1501
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1616
1502
  """
1617
- Specifies the event(s) that this flow depends on.
1618
-
1619
- ```
1620
- @trigger(event='foo')
1621
- ```
1622
- or
1623
- ```
1624
- @trigger(events=['foo', 'bar'])
1625
- ```
1626
-
1627
- Additionally, you can specify the parameter mappings
1628
- to map event payload to Metaflow parameters for the flow.
1629
- ```
1630
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1631
- ```
1632
- or
1633
- ```
1634
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1635
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1636
- ```
1503
+ Specifies the Conda environment for all steps of the flow.
1637
1504
 
1638
- 'parameters' can also be a list of strings and tuples like so:
1639
- ```
1640
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1641
- ```
1642
- This is equivalent to:
1643
- ```
1644
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1645
- ```
1505
+ Use `@conda_base` to set common libraries required by all
1506
+ steps and use `@conda` to specify step-specific additions.
1646
1507
 
1647
1508
  Parameters
1648
1509
  ----------
1649
- event : Union[str, Dict[str, Any]], optional, default None
1650
- Event dependency for this flow.
1651
- events : List[Union[str, Dict[str, Any]]], default []
1652
- Events dependency for this flow.
1653
- options : Dict[str, Any], default {}
1654
- Backend-specific configuration for tuning eventing behavior.
1510
+ packages : Dict[str, str], default {}
1511
+ Packages to use for this flow. The key is the name of the package
1512
+ and the value is the version to use.
1513
+ libraries : Dict[str, str], default {}
1514
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1515
+ python : str, optional, default None
1516
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1517
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1518
+ disabled : bool, default False
1519
+ If set to True, disables Conda.
1520
+ """
1521
+ ...
1522
+
1523
+ @typing.overload
1524
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1525
+ """
1526
+ Specifies the times when the flow should be run when running on a
1527
+ production scheduler.
1655
1528
 
1529
+ Parameters
1530
+ ----------
1531
+ hourly : bool, default False
1532
+ Run the workflow hourly.
1533
+ daily : bool, default True
1534
+ Run the workflow daily.
1535
+ weekly : bool, default False
1536
+ Run the workflow weekly.
1537
+ cron : str, optional, default None
1538
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1539
+ specified by this expression.
1540
+ timezone : str, optional, default None
1541
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1542
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1543
+ """
1544
+ ...
1545
+
1546
+ @typing.overload
1547
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1548
+ ...
1549
+
1550
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1551
+ """
1552
+ Specifies the times when the flow should be run when running on a
1553
+ production scheduler.
1656
1554
 
1555
+ Parameters
1556
+ ----------
1557
+ hourly : bool, default False
1558
+ Run the workflow hourly.
1559
+ daily : bool, default True
1560
+ Run the workflow daily.
1561
+ weekly : bool, default False
1562
+ Run the workflow weekly.
1563
+ cron : str, optional, default None
1564
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1565
+ specified by this expression.
1566
+ timezone : str, optional, default None
1567
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1568
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1657
1569
  """
1658
1570
  ...
1659
1571
 
@@ -1761,51 +1673,97 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1761
1673
  ...
1762
1674
 
1763
1675
  @typing.overload
1764
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1676
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1765
1677
  """
1766
- Specifies the times when the flow should be run when running on a
1767
- production scheduler.
1678
+ Specifies the event(s) that this flow depends on.
1679
+
1680
+ ```
1681
+ @trigger(event='foo')
1682
+ ```
1683
+ or
1684
+ ```
1685
+ @trigger(events=['foo', 'bar'])
1686
+ ```
1687
+
1688
+ Additionally, you can specify the parameter mappings
1689
+ to map event payload to Metaflow parameters for the flow.
1690
+ ```
1691
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1692
+ ```
1693
+ or
1694
+ ```
1695
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1696
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1697
+ ```
1698
+
1699
+ 'parameters' can also be a list of strings and tuples like so:
1700
+ ```
1701
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1702
+ ```
1703
+ This is equivalent to:
1704
+ ```
1705
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1706
+ ```
1768
1707
 
1769
1708
  Parameters
1770
1709
  ----------
1771
- hourly : bool, default False
1772
- Run the workflow hourly.
1773
- daily : bool, default True
1774
- Run the workflow daily.
1775
- weekly : bool, default False
1776
- Run the workflow weekly.
1777
- cron : str, optional, default None
1778
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1779
- specified by this expression.
1780
- timezone : str, optional, default None
1781
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1782
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1710
+ event : Union[str, Dict[str, Any]], optional, default None
1711
+ Event dependency for this flow.
1712
+ events : List[Union[str, Dict[str, Any]]], default []
1713
+ Events dependency for this flow.
1714
+ options : Dict[str, Any], default {}
1715
+ Backend-specific configuration for tuning eventing behavior.
1716
+
1717
+
1783
1718
  """
1784
1719
  ...
1785
1720
 
1786
1721
  @typing.overload
1787
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1722
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1788
1723
  ...
1789
1724
 
1790
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1725
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1791
1726
  """
1792
- Specifies the times when the flow should be run when running on a
1793
- production scheduler.
1727
+ Specifies the event(s) that this flow depends on.
1728
+
1729
+ ```
1730
+ @trigger(event='foo')
1731
+ ```
1732
+ or
1733
+ ```
1734
+ @trigger(events=['foo', 'bar'])
1735
+ ```
1736
+
1737
+ Additionally, you can specify the parameter mappings
1738
+ to map event payload to Metaflow parameters for the flow.
1739
+ ```
1740
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1741
+ ```
1742
+ or
1743
+ ```
1744
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1745
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1746
+ ```
1747
+
1748
+ 'parameters' can also be a list of strings and tuples like so:
1749
+ ```
1750
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1751
+ ```
1752
+ This is equivalent to:
1753
+ ```
1754
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1755
+ ```
1794
1756
 
1795
1757
  Parameters
1796
1758
  ----------
1797
- hourly : bool, default False
1798
- Run the workflow hourly.
1799
- daily : bool, default True
1800
- Run the workflow daily.
1801
- weekly : bool, default False
1802
- Run the workflow weekly.
1803
- cron : str, optional, default None
1804
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1805
- specified by this expression.
1806
- timezone : str, optional, default None
1807
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1808
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1759
+ event : Union[str, Dict[str, Any]], optional, default None
1760
+ Event dependency for this flow.
1761
+ events : List[Union[str, Dict[str, Any]]], default []
1762
+ Events dependency for this flow.
1763
+ options : Dict[str, Any], default {}
1764
+ Backend-specific configuration for tuning eventing behavior.
1765
+
1766
+
1809
1767
  """
1810
1768
  ...
1811
1769
 
@@ -1827,6 +1785,48 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
1827
1785
  """
1828
1786
  ...
1829
1787
 
1788
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1789
+ """
1790
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1791
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1792
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1793
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1794
+ starts only after all sensors finish.
1795
+
1796
+ Parameters
1797
+ ----------
1798
+ timeout : int
1799
+ Time, in seconds before the task times out and fails. (Default: 3600)
1800
+ poke_interval : int
1801
+ Time in seconds that the job should wait in between each try. (Default: 60)
1802
+ mode : str
1803
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1804
+ exponential_backoff : bool
1805
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1806
+ pool : str
1807
+ the slot pool this task should run in,
1808
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1809
+ soft_fail : bool
1810
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1811
+ name : str
1812
+ Name of the sensor on Airflow
1813
+ description : str
1814
+ Description of sensor in the Airflow UI
1815
+ bucket_key : Union[str, List[str]]
1816
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1817
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1818
+ bucket_name : str
1819
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1820
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1821
+ wildcard_match : bool
1822
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1823
+ aws_conn_id : str
1824
+ a reference to the s3 connection on Airflow. (Default: None)
1825
+ verify : bool
1826
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1827
+ """
1828
+ ...
1829
+
1830
1830
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1831
1831
  """
1832
1832
  Switch namespace to the one provided.