metaflow-stubs 2.11.9__py2.py3-none-any.whl → 2.11.11__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. metaflow-stubs/__init__.pyi +468 -466
  2. metaflow-stubs/cards.pyi +6 -6
  3. metaflow-stubs/cli.pyi +3 -3
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +5 -5
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +6 -2
  16. metaflow-stubs/metaflow_current.pyi +4 -4
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +4 -4
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
  37. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +4 -4
  45. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +4 -4
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  56. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  59. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  60. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  61. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  62. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  63. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  64. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  67. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  76. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  79. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  80. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  81. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  82. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  83. metaflow-stubs/plugins/datatools/__init__.pyi +4 -4
  84. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  86. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  87. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  88. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  89. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  90. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  91. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  92. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  93. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  95. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  98. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  99. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +4 -4
  102. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +4 -4
  103. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +6 -4
  105. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  106. metaflow-stubs/plugins/package_cli.pyi +2 -2
  107. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  109. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  111. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  112. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  114. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  115. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  118. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  119. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  121. metaflow-stubs/plugins/tag_cli.pyi +5 -5
  122. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  124. metaflow-stubs/procpoll.pyi +2 -2
  125. metaflow-stubs/pylint_wrapper.pyi +2 -2
  126. metaflow-stubs/tagging_util.pyi +2 -2
  127. metaflow-stubs/version.pyi +2 -2
  128. {metaflow_stubs-2.11.9.dist-info → metaflow_stubs-2.11.11.dist-info}/METADATA +2 -2
  129. metaflow_stubs-2.11.11.dist-info/RECORD +132 -0
  130. metaflow_stubs-2.11.9.dist-info/RECORD +0 -132
  131. {metaflow_stubs-2.11.9.dist-info → metaflow_stubs-2.11.11.dist-info}/WHEEL +0 -0
  132. {metaflow_stubs-2.11.9.dist-info → metaflow_stubs-2.11.11.dist-info}/top_level.txt +0 -0
@@ -1,23 +1,23 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.11.9 #
4
- # Generated on 2024-03-29T22:28:00.911417 #
3
+ # MF version: 2.11.11 #
4
+ # Generated on 2024-05-02T22:04:13.800222 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.datastore.inputs
11
+ import metaflow.client.core
12
+ import datetime
13
+ import metaflow._vendor.click.types
14
+ import typing
12
15
  import metaflow.plugins.datatools.s3.s3
16
+ import metaflow.datastore.inputs
13
17
  import metaflow.metaflow_current
14
- import typing
18
+ import metaflow.parameters
15
19
  import io
16
20
  import metaflow.events
17
- import metaflow.parameters
18
- import datetime
19
- import metaflow.client.core
20
- import metaflow._vendor.click.types
21
21
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
22
22
  StepFlag = typing.NewType("StepFlag", bool)
23
23
 
@@ -726,194 +726,51 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
726
726
  ...
727
727
 
728
728
  @typing.overload
729
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
730
- """
731
- Creates a human-readable report, a Metaflow Card, after this step completes.
732
-
733
- Note that you may add multiple `@card` decorators in a step with different parameters.
734
-
735
- Parameters
736
- ----------
737
- type : str, default 'default'
738
- Card type.
739
- id : str, optional, default None
740
- If multiple cards are present, use this id to identify this card.
741
- options : Dict[str, Any], default {}
742
- Options passed to the card. The contents depend on the card type.
743
- timeout : int, default 45
744
- Interrupt reporting if it takes more than this many seconds.
745
-
746
-
747
- """
748
- ...
749
-
750
- @typing.overload
751
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
752
- ...
753
-
754
- @typing.overload
755
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
756
- ...
757
-
758
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
759
- """
760
- Creates a human-readable report, a Metaflow Card, after this step completes.
761
-
762
- Note that you may add multiple `@card` decorators in a step with different parameters.
763
-
764
- Parameters
765
- ----------
766
- type : str, default 'default'
767
- Card type.
768
- id : str, optional, default None
769
- If multiple cards are present, use this id to identify this card.
770
- options : Dict[str, Any], default {}
771
- Options passed to the card. The contents depend on the card type.
772
- timeout : int, default 45
773
- Interrupt reporting if it takes more than this many seconds.
774
-
775
-
776
- """
777
- ...
778
-
779
- @typing.overload
780
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
781
- """
782
- Specifies environment variables to be set prior to the execution of a step.
783
-
784
- Parameters
785
- ----------
786
- vars : Dict[str, str], default {}
787
- Dictionary of environment variables to set.
788
- """
789
- ...
790
-
791
- @typing.overload
792
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
793
- ...
794
-
795
- @typing.overload
796
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
797
- ...
798
-
799
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
800
- """
801
- Specifies environment variables to be set prior to the execution of a step.
802
-
803
- Parameters
804
- ----------
805
- vars : Dict[str, str], default {}
806
- Dictionary of environment variables to set.
807
- """
808
- ...
809
-
810
- @typing.overload
811
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
812
- """
813
- Specifies secrets to be retrieved and injected as environment variables prior to
814
- the execution of a step.
815
-
816
- Parameters
817
- ----------
818
- sources : List[Union[str, Dict[str, Any]]], default: []
819
- List of secret specs, defining how the secrets are to be retrieved
820
- """
821
- ...
822
-
823
- @typing.overload
824
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
825
- ...
826
-
827
- @typing.overload
828
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
829
- ...
830
-
831
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
832
- """
833
- Specifies secrets to be retrieved and injected as environment variables prior to
834
- the execution of a step.
835
-
836
- Parameters
837
- ----------
838
- sources : List[Union[str, Dict[str, Any]]], default: []
839
- List of secret specs, defining how the secrets are to be retrieved
840
- """
841
- ...
842
-
843
- @typing.overload
844
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
729
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
845
730
  """
846
- Specifies the resources needed when executing this step.
847
-
848
- Use `@resources` to specify the resource requirements
849
- independently of the specific compute layer (`@batch`, `@kubernetes`).
731
+ Specifies the PyPI packages for the step.
850
732
 
851
- You can choose the compute layer on the command line by executing e.g.
852
- ```
853
- python myflow.py run --with batch
854
- ```
855
- or
856
- ```
857
- python myflow.py run --with kubernetes
858
- ```
859
- which executes the flow on the desired system using the
860
- requirements specified in `@resources`.
733
+ Information in this decorator will augment any
734
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
735
+ you can use `@pypi_base` to set packages required by all
736
+ steps and use `@pypi` to specify step-specific overrides.
861
737
 
862
738
  Parameters
863
739
  ----------
864
- cpu : int, default 1
865
- Number of CPUs required for this step.
866
- gpu : int, default 0
867
- Number of GPUs required for this step.
868
- disk : int, optional, default None
869
- Disk size (in MB) required for this step. Only applies on Kubernetes.
870
- memory : int, default 4096
871
- Memory size (in MB) required for this step.
872
- shared_memory : int, optional, default None
873
- The value for the size (in MiB) of the /dev/shm volume for this step.
874
- This parameter maps to the `--shm-size` option in Docker.
740
+ packages : Dict[str, str], default: {}
741
+ Packages to use for this step. The key is the name of the package
742
+ and the value is the version to use.
743
+ python : str, optional, default: None
744
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
745
+ that the version used will correspond to the version of the Python interpreter used to start the run.
875
746
  """
876
747
  ...
877
748
 
878
749
  @typing.overload
879
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
750
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
880
751
  ...
881
752
 
882
753
  @typing.overload
883
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
754
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
884
755
  ...
885
756
 
886
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
757
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
887
758
  """
888
- Specifies the resources needed when executing this step.
889
-
890
- Use `@resources` to specify the resource requirements
891
- independently of the specific compute layer (`@batch`, `@kubernetes`).
759
+ Specifies the PyPI packages for the step.
892
760
 
893
- You can choose the compute layer on the command line by executing e.g.
894
- ```
895
- python myflow.py run --with batch
896
- ```
897
- or
898
- ```
899
- python myflow.py run --with kubernetes
900
- ```
901
- which executes the flow on the desired system using the
902
- requirements specified in `@resources`.
761
+ Information in this decorator will augment any
762
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
763
+ you can use `@pypi_base` to set packages required by all
764
+ steps and use `@pypi` to specify step-specific overrides.
903
765
 
904
766
  Parameters
905
767
  ----------
906
- cpu : int, default 1
907
- Number of CPUs required for this step.
908
- gpu : int, default 0
909
- Number of GPUs required for this step.
910
- disk : int, optional, default None
911
- Disk size (in MB) required for this step. Only applies on Kubernetes.
912
- memory : int, default 4096
913
- Memory size (in MB) required for this step.
914
- shared_memory : int, optional, default None
915
- The value for the size (in MiB) of the /dev/shm volume for this step.
916
- This parameter maps to the `--shm-size` option in Docker.
768
+ packages : Dict[str, str], default: {}
769
+ Packages to use for this step. The key is the name of the package
770
+ and the value is the version to use.
771
+ python : str, optional, default: None
772
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
773
+ that the version used will correspond to the version of the Python interpreter used to start the run.
917
774
  """
918
775
  ...
919
776
 
@@ -1065,51 +922,187 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1065
922
  ...
1066
923
 
1067
924
  @typing.overload
1068
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
925
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1069
926
  """
1070
- Specifies the PyPI packages for the step.
927
+ Specifies the resources needed when executing this step.
1071
928
 
1072
- Information in this decorator will augment any
1073
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1074
- you can use `@pypi_base` to set packages required by all
1075
- steps and use `@pypi` to specify step-specific overrides.
929
+ Use `@resources` to specify the resource requirements
930
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
931
+
932
+ You can choose the compute layer on the command line by executing e.g.
933
+ ```
934
+ python myflow.py run --with batch
935
+ ```
936
+ or
937
+ ```
938
+ python myflow.py run --with kubernetes
939
+ ```
940
+ which executes the flow on the desired system using the
941
+ requirements specified in `@resources`.
1076
942
 
1077
943
  Parameters
1078
944
  ----------
1079
- packages : Dict[str, str], default: {}
1080
- Packages to use for this step. The key is the name of the package
1081
- and the value is the version to use.
1082
- python : str, optional, default: None
1083
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1084
- that the version used will correspond to the version of the Python interpreter used to start the run.
945
+ cpu : int, default 1
946
+ Number of CPUs required for this step.
947
+ gpu : int, default 0
948
+ Number of GPUs required for this step.
949
+ disk : int, optional, default None
950
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
951
+ memory : int, default 4096
952
+ Memory size (in MB) required for this step.
953
+ shared_memory : int, optional, default None
954
+ The value for the size (in MiB) of the /dev/shm volume for this step.
955
+ This parameter maps to the `--shm-size` option in Docker.
1085
956
  """
1086
957
  ...
1087
958
 
1088
959
  @typing.overload
1089
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
960
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1090
961
  ...
1091
962
 
1092
963
  @typing.overload
1093
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
964
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1094
965
  ...
1095
966
 
1096
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
967
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1097
968
  """
1098
- Specifies the PyPI packages for the step.
969
+ Specifies the resources needed when executing this step.
1099
970
 
1100
- Information in this decorator will augment any
1101
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1102
- you can use `@pypi_base` to set packages required by all
1103
- steps and use `@pypi` to specify step-specific overrides.
971
+ Use `@resources` to specify the resource requirements
972
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
973
+
974
+ You can choose the compute layer on the command line by executing e.g.
975
+ ```
976
+ python myflow.py run --with batch
977
+ ```
978
+ or
979
+ ```
980
+ python myflow.py run --with kubernetes
981
+ ```
982
+ which executes the flow on the desired system using the
983
+ requirements specified in `@resources`.
1104
984
 
1105
985
  Parameters
1106
986
  ----------
1107
- packages : Dict[str, str], default: {}
1108
- Packages to use for this step. The key is the name of the package
1109
- and the value is the version to use.
1110
- python : str, optional, default: None
1111
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1112
- that the version used will correspond to the version of the Python interpreter used to start the run.
987
+ cpu : int, default 1
988
+ Number of CPUs required for this step.
989
+ gpu : int, default 0
990
+ Number of GPUs required for this step.
991
+ disk : int, optional, default None
992
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
993
+ memory : int, default 4096
994
+ Memory size (in MB) required for this step.
995
+ shared_memory : int, optional, default None
996
+ The value for the size (in MiB) of the /dev/shm volume for this step.
997
+ This parameter maps to the `--shm-size` option in Docker.
998
+ """
999
+ ...
1000
+
1001
+ @typing.overload
1002
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1003
+ """
1004
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1005
+
1006
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1007
+
1008
+ Parameters
1009
+ ----------
1010
+ type : str, default 'default'
1011
+ Card type.
1012
+ id : str, optional, default None
1013
+ If multiple cards are present, use this id to identify this card.
1014
+ options : Dict[str, Any], default {}
1015
+ Options passed to the card. The contents depend on the card type.
1016
+ timeout : int, default 45
1017
+ Interrupt reporting if it takes more than this many seconds.
1018
+
1019
+
1020
+ """
1021
+ ...
1022
+
1023
+ @typing.overload
1024
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1025
+ ...
1026
+
1027
+ @typing.overload
1028
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1029
+ ...
1030
+
1031
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1032
+ """
1033
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1034
+
1035
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1036
+
1037
+ Parameters
1038
+ ----------
1039
+ type : str, default 'default'
1040
+ Card type.
1041
+ id : str, optional, default None
1042
+ If multiple cards are present, use this id to identify this card.
1043
+ options : Dict[str, Any], default {}
1044
+ Options passed to the card. The contents depend on the card type.
1045
+ timeout : int, default 45
1046
+ Interrupt reporting if it takes more than this many seconds.
1047
+
1048
+
1049
+ """
1050
+ ...
1051
+
1052
+ @typing.overload
1053
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1054
+ """
1055
+ Specifies a timeout for your step.
1056
+
1057
+ This decorator is useful if this step may hang indefinitely.
1058
+
1059
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1060
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1061
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1062
+
1063
+ Note that all the values specified in parameters are added together so if you specify
1064
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1065
+
1066
+ Parameters
1067
+ ----------
1068
+ seconds : int, default 0
1069
+ Number of seconds to wait prior to timing out.
1070
+ minutes : int, default 0
1071
+ Number of minutes to wait prior to timing out.
1072
+ hours : int, default 0
1073
+ Number of hours to wait prior to timing out.
1074
+ """
1075
+ ...
1076
+
1077
+ @typing.overload
1078
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1079
+ ...
1080
+
1081
+ @typing.overload
1082
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1083
+ ...
1084
+
1085
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1086
+ """
1087
+ Specifies a timeout for your step.
1088
+
1089
+ This decorator is useful if this step may hang indefinitely.
1090
+
1091
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1092
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1093
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1094
+
1095
+ Note that all the values specified in parameters are added together so if you specify
1096
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1097
+
1098
+ Parameters
1099
+ ----------
1100
+ seconds : int, default 0
1101
+ Number of seconds to wait prior to timing out.
1102
+ minutes : int, default 0
1103
+ Number of minutes to wait prior to timing out.
1104
+ hours : int, default 0
1105
+ Number of hours to wait prior to timing out.
1113
1106
  """
1114
1107
  ...
1115
1108
 
@@ -1215,7 +1208,128 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1215
1208
  """
1216
1209
  ...
1217
1210
 
1218
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1211
+ @typing.overload
1212
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1213
+ """
1214
+ Specifies the Conda environment for the step.
1215
+
1216
+ Information in this decorator will augment any
1217
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1218
+ you can use `@conda_base` to set packages required by all
1219
+ steps and use `@conda` to specify step-specific overrides.
1220
+
1221
+ Parameters
1222
+ ----------
1223
+ packages : Dict[str, str], default {}
1224
+ Packages to use for this step. The key is the name of the package
1225
+ and the value is the version to use.
1226
+ libraries : Dict[str, str], default {}
1227
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1228
+ python : str, optional, default None
1229
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1230
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1231
+ disabled : bool, default False
1232
+ If set to True, disables @conda.
1233
+ """
1234
+ ...
1235
+
1236
+ @typing.overload
1237
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1238
+ ...
1239
+
1240
+ @typing.overload
1241
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1242
+ ...
1243
+
1244
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1245
+ """
1246
+ Specifies the Conda environment for the step.
1247
+
1248
+ Information in this decorator will augment any
1249
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1250
+ you can use `@conda_base` to set packages required by all
1251
+ steps and use `@conda` to specify step-specific overrides.
1252
+
1253
+ Parameters
1254
+ ----------
1255
+ packages : Dict[str, str], default {}
1256
+ Packages to use for this step. The key is the name of the package
1257
+ and the value is the version to use.
1258
+ libraries : Dict[str, str], default {}
1259
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1260
+ python : str, optional, default None
1261
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1262
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1263
+ disabled : bool, default False
1264
+ If set to True, disables @conda.
1265
+ """
1266
+ ...
1267
+
1268
+ @typing.overload
1269
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1270
+ """
1271
+ Specifies secrets to be retrieved and injected as environment variables prior to
1272
+ the execution of a step.
1273
+
1274
+ Parameters
1275
+ ----------
1276
+ sources : List[Union[str, Dict[str, Any]]], default: []
1277
+ List of secret specs, defining how the secrets are to be retrieved
1278
+ """
1279
+ ...
1280
+
1281
+ @typing.overload
1282
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1283
+ ...
1284
+
1285
+ @typing.overload
1286
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1287
+ ...
1288
+
1289
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1290
+ """
1291
+ Specifies secrets to be retrieved and injected as environment variables prior to
1292
+ the execution of a step.
1293
+
1294
+ Parameters
1295
+ ----------
1296
+ sources : List[Union[str, Dict[str, Any]]], default: []
1297
+ List of secret specs, defining how the secrets are to be retrieved
1298
+ """
1299
+ ...
1300
+
1301
+ @typing.overload
1302
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1303
+ """
1304
+ Specifies environment variables to be set prior to the execution of a step.
1305
+
1306
+ Parameters
1307
+ ----------
1308
+ vars : Dict[str, str], default {}
1309
+ Dictionary of environment variables to set.
1310
+ """
1311
+ ...
1312
+
1313
+ @typing.overload
1314
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1315
+ ...
1316
+
1317
+ @typing.overload
1318
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1319
+ ...
1320
+
1321
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1322
+ """
1323
+ Specifies environment variables to be set prior to the execution of a step.
1324
+
1325
+ Parameters
1326
+ ----------
1327
+ vars : Dict[str, str], default {}
1328
+ Dictionary of environment variables to set.
1329
+ """
1330
+ ...
1331
+
1332
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1219
1333
  """
1220
1334
  Specifies that this step should execute on Kubernetes.
1221
1335
 
@@ -1269,138 +1383,156 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1269
1383
  volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1270
1384
  shared_memory: int, optional
1271
1385
  Shared memory size (in MiB) required for this step
1386
+ port: int, optional
1387
+ Port number to specify in the Kubernetes job object
1272
1388
  """
1273
1389
  ...
1274
1390
 
1275
1391
  @typing.overload
1276
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1392
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1277
1393
  """
1278
- Specifies the Conda environment for the step.
1279
-
1280
- Information in this decorator will augment any
1281
- attributes set in the `@conda_base` flow-level decorator. Hence,
1282
- you can use `@conda_base` to set packages required by all
1283
- steps and use `@conda` to specify step-specific overrides.
1394
+ Specifies the PyPI packages for all steps of the flow.
1284
1395
 
1396
+ Use `@pypi_base` to set common packages required by all
1397
+ steps and use `@pypi` to specify step-specific overrides.
1285
1398
  Parameters
1286
1399
  ----------
1287
- packages : Dict[str, str], default {}
1288
- Packages to use for this step. The key is the name of the package
1400
+ packages : Dict[str, str], default: {}
1401
+ Packages to use for this flow. The key is the name of the package
1289
1402
  and the value is the version to use.
1290
- libraries : Dict[str, str], default {}
1291
- Supported for backward compatibility. When used with packages, packages will take precedence.
1292
- python : str, optional, default None
1403
+ python : str, optional, default: None
1293
1404
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1294
1405
  that the version used will correspond to the version of the Python interpreter used to start the run.
1295
- disabled : bool, default False
1296
- If set to True, disables @conda.
1297
1406
  """
1298
1407
  ...
1299
1408
 
1300
1409
  @typing.overload
1301
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1302
- ...
1303
-
1304
- @typing.overload
1305
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1410
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1306
1411
  ...
1307
1412
 
1308
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1413
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1309
1414
  """
1310
- Specifies the Conda environment for the step.
1311
-
1312
- Information in this decorator will augment any
1313
- attributes set in the `@conda_base` flow-level decorator. Hence,
1314
- you can use `@conda_base` to set packages required by all
1315
- steps and use `@conda` to specify step-specific overrides.
1415
+ Specifies the PyPI packages for all steps of the flow.
1316
1416
 
1417
+ Use `@pypi_base` to set common packages required by all
1418
+ steps and use `@pypi` to specify step-specific overrides.
1317
1419
  Parameters
1318
1420
  ----------
1319
- packages : Dict[str, str], default {}
1320
- Packages to use for this step. The key is the name of the package
1421
+ packages : Dict[str, str], default: {}
1422
+ Packages to use for this flow. The key is the name of the package
1321
1423
  and the value is the version to use.
1322
- libraries : Dict[str, str], default {}
1323
- Supported for backward compatibility. When used with packages, packages will take precedence.
1324
- python : str, optional, default None
1424
+ python : str, optional, default: None
1325
1425
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1326
1426
  that the version used will correspond to the version of the Python interpreter used to start the run.
1327
- disabled : bool, default False
1328
- If set to True, disables @conda.
1329
1427
  """
1330
1428
  ...
1331
1429
 
1332
- @typing.overload
1333
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1430
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1334
1431
  """
1335
- Specifies a timeout for your step.
1336
-
1337
- This decorator is useful if this step may hang indefinitely.
1338
-
1339
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1340
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1341
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1342
-
1343
- Note that all the values specified in parameters are added together so if you specify
1344
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1432
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1433
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1434
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1435
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1436
+ starts only after all sensors finish.
1345
1437
 
1346
1438
  Parameters
1347
1439
  ----------
1348
- seconds : int, default 0
1349
- Number of seconds to wait prior to timing out.
1350
- minutes : int, default 0
1351
- Number of minutes to wait prior to timing out.
1352
- hours : int, default 0
1353
- Number of hours to wait prior to timing out.
1440
+ timeout : int
1441
+ Time, in seconds before the task times out and fails. (Default: 3600)
1442
+ poke_interval : int
1443
+ Time in seconds that the job should wait in between each try. (Default: 60)
1444
+ mode : str
1445
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1446
+ exponential_backoff : bool
1447
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1448
+ pool : str
1449
+ the slot pool this task should run in,
1450
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1451
+ soft_fail : bool
1452
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1453
+ name : str
1454
+ Name of the sensor on Airflow
1455
+ description : str
1456
+ Description of sensor in the Airflow UI
1457
+ bucket_key : Union[str, List[str]]
1458
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1459
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1460
+ bucket_name : str
1461
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1462
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1463
+ wildcard_match : bool
1464
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1465
+ aws_conn_id : str
1466
+ a reference to the s3 connection on Airflow. (Default: None)
1467
+ verify : bool
1468
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1354
1469
  """
1355
1470
  ...
1356
1471
 
1357
- @typing.overload
1358
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1359
- ...
1360
-
1361
- @typing.overload
1362
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1363
- ...
1364
-
1365
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1472
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1366
1473
  """
1367
- Specifies a timeout for your step.
1474
+ Specifies what flows belong to the same project.
1368
1475
 
1369
- This decorator is useful if this step may hang indefinitely.
1476
+ A project-specific namespace is created for all flows that
1477
+ use the same `@project(name)`.
1370
1478
 
1371
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1372
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1373
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1479
+ Parameters
1480
+ ----------
1481
+ name : str
1482
+ Project name. Make sure that the name is unique amongst all
1483
+ projects that use the same production scheduler. The name may
1484
+ contain only lowercase alphanumeric characters and underscores.
1374
1485
 
1375
- Note that all the values specified in parameters are added together so if you specify
1376
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1486
+
1487
+ """
1488
+ ...
1489
+
1490
+ @typing.overload
1491
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1492
+ """
1493
+ Specifies the times when the flow should be run when running on a
1494
+ production scheduler.
1377
1495
 
1378
1496
  Parameters
1379
1497
  ----------
1380
- seconds : int, default 0
1381
- Number of seconds to wait prior to timing out.
1382
- minutes : int, default 0
1383
- Number of minutes to wait prior to timing out.
1384
- hours : int, default 0
1385
- Number of hours to wait prior to timing out.
1498
+ hourly : bool, default False
1499
+ Run the workflow hourly.
1500
+ daily : bool, default True
1501
+ Run the workflow daily.
1502
+ weekly : bool, default False
1503
+ Run the workflow weekly.
1504
+ cron : str, optional, default None
1505
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1506
+ specified by this expression.
1507
+ timezone : str, optional, default None
1508
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1509
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1386
1510
  """
1387
1511
  ...
1388
1512
 
1389
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1513
+ @typing.overload
1514
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1515
+ ...
1516
+
1517
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1390
1518
  """
1391
- Specifies what flows belong to the same project.
1392
-
1393
- A project-specific namespace is created for all flows that
1394
- use the same `@project(name)`.
1519
+ Specifies the times when the flow should be run when running on a
1520
+ production scheduler.
1395
1521
 
1396
1522
  Parameters
1397
1523
  ----------
1398
- name : str
1399
- Project name. Make sure that the name is unique amongst all
1400
- projects that use the same production scheduler. The name may
1401
- contain only lowercase alphanumeric characters and underscores.
1402
-
1403
-
1524
+ hourly : bool, default False
1525
+ Run the workflow hourly.
1526
+ daily : bool, default True
1527
+ Run the workflow daily.
1528
+ weekly : bool, default False
1529
+ Run the workflow weekly.
1530
+ cron : str, optional, default None
1531
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1532
+ specified by this expression.
1533
+ timezone : str, optional, default None
1534
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1535
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1404
1536
  """
1405
1537
  ...
1406
1538
 
@@ -1499,52 +1631,10 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1499
1631
  """
1500
1632
  ...
1501
1633
 
1502
- @typing.overload
1503
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1504
- """
1505
- Specifies the PyPI packages for all steps of the flow.
1506
-
1507
- Use `@pypi_base` to set common packages required by all
1508
- steps and use `@pypi` to specify step-specific overrides.
1509
- Parameters
1510
- ----------
1511
- packages : Dict[str, str], default: {}
1512
- Packages to use for this flow. The key is the name of the package
1513
- and the value is the version to use.
1514
- python : str, optional, default: None
1515
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1516
- that the version used will correspond to the version of the Python interpreter used to start the run.
1517
- """
1518
- ...
1519
-
1520
- @typing.overload
1521
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1522
- ...
1523
-
1524
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1525
- """
1526
- Specifies the PyPI packages for all steps of the flow.
1527
-
1528
- Use `@pypi_base` to set common packages required by all
1529
- steps and use `@pypi` to specify step-specific overrides.
1530
- Parameters
1531
- ----------
1532
- packages : Dict[str, str], default: {}
1533
- Packages to use for this flow. The key is the name of the package
1534
- and the value is the version to use.
1535
- python : str, optional, default: None
1536
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1537
- that the version used will correspond to the version of the Python interpreter used to start the run.
1538
- """
1539
- ...
1540
-
1541
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1634
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1542
1635
  """
1543
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1544
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1545
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1546
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1547
- starts only after all sensors finish.
1636
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1637
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1548
1638
 
1549
1639
  Parameters
1550
1640
  ----------
@@ -1565,67 +1655,21 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1565
1655
  Name of the sensor on Airflow
1566
1656
  description : str
1567
1657
  Description of sensor in the Airflow UI
1568
- bucket_key : Union[str, List[str]]
1569
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1570
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1571
- bucket_name : str
1572
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1573
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1574
- wildcard_match : bool
1575
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1576
- aws_conn_id : str
1577
- a reference to the s3 connection on Airflow. (Default: None)
1578
- verify : bool
1579
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1580
- """
1581
- ...
1582
-
1583
- @typing.overload
1584
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1585
- """
1586
- Specifies the Conda environment for all steps of the flow.
1587
-
1588
- Use `@conda_base` to set common libraries required by all
1589
- steps and use `@conda` to specify step-specific additions.
1590
-
1591
- Parameters
1592
- ----------
1593
- packages : Dict[str, str], default {}
1594
- Packages to use for this flow. The key is the name of the package
1595
- and the value is the version to use.
1596
- libraries : Dict[str, str], default {}
1597
- Supported for backward compatibility. When used with packages, packages will take precedence.
1598
- python : str, optional, default None
1599
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1600
- that the version used will correspond to the version of the Python interpreter used to start the run.
1601
- disabled : bool, default False
1602
- If set to True, disables Conda.
1603
- """
1604
- ...
1605
-
1606
- @typing.overload
1607
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1608
- ...
1609
-
1610
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1611
- """
1612
- Specifies the Conda environment for all steps of the flow.
1613
-
1614
- Use `@conda_base` to set common libraries required by all
1615
- steps and use `@conda` to specify step-specific additions.
1616
-
1617
- Parameters
1618
- ----------
1619
- packages : Dict[str, str], default {}
1620
- Packages to use for this flow. The key is the name of the package
1621
- and the value is the version to use.
1622
- libraries : Dict[str, str], default {}
1623
- Supported for backward compatibility. When used with packages, packages will take precedence.
1624
- python : str, optional, default None
1625
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1626
- that the version used will correspond to the version of the Python interpreter used to start the run.
1627
- disabled : bool, default False
1628
- If set to True, disables Conda.
1658
+ external_dag_id : str
1659
+ The dag_id that contains the task you want to wait for.
1660
+ external_task_ids : List[str]
1661
+ The list of task_ids that you want to wait for.
1662
+ If None (default value) the sensor waits for the DAG. (Default: None)
1663
+ allowed_states : List[str]
1664
+ Iterable of allowed states, (Default: ['success'])
1665
+ failed_states : List[str]
1666
+ Iterable of failed or dis-allowed states. (Default: None)
1667
+ execution_delta : datetime.timedelta
1668
+ time difference with the previous execution to look at,
1669
+ the default is the same logical date as the current task or DAG. (Default: None)
1670
+ check_existence: bool
1671
+ Set to True to check if the external task exists or check if
1672
+ the DAG to wait for exists. (Default: True)
1629
1673
  """
1630
1674
  ...
1631
1675
 
@@ -1733,93 +1777,51 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1733
1777
  ...
1734
1778
 
1735
1779
  @typing.overload
1736
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1780
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1737
1781
  """
1738
- Specifies the times when the flow should be run when running on a
1739
- production scheduler.
1782
+ Specifies the Conda environment for all steps of the flow.
1783
+
1784
+ Use `@conda_base` to set common libraries required by all
1785
+ steps and use `@conda` to specify step-specific additions.
1740
1786
 
1741
1787
  Parameters
1742
1788
  ----------
1743
- hourly : bool, default False
1744
- Run the workflow hourly.
1745
- daily : bool, default True
1746
- Run the workflow daily.
1747
- weekly : bool, default False
1748
- Run the workflow weekly.
1749
- cron : str, optional, default None
1750
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1751
- specified by this expression.
1752
- timezone : str, optional, default None
1753
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1754
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1789
+ packages : Dict[str, str], default {}
1790
+ Packages to use for this flow. The key is the name of the package
1791
+ and the value is the version to use.
1792
+ libraries : Dict[str, str], default {}
1793
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1794
+ python : str, optional, default None
1795
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1796
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1797
+ disabled : bool, default False
1798
+ If set to True, disables Conda.
1755
1799
  """
1756
1800
  ...
1757
1801
 
1758
1802
  @typing.overload
1759
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1803
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1760
1804
  ...
1761
1805
 
1762
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1806
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1763
1807
  """
1764
- Specifies the times when the flow should be run when running on a
1765
- production scheduler.
1808
+ Specifies the Conda environment for all steps of the flow.
1766
1809
 
1767
- Parameters
1768
- ----------
1769
- hourly : bool, default False
1770
- Run the workflow hourly.
1771
- daily : bool, default True
1772
- Run the workflow daily.
1773
- weekly : bool, default False
1774
- Run the workflow weekly.
1775
- cron : str, optional, default None
1776
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1777
- specified by this expression.
1778
- timezone : str, optional, default None
1779
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1780
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1781
- """
1782
- ...
1783
-
1784
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1785
- """
1786
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1787
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1810
+ Use `@conda_base` to set common libraries required by all
1811
+ steps and use `@conda` to specify step-specific additions.
1788
1812
 
1789
1813
  Parameters
1790
1814
  ----------
1791
- timeout : int
1792
- Time, in seconds before the task times out and fails. (Default: 3600)
1793
- poke_interval : int
1794
- Time in seconds that the job should wait in between each try. (Default: 60)
1795
- mode : str
1796
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1797
- exponential_backoff : bool
1798
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1799
- pool : str
1800
- the slot pool this task should run in,
1801
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1802
- soft_fail : bool
1803
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1804
- name : str
1805
- Name of the sensor on Airflow
1806
- description : str
1807
- Description of sensor in the Airflow UI
1808
- external_dag_id : str
1809
- The dag_id that contains the task you want to wait for.
1810
- external_task_ids : List[str]
1811
- The list of task_ids that you want to wait for.
1812
- If None (default value) the sensor waits for the DAG. (Default: None)
1813
- allowed_states : List[str]
1814
- Iterable of allowed states, (Default: ['success'])
1815
- failed_states : List[str]
1816
- Iterable of failed or dis-allowed states. (Default: None)
1817
- execution_delta : datetime.timedelta
1818
- time difference with the previous execution to look at,
1819
- the default is the same logical date as the current task or DAG. (Default: None)
1820
- check_existence: bool
1821
- Set to True to check if the external task exists or check if
1822
- the DAG to wait for exists. (Default: True)
1815
+ packages : Dict[str, str], default {}
1816
+ Packages to use for this flow. The key is the name of the package
1817
+ and the value is the version to use.
1818
+ libraries : Dict[str, str], default {}
1819
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1820
+ python : str, optional, default None
1821
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1822
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1823
+ disabled : bool, default False
1824
+ If set to True, disables Conda.
1823
1825
  """
1824
1826
  ...
1825
1827