metaflow-stubs 2.12.5__py2.py3-none-any.whl → 2.12.7__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. metaflow-stubs/__init__.pyi +386 -386
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +3 -3
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +17 -17
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +6 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +4 -4
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  37. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  45. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  47. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +4 -4
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  56. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  60. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  61. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  62. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  63. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  65. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  78. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  81. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  82. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  83. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  84. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/s3/s3.pyi +7 -7
  89. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  91. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  92. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  93. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  95. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  97. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  99. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  106. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  107. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  108. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  110. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  111. metaflow-stubs/plugins/package_cli.pyi +2 -2
  112. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  114. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  117. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  119. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  120. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  123. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  124. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  126. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  127. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  128. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  129. metaflow-stubs/procpoll.pyi +2 -2
  130. metaflow-stubs/pylint_wrapper.pyi +2 -2
  131. metaflow-stubs/runner/__init__.pyi +2 -2
  132. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  133. metaflow-stubs/runner/nbrun.pyi +2 -2
  134. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  135. metaflow-stubs/system/__init__.pyi +112 -0
  136. metaflow-stubs/system/system_logger.pyi +51 -0
  137. metaflow-stubs/system/system_monitor.pyi +73 -0
  138. metaflow-stubs/tagging_util.pyi +2 -2
  139. metaflow-stubs/tuple_util.pyi +2 -2
  140. metaflow-stubs/version.pyi +2 -2
  141. {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.7.dist-info}/METADATA +2 -2
  142. metaflow_stubs-2.12.7.dist-info/RECORD +145 -0
  143. {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.7.dist-info}/WHEEL +1 -1
  144. metaflow_stubs-2.12.5.dist-info/RECORD +0 -142
  145. {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.7.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.5 #
4
- # Generated on 2024-06-20T19:51:33.982232 #
3
+ # MF version: 2.12.7 #
4
+ # Generated on 2024-07-03T19:20:47.569890 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,17 +9,17 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.client.core
12
- import metaflow.runner.metaflow_runner
12
+ import metaflow.datastore.inputs
13
+ import metaflow.parameters
14
+ import metaflow.metaflow_current
13
15
  import metaflow._vendor.click.types
14
16
  import typing
15
- import metaflow.parameters
16
- import metaflow.flowspec
17
+ import metaflow.runner.metaflow_runner
18
+ import io
17
19
  import metaflow.plugins.datatools.s3.s3
18
20
  import datetime
19
- import io
20
- import metaflow.metaflow_current
21
- import metaflow.datastore.inputs
22
21
  import metaflow.events
22
+ import metaflow.flowspec
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -728,35 +728,110 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
728
728
  ...
729
729
 
730
730
  @typing.overload
731
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
731
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
732
  """
733
- Specifies secrets to be retrieved and injected as environment variables prior to
734
- the execution of a step.
733
+ Specifies the Conda environment for the step.
734
+
735
+ Information in this decorator will augment any
736
+ attributes set in the `@conda_base` flow-level decorator. Hence,
737
+ you can use `@conda_base` to set packages required by all
738
+ steps and use `@conda` to specify step-specific overrides.
735
739
 
736
740
  Parameters
737
741
  ----------
738
- sources : List[Union[str, Dict[str, Any]]], default: []
739
- List of secret specs, defining how the secrets are to be retrieved
742
+ packages : Dict[str, str], default {}
743
+ Packages to use for this step. The key is the name of the package
744
+ and the value is the version to use.
745
+ libraries : Dict[str, str], default {}
746
+ Supported for backward compatibility. When used with packages, packages will take precedence.
747
+ python : str, optional, default None
748
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
749
+ that the version used will correspond to the version of the Python interpreter used to start the run.
750
+ disabled : bool, default False
751
+ If set to True, disables @conda.
740
752
  """
741
753
  ...
742
754
 
743
755
  @typing.overload
744
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
756
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
745
757
  ...
746
758
 
747
759
  @typing.overload
748
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
760
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
749
761
  ...
750
762
 
751
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
763
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
752
764
  """
753
- Specifies secrets to be retrieved and injected as environment variables prior to
754
- the execution of a step.
765
+ Specifies the Conda environment for the step.
766
+
767
+ Information in this decorator will augment any
768
+ attributes set in the `@conda_base` flow-level decorator. Hence,
769
+ you can use `@conda_base` to set packages required by all
770
+ steps and use `@conda` to specify step-specific overrides.
755
771
 
756
772
  Parameters
757
773
  ----------
758
- sources : List[Union[str, Dict[str, Any]]], default: []
759
- List of secret specs, defining how the secrets are to be retrieved
774
+ packages : Dict[str, str], default {}
775
+ Packages to use for this step. The key is the name of the package
776
+ and the value is the version to use.
777
+ libraries : Dict[str, str], default {}
778
+ Supported for backward compatibility. When used with packages, packages will take precedence.
779
+ python : str, optional, default None
780
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
781
+ that the version used will correspond to the version of the Python interpreter used to start the run.
782
+ disabled : bool, default False
783
+ If set to True, disables @conda.
784
+ """
785
+ ...
786
+
787
+ @typing.overload
788
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
789
+ """
790
+ Creates a human-readable report, a Metaflow Card, after this step completes.
791
+
792
+ Note that you may add multiple `@card` decorators in a step with different parameters.
793
+
794
+ Parameters
795
+ ----------
796
+ type : str, default 'default'
797
+ Card type.
798
+ id : str, optional, default None
799
+ If multiple cards are present, use this id to identify this card.
800
+ options : Dict[str, Any], default {}
801
+ Options passed to the card. The contents depend on the card type.
802
+ timeout : int, default 45
803
+ Interrupt reporting if it takes more than this many seconds.
804
+
805
+
806
+ """
807
+ ...
808
+
809
+ @typing.overload
810
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
811
+ ...
812
+
813
+ @typing.overload
814
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
815
+ ...
816
+
817
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
818
+ """
819
+ Creates a human-readable report, a Metaflow Card, after this step completes.
820
+
821
+ Note that you may add multiple `@card` decorators in a step with different parameters.
822
+
823
+ Parameters
824
+ ----------
825
+ type : str, default 'default'
826
+ Card type.
827
+ id : str, optional, default None
828
+ If multiple cards are present, use this id to identify this card.
829
+ options : Dict[str, Any], default {}
830
+ Options passed to the card. The contents depend on the card type.
831
+ timeout : int, default 45
832
+ Interrupt reporting if it takes more than this many seconds.
833
+
834
+
760
835
  """
761
836
  ...
762
837
 
@@ -907,201 +982,101 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
907
982
  """
908
983
  ...
909
984
 
910
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
911
- """
912
- Specifies that this step should execute on Kubernetes.
913
-
914
- Parameters
915
- ----------
916
- cpu : int, default 1
917
- Number of CPUs required for this step. If `@resources` is
918
- also present, the maximum value from all decorators is used.
919
- memory : int, default 4096
920
- Memory size (in MB) required for this step. If
921
- `@resources` is also present, the maximum value from all decorators is
922
- used.
923
- disk : int, default 10240
924
- Disk size (in MB) required for this step. If
925
- `@resources` is also present, the maximum value from all decorators is
926
- used.
927
- image : str, optional, default None
928
- Docker image to use when launching on Kubernetes. If not specified, and
929
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
930
- not, a default Docker image mapping to the current version of Python is used.
931
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
932
- If given, the imagePullPolicy to be applied to the Docker image of the step.
933
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
934
- Kubernetes service account to use when launching pod in Kubernetes.
935
- secrets : List[str], optional, default None
936
- Kubernetes secrets to use when launching pod in Kubernetes. These
937
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
938
- in Metaflow configuration.
939
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
940
- Kubernetes namespace to use when launching pod in Kubernetes.
941
- gpu : int, optional, default None
942
- Number of GPUs required for this step. A value of zero implies that
943
- the scheduled node should not have GPUs.
944
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
945
- The vendor of the GPUs to be used for this step.
946
- tolerations : List[str], default []
947
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
948
- Kubernetes tolerations to use when launching pod in Kubernetes.
949
- use_tmpfs : bool, default False
950
- This enables an explicit tmpfs mount for this step.
951
- tmpfs_tempdir : bool, default True
952
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
953
- tmpfs_size : int, optional, default: None
954
- The value for the size (in MiB) of the tmpfs mount for this step.
955
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
956
- memory allocated for this step.
957
- tmpfs_path : str, optional, default /metaflow_temp
958
- Path to tmpfs mount for this step.
959
- persistent_volume_claims : Dict[str, str], optional, default None
960
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
961
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
962
- shared_memory: int, optional
963
- Shared memory size (in MiB) required for this step
964
- port: int, optional
965
- Port number to specify in the Kubernetes job object
966
- """
967
- ...
968
-
969
985
  @typing.overload
970
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
986
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
971
987
  """
972
- Specifies the Conda environment for the step.
988
+ Specifies the PyPI packages for the step.
973
989
 
974
990
  Information in this decorator will augment any
975
- attributes set in the `@conda_base` flow-level decorator. Hence,
976
- you can use `@conda_base` to set packages required by all
977
- steps and use `@conda` to specify step-specific overrides.
991
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
992
+ you can use `@pypi_base` to set packages required by all
993
+ steps and use `@pypi` to specify step-specific overrides.
978
994
 
979
995
  Parameters
980
996
  ----------
981
- packages : Dict[str, str], default {}
997
+ packages : Dict[str, str], default: {}
982
998
  Packages to use for this step. The key is the name of the package
983
999
  and the value is the version to use.
984
- libraries : Dict[str, str], default {}
985
- Supported for backward compatibility. When used with packages, packages will take precedence.
986
- python : str, optional, default None
1000
+ python : str, optional, default: None
987
1001
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
988
1002
  that the version used will correspond to the version of the Python interpreter used to start the run.
989
- disabled : bool, default False
990
- If set to True, disables @conda.
991
1003
  """
992
1004
  ...
993
1005
 
994
1006
  @typing.overload
995
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1007
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
996
1008
  ...
997
1009
 
998
1010
  @typing.overload
999
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1011
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1000
1012
  ...
1001
1013
 
1002
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1014
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1003
1015
  """
1004
- Specifies the Conda environment for the step.
1016
+ Specifies the PyPI packages for the step.
1005
1017
 
1006
1018
  Information in this decorator will augment any
1007
- attributes set in the `@conda_base` flow-level decorator. Hence,
1008
- you can use `@conda_base` to set packages required by all
1009
- steps and use `@conda` to specify step-specific overrides.
1019
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1020
+ you can use `@pypi_base` to set packages required by all
1021
+ steps and use `@pypi` to specify step-specific overrides.
1010
1022
 
1011
1023
  Parameters
1012
1024
  ----------
1013
- packages : Dict[str, str], default {}
1025
+ packages : Dict[str, str], default: {}
1014
1026
  Packages to use for this step. The key is the name of the package
1015
1027
  and the value is the version to use.
1016
- libraries : Dict[str, str], default {}
1017
- Supported for backward compatibility. When used with packages, packages will take precedence.
1018
- python : str, optional, default None
1028
+ python : str, optional, default: None
1019
1029
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1020
1030
  that the version used will correspond to the version of the Python interpreter used to start the run.
1021
- disabled : bool, default False
1022
- If set to True, disables @conda.
1023
1031
  """
1024
1032
  ...
1025
1033
 
1026
1034
  @typing.overload
1027
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1035
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1028
1036
  """
1029
- Creates a human-readable report, a Metaflow Card, after this step completes.
1037
+ Specifies that the step will success under all circumstances.
1030
1038
 
1031
- Note that you may add multiple `@card` decorators in a step with different parameters.
1039
+ The decorator will create an optional artifact, specified by `var`, which
1040
+ contains the exception raised. You can use it to detect the presence
1041
+ of errors, indicating that all happy-path artifacts produced by the step
1042
+ are missing.
1032
1043
 
1033
1044
  Parameters
1034
1045
  ----------
1035
- type : str, default 'default'
1036
- Card type.
1037
- id : str, optional, default None
1038
- If multiple cards are present, use this id to identify this card.
1039
- options : Dict[str, Any], default {}
1040
- Options passed to the card. The contents depend on the card type.
1041
- timeout : int, default 45
1042
- Interrupt reporting if it takes more than this many seconds.
1043
-
1044
-
1046
+ var : str, optional, default None
1047
+ Name of the artifact in which to store the caught exception.
1048
+ If not specified, the exception is not stored.
1049
+ print_exception : bool, default True
1050
+ Determines whether or not the exception is printed to
1051
+ stdout when caught.
1045
1052
  """
1046
1053
  ...
1047
1054
 
1048
1055
  @typing.overload
1049
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1056
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1050
1057
  ...
1051
1058
 
1052
1059
  @typing.overload
1053
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1060
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1054
1061
  ...
1055
1062
 
1056
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1063
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1057
1064
  """
1058
- Creates a human-readable report, a Metaflow Card, after this step completes.
1065
+ Specifies that the step will success under all circumstances.
1059
1066
 
1060
- Note that you may add multiple `@card` decorators in a step with different parameters.
1067
+ The decorator will create an optional artifact, specified by `var`, which
1068
+ contains the exception raised. You can use it to detect the presence
1069
+ of errors, indicating that all happy-path artifacts produced by the step
1070
+ are missing.
1061
1071
 
1062
1072
  Parameters
1063
1073
  ----------
1064
- type : str, default 'default'
1065
- Card type.
1066
- id : str, optional, default None
1067
- If multiple cards are present, use this id to identify this card.
1068
- options : Dict[str, Any], default {}
1069
- Options passed to the card. The contents depend on the card type.
1070
- timeout : int, default 45
1071
- Interrupt reporting if it takes more than this many seconds.
1072
-
1073
-
1074
- """
1075
- ...
1076
-
1077
- @typing.overload
1078
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1079
- """
1080
- Specifies environment variables to be set prior to the execution of a step.
1081
-
1082
- Parameters
1083
- ----------
1084
- vars : Dict[str, str], default {}
1085
- Dictionary of environment variables to set.
1086
- """
1087
- ...
1088
-
1089
- @typing.overload
1090
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1091
- ...
1092
-
1093
- @typing.overload
1094
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1095
- ...
1096
-
1097
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1098
- """
1099
- Specifies environment variables to be set prior to the execution of a step.
1100
-
1101
- Parameters
1102
- ----------
1103
- vars : Dict[str, str], default {}
1104
- Dictionary of environment variables to set.
1074
+ var : str, optional, default None
1075
+ Name of the artifact in which to store the caught exception.
1076
+ If not specified, the exception is not stored.
1077
+ print_exception : bool, default True
1078
+ Determines whether or not the exception is printed to
1079
+ stdout when caught.
1105
1080
  """
1106
1081
  ...
1107
1082
 
@@ -1159,51 +1134,151 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1159
1134
  ...
1160
1135
 
1161
1136
  @typing.overload
1162
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1137
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1163
1138
  """
1164
- Specifies that the step will success under all circumstances.
1139
+ Specifies a timeout for your step.
1165
1140
 
1166
- The decorator will create an optional artifact, specified by `var`, which
1167
- contains the exception raised. You can use it to detect the presence
1168
- of errors, indicating that all happy-path artifacts produced by the step
1169
- are missing.
1141
+ This decorator is useful if this step may hang indefinitely.
1142
+
1143
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1144
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1145
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1146
+
1147
+ Note that all the values specified in parameters are added together so if you specify
1148
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1170
1149
 
1171
1150
  Parameters
1172
1151
  ----------
1173
- var : str, optional, default None
1174
- Name of the artifact in which to store the caught exception.
1175
- If not specified, the exception is not stored.
1176
- print_exception : bool, default True
1177
- Determines whether or not the exception is printed to
1178
- stdout when caught.
1152
+ seconds : int, default 0
1153
+ Number of seconds to wait prior to timing out.
1154
+ minutes : int, default 0
1155
+ Number of minutes to wait prior to timing out.
1156
+ hours : int, default 0
1157
+ Number of hours to wait prior to timing out.
1179
1158
  """
1180
1159
  ...
1181
1160
 
1182
1161
  @typing.overload
1183
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1162
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1184
1163
  ...
1185
1164
 
1186
1165
  @typing.overload
1187
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1166
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1188
1167
  ...
1189
1168
 
1190
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1169
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1191
1170
  """
1192
- Specifies that the step will success under all circumstances.
1171
+ Specifies a timeout for your step.
1193
1172
 
1194
- The decorator will create an optional artifact, specified by `var`, which
1195
- contains the exception raised. You can use it to detect the presence
1196
- of errors, indicating that all happy-path artifacts produced by the step
1197
- are missing.
1173
+ This decorator is useful if this step may hang indefinitely.
1174
+
1175
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1176
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1177
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1178
+
1179
+ Note that all the values specified in parameters are added together so if you specify
1180
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1198
1181
 
1199
1182
  Parameters
1200
1183
  ----------
1201
- var : str, optional, default None
1202
- Name of the artifact in which to store the caught exception.
1203
- If not specified, the exception is not stored.
1204
- print_exception : bool, default True
1205
- Determines whether or not the exception is printed to
1206
- stdout when caught.
1184
+ seconds : int, default 0
1185
+ Number of seconds to wait prior to timing out.
1186
+ minutes : int, default 0
1187
+ Number of minutes to wait prior to timing out.
1188
+ hours : int, default 0
1189
+ Number of hours to wait prior to timing out.
1190
+ """
1191
+ ...
1192
+
1193
+ @typing.overload
1194
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1195
+ """
1196
+ Specifies secrets to be retrieved and injected as environment variables prior to
1197
+ the execution of a step.
1198
+
1199
+ Parameters
1200
+ ----------
1201
+ sources : List[Union[str, Dict[str, Any]]], default: []
1202
+ List of secret specs, defining how the secrets are to be retrieved
1203
+ """
1204
+ ...
1205
+
1206
+ @typing.overload
1207
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1208
+ ...
1209
+
1210
+ @typing.overload
1211
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1212
+ ...
1213
+
1214
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1215
+ """
1216
+ Specifies secrets to be retrieved and injected as environment variables prior to
1217
+ the execution of a step.
1218
+
1219
+ Parameters
1220
+ ----------
1221
+ sources : List[Union[str, Dict[str, Any]]], default: []
1222
+ List of secret specs, defining how the secrets are to be retrieved
1223
+ """
1224
+ ...
1225
+
1226
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1227
+ """
1228
+ Specifies that this step should execute on Kubernetes.
1229
+
1230
+ Parameters
1231
+ ----------
1232
+ cpu : int, default 1
1233
+ Number of CPUs required for this step. If `@resources` is
1234
+ also present, the maximum value from all decorators is used.
1235
+ memory : int, default 4096
1236
+ Memory size (in MB) required for this step. If
1237
+ `@resources` is also present, the maximum value from all decorators is
1238
+ used.
1239
+ disk : int, default 10240
1240
+ Disk size (in MB) required for this step. If
1241
+ `@resources` is also present, the maximum value from all decorators is
1242
+ used.
1243
+ image : str, optional, default None
1244
+ Docker image to use when launching on Kubernetes. If not specified, and
1245
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1246
+ not, a default Docker image mapping to the current version of Python is used.
1247
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1248
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1249
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1250
+ Kubernetes service account to use when launching pod in Kubernetes.
1251
+ secrets : List[str], optional, default None
1252
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1253
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1254
+ in Metaflow configuration.
1255
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1256
+ Kubernetes namespace to use when launching pod in Kubernetes.
1257
+ gpu : int, optional, default None
1258
+ Number of GPUs required for this step. A value of zero implies that
1259
+ the scheduled node should not have GPUs.
1260
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1261
+ The vendor of the GPUs to be used for this step.
1262
+ tolerations : List[str], default []
1263
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1264
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1265
+ use_tmpfs : bool, default False
1266
+ This enables an explicit tmpfs mount for this step.
1267
+ tmpfs_tempdir : bool, default True
1268
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1269
+ tmpfs_size : int, optional, default: None
1270
+ The value for the size (in MiB) of the tmpfs mount for this step.
1271
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1272
+ memory allocated for this step.
1273
+ tmpfs_path : str, optional, default /metaflow_temp
1274
+ Path to tmpfs mount for this step.
1275
+ persistent_volume_claims : Dict[str, str], optional, default None
1276
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1277
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1278
+ shared_memory: int, optional
1279
+ Shared memory size (in MiB) required for this step
1280
+ port: int, optional
1281
+ Port number to specify in the Kubernetes job object
1207
1282
  """
1208
1283
  ...
1209
1284
 
@@ -1285,157 +1360,93 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1285
1360
  ...
1286
1361
 
1287
1362
  @typing.overload
1288
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1363
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1289
1364
  """
1290
- Specifies a timeout for your step.
1291
-
1292
- This decorator is useful if this step may hang indefinitely.
1293
-
1294
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1295
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1296
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1297
-
1298
- Note that all the values specified in parameters are added together so if you specify
1299
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1365
+ Specifies environment variables to be set prior to the execution of a step.
1300
1366
 
1301
1367
  Parameters
1302
1368
  ----------
1303
- seconds : int, default 0
1304
- Number of seconds to wait prior to timing out.
1305
- minutes : int, default 0
1306
- Number of minutes to wait prior to timing out.
1307
- hours : int, default 0
1308
- Number of hours to wait prior to timing out.
1369
+ vars : Dict[str, str], default {}
1370
+ Dictionary of environment variables to set.
1309
1371
  """
1310
1372
  ...
1311
1373
 
1312
1374
  @typing.overload
1313
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1375
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1314
1376
  ...
1315
1377
 
1316
1378
  @typing.overload
1317
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1379
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1318
1380
  ...
1319
1381
 
1320
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1382
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1321
1383
  """
1322
- Specifies a timeout for your step.
1323
-
1324
- This decorator is useful if this step may hang indefinitely.
1325
-
1326
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1327
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1328
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1329
-
1330
- Note that all the values specified in parameters are added together so if you specify
1331
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1384
+ Specifies environment variables to be set prior to the execution of a step.
1332
1385
 
1333
1386
  Parameters
1334
1387
  ----------
1335
- seconds : int, default 0
1336
- Number of seconds to wait prior to timing out.
1337
- minutes : int, default 0
1338
- Number of minutes to wait prior to timing out.
1339
- hours : int, default 0
1340
- Number of hours to wait prior to timing out.
1388
+ vars : Dict[str, str], default {}
1389
+ Dictionary of environment variables to set.
1341
1390
  """
1342
1391
  ...
1343
1392
 
1344
- @typing.overload
1345
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1393
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1346
1394
  """
1347
- Specifies the PyPI packages for the step.
1348
-
1349
- Information in this decorator will augment any
1350
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1351
- you can use `@pypi_base` to set packages required by all
1352
- steps and use `@pypi` to specify step-specific overrides.
1395
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1396
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1353
1397
 
1354
1398
  Parameters
1355
1399
  ----------
1356
- packages : Dict[str, str], default: {}
1357
- Packages to use for this step. The key is the name of the package
1358
- and the value is the version to use.
1359
- python : str, optional, default: None
1360
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1361
- that the version used will correspond to the version of the Python interpreter used to start the run.
1400
+ timeout : int
1401
+ Time, in seconds before the task times out and fails. (Default: 3600)
1402
+ poke_interval : int
1403
+ Time in seconds that the job should wait in between each try. (Default: 60)
1404
+ mode : str
1405
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1406
+ exponential_backoff : bool
1407
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1408
+ pool : str
1409
+ the slot pool this task should run in,
1410
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1411
+ soft_fail : bool
1412
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1413
+ name : str
1414
+ Name of the sensor on Airflow
1415
+ description : str
1416
+ Description of sensor in the Airflow UI
1417
+ external_dag_id : str
1418
+ The dag_id that contains the task you want to wait for.
1419
+ external_task_ids : List[str]
1420
+ The list of task_ids that you want to wait for.
1421
+ If None (default value) the sensor waits for the DAG. (Default: None)
1422
+ allowed_states : List[str]
1423
+ Iterable of allowed states, (Default: ['success'])
1424
+ failed_states : List[str]
1425
+ Iterable of failed or dis-allowed states. (Default: None)
1426
+ execution_delta : datetime.timedelta
1427
+ time difference with the previous execution to look at,
1428
+ the default is the same logical date as the current task or DAG. (Default: None)
1429
+ check_existence: bool
1430
+ Set to True to check if the external task exists or check if
1431
+ the DAG to wait for exists. (Default: True)
1362
1432
  """
1363
1433
  ...
1364
1434
 
1365
- @typing.overload
1366
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1367
- ...
1368
-
1369
- @typing.overload
1370
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1371
- ...
1372
-
1373
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1435
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1374
1436
  """
1375
- Specifies the PyPI packages for the step.
1437
+ Specifies what flows belong to the same project.
1376
1438
 
1377
- Information in this decorator will augment any
1378
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1379
- you can use `@pypi_base` to set packages required by all
1380
- steps and use `@pypi` to specify step-specific overrides.
1439
+ A project-specific namespace is created for all flows that
1440
+ use the same `@project(name)`.
1381
1441
 
1382
1442
  Parameters
1383
1443
  ----------
1384
- packages : Dict[str, str], default: {}
1385
- Packages to use for this step. The key is the name of the package
1386
- and the value is the version to use.
1387
- python : str, optional, default: None
1388
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1389
- that the version used will correspond to the version of the Python interpreter used to start the run.
1390
- """
1391
- ...
1392
-
1393
- @typing.overload
1394
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1395
- """
1396
- Specifies the times when the flow should be run when running on a
1397
- production scheduler.
1444
+ name : str
1445
+ Project name. Make sure that the name is unique amongst all
1446
+ projects that use the same production scheduler. The name may
1447
+ contain only lowercase alphanumeric characters and underscores.
1398
1448
 
1399
- Parameters
1400
- ----------
1401
- hourly : bool, default False
1402
- Run the workflow hourly.
1403
- daily : bool, default True
1404
- Run the workflow daily.
1405
- weekly : bool, default False
1406
- Run the workflow weekly.
1407
- cron : str, optional, default None
1408
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1409
- specified by this expression.
1410
- timezone : str, optional, default None
1411
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1412
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1413
- """
1414
- ...
1415
-
1416
- @typing.overload
1417
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1418
- ...
1419
-
1420
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1421
- """
1422
- Specifies the times when the flow should be run when running on a
1423
- production scheduler.
1424
1449
 
1425
- Parameters
1426
- ----------
1427
- hourly : bool, default False
1428
- Run the workflow hourly.
1429
- daily : bool, default True
1430
- Run the workflow daily.
1431
- weekly : bool, default False
1432
- Run the workflow weekly.
1433
- cron : str, optional, default None
1434
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1435
- specified by this expression.
1436
- timezone : str, optional, default None
1437
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1438
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1439
1450
  """
1440
1451
  ...
1441
1452
 
@@ -1534,24 +1545,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1534
1545
  """
1535
1546
  ...
1536
1547
 
1537
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1538
- """
1539
- Specifies what flows belong to the same project.
1540
-
1541
- A project-specific namespace is created for all flows that
1542
- use the same `@project(name)`.
1543
-
1544
- Parameters
1545
- ----------
1546
- name : str
1547
- Project name. Make sure that the name is unique amongst all
1548
- projects that use the same production scheduler. The name may
1549
- contain only lowercase alphanumeric characters and underscores.
1550
-
1551
-
1552
- """
1553
- ...
1554
-
1555
1548
  @typing.overload
1556
1549
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1557
1550
  """
@@ -1601,6 +1594,45 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1601
1594
  """
1602
1595
  ...
1603
1596
 
1597
+ @typing.overload
1598
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1599
+ """
1600
+ Specifies the PyPI packages for all steps of the flow.
1601
+
1602
+ Use `@pypi_base` to set common packages required by all
1603
+ steps and use `@pypi` to specify step-specific overrides.
1604
+ Parameters
1605
+ ----------
1606
+ packages : Dict[str, str], default: {}
1607
+ Packages to use for this flow. The key is the name of the package
1608
+ and the value is the version to use.
1609
+ python : str, optional, default: None
1610
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1611
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1612
+ """
1613
+ ...
1614
+
1615
+ @typing.overload
1616
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1617
+ ...
1618
+
1619
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1620
+ """
1621
+ Specifies the PyPI packages for all steps of the flow.
1622
+
1623
+ Use `@pypi_base` to set common packages required by all
1624
+ steps and use `@pypi` to specify step-specific overrides.
1625
+ Parameters
1626
+ ----------
1627
+ packages : Dict[str, str], default: {}
1628
+ Packages to use for this flow. The key is the name of the package
1629
+ and the value is the version to use.
1630
+ python : str, optional, default: None
1631
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1632
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1633
+ """
1634
+ ...
1635
+
1604
1636
  def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1605
1637
  """
1606
1638
  The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
@@ -1643,84 +1675,52 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1643
1675
  """
1644
1676
  ...
1645
1677
 
1646
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1647
- """
1648
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1649
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1650
-
1651
- Parameters
1652
- ----------
1653
- timeout : int
1654
- Time, in seconds before the task times out and fails. (Default: 3600)
1655
- poke_interval : int
1656
- Time in seconds that the job should wait in between each try. (Default: 60)
1657
- mode : str
1658
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1659
- exponential_backoff : bool
1660
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1661
- pool : str
1662
- the slot pool this task should run in,
1663
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1664
- soft_fail : bool
1665
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1666
- name : str
1667
- Name of the sensor on Airflow
1668
- description : str
1669
- Description of sensor in the Airflow UI
1670
- external_dag_id : str
1671
- The dag_id that contains the task you want to wait for.
1672
- external_task_ids : List[str]
1673
- The list of task_ids that you want to wait for.
1674
- If None (default value) the sensor waits for the DAG. (Default: None)
1675
- allowed_states : List[str]
1676
- Iterable of allowed states, (Default: ['success'])
1677
- failed_states : List[str]
1678
- Iterable of failed or dis-allowed states. (Default: None)
1679
- execution_delta : datetime.timedelta
1680
- time difference with the previous execution to look at,
1681
- the default is the same logical date as the current task or DAG. (Default: None)
1682
- check_existence: bool
1683
- Set to True to check if the external task exists or check if
1684
- the DAG to wait for exists. (Default: True)
1685
- """
1686
- ...
1687
-
1688
1678
  @typing.overload
1689
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1679
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1690
1680
  """
1691
- Specifies the PyPI packages for all steps of the flow.
1681
+ Specifies the times when the flow should be run when running on a
1682
+ production scheduler.
1692
1683
 
1693
- Use `@pypi_base` to set common packages required by all
1694
- steps and use `@pypi` to specify step-specific overrides.
1695
1684
  Parameters
1696
1685
  ----------
1697
- packages : Dict[str, str], default: {}
1698
- Packages to use for this flow. The key is the name of the package
1699
- and the value is the version to use.
1700
- python : str, optional, default: None
1701
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1702
- that the version used will correspond to the version of the Python interpreter used to start the run.
1686
+ hourly : bool, default False
1687
+ Run the workflow hourly.
1688
+ daily : bool, default True
1689
+ Run the workflow daily.
1690
+ weekly : bool, default False
1691
+ Run the workflow weekly.
1692
+ cron : str, optional, default None
1693
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1694
+ specified by this expression.
1695
+ timezone : str, optional, default None
1696
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1697
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1703
1698
  """
1704
1699
  ...
1705
1700
 
1706
1701
  @typing.overload
1707
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1702
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1708
1703
  ...
1709
1704
 
1710
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1705
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1711
1706
  """
1712
- Specifies the PyPI packages for all steps of the flow.
1707
+ Specifies the times when the flow should be run when running on a
1708
+ production scheduler.
1713
1709
 
1714
- Use `@pypi_base` to set common packages required by all
1715
- steps and use `@pypi` to specify step-specific overrides.
1716
1710
  Parameters
1717
1711
  ----------
1718
- packages : Dict[str, str], default: {}
1719
- Packages to use for this flow. The key is the name of the package
1720
- and the value is the version to use.
1721
- python : str, optional, default: None
1722
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1723
- that the version used will correspond to the version of the Python interpreter used to start the run.
1712
+ hourly : bool, default False
1713
+ Run the workflow hourly.
1714
+ daily : bool, default True
1715
+ Run the workflow daily.
1716
+ weekly : bool, default False
1717
+ Run the workflow weekly.
1718
+ cron : str, optional, default None
1719
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1720
+ specified by this expression.
1721
+ timezone : str, optional, default None
1722
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1723
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1724
1724
  """
1725
1725
  ...
1726
1726