metaflow-stubs 2.12.21__py2.py3-none-any.whl → 2.12.22__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. metaflow-stubs/__init__.pyi +422 -422
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +6 -6
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +2 -2
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +22 -22
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +4 -4
  21. metaflow-stubs/plugins/__init__.pyi +3 -3
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow.pyi +4 -4
  24. metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
  25. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  32. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  35. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  36. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  37. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
  38. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  39. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  46. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  47. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  49. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  59. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  61. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  62. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  63. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  64. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  65. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  66. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  68. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  72. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  74. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  81. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  84. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  85. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  86. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  87. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  89. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  92. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  94. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  95. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  96. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  98. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -4
  100. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  102. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  104. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  106. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  110. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  112. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  114. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  115. metaflow-stubs/plugins/package_cli.pyi +2 -2
  116. metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
  117. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  118. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  119. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  121. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  124. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  126. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  128. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  129. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  130. metaflow-stubs/plugins/tag_cli.pyi +5 -5
  131. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  132. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  133. metaflow-stubs/procpoll.pyi +2 -2
  134. metaflow-stubs/pylint_wrapper.pyi +2 -2
  135. metaflow-stubs/runner/__init__.pyi +2 -2
  136. metaflow-stubs/runner/deployer.pyi +3 -3
  137. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  138. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  139. metaflow-stubs/runner/nbrun.pyi +2 -2
  140. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  141. metaflow-stubs/runner/utils.pyi +2 -2
  142. metaflow-stubs/system/__init__.pyi +4 -4
  143. metaflow-stubs/system/system_logger.pyi +3 -3
  144. metaflow-stubs/system/system_monitor.pyi +3 -3
  145. metaflow-stubs/tagging_util.pyi +2 -2
  146. metaflow-stubs/tuple_util.pyi +2 -2
  147. metaflow-stubs/version.pyi +2 -2
  148. {metaflow_stubs-2.12.21.dist-info → metaflow_stubs-2.12.22.dist-info}/METADATA +2 -2
  149. metaflow_stubs-2.12.22.dist-info/RECORD +152 -0
  150. metaflow_stubs-2.12.21.dist-info/RECORD +0 -152
  151. {metaflow_stubs-2.12.21.dist-info → metaflow_stubs-2.12.22.dist-info}/WHEEL +0 -0
  152. {metaflow_stubs-2.12.21.dist-info → metaflow_stubs-2.12.22.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.21 #
4
- # Generated on 2024-09-19T17:04:54.824903 #
3
+ # MF version: 2.12.22 #
4
+ # Generated on 2024-09-20T00:45:49.586219 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.datastore.inputs
12
- import datetime
13
- import metaflow.plugins.datatools.s3.s3
11
+ import metaflow.flowspec
14
12
  import metaflow.runner.metaflow_runner
15
- import metaflow.client.core
16
13
  import metaflow.events
14
+ import metaflow.client.core
15
+ import metaflow.plugins.datatools.s3.s3
17
16
  import metaflow.metaflow_current
18
- import metaflow.flowspec
19
- import metaflow.parameters
20
- import metaflow._vendor.click.types
17
+ import datetime
18
+ import metaflow.datastore.inputs
21
19
  import typing
22
20
  import io
21
+ import metaflow._vendor.click.types
22
+ import metaflow.parameters
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -724,123 +724,79 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
724
724
  ...
725
725
 
726
726
  @typing.overload
727
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
728
- """
729
- Specifies secrets to be retrieved and injected as environment variables prior to
730
- the execution of a step.
731
-
732
- Parameters
733
- ----------
734
- sources : List[Union[str, Dict[str, Any]]], default: []
735
- List of secret specs, defining how the secrets are to be retrieved
736
- """
737
- ...
738
-
739
- @typing.overload
740
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
741
- ...
742
-
743
- @typing.overload
744
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
745
- ...
746
-
747
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
748
- """
749
- Specifies secrets to be retrieved and injected as environment variables prior to
750
- the execution of a step.
751
-
752
- Parameters
753
- ----------
754
- sources : List[Union[str, Dict[str, Any]]], default: []
755
- List of secret specs, defining how the secrets are to be retrieved
756
- """
757
- ...
758
-
759
- @typing.overload
760
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
727
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
761
728
  """
762
- Specifies a timeout for your step.
763
-
764
- This decorator is useful if this step may hang indefinitely.
729
+ Specifies the resources needed when executing this step.
765
730
 
766
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
767
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
768
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
731
+ Use `@resources` to specify the resource requirements
732
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
769
733
 
770
- Note that all the values specified in parameters are added together so if you specify
771
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
734
+ You can choose the compute layer on the command line by executing e.g.
735
+ ```
736
+ python myflow.py run --with batch
737
+ ```
738
+ or
739
+ ```
740
+ python myflow.py run --with kubernetes
741
+ ```
742
+ which executes the flow on the desired system using the
743
+ requirements specified in `@resources`.
772
744
 
773
745
  Parameters
774
746
  ----------
775
- seconds : int, default 0
776
- Number of seconds to wait prior to timing out.
777
- minutes : int, default 0
778
- Number of minutes to wait prior to timing out.
779
- hours : int, default 0
780
- Number of hours to wait prior to timing out.
747
+ cpu : int, default 1
748
+ Number of CPUs required for this step.
749
+ gpu : int, default 0
750
+ Number of GPUs required for this step.
751
+ disk : int, optional, default None
752
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
753
+ memory : int, default 4096
754
+ Memory size (in MB) required for this step.
755
+ shared_memory : int, optional, default None
756
+ The value for the size (in MiB) of the /dev/shm volume for this step.
757
+ This parameter maps to the `--shm-size` option in Docker.
781
758
  """
782
759
  ...
783
760
 
784
761
  @typing.overload
785
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
762
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
786
763
  ...
787
764
 
788
765
  @typing.overload
789
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
766
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
790
767
  ...
791
768
 
792
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
769
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
793
770
  """
794
- Specifies a timeout for your step.
795
-
796
- This decorator is useful if this step may hang indefinitely.
797
-
798
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
799
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
800
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
801
-
802
- Note that all the values specified in parameters are added together so if you specify
803
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
771
+ Specifies the resources needed when executing this step.
804
772
 
805
- Parameters
806
- ----------
807
- seconds : int, default 0
808
- Number of seconds to wait prior to timing out.
809
- minutes : int, default 0
810
- Number of minutes to wait prior to timing out.
811
- hours : int, default 0
812
- Number of hours to wait prior to timing out.
813
- """
814
- ...
815
-
816
- @typing.overload
817
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
818
- """
819
- Specifies environment variables to be set prior to the execution of a step.
773
+ Use `@resources` to specify the resource requirements
774
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
820
775
 
821
- Parameters
822
- ----------
823
- vars : Dict[str, str], default {}
824
- Dictionary of environment variables to set.
825
- """
826
- ...
827
-
828
- @typing.overload
829
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
830
- ...
831
-
832
- @typing.overload
833
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
834
- ...
835
-
836
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
837
- """
838
- Specifies environment variables to be set prior to the execution of a step.
776
+ You can choose the compute layer on the command line by executing e.g.
777
+ ```
778
+ python myflow.py run --with batch
779
+ ```
780
+ or
781
+ ```
782
+ python myflow.py run --with kubernetes
783
+ ```
784
+ which executes the flow on the desired system using the
785
+ requirements specified in `@resources`.
839
786
 
840
787
  Parameters
841
788
  ----------
842
- vars : Dict[str, str], default {}
843
- Dictionary of environment variables to set.
789
+ cpu : int, default 1
790
+ Number of CPUs required for this step.
791
+ gpu : int, default 0
792
+ Number of GPUs required for this step.
793
+ disk : int, optional, default None
794
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
795
+ memory : int, default 4096
796
+ Memory size (in MB) required for this step.
797
+ shared_memory : int, optional, default None
798
+ The value for the size (in MiB) of the /dev/shm volume for this step.
799
+ This parameter maps to the `--shm-size` option in Docker.
844
800
  """
845
801
  ...
846
802
 
@@ -947,99 +903,142 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
947
903
  ...
948
904
 
949
905
  @typing.overload
950
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
906
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
951
907
  """
952
- Specifies the resources needed when executing this step.
953
-
954
- Use `@resources` to specify the resource requirements
955
- independently of the specific compute layer (`@batch`, `@kubernetes`).
956
-
957
- You can choose the compute layer on the command line by executing e.g.
958
- ```
959
- python myflow.py run --with batch
960
- ```
961
- or
962
- ```
963
- python myflow.py run --with kubernetes
964
- ```
965
- which executes the flow on the desired system using the
966
- requirements specified in `@resources`.
908
+ Specifies secrets to be retrieved and injected as environment variables prior to
909
+ the execution of a step.
967
910
 
968
911
  Parameters
969
912
  ----------
970
- cpu : int, default 1
971
- Number of CPUs required for this step.
972
- gpu : int, default 0
973
- Number of GPUs required for this step.
974
- disk : int, optional, default None
975
- Disk size (in MB) required for this step. Only applies on Kubernetes.
976
- memory : int, default 4096
977
- Memory size (in MB) required for this step.
978
- shared_memory : int, optional, default None
979
- The value for the size (in MiB) of the /dev/shm volume for this step.
980
- This parameter maps to the `--shm-size` option in Docker.
913
+ sources : List[Union[str, Dict[str, Any]]], default: []
914
+ List of secret specs, defining how the secrets are to be retrieved
981
915
  """
982
916
  ...
983
917
 
984
918
  @typing.overload
985
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
919
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
986
920
  ...
987
921
 
988
922
  @typing.overload
989
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
923
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
990
924
  ...
991
925
 
992
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
926
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
993
927
  """
994
- Specifies the resources needed when executing this step.
995
-
996
- Use `@resources` to specify the resource requirements
997
- independently of the specific compute layer (`@batch`, `@kubernetes`).
998
-
999
- You can choose the compute layer on the command line by executing e.g.
1000
- ```
1001
- python myflow.py run --with batch
1002
- ```
1003
- or
1004
- ```
1005
- python myflow.py run --with kubernetes
1006
- ```
1007
- which executes the flow on the desired system using the
1008
- requirements specified in `@resources`.
928
+ Specifies secrets to be retrieved and injected as environment variables prior to
929
+ the execution of a step.
1009
930
 
1010
931
  Parameters
1011
932
  ----------
1012
- cpu : int, default 1
1013
- Number of CPUs required for this step.
1014
- gpu : int, default 0
1015
- Number of GPUs required for this step.
1016
- disk : int, optional, default None
1017
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1018
- memory : int, default 4096
1019
- Memory size (in MB) required for this step.
1020
- shared_memory : int, optional, default None
1021
- The value for the size (in MiB) of the /dev/shm volume for this step.
1022
- This parameter maps to the `--shm-size` option in Docker.
933
+ sources : List[Union[str, Dict[str, Any]]], default: []
934
+ List of secret specs, defining how the secrets are to be retrieved
1023
935
  """
1024
936
  ...
1025
937
 
1026
938
  @typing.overload
1027
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
939
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1028
940
  """
1029
- Decorator prototype for all step decorators. This function gets specialized
1030
- and imported for all decorators types by _import_plugin_decorators().
941
+ Specifies the PyPI packages for the step.
942
+
943
+ Information in this decorator will augment any
944
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
945
+ you can use `@pypi_base` to set packages required by all
946
+ steps and use `@pypi` to specify step-specific overrides.
947
+
948
+ Parameters
949
+ ----------
950
+ packages : Dict[str, str], default: {}
951
+ Packages to use for this step. The key is the name of the package
952
+ and the value is the version to use.
953
+ python : str, optional, default: None
954
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
955
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1031
956
  """
1032
957
  ...
1033
958
 
1034
959
  @typing.overload
1035
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
960
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1036
961
  ...
1037
962
 
1038
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1039
- """
1040
- Decorator prototype for all step decorators. This function gets specialized
1041
- and imported for all decorators types by _import_plugin_decorators().
1042
- """
963
+ @typing.overload
964
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
965
+ ...
966
+
967
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
968
+ """
969
+ Specifies the PyPI packages for the step.
970
+
971
+ Information in this decorator will augment any
972
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
973
+ you can use `@pypi_base` to set packages required by all
974
+ steps and use `@pypi` to specify step-specific overrides.
975
+
976
+ Parameters
977
+ ----------
978
+ packages : Dict[str, str], default: {}
979
+ Packages to use for this step. The key is the name of the package
980
+ and the value is the version to use.
981
+ python : str, optional, default: None
982
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
983
+ that the version used will correspond to the version of the Python interpreter used to start the run.
984
+ """
985
+ ...
986
+
987
+ @typing.overload
988
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
989
+ """
990
+ Specifies the Conda environment for the step.
991
+
992
+ Information in this decorator will augment any
993
+ attributes set in the `@conda_base` flow-level decorator. Hence,
994
+ you can use `@conda_base` to set packages required by all
995
+ steps and use `@conda` to specify step-specific overrides.
996
+
997
+ Parameters
998
+ ----------
999
+ packages : Dict[str, str], default {}
1000
+ Packages to use for this step. The key is the name of the package
1001
+ and the value is the version to use.
1002
+ libraries : Dict[str, str], default {}
1003
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1004
+ python : str, optional, default None
1005
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1006
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1007
+ disabled : bool, default False
1008
+ If set to True, disables @conda.
1009
+ """
1010
+ ...
1011
+
1012
+ @typing.overload
1013
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1014
+ ...
1015
+
1016
+ @typing.overload
1017
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1018
+ ...
1019
+
1020
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1021
+ """
1022
+ Specifies the Conda environment for the step.
1023
+
1024
+ Information in this decorator will augment any
1025
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1026
+ you can use `@conda_base` to set packages required by all
1027
+ steps and use `@conda` to specify step-specific overrides.
1028
+
1029
+ Parameters
1030
+ ----------
1031
+ packages : Dict[str, str], default {}
1032
+ Packages to use for this step. The key is the name of the package
1033
+ and the value is the version to use.
1034
+ libraries : Dict[str, str], default {}
1035
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1036
+ python : str, optional, default None
1037
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1038
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1039
+ disabled : bool, default False
1040
+ If set to True, disables @conda.
1041
+ """
1043
1042
  ...
1044
1043
 
1045
1044
  @typing.overload
@@ -1190,159 +1189,141 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1190
1189
  ...
1191
1190
 
1192
1191
  @typing.overload
1193
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1192
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1194
1193
  """
1195
- Creates a human-readable report, a Metaflow Card, after this step completes.
1196
-
1197
- Note that you may add multiple `@card` decorators in a step with different parameters.
1194
+ Specifies environment variables to be set prior to the execution of a step.
1198
1195
 
1199
1196
  Parameters
1200
1197
  ----------
1201
- type : str, default 'default'
1202
- Card type.
1203
- id : str, optional, default None
1204
- If multiple cards are present, use this id to identify this card.
1205
- options : Dict[str, Any], default {}
1206
- Options passed to the card. The contents depend on the card type.
1207
- timeout : int, default 45
1208
- Interrupt reporting if it takes more than this many seconds.
1209
-
1210
-
1198
+ vars : Dict[str, str], default {}
1199
+ Dictionary of environment variables to set.
1211
1200
  """
1212
1201
  ...
1213
1202
 
1214
1203
  @typing.overload
1215
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1204
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1216
1205
  ...
1217
1206
 
1218
1207
  @typing.overload
1219
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1208
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1220
1209
  ...
1221
1210
 
1222
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1211
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1223
1212
  """
1224
- Creates a human-readable report, a Metaflow Card, after this step completes.
1225
-
1226
- Note that you may add multiple `@card` decorators in a step with different parameters.
1213
+ Specifies environment variables to be set prior to the execution of a step.
1227
1214
 
1228
1215
  Parameters
1229
1216
  ----------
1230
- type : str, default 'default'
1231
- Card type.
1232
- id : str, optional, default None
1233
- If multiple cards are present, use this id to identify this card.
1234
- options : Dict[str, Any], default {}
1235
- Options passed to the card. The contents depend on the card type.
1236
- timeout : int, default 45
1237
- Interrupt reporting if it takes more than this many seconds.
1238
-
1239
-
1217
+ vars : Dict[str, str], default {}
1218
+ Dictionary of environment variables to set.
1240
1219
  """
1241
1220
  ...
1242
1221
 
1243
1222
  @typing.overload
1244
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1223
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1245
1224
  """
1246
- Specifies the PyPI packages for the step.
1225
+ Specifies a timeout for your step.
1247
1226
 
1248
- Information in this decorator will augment any
1249
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1250
- you can use `@pypi_base` to set packages required by all
1251
- steps and use `@pypi` to specify step-specific overrides.
1227
+ This decorator is useful if this step may hang indefinitely.
1228
+
1229
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1230
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1231
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1232
+
1233
+ Note that all the values specified in parameters are added together so if you specify
1234
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1252
1235
 
1253
1236
  Parameters
1254
1237
  ----------
1255
- packages : Dict[str, str], default: {}
1256
- Packages to use for this step. The key is the name of the package
1257
- and the value is the version to use.
1258
- python : str, optional, default: None
1259
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1260
- that the version used will correspond to the version of the Python interpreter used to start the run.
1238
+ seconds : int, default 0
1239
+ Number of seconds to wait prior to timing out.
1240
+ minutes : int, default 0
1241
+ Number of minutes to wait prior to timing out.
1242
+ hours : int, default 0
1243
+ Number of hours to wait prior to timing out.
1261
1244
  """
1262
1245
  ...
1263
1246
 
1264
1247
  @typing.overload
1265
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1248
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1266
1249
  ...
1267
1250
 
1268
1251
  @typing.overload
1269
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1252
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1270
1253
  ...
1271
1254
 
1272
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1255
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1273
1256
  """
1274
- Specifies the PyPI packages for the step.
1257
+ Specifies a timeout for your step.
1275
1258
 
1276
- Information in this decorator will augment any
1277
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1278
- you can use `@pypi_base` to set packages required by all
1279
- steps and use `@pypi` to specify step-specific overrides.
1259
+ This decorator is useful if this step may hang indefinitely.
1260
+
1261
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1262
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1263
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1264
+
1265
+ Note that all the values specified in parameters are added together so if you specify
1266
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1280
1267
 
1281
1268
  Parameters
1282
1269
  ----------
1283
- packages : Dict[str, str], default: {}
1284
- Packages to use for this step. The key is the name of the package
1285
- and the value is the version to use.
1286
- python : str, optional, default: None
1287
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1288
- that the version used will correspond to the version of the Python interpreter used to start the run.
1270
+ seconds : int, default 0
1271
+ Number of seconds to wait prior to timing out.
1272
+ minutes : int, default 0
1273
+ Number of minutes to wait prior to timing out.
1274
+ hours : int, default 0
1275
+ Number of hours to wait prior to timing out.
1289
1276
  """
1290
1277
  ...
1291
1278
 
1292
1279
  @typing.overload
1293
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1280
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1294
1281
  """
1295
- Specifies the Conda environment for the step.
1282
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1296
1283
 
1297
- Information in this decorator will augment any
1298
- attributes set in the `@conda_base` flow-level decorator. Hence,
1299
- you can use `@conda_base` to set packages required by all
1300
- steps and use `@conda` to specify step-specific overrides.
1284
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1301
1285
 
1302
1286
  Parameters
1303
1287
  ----------
1304
- packages : Dict[str, str], default {}
1305
- Packages to use for this step. The key is the name of the package
1306
- and the value is the version to use.
1307
- libraries : Dict[str, str], default {}
1308
- Supported for backward compatibility. When used with packages, packages will take precedence.
1309
- python : str, optional, default None
1310
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1311
- that the version used will correspond to the version of the Python interpreter used to start the run.
1312
- disabled : bool, default False
1313
- If set to True, disables @conda.
1288
+ type : str, default 'default'
1289
+ Card type.
1290
+ id : str, optional, default None
1291
+ If multiple cards are present, use this id to identify this card.
1292
+ options : Dict[str, Any], default {}
1293
+ Options passed to the card. The contents depend on the card type.
1294
+ timeout : int, default 45
1295
+ Interrupt reporting if it takes more than this many seconds.
1296
+
1297
+
1314
1298
  """
1315
1299
  ...
1316
1300
 
1317
1301
  @typing.overload
1318
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1302
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1319
1303
  ...
1320
1304
 
1321
1305
  @typing.overload
1322
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1306
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1323
1307
  ...
1324
1308
 
1325
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1309
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1326
1310
  """
1327
- Specifies the Conda environment for the step.
1311
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1328
1312
 
1329
- Information in this decorator will augment any
1330
- attributes set in the `@conda_base` flow-level decorator. Hence,
1331
- you can use `@conda_base` to set packages required by all
1332
- steps and use `@conda` to specify step-specific overrides.
1313
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1333
1314
 
1334
1315
  Parameters
1335
1316
  ----------
1336
- packages : Dict[str, str], default {}
1337
- Packages to use for this step. The key is the name of the package
1338
- and the value is the version to use.
1339
- libraries : Dict[str, str], default {}
1340
- Supported for backward compatibility. When used with packages, packages will take precedence.
1341
- python : str, optional, default None
1342
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1343
- that the version used will correspond to the version of the Python interpreter used to start the run.
1344
- disabled : bool, default False
1345
- If set to True, disables @conda.
1317
+ type : str, default 'default'
1318
+ Card type.
1319
+ id : str, optional, default None
1320
+ If multiple cards are present, use this id to identify this card.
1321
+ options : Dict[str, Any], default {}
1322
+ Options passed to the card. The contents depend on the card type.
1323
+ timeout : int, default 45
1324
+ Interrupt reporting if it takes more than this many seconds.
1325
+
1326
+
1346
1327
  """
1347
1328
  ...
1348
1329
 
@@ -1412,45 +1393,22 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1412
1393
  """
1413
1394
  ...
1414
1395
 
1415
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1396
+ @typing.overload
1397
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1416
1398
  """
1417
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1418
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1419
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1420
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1421
- starts only after all sensors finish.
1422
-
1423
- Parameters
1424
- ----------
1425
- timeout : int
1426
- Time, in seconds before the task times out and fails. (Default: 3600)
1427
- poke_interval : int
1428
- Time in seconds that the job should wait in between each try. (Default: 60)
1429
- mode : str
1430
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1431
- exponential_backoff : bool
1432
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1433
- pool : str
1434
- the slot pool this task should run in,
1435
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1436
- soft_fail : bool
1437
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1438
- name : str
1439
- Name of the sensor on Airflow
1440
- description : str
1441
- Description of sensor in the Airflow UI
1442
- bucket_key : Union[str, List[str]]
1443
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1444
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1445
- bucket_name : str
1446
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1447
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1448
- wildcard_match : bool
1449
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1450
- aws_conn_id : str
1451
- a reference to the s3 connection on Airflow. (Default: None)
1452
- verify : bool
1453
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1399
+ Decorator prototype for all step decorators. This function gets specialized
1400
+ and imported for all decorators types by _import_plugin_decorators().
1401
+ """
1402
+ ...
1403
+
1404
+ @typing.overload
1405
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1406
+ ...
1407
+
1408
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1409
+ """
1410
+ Decorator prototype for all step decorators. This function gets specialized
1411
+ and imported for all decorators types by _import_plugin_decorators().
1454
1412
  """
1455
1413
  ...
1456
1414
 
@@ -1557,21 +1515,84 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1557
1515
  """
1558
1516
  ...
1559
1517
 
1560
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1518
+ @typing.overload
1519
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1561
1520
  """
1562
- Specifies what flows belong to the same project.
1563
-
1564
- A project-specific namespace is created for all flows that
1565
- use the same `@project(name)`.
1521
+ Specifies the PyPI packages for all steps of the flow.
1566
1522
 
1523
+ Use `@pypi_base` to set common packages required by all
1524
+ steps and use `@pypi` to specify step-specific overrides.
1567
1525
  Parameters
1568
1526
  ----------
1569
- name : str
1570
- Project name. Make sure that the name is unique amongst all
1571
- projects that use the same production scheduler. The name may
1572
- contain only lowercase alphanumeric characters and underscores.
1527
+ packages : Dict[str, str], default: {}
1528
+ Packages to use for this flow. The key is the name of the package
1529
+ and the value is the version to use.
1530
+ python : str, optional, default: None
1531
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1532
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1533
+ """
1534
+ ...
1535
+
1536
+ @typing.overload
1537
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1538
+ ...
1539
+
1540
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1541
+ """
1542
+ Specifies the PyPI packages for all steps of the flow.
1573
1543
 
1544
+ Use `@pypi_base` to set common packages required by all
1545
+ steps and use `@pypi` to specify step-specific overrides.
1546
+ Parameters
1547
+ ----------
1548
+ packages : Dict[str, str], default: {}
1549
+ Packages to use for this flow. The key is the name of the package
1550
+ and the value is the version to use.
1551
+ python : str, optional, default: None
1552
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1553
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1554
+ """
1555
+ ...
1556
+
1557
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1558
+ """
1559
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1560
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1574
1561
 
1562
+ Parameters
1563
+ ----------
1564
+ timeout : int
1565
+ Time, in seconds before the task times out and fails. (Default: 3600)
1566
+ poke_interval : int
1567
+ Time in seconds that the job should wait in between each try. (Default: 60)
1568
+ mode : str
1569
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1570
+ exponential_backoff : bool
1571
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1572
+ pool : str
1573
+ the slot pool this task should run in,
1574
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1575
+ soft_fail : bool
1576
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1577
+ name : str
1578
+ Name of the sensor on Airflow
1579
+ description : str
1580
+ Description of sensor in the Airflow UI
1581
+ external_dag_id : str
1582
+ The dag_id that contains the task you want to wait for.
1583
+ external_task_ids : List[str]
1584
+ The list of task_ids that you want to wait for.
1585
+ If None (default value) the sensor waits for the DAG. (Default: None)
1586
+ allowed_states : List[str]
1587
+ Iterable of allowed states, (Default: ['success'])
1588
+ failed_states : List[str]
1589
+ Iterable of failed or dis-allowed states. (Default: None)
1590
+ execution_delta : datetime.timedelta
1591
+ time difference with the previous execution to look at,
1592
+ the default is the same logical date as the current task or DAG. (Default: None)
1593
+ check_existence: bool
1594
+ Set to True to check if the external task exists or check if
1595
+ the DAG to wait for exists. (Default: True)
1575
1596
  """
1576
1597
  ...
1577
1598
 
@@ -1624,91 +1645,45 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1624
1645
  """
1625
1646
  ...
1626
1647
 
1627
- @typing.overload
1628
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1629
- """
1630
- Specifies the PyPI packages for all steps of the flow.
1631
-
1632
- Use `@pypi_base` to set common packages required by all
1633
- steps and use `@pypi` to specify step-specific overrides.
1634
- Parameters
1635
- ----------
1636
- packages : Dict[str, str], default: {}
1637
- Packages to use for this flow. The key is the name of the package
1638
- and the value is the version to use.
1639
- python : str, optional, default: None
1640
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1641
- that the version used will correspond to the version of the Python interpreter used to start the run.
1642
- """
1643
- ...
1644
-
1645
- @typing.overload
1646
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1647
- ...
1648
-
1649
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1650
- """
1651
- Specifies the PyPI packages for all steps of the flow.
1652
-
1653
- Use `@pypi_base` to set common packages required by all
1654
- steps and use `@pypi` to specify step-specific overrides.
1655
- Parameters
1656
- ----------
1657
- packages : Dict[str, str], default: {}
1658
- Packages to use for this flow. The key is the name of the package
1659
- and the value is the version to use.
1660
- python : str, optional, default: None
1661
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1662
- that the version used will correspond to the version of the Python interpreter used to start the run.
1663
- """
1664
- ...
1665
-
1666
- @typing.overload
1667
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1668
- """
1669
- Specifies the Conda environment for all steps of the flow.
1670
-
1671
- Use `@conda_base` to set common libraries required by all
1672
- steps and use `@conda` to specify step-specific additions.
1673
-
1674
- Parameters
1675
- ----------
1676
- packages : Dict[str, str], default {}
1677
- Packages to use for this flow. The key is the name of the package
1678
- and the value is the version to use.
1679
- libraries : Dict[str, str], default {}
1680
- Supported for backward compatibility. When used with packages, packages will take precedence.
1681
- python : str, optional, default None
1682
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1683
- that the version used will correspond to the version of the Python interpreter used to start the run.
1684
- disabled : bool, default False
1685
- If set to True, disables Conda.
1686
- """
1687
- ...
1688
-
1689
- @typing.overload
1690
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1691
- ...
1692
-
1693
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1648
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1694
1649
  """
1695
- Specifies the Conda environment for all steps of the flow.
1696
-
1697
- Use `@conda_base` to set common libraries required by all
1698
- steps and use `@conda` to specify step-specific additions.
1650
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1651
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1652
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1653
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1654
+ starts only after all sensors finish.
1699
1655
 
1700
1656
  Parameters
1701
1657
  ----------
1702
- packages : Dict[str, str], default {}
1703
- Packages to use for this flow. The key is the name of the package
1704
- and the value is the version to use.
1705
- libraries : Dict[str, str], default {}
1706
- Supported for backward compatibility. When used with packages, packages will take precedence.
1707
- python : str, optional, default None
1708
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1709
- that the version used will correspond to the version of the Python interpreter used to start the run.
1710
- disabled : bool, default False
1711
- If set to True, disables Conda.
1658
+ timeout : int
1659
+ Time, in seconds before the task times out and fails. (Default: 3600)
1660
+ poke_interval : int
1661
+ Time in seconds that the job should wait in between each try. (Default: 60)
1662
+ mode : str
1663
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1664
+ exponential_backoff : bool
1665
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1666
+ pool : str
1667
+ the slot pool this task should run in,
1668
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1669
+ soft_fail : bool
1670
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1671
+ name : str
1672
+ Name of the sensor on Airflow
1673
+ description : str
1674
+ Description of sensor in the Airflow UI
1675
+ bucket_key : Union[str, List[str]]
1676
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1677
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1678
+ bucket_name : str
1679
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1680
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1681
+ wildcard_match : bool
1682
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1683
+ aws_conn_id : str
1684
+ a reference to the s3 connection on Airflow. (Default: None)
1685
+ verify : bool
1686
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1712
1687
  """
1713
1688
  ...
1714
1689
 
@@ -1807,45 +1782,70 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1807
1782
  """
1808
1783
  ...
1809
1784
 
1810
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1785
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1811
1786
  """
1812
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1813
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1787
+ Specifies what flows belong to the same project.
1788
+
1789
+ A project-specific namespace is created for all flows that
1790
+ use the same `@project(name)`.
1814
1791
 
1815
1792
  Parameters
1816
1793
  ----------
1817
- timeout : int
1818
- Time, in seconds before the task times out and fails. (Default: 3600)
1819
- poke_interval : int
1820
- Time in seconds that the job should wait in between each try. (Default: 60)
1821
- mode : str
1822
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1823
- exponential_backoff : bool
1824
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1825
- pool : str
1826
- the slot pool this task should run in,
1827
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1828
- soft_fail : bool
1829
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1830
1794
  name : str
1831
- Name of the sensor on Airflow
1832
- description : str
1833
- Description of sensor in the Airflow UI
1834
- external_dag_id : str
1835
- The dag_id that contains the task you want to wait for.
1836
- external_task_ids : List[str]
1837
- The list of task_ids that you want to wait for.
1838
- If None (default value) the sensor waits for the DAG. (Default: None)
1839
- allowed_states : List[str]
1840
- Iterable of allowed states, (Default: ['success'])
1841
- failed_states : List[str]
1842
- Iterable of failed or dis-allowed states. (Default: None)
1843
- execution_delta : datetime.timedelta
1844
- time difference with the previous execution to look at,
1845
- the default is the same logical date as the current task or DAG. (Default: None)
1846
- check_existence: bool
1847
- Set to True to check if the external task exists or check if
1848
- the DAG to wait for exists. (Default: True)
1795
+ Project name. Make sure that the name is unique amongst all
1796
+ projects that use the same production scheduler. The name may
1797
+ contain only lowercase alphanumeric characters and underscores.
1798
+
1799
+
1800
+ """
1801
+ ...
1802
+
1803
+ @typing.overload
1804
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1805
+ """
1806
+ Specifies the Conda environment for all steps of the flow.
1807
+
1808
+ Use `@conda_base` to set common libraries required by all
1809
+ steps and use `@conda` to specify step-specific additions.
1810
+
1811
+ Parameters
1812
+ ----------
1813
+ packages : Dict[str, str], default {}
1814
+ Packages to use for this flow. The key is the name of the package
1815
+ and the value is the version to use.
1816
+ libraries : Dict[str, str], default {}
1817
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1818
+ python : str, optional, default None
1819
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1820
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1821
+ disabled : bool, default False
1822
+ If set to True, disables Conda.
1823
+ """
1824
+ ...
1825
+
1826
+ @typing.overload
1827
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1828
+ ...
1829
+
1830
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1831
+ """
1832
+ Specifies the Conda environment for all steps of the flow.
1833
+
1834
+ Use `@conda_base` to set common libraries required by all
1835
+ steps and use `@conda` to specify step-specific additions.
1836
+
1837
+ Parameters
1838
+ ----------
1839
+ packages : Dict[str, str], default {}
1840
+ Packages to use for this flow. The key is the name of the package
1841
+ and the value is the version to use.
1842
+ libraries : Dict[str, str], default {}
1843
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1844
+ python : str, optional, default None
1845
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1846
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1847
+ disabled : bool, default False
1848
+ If set to True, disables Conda.
1849
1849
  """
1850
1850
  ...
1851
1851