metaflow-stubs 2.12.18__py2.py3-none-any.whl → 2.12.20__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. metaflow-stubs/__init__.pyi +643 -639
  2. metaflow-stubs/cards.pyi +4 -4
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +5 -5
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +26 -24
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +4 -4
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +6 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +7 -7
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
  37. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +6 -4
  38. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  40. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  46. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  48. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +5 -3
  58. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  61. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  62. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  63. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  64. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  65. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  67. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  68. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  73. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  80. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  81. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  83. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  84. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  85. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  86. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  87. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  90. metaflow-stubs/plugins/datatools/s3/s3.pyi +7 -7
  91. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  92. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  93. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  94. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  95. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  96. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  99. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  101. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  104. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  106. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +18 -0
  107. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +7 -3
  108. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +26 -4
  109. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +8 -2
  110. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  111. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  113. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  114. metaflow-stubs/plugins/package_cli.pyi +2 -2
  115. metaflow-stubs/plugins/parallel_decorator.pyi +4 -4
  116. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/conda_decorator.pyi +6 -2
  119. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  120. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +4 -2
  121. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  122. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  123. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  124. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  126. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  127. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  128. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  129. metaflow-stubs/plugins/tag_cli.pyi +5 -5
  130. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  131. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  132. metaflow-stubs/procpoll.pyi +2 -2
  133. metaflow-stubs/pylint_wrapper.pyi +2 -2
  134. metaflow-stubs/runner/__init__.pyi +2 -2
  135. metaflow-stubs/runner/deployer.pyi +8 -6
  136. metaflow-stubs/runner/metaflow_runner.pyi +8 -8
  137. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  138. metaflow-stubs/runner/nbrun.pyi +2 -2
  139. metaflow-stubs/runner/subprocess_manager.pyi +4 -4
  140. metaflow-stubs/runner/utils.pyi +5 -3
  141. metaflow-stubs/system/__init__.pyi +3 -3
  142. metaflow-stubs/system/system_logger.pyi +2 -2
  143. metaflow-stubs/system/system_monitor.pyi +3 -3
  144. metaflow-stubs/tagging_util.pyi +2 -2
  145. metaflow-stubs/tuple_util.pyi +2 -2
  146. metaflow-stubs/version.pyi +2 -2
  147. {metaflow_stubs-2.12.18.dist-info → metaflow_stubs-2.12.20.dist-info}/METADATA +2 -2
  148. metaflow_stubs-2.12.20.dist-info/RECORD +151 -0
  149. {metaflow_stubs-2.12.18.dist-info → metaflow_stubs-2.12.20.dist-info}/WHEEL +1 -1
  150. metaflow_stubs-2.12.18.dist-info/RECORD +0 -150
  151. {metaflow_stubs-2.12.18.dist-info → metaflow_stubs-2.12.20.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.18 #
4
- # Generated on 2024-08-28T16:18:32.928126 #
3
+ # MF version: 2.12.20 #
4
+ # Generated on 2024-09-16T18:11:29.503408 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.metaflow_current
12
- import metaflow._vendor.click.types
13
- import io
14
- import metaflow.client.core
15
- import metaflow.runner.metaflow_runner
16
- import typing
17
- import metaflow.flowspec
18
- import metaflow.plugins.datatools.s3.s3
19
11
  import metaflow.parameters
20
- import metaflow.events
21
12
  import datetime
13
+ import metaflow.plugins.datatools.s3.s3
14
+ import metaflow.flowspec
15
+ import typing
16
+ import metaflow._vendor.click.types
17
+ import metaflow.metaflow_current
18
+ import metaflow.events
19
+ import metaflow.runner.metaflow_runner
20
+ import metaflow.client.core
22
21
  import metaflow.datastore.inputs
22
+ import io
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -728,190 +728,110 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
728
728
  ...
729
729
 
730
730
  @typing.overload
731
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
731
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
732
  """
733
- Specifies a timeout for your step.
734
-
735
- This decorator is useful if this step may hang indefinitely.
736
-
737
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
738
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
739
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
740
-
741
- Note that all the values specified in parameters are added together so if you specify
742
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
733
+ Specifies environment variables to be set prior to the execution of a step.
743
734
 
744
735
  Parameters
745
736
  ----------
746
- seconds : int, default 0
747
- Number of seconds to wait prior to timing out.
748
- minutes : int, default 0
749
- Number of minutes to wait prior to timing out.
750
- hours : int, default 0
751
- Number of hours to wait prior to timing out.
737
+ vars : Dict[str, str], default {}
738
+ Dictionary of environment variables to set.
752
739
  """
753
740
  ...
754
741
 
755
742
  @typing.overload
756
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
743
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
757
744
  ...
758
745
 
759
746
  @typing.overload
760
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
761
- ...
762
-
763
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
764
- """
765
- Specifies a timeout for your step.
766
-
767
- This decorator is useful if this step may hang indefinitely.
768
-
769
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
770
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
771
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
772
-
773
- Note that all the values specified in parameters are added together so if you specify
774
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
775
-
776
- Parameters
777
- ----------
778
- seconds : int, default 0
779
- Number of seconds to wait prior to timing out.
780
- minutes : int, default 0
781
- Number of minutes to wait prior to timing out.
782
- hours : int, default 0
783
- Number of hours to wait prior to timing out.
784
- """
747
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
785
748
  ...
786
749
 
787
- @typing.overload
788
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
750
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
789
751
  """
790
- Creates a human-readable report, a Metaflow Card, after this step completes.
791
-
792
- Note that you may add multiple `@card` decorators in a step with different parameters.
752
+ Specifies environment variables to be set prior to the execution of a step.
793
753
 
794
754
  Parameters
795
755
  ----------
796
- type : str, default 'default'
797
- Card type.
798
- id : str, optional, default None
799
- If multiple cards are present, use this id to identify this card.
800
- options : Dict[str, Any], default {}
801
- Options passed to the card. The contents depend on the card type.
802
- timeout : int, default 45
803
- Interrupt reporting if it takes more than this many seconds.
804
-
805
-
756
+ vars : Dict[str, str], default {}
757
+ Dictionary of environment variables to set.
806
758
  """
807
759
  ...
808
760
 
809
761
  @typing.overload
810
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
811
- ...
812
-
813
- @typing.overload
814
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
815
- ...
816
-
817
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
762
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
818
763
  """
819
- Creates a human-readable report, a Metaflow Card, after this step completes.
820
-
821
- Note that you may add multiple `@card` decorators in a step with different parameters.
822
-
823
- Parameters
824
- ----------
825
- type : str, default 'default'
826
- Card type.
827
- id : str, optional, default None
828
- If multiple cards are present, use this id to identify this card.
829
- options : Dict[str, Any], default {}
830
- Options passed to the card. The contents depend on the card type.
831
- timeout : int, default 45
832
- Interrupt reporting if it takes more than this many seconds.
833
-
764
+ Specifies the resources needed when executing this step.
834
765
 
835
- """
836
- ...
837
-
838
- @typing.overload
839
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
840
- """
841
- Specifies that the step will success under all circumstances.
766
+ Use `@resources` to specify the resource requirements
767
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
842
768
 
843
- The decorator will create an optional artifact, specified by `var`, which
844
- contains the exception raised. You can use it to detect the presence
845
- of errors, indicating that all happy-path artifacts produced by the step
846
- are missing.
769
+ You can choose the compute layer on the command line by executing e.g.
770
+ ```
771
+ python myflow.py run --with batch
772
+ ```
773
+ or
774
+ ```
775
+ python myflow.py run --with kubernetes
776
+ ```
777
+ which executes the flow on the desired system using the
778
+ requirements specified in `@resources`.
847
779
 
848
780
  Parameters
849
781
  ----------
850
- var : str, optional, default None
851
- Name of the artifact in which to store the caught exception.
852
- If not specified, the exception is not stored.
853
- print_exception : bool, default True
854
- Determines whether or not the exception is printed to
855
- stdout when caught.
782
+ cpu : int, default 1
783
+ Number of CPUs required for this step.
784
+ gpu : int, default 0
785
+ Number of GPUs required for this step.
786
+ disk : int, optional, default None
787
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
788
+ memory : int, default 4096
789
+ Memory size (in MB) required for this step.
790
+ shared_memory : int, optional, default None
791
+ The value for the size (in MiB) of the /dev/shm volume for this step.
792
+ This parameter maps to the `--shm-size` option in Docker.
856
793
  """
857
794
  ...
858
795
 
859
796
  @typing.overload
860
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
797
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
861
798
  ...
862
799
 
863
800
  @typing.overload
864
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
801
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
865
802
  ...
866
803
 
867
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
804
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
868
805
  """
869
- Specifies that the step will success under all circumstances.
870
-
871
- The decorator will create an optional artifact, specified by `var`, which
872
- contains the exception raised. You can use it to detect the presence
873
- of errors, indicating that all happy-path artifacts produced by the step
874
- are missing.
806
+ Specifies the resources needed when executing this step.
875
807
 
876
- Parameters
877
- ----------
878
- var : str, optional, default None
879
- Name of the artifact in which to store the caught exception.
880
- If not specified, the exception is not stored.
881
- print_exception : bool, default True
882
- Determines whether or not the exception is printed to
883
- stdout when caught.
884
- """
885
- ...
886
-
887
- @typing.overload
888
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
889
- """
890
- Specifies environment variables to be set prior to the execution of a step.
808
+ Use `@resources` to specify the resource requirements
809
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
891
810
 
892
- Parameters
893
- ----------
894
- vars : Dict[str, str], default {}
895
- Dictionary of environment variables to set.
896
- """
897
- ...
898
-
899
- @typing.overload
900
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
901
- ...
902
-
903
- @typing.overload
904
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
905
- ...
906
-
907
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
908
- """
909
- Specifies environment variables to be set prior to the execution of a step.
811
+ You can choose the compute layer on the command line by executing e.g.
812
+ ```
813
+ python myflow.py run --with batch
814
+ ```
815
+ or
816
+ ```
817
+ python myflow.py run --with kubernetes
818
+ ```
819
+ which executes the flow on the desired system using the
820
+ requirements specified in `@resources`.
910
821
 
911
822
  Parameters
912
823
  ----------
913
- vars : Dict[str, str], default {}
914
- Dictionary of environment variables to set.
824
+ cpu : int, default 1
825
+ Number of CPUs required for this step.
826
+ gpu : int, default 0
827
+ Number of GPUs required for this step.
828
+ disk : int, optional, default None
829
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
830
+ memory : int, default 4096
831
+ Memory size (in MB) required for this step.
832
+ shared_memory : int, optional, default None
833
+ The value for the size (in MiB) of the /dev/shm volume for this step.
834
+ This parameter maps to the `--shm-size` option in Docker.
915
835
  """
916
836
  ...
917
837
 
@@ -935,291 +855,9 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
935
855
  ...
936
856
 
937
857
  @typing.overload
938
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
939
- """
940
- Specifies secrets to be retrieved and injected as environment variables prior to
941
- the execution of a step.
942
-
943
- Parameters
944
- ----------
945
- sources : List[Union[str, Dict[str, Any]]], default: []
946
- List of secret specs, defining how the secrets are to be retrieved
947
- """
948
- ...
949
-
950
- @typing.overload
951
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
952
- ...
953
-
954
- @typing.overload
955
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
956
- ...
957
-
958
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
858
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
959
859
  """
960
- Specifies secrets to be retrieved and injected as environment variables prior to
961
- the execution of a step.
962
-
963
- Parameters
964
- ----------
965
- sources : List[Union[str, Dict[str, Any]]], default: []
966
- List of secret specs, defining how the secrets are to be retrieved
967
- """
968
- ...
969
-
970
- @typing.overload
971
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
972
- """
973
- Specifies the number of times the task corresponding
974
- to a step needs to be retried.
975
-
976
- This decorator is useful for handling transient errors, such as networking issues.
977
- If your task contains operations that can't be retried safely, e.g. database updates,
978
- it is advisable to annotate it with `@retry(times=0)`.
979
-
980
- This can be used in conjunction with the `@catch` decorator. The `@catch`
981
- decorator will execute a no-op task after all retries have been exhausted,
982
- ensuring that the flow execution can continue.
983
-
984
- Parameters
985
- ----------
986
- times : int, default 3
987
- Number of times to retry this task.
988
- minutes_between_retries : int, default 2
989
- Number of minutes between retries.
990
- """
991
- ...
992
-
993
- @typing.overload
994
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
995
- ...
996
-
997
- @typing.overload
998
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
999
- ...
1000
-
1001
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1002
- """
1003
- Specifies the number of times the task corresponding
1004
- to a step needs to be retried.
1005
-
1006
- This decorator is useful for handling transient errors, such as networking issues.
1007
- If your task contains operations that can't be retried safely, e.g. database updates,
1008
- it is advisable to annotate it with `@retry(times=0)`.
1009
-
1010
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1011
- decorator will execute a no-op task after all retries have been exhausted,
1012
- ensuring that the flow execution can continue.
1013
-
1014
- Parameters
1015
- ----------
1016
- times : int, default 3
1017
- Number of times to retry this task.
1018
- minutes_between_retries : int, default 2
1019
- Number of minutes between retries.
1020
- """
1021
- ...
1022
-
1023
- @typing.overload
1024
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1025
- """
1026
- Specifies the resources needed when executing this step.
1027
-
1028
- Use `@resources` to specify the resource requirements
1029
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1030
-
1031
- You can choose the compute layer on the command line by executing e.g.
1032
- ```
1033
- python myflow.py run --with batch
1034
- ```
1035
- or
1036
- ```
1037
- python myflow.py run --with kubernetes
1038
- ```
1039
- which executes the flow on the desired system using the
1040
- requirements specified in `@resources`.
1041
-
1042
- Parameters
1043
- ----------
1044
- cpu : int, default 1
1045
- Number of CPUs required for this step.
1046
- gpu : int, default 0
1047
- Number of GPUs required for this step.
1048
- disk : int, optional, default None
1049
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1050
- memory : int, default 4096
1051
- Memory size (in MB) required for this step.
1052
- shared_memory : int, optional, default None
1053
- The value for the size (in MiB) of the /dev/shm volume for this step.
1054
- This parameter maps to the `--shm-size` option in Docker.
1055
- """
1056
- ...
1057
-
1058
- @typing.overload
1059
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1060
- ...
1061
-
1062
- @typing.overload
1063
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1064
- ...
1065
-
1066
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1067
- """
1068
- Specifies the resources needed when executing this step.
1069
-
1070
- Use `@resources` to specify the resource requirements
1071
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1072
-
1073
- You can choose the compute layer on the command line by executing e.g.
1074
- ```
1075
- python myflow.py run --with batch
1076
- ```
1077
- or
1078
- ```
1079
- python myflow.py run --with kubernetes
1080
- ```
1081
- which executes the flow on the desired system using the
1082
- requirements specified in `@resources`.
1083
-
1084
- Parameters
1085
- ----------
1086
- cpu : int, default 1
1087
- Number of CPUs required for this step.
1088
- gpu : int, default 0
1089
- Number of GPUs required for this step.
1090
- disk : int, optional, default None
1091
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1092
- memory : int, default 4096
1093
- Memory size (in MB) required for this step.
1094
- shared_memory : int, optional, default None
1095
- The value for the size (in MiB) of the /dev/shm volume for this step.
1096
- This parameter maps to the `--shm-size` option in Docker.
1097
- """
1098
- ...
1099
-
1100
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1101
- """
1102
- Specifies that this step should execute on Kubernetes.
1103
-
1104
- Parameters
1105
- ----------
1106
- cpu : int, default 1
1107
- Number of CPUs required for this step. If `@resources` is
1108
- also present, the maximum value from all decorators is used.
1109
- memory : int, default 4096
1110
- Memory size (in MB) required for this step. If
1111
- `@resources` is also present, the maximum value from all decorators is
1112
- used.
1113
- disk : int, default 10240
1114
- Disk size (in MB) required for this step. If
1115
- `@resources` is also present, the maximum value from all decorators is
1116
- used.
1117
- image : str, optional, default None
1118
- Docker image to use when launching on Kubernetes. If not specified, and
1119
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1120
- not, a default Docker image mapping to the current version of Python is used.
1121
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1122
- If given, the imagePullPolicy to be applied to the Docker image of the step.
1123
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1124
- Kubernetes service account to use when launching pod in Kubernetes.
1125
- secrets : List[str], optional, default None
1126
- Kubernetes secrets to use when launching pod in Kubernetes. These
1127
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1128
- in Metaflow configuration.
1129
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1130
- Kubernetes namespace to use when launching pod in Kubernetes.
1131
- gpu : int, optional, default None
1132
- Number of GPUs required for this step. A value of zero implies that
1133
- the scheduled node should not have GPUs.
1134
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1135
- The vendor of the GPUs to be used for this step.
1136
- tolerations : List[str], default []
1137
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1138
- Kubernetes tolerations to use when launching pod in Kubernetes.
1139
- use_tmpfs : bool, default False
1140
- This enables an explicit tmpfs mount for this step.
1141
- tmpfs_tempdir : bool, default True
1142
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1143
- tmpfs_size : int, optional, default: None
1144
- The value for the size (in MiB) of the tmpfs mount for this step.
1145
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1146
- memory allocated for this step.
1147
- tmpfs_path : str, optional, default /metaflow_temp
1148
- Path to tmpfs mount for this step.
1149
- persistent_volume_claims : Dict[str, str], optional, default None
1150
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1151
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1152
- shared_memory: int, optional
1153
- Shared memory size (in MiB) required for this step
1154
- port: int, optional
1155
- Port number to specify in the Kubernetes job object
1156
- compute_pool : str, optional, default None
1157
- Compute pool to be used for for this step.
1158
- If not specified, any accessible compute pool within the perimeter is used.
1159
- """
1160
- ...
1161
-
1162
- @typing.overload
1163
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1164
- """
1165
- Specifies the Conda environment for the step.
1166
-
1167
- Information in this decorator will augment any
1168
- attributes set in the `@conda_base` flow-level decorator. Hence,
1169
- you can use `@conda_base` to set packages required by all
1170
- steps and use `@conda` to specify step-specific overrides.
1171
-
1172
- Parameters
1173
- ----------
1174
- packages : Dict[str, str], default {}
1175
- Packages to use for this step. The key is the name of the package
1176
- and the value is the version to use.
1177
- libraries : Dict[str, str], default {}
1178
- Supported for backward compatibility. When used with packages, packages will take precedence.
1179
- python : str, optional, default None
1180
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1181
- that the version used will correspond to the version of the Python interpreter used to start the run.
1182
- disabled : bool, default False
1183
- If set to True, disables @conda.
1184
- """
1185
- ...
1186
-
1187
- @typing.overload
1188
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1189
- ...
1190
-
1191
- @typing.overload
1192
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1193
- ...
1194
-
1195
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1196
- """
1197
- Specifies the Conda environment for the step.
1198
-
1199
- Information in this decorator will augment any
1200
- attributes set in the `@conda_base` flow-level decorator. Hence,
1201
- you can use `@conda_base` to set packages required by all
1202
- steps and use `@conda` to specify step-specific overrides.
1203
-
1204
- Parameters
1205
- ----------
1206
- packages : Dict[str, str], default {}
1207
- Packages to use for this step. The key is the name of the package
1208
- and the value is the version to use.
1209
- libraries : Dict[str, str], default {}
1210
- Supported for backward compatibility. When used with packages, packages will take precedence.
1211
- python : str, optional, default None
1212
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1213
- that the version used will correspond to the version of the Python interpreter used to start the run.
1214
- disabled : bool, default False
1215
- If set to True, disables @conda.
1216
- """
1217
- ...
1218
-
1219
- @typing.overload
1220
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1221
- """
1222
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
860
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
1223
861
 
1224
862
  Parameters
1225
863
  ----------
@@ -1364,9 +1002,42 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1364
1002
  ...
1365
1003
 
1366
1004
  @typing.overload
1367
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1005
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1368
1006
  """
1369
- Specifies the PyPI packages for the step.
1007
+ Specifies secrets to be retrieved and injected as environment variables prior to
1008
+ the execution of a step.
1009
+
1010
+ Parameters
1011
+ ----------
1012
+ sources : List[Union[str, Dict[str, Any]]], default: []
1013
+ List of secret specs, defining how the secrets are to be retrieved
1014
+ """
1015
+ ...
1016
+
1017
+ @typing.overload
1018
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1019
+ ...
1020
+
1021
+ @typing.overload
1022
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1023
+ ...
1024
+
1025
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1026
+ """
1027
+ Specifies secrets to be retrieved and injected as environment variables prior to
1028
+ the execution of a step.
1029
+
1030
+ Parameters
1031
+ ----------
1032
+ sources : List[Union[str, Dict[str, Any]]], default: []
1033
+ List of secret specs, defining how the secrets are to be retrieved
1034
+ """
1035
+ ...
1036
+
1037
+ @typing.overload
1038
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1039
+ """
1040
+ Specifies the PyPI packages for the step.
1370
1041
 
1371
1042
  Information in this decorator will augment any
1372
1043
  attributes set in the `@pyi_base` flow-level decorator. Hence,
@@ -1413,277 +1084,438 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1413
1084
  ...
1414
1085
 
1415
1086
  @typing.overload
1416
- def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1087
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1417
1088
  """
1418
- Specifies the flow(s) that this flow depends on.
1089
+ Specifies that the step will success under all circumstances.
1419
1090
 
1420
- ```
1421
- @trigger_on_finish(flow='FooFlow')
1422
- ```
1423
- or
1424
- ```
1425
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1426
- ```
1427
- This decorator respects the @project decorator and triggers the flow
1428
- when upstream runs within the same namespace complete successfully
1091
+ The decorator will create an optional artifact, specified by `var`, which
1092
+ contains the exception raised. You can use it to detect the presence
1093
+ of errors, indicating that all happy-path artifacts produced by the step
1094
+ are missing.
1429
1095
 
1430
- Additionally, you can specify project aware upstream flow dependencies
1431
- by specifying the fully qualified project_flow_name.
1432
- ```
1433
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1434
- ```
1435
- or
1436
- ```
1437
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1438
- ```
1096
+ Parameters
1097
+ ----------
1098
+ var : str, optional, default None
1099
+ Name of the artifact in which to store the caught exception.
1100
+ If not specified, the exception is not stored.
1101
+ print_exception : bool, default True
1102
+ Determines whether or not the exception is printed to
1103
+ stdout when caught.
1104
+ """
1105
+ ...
1106
+
1107
+ @typing.overload
1108
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1109
+ ...
1110
+
1111
+ @typing.overload
1112
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1113
+ ...
1114
+
1115
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1116
+ """
1117
+ Specifies that the step will success under all circumstances.
1439
1118
 
1440
- You can also specify just the project or project branch (other values will be
1441
- inferred from the current project or project branch):
1442
- ```
1443
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1444
- ```
1119
+ The decorator will create an optional artifact, specified by `var`, which
1120
+ contains the exception raised. You can use it to detect the presence
1121
+ of errors, indicating that all happy-path artifacts produced by the step
1122
+ are missing.
1445
1123
 
1446
- Note that `branch` is typically one of:
1447
- - `prod`
1448
- - `user.bob`
1449
- - `test.my_experiment`
1450
- - `prod.staging`
1124
+ Parameters
1125
+ ----------
1126
+ var : str, optional, default None
1127
+ Name of the artifact in which to store the caught exception.
1128
+ If not specified, the exception is not stored.
1129
+ print_exception : bool, default True
1130
+ Determines whether or not the exception is printed to
1131
+ stdout when caught.
1132
+ """
1133
+ ...
1134
+
1135
+ @typing.overload
1136
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1137
+ """
1138
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1139
+
1140
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1451
1141
 
1452
1142
  Parameters
1453
1143
  ----------
1454
- flow : Union[str, Dict[str, str]], optional, default None
1455
- Upstream flow dependency for this flow.
1456
- flows : List[Union[str, Dict[str, str]]], default []
1457
- Upstream flow dependencies for this flow.
1144
+ type : str, default 'default'
1145
+ Card type.
1146
+ id : str, optional, default None
1147
+ If multiple cards are present, use this id to identify this card.
1458
1148
  options : Dict[str, Any], default {}
1459
- Backend-specific configuration for tuning eventing behavior.
1149
+ Options passed to the card. The contents depend on the card type.
1150
+ timeout : int, default 45
1151
+ Interrupt reporting if it takes more than this many seconds.
1460
1152
 
1461
1153
 
1462
1154
  """
1463
1155
  ...
1464
1156
 
1465
1157
  @typing.overload
1466
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1158
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1159
+ ...
1160
+
1161
+ @typing.overload
1162
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1467
1163
  ...
1468
1164
 
1469
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1165
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1470
1166
  """
1471
- Specifies the flow(s) that this flow depends on.
1472
-
1473
- ```
1474
- @trigger_on_finish(flow='FooFlow')
1475
- ```
1476
- or
1477
- ```
1478
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1479
- ```
1480
- This decorator respects the @project decorator and triggers the flow
1481
- when upstream runs within the same namespace complete successfully
1482
-
1483
- Additionally, you can specify project aware upstream flow dependencies
1484
- by specifying the fully qualified project_flow_name.
1485
- ```
1486
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1487
- ```
1488
- or
1489
- ```
1490
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1491
- ```
1492
-
1493
- You can also specify just the project or project branch (other values will be
1494
- inferred from the current project or project branch):
1495
- ```
1496
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1497
- ```
1167
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1498
1168
 
1499
- Note that `branch` is typically one of:
1500
- - `prod`
1501
- - `user.bob`
1502
- - `test.my_experiment`
1503
- - `prod.staging`
1169
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1504
1170
 
1505
1171
  Parameters
1506
1172
  ----------
1507
- flow : Union[str, Dict[str, str]], optional, default None
1508
- Upstream flow dependency for this flow.
1509
- flows : List[Union[str, Dict[str, str]]], default []
1510
- Upstream flow dependencies for this flow.
1173
+ type : str, default 'default'
1174
+ Card type.
1175
+ id : str, optional, default None
1176
+ If multiple cards are present, use this id to identify this card.
1511
1177
  options : Dict[str, Any], default {}
1512
- Backend-specific configuration for tuning eventing behavior.
1178
+ Options passed to the card. The contents depend on the card type.
1179
+ timeout : int, default 45
1180
+ Interrupt reporting if it takes more than this many seconds.
1513
1181
 
1514
1182
 
1515
1183
  """
1516
1184
  ...
1517
1185
 
1518
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1186
+ @typing.overload
1187
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1519
1188
  """
1520
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1521
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1189
+ Specifies a timeout for your step.
1190
+
1191
+ This decorator is useful if this step may hang indefinitely.
1192
+
1193
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1194
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1195
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1196
+
1197
+ Note that all the values specified in parameters are added together so if you specify
1198
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1522
1199
 
1523
1200
  Parameters
1524
1201
  ----------
1525
- timeout : int
1526
- Time, in seconds before the task times out and fails. (Default: 3600)
1527
- poke_interval : int
1528
- Time in seconds that the job should wait in between each try. (Default: 60)
1529
- mode : str
1530
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1531
- exponential_backoff : bool
1532
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1533
- pool : str
1534
- the slot pool this task should run in,
1535
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1536
- soft_fail : bool
1537
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1538
- name : str
1539
- Name of the sensor on Airflow
1540
- description : str
1541
- Description of sensor in the Airflow UI
1542
- external_dag_id : str
1543
- The dag_id that contains the task you want to wait for.
1544
- external_task_ids : List[str]
1545
- The list of task_ids that you want to wait for.
1546
- If None (default value) the sensor waits for the DAG. (Default: None)
1547
- allowed_states : List[str]
1548
- Iterable of allowed states, (Default: ['success'])
1549
- failed_states : List[str]
1550
- Iterable of failed or dis-allowed states. (Default: None)
1551
- execution_delta : datetime.timedelta
1552
- time difference with the previous execution to look at,
1553
- the default is the same logical date as the current task or DAG. (Default: None)
1554
- check_existence: bool
1555
- Set to True to check if the external task exists or check if
1556
- the DAG to wait for exists. (Default: True)
1202
+ seconds : int, default 0
1203
+ Number of seconds to wait prior to timing out.
1204
+ minutes : int, default 0
1205
+ Number of minutes to wait prior to timing out.
1206
+ hours : int, default 0
1207
+ Number of hours to wait prior to timing out.
1557
1208
  """
1558
1209
  ...
1559
1210
 
1560
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1211
+ @typing.overload
1212
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1213
+ ...
1214
+
1215
+ @typing.overload
1216
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1217
+ ...
1218
+
1219
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1561
1220
  """
1562
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1563
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1564
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1565
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1566
- starts only after all sensors finish.
1221
+ Specifies a timeout for your step.
1222
+
1223
+ This decorator is useful if this step may hang indefinitely.
1224
+
1225
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1226
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1227
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1228
+
1229
+ Note that all the values specified in parameters are added together so if you specify
1230
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1567
1231
 
1568
1232
  Parameters
1569
1233
  ----------
1570
- timeout : int
1571
- Time, in seconds before the task times out and fails. (Default: 3600)
1572
- poke_interval : int
1573
- Time in seconds that the job should wait in between each try. (Default: 60)
1574
- mode : str
1575
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1576
- exponential_backoff : bool
1577
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1578
- pool : str
1579
- the slot pool this task should run in,
1580
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1581
- soft_fail : bool
1582
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1583
- name : str
1584
- Name of the sensor on Airflow
1585
- description : str
1586
- Description of sensor in the Airflow UI
1587
- bucket_key : Union[str, List[str]]
1588
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1589
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1590
- bucket_name : str
1591
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1592
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1593
- wildcard_match : bool
1594
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1595
- aws_conn_id : str
1596
- a reference to the s3 connection on Airflow. (Default: None)
1597
- verify : bool
1598
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1234
+ seconds : int, default 0
1235
+ Number of seconds to wait prior to timing out.
1236
+ minutes : int, default 0
1237
+ Number of minutes to wait prior to timing out.
1238
+ hours : int, default 0
1239
+ Number of hours to wait prior to timing out.
1599
1240
  """
1600
1241
  ...
1601
1242
 
1602
1243
  @typing.overload
1603
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1244
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1604
1245
  """
1605
- Specifies the PyPI packages for all steps of the flow.
1246
+ Specifies the Conda environment for the step.
1247
+
1248
+ Information in this decorator will augment any
1249
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1250
+ you can use `@conda_base` to set packages required by all
1251
+ steps and use `@conda` to specify step-specific overrides.
1606
1252
 
1607
- Use `@pypi_base` to set common packages required by all
1608
- steps and use `@pypi` to specify step-specific overrides.
1609
1253
  Parameters
1610
1254
  ----------
1611
- packages : Dict[str, str], default: {}
1612
- Packages to use for this flow. The key is the name of the package
1255
+ packages : Dict[str, str], default {}
1256
+ Packages to use for this step. The key is the name of the package
1613
1257
  and the value is the version to use.
1614
- python : str, optional, default: None
1258
+ libraries : Dict[str, str], default {}
1259
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1260
+ python : str, optional, default None
1615
1261
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1616
1262
  that the version used will correspond to the version of the Python interpreter used to start the run.
1263
+ disabled : bool, default False
1264
+ If set to True, disables @conda.
1617
1265
  """
1618
1266
  ...
1619
1267
 
1620
1268
  @typing.overload
1621
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1269
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1622
1270
  ...
1623
1271
 
1624
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1272
+ @typing.overload
1273
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1274
+ ...
1275
+
1276
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1625
1277
  """
1626
- Specifies the PyPI packages for all steps of the flow.
1278
+ Specifies the Conda environment for the step.
1279
+
1280
+ Information in this decorator will augment any
1281
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1282
+ you can use `@conda_base` to set packages required by all
1283
+ steps and use `@conda` to specify step-specific overrides.
1627
1284
 
1628
- Use `@pypi_base` to set common packages required by all
1629
- steps and use `@pypi` to specify step-specific overrides.
1630
1285
  Parameters
1631
1286
  ----------
1632
- packages : Dict[str, str], default: {}
1633
- Packages to use for this flow. The key is the name of the package
1287
+ packages : Dict[str, str], default {}
1288
+ Packages to use for this step. The key is the name of the package
1634
1289
  and the value is the version to use.
1635
- python : str, optional, default: None
1290
+ libraries : Dict[str, str], default {}
1291
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1292
+ python : str, optional, default None
1636
1293
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1637
1294
  that the version used will correspond to the version of the Python interpreter used to start the run.
1295
+ disabled : bool, default False
1296
+ If set to True, disables @conda.
1297
+ """
1298
+ ...
1299
+
1300
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1301
+ """
1302
+ Specifies that this step should execute on Kubernetes.
1303
+
1304
+ Parameters
1305
+ ----------
1306
+ cpu : int, default 1
1307
+ Number of CPUs required for this step. If `@resources` is
1308
+ also present, the maximum value from all decorators is used.
1309
+ memory : int, default 4096
1310
+ Memory size (in MB) required for this step. If
1311
+ `@resources` is also present, the maximum value from all decorators is
1312
+ used.
1313
+ disk : int, default 10240
1314
+ Disk size (in MB) required for this step. If
1315
+ `@resources` is also present, the maximum value from all decorators is
1316
+ used.
1317
+ image : str, optional, default None
1318
+ Docker image to use when launching on Kubernetes. If not specified, and
1319
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1320
+ not, a default Docker image mapping to the current version of Python is used.
1321
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1322
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1323
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1324
+ Kubernetes service account to use when launching pod in Kubernetes.
1325
+ secrets : List[str], optional, default None
1326
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1327
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1328
+ in Metaflow configuration.
1329
+ node_selector: Union[Dict[str,str], str], optional, default None
1330
+ Kubernetes node selector(s) to apply to the pod running the task.
1331
+ Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
1332
+ or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
1333
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1334
+ Kubernetes namespace to use when launching pod in Kubernetes.
1335
+ gpu : int, optional, default None
1336
+ Number of GPUs required for this step. A value of zero implies that
1337
+ the scheduled node should not have GPUs.
1338
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1339
+ The vendor of the GPUs to be used for this step.
1340
+ tolerations : List[str], default []
1341
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1342
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1343
+ use_tmpfs : bool, default False
1344
+ This enables an explicit tmpfs mount for this step.
1345
+ tmpfs_tempdir : bool, default True
1346
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1347
+ tmpfs_size : int, optional, default: None
1348
+ The value for the size (in MiB) of the tmpfs mount for this step.
1349
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1350
+ memory allocated for this step.
1351
+ tmpfs_path : str, optional, default /metaflow_temp
1352
+ Path to tmpfs mount for this step.
1353
+ persistent_volume_claims : Dict[str, str], optional, default None
1354
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1355
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1356
+ shared_memory: int, optional
1357
+ Shared memory size (in MiB) required for this step
1358
+ port: int, optional
1359
+ Port number to specify in the Kubernetes job object
1360
+ compute_pool : str, optional, default None
1361
+ Compute pool to be used for for this step.
1362
+ If not specified, any accessible compute pool within the perimeter is used.
1638
1363
  """
1639
1364
  ...
1640
1365
 
1641
1366
  @typing.overload
1642
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1367
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1643
1368
  """
1644
- Specifies the times when the flow should be run when running on a
1645
- production scheduler.
1369
+ Specifies the number of times the task corresponding
1370
+ to a step needs to be retried.
1371
+
1372
+ This decorator is useful for handling transient errors, such as networking issues.
1373
+ If your task contains operations that can't be retried safely, e.g. database updates,
1374
+ it is advisable to annotate it with `@retry(times=0)`.
1375
+
1376
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1377
+ decorator will execute a no-op task after all retries have been exhausted,
1378
+ ensuring that the flow execution can continue.
1646
1379
 
1647
1380
  Parameters
1648
1381
  ----------
1649
- hourly : bool, default False
1650
- Run the workflow hourly.
1651
- daily : bool, default True
1652
- Run the workflow daily.
1653
- weekly : bool, default False
1654
- Run the workflow weekly.
1655
- cron : str, optional, default None
1656
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1657
- specified by this expression.
1658
- timezone : str, optional, default None
1659
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1660
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1382
+ times : int, default 3
1383
+ Number of times to retry this task.
1384
+ minutes_between_retries : int, default 2
1385
+ Number of minutes between retries.
1386
+ """
1387
+ ...
1388
+
1389
+ @typing.overload
1390
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1391
+ ...
1392
+
1393
+ @typing.overload
1394
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1395
+ ...
1396
+
1397
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1398
+ """
1399
+ Specifies the number of times the task corresponding
1400
+ to a step needs to be retried.
1401
+
1402
+ This decorator is useful for handling transient errors, such as networking issues.
1403
+ If your task contains operations that can't be retried safely, e.g. database updates,
1404
+ it is advisable to annotate it with `@retry(times=0)`.
1405
+
1406
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1407
+ decorator will execute a no-op task after all retries have been exhausted,
1408
+ ensuring that the flow execution can continue.
1409
+
1410
+ Parameters
1411
+ ----------
1412
+ times : int, default 3
1413
+ Number of times to retry this task.
1414
+ minutes_between_retries : int, default 2
1415
+ Number of minutes between retries.
1416
+ """
1417
+ ...
1418
+
1419
+ @typing.overload
1420
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1421
+ """
1422
+ Specifies the flow(s) that this flow depends on.
1423
+
1424
+ ```
1425
+ @trigger_on_finish(flow='FooFlow')
1426
+ ```
1427
+ or
1428
+ ```
1429
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1430
+ ```
1431
+ This decorator respects the @project decorator and triggers the flow
1432
+ when upstream runs within the same namespace complete successfully
1433
+
1434
+ Additionally, you can specify project aware upstream flow dependencies
1435
+ by specifying the fully qualified project_flow_name.
1436
+ ```
1437
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1438
+ ```
1439
+ or
1440
+ ```
1441
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1442
+ ```
1443
+
1444
+ You can also specify just the project or project branch (other values will be
1445
+ inferred from the current project or project branch):
1446
+ ```
1447
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1448
+ ```
1449
+
1450
+ Note that `branch` is typically one of:
1451
+ - `prod`
1452
+ - `user.bob`
1453
+ - `test.my_experiment`
1454
+ - `prod.staging`
1455
+
1456
+ Parameters
1457
+ ----------
1458
+ flow : Union[str, Dict[str, str]], optional, default None
1459
+ Upstream flow dependency for this flow.
1460
+ flows : List[Union[str, Dict[str, str]]], default []
1461
+ Upstream flow dependencies for this flow.
1462
+ options : Dict[str, Any], default {}
1463
+ Backend-specific configuration for tuning eventing behavior.
1464
+
1465
+
1661
1466
  """
1662
1467
  ...
1663
1468
 
1664
1469
  @typing.overload
1665
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1470
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1666
1471
  ...
1667
1472
 
1668
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1473
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1669
1474
  """
1670
- Specifies the times when the flow should be run when running on a
1671
- production scheduler.
1475
+ Specifies the flow(s) that this flow depends on.
1476
+
1477
+ ```
1478
+ @trigger_on_finish(flow='FooFlow')
1479
+ ```
1480
+ or
1481
+ ```
1482
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1483
+ ```
1484
+ This decorator respects the @project decorator and triggers the flow
1485
+ when upstream runs within the same namespace complete successfully
1486
+
1487
+ Additionally, you can specify project aware upstream flow dependencies
1488
+ by specifying the fully qualified project_flow_name.
1489
+ ```
1490
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1491
+ ```
1492
+ or
1493
+ ```
1494
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1495
+ ```
1496
+
1497
+ You can also specify just the project or project branch (other values will be
1498
+ inferred from the current project or project branch):
1499
+ ```
1500
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1501
+ ```
1502
+
1503
+ Note that `branch` is typically one of:
1504
+ - `prod`
1505
+ - `user.bob`
1506
+ - `test.my_experiment`
1507
+ - `prod.staging`
1672
1508
 
1673
1509
  Parameters
1674
1510
  ----------
1675
- hourly : bool, default False
1676
- Run the workflow hourly.
1677
- daily : bool, default True
1678
- Run the workflow daily.
1679
- weekly : bool, default False
1680
- Run the workflow weekly.
1681
- cron : str, optional, default None
1682
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1683
- specified by this expression.
1684
- timezone : str, optional, default None
1685
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1686
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1511
+ flow : Union[str, Dict[str, str]], optional, default None
1512
+ Upstream flow dependency for this flow.
1513
+ flows : List[Union[str, Dict[str, str]]], default []
1514
+ Upstream flow dependencies for this flow.
1515
+ options : Dict[str, Any], default {}
1516
+ Backend-specific configuration for tuning eventing behavior.
1517
+
1518
+
1687
1519
  """
1688
1520
  ...
1689
1521
 
@@ -1736,21 +1568,42 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1736
1568
  """
1737
1569
  ...
1738
1570
 
1739
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1571
+ @typing.overload
1572
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1740
1573
  """
1741
- Specifies what flows belong to the same project.
1742
-
1743
- A project-specific namespace is created for all flows that
1744
- use the same `@project(name)`.
1574
+ Specifies the PyPI packages for all steps of the flow.
1745
1575
 
1576
+ Use `@pypi_base` to set common packages required by all
1577
+ steps and use `@pypi` to specify step-specific overrides.
1746
1578
  Parameters
1747
1579
  ----------
1748
- name : str
1749
- Project name. Make sure that the name is unique amongst all
1750
- projects that use the same production scheduler. The name may
1751
- contain only lowercase alphanumeric characters and underscores.
1752
-
1580
+ packages : Dict[str, str], default: {}
1581
+ Packages to use for this flow. The key is the name of the package
1582
+ and the value is the version to use.
1583
+ python : str, optional, default: None
1584
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1585
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1586
+ """
1587
+ ...
1588
+
1589
+ @typing.overload
1590
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1591
+ ...
1592
+
1593
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1594
+ """
1595
+ Specifies the PyPI packages for all steps of the flow.
1753
1596
 
1597
+ Use `@pypi_base` to set common packages required by all
1598
+ steps and use `@pypi` to specify step-specific overrides.
1599
+ Parameters
1600
+ ----------
1601
+ packages : Dict[str, str], default: {}
1602
+ Packages to use for this flow. The key is the name of the package
1603
+ and the value is the version to use.
1604
+ python : str, optional, default: None
1605
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1606
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1754
1607
  """
1755
1608
  ...
1756
1609
 
@@ -1849,6 +1702,157 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1849
1702
  """
1850
1703
  ...
1851
1704
 
1705
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1706
+ """
1707
+ Specifies what flows belong to the same project.
1708
+
1709
+ A project-specific namespace is created for all flows that
1710
+ use the same `@project(name)`.
1711
+
1712
+ Parameters
1713
+ ----------
1714
+ name : str
1715
+ Project name. Make sure that the name is unique amongst all
1716
+ projects that use the same production scheduler. The name may
1717
+ contain only lowercase alphanumeric characters and underscores.
1718
+
1719
+
1720
+ """
1721
+ ...
1722
+
1723
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1724
+ """
1725
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1726
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1727
+
1728
+ Parameters
1729
+ ----------
1730
+ timeout : int
1731
+ Time, in seconds before the task times out and fails. (Default: 3600)
1732
+ poke_interval : int
1733
+ Time in seconds that the job should wait in between each try. (Default: 60)
1734
+ mode : str
1735
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1736
+ exponential_backoff : bool
1737
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1738
+ pool : str
1739
+ the slot pool this task should run in,
1740
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1741
+ soft_fail : bool
1742
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1743
+ name : str
1744
+ Name of the sensor on Airflow
1745
+ description : str
1746
+ Description of sensor in the Airflow UI
1747
+ external_dag_id : str
1748
+ The dag_id that contains the task you want to wait for.
1749
+ external_task_ids : List[str]
1750
+ The list of task_ids that you want to wait for.
1751
+ If None (default value) the sensor waits for the DAG. (Default: None)
1752
+ allowed_states : List[str]
1753
+ Iterable of allowed states, (Default: ['success'])
1754
+ failed_states : List[str]
1755
+ Iterable of failed or dis-allowed states. (Default: None)
1756
+ execution_delta : datetime.timedelta
1757
+ time difference with the previous execution to look at,
1758
+ the default is the same logical date as the current task or DAG. (Default: None)
1759
+ check_existence: bool
1760
+ Set to True to check if the external task exists or check if
1761
+ the DAG to wait for exists. (Default: True)
1762
+ """
1763
+ ...
1764
+
1765
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1766
+ """
1767
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1768
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1769
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1770
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1771
+ starts only after all sensors finish.
1772
+
1773
+ Parameters
1774
+ ----------
1775
+ timeout : int
1776
+ Time, in seconds before the task times out and fails. (Default: 3600)
1777
+ poke_interval : int
1778
+ Time in seconds that the job should wait in between each try. (Default: 60)
1779
+ mode : str
1780
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1781
+ exponential_backoff : bool
1782
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1783
+ pool : str
1784
+ the slot pool this task should run in,
1785
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1786
+ soft_fail : bool
1787
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1788
+ name : str
1789
+ Name of the sensor on Airflow
1790
+ description : str
1791
+ Description of sensor in the Airflow UI
1792
+ bucket_key : Union[str, List[str]]
1793
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1794
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1795
+ bucket_name : str
1796
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1797
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1798
+ wildcard_match : bool
1799
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1800
+ aws_conn_id : str
1801
+ a reference to the s3 connection on Airflow. (Default: None)
1802
+ verify : bool
1803
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1804
+ """
1805
+ ...
1806
+
1807
+ @typing.overload
1808
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1809
+ """
1810
+ Specifies the times when the flow should be run when running on a
1811
+ production scheduler.
1812
+
1813
+ Parameters
1814
+ ----------
1815
+ hourly : bool, default False
1816
+ Run the workflow hourly.
1817
+ daily : bool, default True
1818
+ Run the workflow daily.
1819
+ weekly : bool, default False
1820
+ Run the workflow weekly.
1821
+ cron : str, optional, default None
1822
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1823
+ specified by this expression.
1824
+ timezone : str, optional, default None
1825
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1826
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1827
+ """
1828
+ ...
1829
+
1830
+ @typing.overload
1831
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1832
+ ...
1833
+
1834
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1835
+ """
1836
+ Specifies the times when the flow should be run when running on a
1837
+ production scheduler.
1838
+
1839
+ Parameters
1840
+ ----------
1841
+ hourly : bool, default False
1842
+ Run the workflow hourly.
1843
+ daily : bool, default True
1844
+ Run the workflow daily.
1845
+ weekly : bool, default False
1846
+ Run the workflow weekly.
1847
+ cron : str, optional, default None
1848
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1849
+ specified by this expression.
1850
+ timezone : str, optional, default None
1851
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1852
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1853
+ """
1854
+ ...
1855
+
1852
1856
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1853
1857
  """
1854
1858
  Switch namespace to the one provided.