metaflow-stubs 2.12.7__py2.py3-none-any.whl → 2.12.8__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. metaflow-stubs/__init__.pyi +441 -441
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +16 -16
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +3 -3
  20. metaflow-stubs/plugins/__init__.pyi +3 -3
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +4 -4
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  37. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  45. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +4 -4
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  56. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  61. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  62. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  63. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
  65. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  78. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  81. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  82. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  83. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  84. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  86. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  89. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  91. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  92. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  93. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  95. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  99. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  106. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  107. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  110. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  111. metaflow-stubs/plugins/package_cli.pyi +2 -2
  112. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  117. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  119. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  120. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  123. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  124. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  126. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  127. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  128. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  129. metaflow-stubs/procpoll.pyi +2 -2
  130. metaflow-stubs/pylint_wrapper.pyi +2 -2
  131. metaflow-stubs/runner/__init__.pyi +2 -2
  132. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  133. metaflow-stubs/runner/nbrun.pyi +2 -2
  134. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  135. metaflow-stubs/system/__init__.pyi +3 -3
  136. metaflow-stubs/system/system_logger.pyi +3 -3
  137. metaflow-stubs/system/system_monitor.pyi +2 -2
  138. metaflow-stubs/tagging_util.pyi +2 -2
  139. metaflow-stubs/tuple_util.pyi +2 -2
  140. metaflow-stubs/version.pyi +2 -2
  141. {metaflow_stubs-2.12.7.dist-info → metaflow_stubs-2.12.8.dist-info}/METADATA +2 -2
  142. metaflow_stubs-2.12.8.dist-info/RECORD +145 -0
  143. {metaflow_stubs-2.12.7.dist-info → metaflow_stubs-2.12.8.dist-info}/WHEEL +1 -1
  144. metaflow_stubs-2.12.7.dist-info/RECORD +0 -145
  145. {metaflow_stubs-2.12.7.dist-info → metaflow_stubs-2.12.8.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.7 #
4
- # Generated on 2024-07-03T19:20:47.569890 #
3
+ # MF version: 2.12.8 #
4
+ # Generated on 2024-07-16T15:51:55.346854 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.client.core
11
+ import metaflow.plugins.datatools.s3.s3
12
+ import metaflow.runner.metaflow_runner
12
13
  import metaflow.datastore.inputs
13
- import metaflow.parameters
14
- import metaflow.metaflow_current
14
+ import metaflow.flowspec
15
15
  import metaflow._vendor.click.types
16
- import typing
17
- import metaflow.runner.metaflow_runner
18
- import io
19
- import metaflow.plugins.datatools.s3.s3
20
16
  import datetime
17
+ import metaflow.metaflow_current
21
18
  import metaflow.events
22
- import metaflow.flowspec
19
+ import typing
20
+ import metaflow.client.core
21
+ import metaflow.parameters
22
+ import io
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -728,110 +728,185 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
728
728
  ...
729
729
 
730
730
  @typing.overload
731
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
731
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
732
  """
733
- Specifies the Conda environment for the step.
733
+ Specifies the resources needed when executing this step.
734
734
 
735
- Information in this decorator will augment any
736
- attributes set in the `@conda_base` flow-level decorator. Hence,
737
- you can use `@conda_base` to set packages required by all
738
- steps and use `@conda` to specify step-specific overrides.
735
+ Use `@resources` to specify the resource requirements
736
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
737
+
738
+ You can choose the compute layer on the command line by executing e.g.
739
+ ```
740
+ python myflow.py run --with batch
741
+ ```
742
+ or
743
+ ```
744
+ python myflow.py run --with kubernetes
745
+ ```
746
+ which executes the flow on the desired system using the
747
+ requirements specified in `@resources`.
739
748
 
740
749
  Parameters
741
750
  ----------
742
- packages : Dict[str, str], default {}
743
- Packages to use for this step. The key is the name of the package
744
- and the value is the version to use.
745
- libraries : Dict[str, str], default {}
746
- Supported for backward compatibility. When used with packages, packages will take precedence.
747
- python : str, optional, default None
748
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
749
- that the version used will correspond to the version of the Python interpreter used to start the run.
750
- disabled : bool, default False
751
- If set to True, disables @conda.
751
+ cpu : int, default 1
752
+ Number of CPUs required for this step.
753
+ gpu : int, default 0
754
+ Number of GPUs required for this step.
755
+ disk : int, optional, default None
756
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
757
+ memory : int, default 4096
758
+ Memory size (in MB) required for this step.
759
+ shared_memory : int, optional, default None
760
+ The value for the size (in MiB) of the /dev/shm volume for this step.
761
+ This parameter maps to the `--shm-size` option in Docker.
752
762
  """
753
763
  ...
754
764
 
755
765
  @typing.overload
756
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
766
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
757
767
  ...
758
768
 
759
769
  @typing.overload
760
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
770
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
761
771
  ...
762
772
 
763
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
773
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
764
774
  """
765
- Specifies the Conda environment for the step.
775
+ Specifies the resources needed when executing this step.
766
776
 
767
- Information in this decorator will augment any
768
- attributes set in the `@conda_base` flow-level decorator. Hence,
769
- you can use `@conda_base` to set packages required by all
770
- steps and use `@conda` to specify step-specific overrides.
777
+ Use `@resources` to specify the resource requirements
778
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
779
+
780
+ You can choose the compute layer on the command line by executing e.g.
781
+ ```
782
+ python myflow.py run --with batch
783
+ ```
784
+ or
785
+ ```
786
+ python myflow.py run --with kubernetes
787
+ ```
788
+ which executes the flow on the desired system using the
789
+ requirements specified in `@resources`.
771
790
 
772
791
  Parameters
773
792
  ----------
774
- packages : Dict[str, str], default {}
775
- Packages to use for this step. The key is the name of the package
776
- and the value is the version to use.
777
- libraries : Dict[str, str], default {}
778
- Supported for backward compatibility. When used with packages, packages will take precedence.
779
- python : str, optional, default None
780
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
781
- that the version used will correspond to the version of the Python interpreter used to start the run.
782
- disabled : bool, default False
783
- If set to True, disables @conda.
793
+ cpu : int, default 1
794
+ Number of CPUs required for this step.
795
+ gpu : int, default 0
796
+ Number of GPUs required for this step.
797
+ disk : int, optional, default None
798
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
799
+ memory : int, default 4096
800
+ Memory size (in MB) required for this step.
801
+ shared_memory : int, optional, default None
802
+ The value for the size (in MiB) of the /dev/shm volume for this step.
803
+ This parameter maps to the `--shm-size` option in Docker.
784
804
  """
785
805
  ...
786
806
 
787
807
  @typing.overload
788
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
808
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
789
809
  """
790
- Creates a human-readable report, a Metaflow Card, after this step completes.
810
+ Specifies a timeout for your step.
791
811
 
792
- Note that you may add multiple `@card` decorators in a step with different parameters.
812
+ This decorator is useful if this step may hang indefinitely.
793
813
 
794
- Parameters
795
- ----------
796
- type : str, default 'default'
797
- Card type.
798
- id : str, optional, default None
799
- If multiple cards are present, use this id to identify this card.
800
- options : Dict[str, Any], default {}
801
- Options passed to the card. The contents depend on the card type.
802
- timeout : int, default 45
803
- Interrupt reporting if it takes more than this many seconds.
814
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
815
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
816
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
804
817
 
818
+ Note that all the values specified in parameters are added together so if you specify
819
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
805
820
 
821
+ Parameters
822
+ ----------
823
+ seconds : int, default 0
824
+ Number of seconds to wait prior to timing out.
825
+ minutes : int, default 0
826
+ Number of minutes to wait prior to timing out.
827
+ hours : int, default 0
828
+ Number of hours to wait prior to timing out.
806
829
  """
807
830
  ...
808
831
 
809
832
  @typing.overload
810
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
833
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
811
834
  ...
812
835
 
813
836
  @typing.overload
814
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
837
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
815
838
  ...
816
839
 
817
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
840
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
818
841
  """
819
- Creates a human-readable report, a Metaflow Card, after this step completes.
842
+ Specifies a timeout for your step.
820
843
 
821
- Note that you may add multiple `@card` decorators in a step with different parameters.
844
+ This decorator is useful if this step may hang indefinitely.
845
+
846
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
847
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
848
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
849
+
850
+ Note that all the values specified in parameters are added together so if you specify
851
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
822
852
 
823
853
  Parameters
824
854
  ----------
825
- type : str, default 'default'
826
- Card type.
827
- id : str, optional, default None
828
- If multiple cards are present, use this id to identify this card.
829
- options : Dict[str, Any], default {}
830
- Options passed to the card. The contents depend on the card type.
831
- timeout : int, default 45
832
- Interrupt reporting if it takes more than this many seconds.
855
+ seconds : int, default 0
856
+ Number of seconds to wait prior to timing out.
857
+ minutes : int, default 0
858
+ Number of minutes to wait prior to timing out.
859
+ hours : int, default 0
860
+ Number of hours to wait prior to timing out.
861
+ """
862
+ ...
863
+
864
+ @typing.overload
865
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
866
+ """
867
+ Specifies that the step will success under all circumstances.
868
+
869
+ The decorator will create an optional artifact, specified by `var`, which
870
+ contains the exception raised. You can use it to detect the presence
871
+ of errors, indicating that all happy-path artifacts produced by the step
872
+ are missing.
873
+
874
+ Parameters
875
+ ----------
876
+ var : str, optional, default None
877
+ Name of the artifact in which to store the caught exception.
878
+ If not specified, the exception is not stored.
879
+ print_exception : bool, default True
880
+ Determines whether or not the exception is printed to
881
+ stdout when caught.
882
+ """
883
+ ...
884
+
885
+ @typing.overload
886
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
887
+ ...
888
+
889
+ @typing.overload
890
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
891
+ ...
892
+
893
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
894
+ """
895
+ Specifies that the step will success under all circumstances.
833
896
 
897
+ The decorator will create an optional artifact, specified by `var`, which
898
+ contains the exception raised. You can use it to detect the presence
899
+ of errors, indicating that all happy-path artifacts produced by the step
900
+ are missing.
834
901
 
902
+ Parameters
903
+ ----------
904
+ var : str, optional, default None
905
+ Name of the artifact in which to store the caught exception.
906
+ If not specified, the exception is not stored.
907
+ print_exception : bool, default True
908
+ Determines whether or not the exception is printed to
909
+ stdout when caught.
835
910
  """
836
911
  ...
837
912
 
@@ -982,6 +1057,37 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
982
1057
  """
983
1058
  ...
984
1059
 
1060
+ @typing.overload
1061
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1062
+ """
1063
+ Specifies environment variables to be set prior to the execution of a step.
1064
+
1065
+ Parameters
1066
+ ----------
1067
+ vars : Dict[str, str], default {}
1068
+ Dictionary of environment variables to set.
1069
+ """
1070
+ ...
1071
+
1072
+ @typing.overload
1073
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1074
+ ...
1075
+
1076
+ @typing.overload
1077
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1078
+ ...
1079
+
1080
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1081
+ """
1082
+ Specifies environment variables to be set prior to the execution of a step.
1083
+
1084
+ Parameters
1085
+ ----------
1086
+ vars : Dict[str, str], default {}
1087
+ Dictionary of environment variables to set.
1088
+ """
1089
+ ...
1090
+
985
1091
  @typing.overload
986
1092
  def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
987
1093
  """
@@ -1032,161 +1138,110 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1032
1138
  ...
1033
1139
 
1034
1140
  @typing.overload
1035
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1141
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1036
1142
  """
1037
- Specifies that the step will success under all circumstances.
1143
+ Specifies the Conda environment for the step.
1038
1144
 
1039
- The decorator will create an optional artifact, specified by `var`, which
1040
- contains the exception raised. You can use it to detect the presence
1041
- of errors, indicating that all happy-path artifacts produced by the step
1042
- are missing.
1145
+ Information in this decorator will augment any
1146
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1147
+ you can use `@conda_base` to set packages required by all
1148
+ steps and use `@conda` to specify step-specific overrides.
1043
1149
 
1044
1150
  Parameters
1045
1151
  ----------
1046
- var : str, optional, default None
1047
- Name of the artifact in which to store the caught exception.
1048
- If not specified, the exception is not stored.
1049
- print_exception : bool, default True
1050
- Determines whether or not the exception is printed to
1051
- stdout when caught.
1152
+ packages : Dict[str, str], default {}
1153
+ Packages to use for this step. The key is the name of the package
1154
+ and the value is the version to use.
1155
+ libraries : Dict[str, str], default {}
1156
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1157
+ python : str, optional, default None
1158
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1159
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1160
+ disabled : bool, default False
1161
+ If set to True, disables @conda.
1052
1162
  """
1053
1163
  ...
1054
1164
 
1055
1165
  @typing.overload
1056
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1166
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1057
1167
  ...
1058
1168
 
1059
1169
  @typing.overload
1060
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1061
- ...
1062
-
1063
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1064
- """
1065
- Specifies that the step will success under all circumstances.
1066
-
1067
- The decorator will create an optional artifact, specified by `var`, which
1068
- contains the exception raised. You can use it to detect the presence
1069
- of errors, indicating that all happy-path artifacts produced by the step
1070
- are missing.
1071
-
1072
- Parameters
1073
- ----------
1074
- var : str, optional, default None
1075
- Name of the artifact in which to store the caught exception.
1076
- If not specified, the exception is not stored.
1077
- print_exception : bool, default True
1078
- Determines whether or not the exception is printed to
1079
- stdout when caught.
1080
- """
1170
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1081
1171
  ...
1082
1172
 
1083
- @typing.overload
1084
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1173
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1085
1174
  """
1086
- Specifies the number of times the task corresponding
1087
- to a step needs to be retried.
1088
-
1089
- This decorator is useful for handling transient errors, such as networking issues.
1090
- If your task contains operations that can't be retried safely, e.g. database updates,
1091
- it is advisable to annotate it with `@retry(times=0)`.
1175
+ Specifies the Conda environment for the step.
1092
1176
 
1093
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1094
- decorator will execute a no-op task after all retries have been exhausted,
1095
- ensuring that the flow execution can continue.
1177
+ Information in this decorator will augment any
1178
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1179
+ you can use `@conda_base` to set packages required by all
1180
+ steps and use `@conda` to specify step-specific overrides.
1096
1181
 
1097
1182
  Parameters
1098
1183
  ----------
1099
- times : int, default 3
1100
- Number of times to retry this task.
1101
- minutes_between_retries : int, default 2
1102
- Number of minutes between retries.
1184
+ packages : Dict[str, str], default {}
1185
+ Packages to use for this step. The key is the name of the package
1186
+ and the value is the version to use.
1187
+ libraries : Dict[str, str], default {}
1188
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1189
+ python : str, optional, default None
1190
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1191
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1192
+ disabled : bool, default False
1193
+ If set to True, disables @conda.
1103
1194
  """
1104
1195
  ...
1105
1196
 
1106
1197
  @typing.overload
1107
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1108
- ...
1109
-
1110
- @typing.overload
1111
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1112
- ...
1113
-
1114
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1198
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1115
1199
  """
1116
- Specifies the number of times the task corresponding
1117
- to a step needs to be retried.
1118
-
1119
- This decorator is useful for handling transient errors, such as networking issues.
1120
- If your task contains operations that can't be retried safely, e.g. database updates,
1121
- it is advisable to annotate it with `@retry(times=0)`.
1200
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1122
1201
 
1123
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1124
- decorator will execute a no-op task after all retries have been exhausted,
1125
- ensuring that the flow execution can continue.
1202
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1126
1203
 
1127
1204
  Parameters
1128
1205
  ----------
1129
- times : int, default 3
1130
- Number of times to retry this task.
1131
- minutes_between_retries : int, default 2
1132
- Number of minutes between retries.
1133
- """
1134
- ...
1135
-
1136
- @typing.overload
1137
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1138
- """
1139
- Specifies a timeout for your step.
1140
-
1141
- This decorator is useful if this step may hang indefinitely.
1142
-
1143
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1144
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1145
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1206
+ type : str, default 'default'
1207
+ Card type.
1208
+ id : str, optional, default None
1209
+ If multiple cards are present, use this id to identify this card.
1210
+ options : Dict[str, Any], default {}
1211
+ Options passed to the card. The contents depend on the card type.
1212
+ timeout : int, default 45
1213
+ Interrupt reporting if it takes more than this many seconds.
1146
1214
 
1147
- Note that all the values specified in parameters are added together so if you specify
1148
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1149
1215
 
1150
- Parameters
1151
- ----------
1152
- seconds : int, default 0
1153
- Number of seconds to wait prior to timing out.
1154
- minutes : int, default 0
1155
- Number of minutes to wait prior to timing out.
1156
- hours : int, default 0
1157
- Number of hours to wait prior to timing out.
1158
1216
  """
1159
1217
  ...
1160
1218
 
1161
1219
  @typing.overload
1162
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1220
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1163
1221
  ...
1164
1222
 
1165
1223
  @typing.overload
1166
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1224
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1167
1225
  ...
1168
1226
 
1169
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1227
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1170
1228
  """
1171
- Specifies a timeout for your step.
1172
-
1173
- This decorator is useful if this step may hang indefinitely.
1174
-
1175
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1176
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1177
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1229
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1178
1230
 
1179
- Note that all the values specified in parameters are added together so if you specify
1180
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1231
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1181
1232
 
1182
1233
  Parameters
1183
1234
  ----------
1184
- seconds : int, default 0
1185
- Number of seconds to wait prior to timing out.
1186
- minutes : int, default 0
1187
- Number of minutes to wait prior to timing out.
1188
- hours : int, default 0
1189
- Number of hours to wait prior to timing out.
1235
+ type : str, default 'default'
1236
+ Card type.
1237
+ id : str, optional, default None
1238
+ If multiple cards are present, use this id to identify this card.
1239
+ options : Dict[str, Any], default {}
1240
+ Options passed to the card. The contents depend on the card type.
1241
+ timeout : int, default 45
1242
+ Interrupt reporting if it takes more than this many seconds.
1243
+
1244
+
1190
1245
  """
1191
1246
  ...
1192
1247
 
@@ -1283,168 +1338,205 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1283
1338
  ...
1284
1339
 
1285
1340
  @typing.overload
1286
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1341
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1287
1342
  """
1288
- Specifies the resources needed when executing this step.
1343
+ Specifies the number of times the task corresponding
1344
+ to a step needs to be retried.
1289
1345
 
1290
- Use `@resources` to specify the resource requirements
1291
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1346
+ This decorator is useful for handling transient errors, such as networking issues.
1347
+ If your task contains operations that can't be retried safely, e.g. database updates,
1348
+ it is advisable to annotate it with `@retry(times=0)`.
1292
1349
 
1293
- You can choose the compute layer on the command line by executing e.g.
1294
- ```
1295
- python myflow.py run --with batch
1296
- ```
1297
- or
1298
- ```
1299
- python myflow.py run --with kubernetes
1300
- ```
1301
- which executes the flow on the desired system using the
1302
- requirements specified in `@resources`.
1350
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1351
+ decorator will execute a no-op task after all retries have been exhausted,
1352
+ ensuring that the flow execution can continue.
1303
1353
 
1304
1354
  Parameters
1305
1355
  ----------
1306
- cpu : int, default 1
1307
- Number of CPUs required for this step.
1308
- gpu : int, default 0
1309
- Number of GPUs required for this step.
1310
- disk : int, optional, default None
1311
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1312
- memory : int, default 4096
1313
- Memory size (in MB) required for this step.
1314
- shared_memory : int, optional, default None
1315
- The value for the size (in MiB) of the /dev/shm volume for this step.
1316
- This parameter maps to the `--shm-size` option in Docker.
1356
+ times : int, default 3
1357
+ Number of times to retry this task.
1358
+ minutes_between_retries : int, default 2
1359
+ Number of minutes between retries.
1317
1360
  """
1318
1361
  ...
1319
1362
 
1320
1363
  @typing.overload
1321
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1364
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1322
1365
  ...
1323
1366
 
1324
1367
  @typing.overload
1325
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1368
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1326
1369
  ...
1327
1370
 
1328
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1371
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1329
1372
  """
1330
- Specifies the resources needed when executing this step.
1373
+ Specifies the number of times the task corresponding
1374
+ to a step needs to be retried.
1331
1375
 
1332
- Use `@resources` to specify the resource requirements
1333
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1376
+ This decorator is useful for handling transient errors, such as networking issues.
1377
+ If your task contains operations that can't be retried safely, e.g. database updates,
1378
+ it is advisable to annotate it with `@retry(times=0)`.
1334
1379
 
1335
- You can choose the compute layer on the command line by executing e.g.
1336
- ```
1337
- python myflow.py run --with batch
1338
- ```
1339
- or
1340
- ```
1341
- python myflow.py run --with kubernetes
1342
- ```
1343
- which executes the flow on the desired system using the
1344
- requirements specified in `@resources`.
1380
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1381
+ decorator will execute a no-op task after all retries have been exhausted,
1382
+ ensuring that the flow execution can continue.
1345
1383
 
1346
1384
  Parameters
1347
1385
  ----------
1348
- cpu : int, default 1
1349
- Number of CPUs required for this step.
1350
- gpu : int, default 0
1351
- Number of GPUs required for this step.
1352
- disk : int, optional, default None
1353
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1354
- memory : int, default 4096
1355
- Memory size (in MB) required for this step.
1356
- shared_memory : int, optional, default None
1357
- The value for the size (in MiB) of the /dev/shm volume for this step.
1358
- This parameter maps to the `--shm-size` option in Docker.
1386
+ times : int, default 3
1387
+ Number of times to retry this task.
1388
+ minutes_between_retries : int, default 2
1389
+ Number of minutes between retries.
1359
1390
  """
1360
1391
  ...
1361
1392
 
1362
1393
  @typing.overload
1363
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1394
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1364
1395
  """
1365
- Specifies environment variables to be set prior to the execution of a step.
1396
+ Specifies the times when the flow should be run when running on a
1397
+ production scheduler.
1366
1398
 
1367
1399
  Parameters
1368
1400
  ----------
1369
- vars : Dict[str, str], default {}
1370
- Dictionary of environment variables to set.
1401
+ hourly : bool, default False
1402
+ Run the workflow hourly.
1403
+ daily : bool, default True
1404
+ Run the workflow daily.
1405
+ weekly : bool, default False
1406
+ Run the workflow weekly.
1407
+ cron : str, optional, default None
1408
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1409
+ specified by this expression.
1410
+ timezone : str, optional, default None
1411
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1412
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1371
1413
  """
1372
1414
  ...
1373
1415
 
1374
1416
  @typing.overload
1375
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1376
- ...
1377
-
1378
- @typing.overload
1379
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1417
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1380
1418
  ...
1381
1419
 
1382
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1420
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1383
1421
  """
1384
- Specifies environment variables to be set prior to the execution of a step.
1422
+ Specifies the times when the flow should be run when running on a
1423
+ production scheduler.
1385
1424
 
1386
1425
  Parameters
1387
1426
  ----------
1388
- vars : Dict[str, str], default {}
1389
- Dictionary of environment variables to set.
1427
+ hourly : bool, default False
1428
+ Run the workflow hourly.
1429
+ daily : bool, default True
1430
+ Run the workflow daily.
1431
+ weekly : bool, default False
1432
+ Run the workflow weekly.
1433
+ cron : str, optional, default None
1434
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1435
+ specified by this expression.
1436
+ timezone : str, optional, default None
1437
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1438
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1390
1439
  """
1391
1440
  ...
1392
1441
 
1393
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1442
+ @typing.overload
1443
+ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1394
1444
  """
1395
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1396
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1445
+ Specifies the flow(s) that this flow depends on.
1446
+
1447
+ ```
1448
+ @trigger_on_finish(flow='FooFlow')
1449
+ ```
1450
+ or
1451
+ ```
1452
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1453
+ ```
1454
+ This decorator respects the @project decorator and triggers the flow
1455
+ when upstream runs within the same namespace complete successfully
1456
+
1457
+ Additionally, you can specify project aware upstream flow dependencies
1458
+ by specifying the fully qualified project_flow_name.
1459
+ ```
1460
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1461
+ ```
1462
+ or
1463
+ ```
1464
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1465
+ ```
1466
+
1467
+ You can also specify just the project or project branch (other values will be
1468
+ inferred from the current project or project branch):
1469
+ ```
1470
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1471
+ ```
1472
+
1473
+ Note that `branch` is typically one of:
1474
+ - `prod`
1475
+ - `user.bob`
1476
+ - `test.my_experiment`
1477
+ - `prod.staging`
1397
1478
 
1398
1479
  Parameters
1399
1480
  ----------
1400
- timeout : int
1401
- Time, in seconds before the task times out and fails. (Default: 3600)
1402
- poke_interval : int
1403
- Time in seconds that the job should wait in between each try. (Default: 60)
1404
- mode : str
1405
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1406
- exponential_backoff : bool
1407
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1408
- pool : str
1409
- the slot pool this task should run in,
1410
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1411
- soft_fail : bool
1412
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1413
- name : str
1414
- Name of the sensor on Airflow
1415
- description : str
1416
- Description of sensor in the Airflow UI
1417
- external_dag_id : str
1418
- The dag_id that contains the task you want to wait for.
1419
- external_task_ids : List[str]
1420
- The list of task_ids that you want to wait for.
1421
- If None (default value) the sensor waits for the DAG. (Default: None)
1422
- allowed_states : List[str]
1423
- Iterable of allowed states, (Default: ['success'])
1424
- failed_states : List[str]
1425
- Iterable of failed or dis-allowed states. (Default: None)
1426
- execution_delta : datetime.timedelta
1427
- time difference with the previous execution to look at,
1428
- the default is the same logical date as the current task or DAG. (Default: None)
1429
- check_existence: bool
1430
- Set to True to check if the external task exists or check if
1431
- the DAG to wait for exists. (Default: True)
1481
+ flow : Union[str, Dict[str, str]], optional, default None
1482
+ Upstream flow dependency for this flow.
1483
+ flows : List[Union[str, Dict[str, str]]], default []
1484
+ Upstream flow dependencies for this flow.
1485
+ options : Dict[str, Any], default {}
1486
+ Backend-specific configuration for tuning eventing behavior.
1487
+
1488
+
1432
1489
  """
1433
1490
  ...
1434
1491
 
1435
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1492
+ @typing.overload
1493
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1494
+ ...
1495
+
1496
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1436
1497
  """
1437
- Specifies what flows belong to the same project.
1498
+ Specifies the flow(s) that this flow depends on.
1438
1499
 
1439
- A project-specific namespace is created for all flows that
1440
- use the same `@project(name)`.
1500
+ ```
1501
+ @trigger_on_finish(flow='FooFlow')
1502
+ ```
1503
+ or
1504
+ ```
1505
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1506
+ ```
1507
+ This decorator respects the @project decorator and triggers the flow
1508
+ when upstream runs within the same namespace complete successfully
1509
+
1510
+ Additionally, you can specify project aware upstream flow dependencies
1511
+ by specifying the fully qualified project_flow_name.
1512
+ ```
1513
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1514
+ ```
1515
+ or
1516
+ ```
1517
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1518
+ ```
1519
+
1520
+ You can also specify just the project or project branch (other values will be
1521
+ inferred from the current project or project branch):
1522
+ ```
1523
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1524
+ ```
1525
+
1526
+ Note that `branch` is typically one of:
1527
+ - `prod`
1528
+ - `user.bob`
1529
+ - `test.my_experiment`
1530
+ - `prod.staging`
1441
1531
 
1442
1532
  Parameters
1443
1533
  ----------
1444
- name : str
1445
- Project name. Make sure that the name is unique amongst all
1446
- projects that use the same production scheduler. The name may
1447
- contain only lowercase alphanumeric characters and underscores.
1534
+ flow : Union[str, Dict[str, str]], optional, default None
1535
+ Upstream flow dependency for this flow.
1536
+ flows : List[Union[str, Dict[str, str]]], default []
1537
+ Upstream flow dependencies for this flow.
1538
+ options : Dict[str, Any], default {}
1539
+ Backend-specific configuration for tuning eventing behavior.
1448
1540
 
1449
1541
 
1450
1542
  """
@@ -1545,6 +1637,48 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1545
1637
  """
1546
1638
  ...
1547
1639
 
1640
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1641
+ """
1642
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1643
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1644
+
1645
+ Parameters
1646
+ ----------
1647
+ timeout : int
1648
+ Time, in seconds before the task times out and fails. (Default: 3600)
1649
+ poke_interval : int
1650
+ Time in seconds that the job should wait in between each try. (Default: 60)
1651
+ mode : str
1652
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1653
+ exponential_backoff : bool
1654
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1655
+ pool : str
1656
+ the slot pool this task should run in,
1657
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1658
+ soft_fail : bool
1659
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1660
+ name : str
1661
+ Name of the sensor on Airflow
1662
+ description : str
1663
+ Description of sensor in the Airflow UI
1664
+ external_dag_id : str
1665
+ The dag_id that contains the task you want to wait for.
1666
+ external_task_ids : List[str]
1667
+ The list of task_ids that you want to wait for.
1668
+ If None (default value) the sensor waits for the DAG. (Default: None)
1669
+ allowed_states : List[str]
1670
+ Iterable of allowed states, (Default: ['success'])
1671
+ failed_states : List[str]
1672
+ Iterable of failed or dis-allowed states. (Default: None)
1673
+ execution_delta : datetime.timedelta
1674
+ time difference with the previous execution to look at,
1675
+ the default is the same logical date as the current task or DAG. (Default: None)
1676
+ check_existence: bool
1677
+ Set to True to check if the external task exists or check if
1678
+ the DAG to wait for exists. (Default: True)
1679
+ """
1680
+ ...
1681
+
1548
1682
  @typing.overload
1549
1683
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1550
1684
  """
@@ -1594,6 +1728,24 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1594
1728
  """
1595
1729
  ...
1596
1730
 
1731
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1732
+ """
1733
+ Specifies what flows belong to the same project.
1734
+
1735
+ A project-specific namespace is created for all flows that
1736
+ use the same `@project(name)`.
1737
+
1738
+ Parameters
1739
+ ----------
1740
+ name : str
1741
+ Project name. Make sure that the name is unique amongst all
1742
+ projects that use the same production scheduler. The name may
1743
+ contain only lowercase alphanumeric characters and underscores.
1744
+
1745
+
1746
+ """
1747
+ ...
1748
+
1597
1749
  @typing.overload
1598
1750
  def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1599
1751
  """
@@ -1675,158 +1827,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1675
1827
  """
1676
1828
  ...
1677
1829
 
1678
- @typing.overload
1679
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1680
- """
1681
- Specifies the times when the flow should be run when running on a
1682
- production scheduler.
1683
-
1684
- Parameters
1685
- ----------
1686
- hourly : bool, default False
1687
- Run the workflow hourly.
1688
- daily : bool, default True
1689
- Run the workflow daily.
1690
- weekly : bool, default False
1691
- Run the workflow weekly.
1692
- cron : str, optional, default None
1693
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1694
- specified by this expression.
1695
- timezone : str, optional, default None
1696
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1697
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1698
- """
1699
- ...
1700
-
1701
- @typing.overload
1702
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1703
- ...
1704
-
1705
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1706
- """
1707
- Specifies the times when the flow should be run when running on a
1708
- production scheduler.
1709
-
1710
- Parameters
1711
- ----------
1712
- hourly : bool, default False
1713
- Run the workflow hourly.
1714
- daily : bool, default True
1715
- Run the workflow daily.
1716
- weekly : bool, default False
1717
- Run the workflow weekly.
1718
- cron : str, optional, default None
1719
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1720
- specified by this expression.
1721
- timezone : str, optional, default None
1722
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1723
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1724
- """
1725
- ...
1726
-
1727
- @typing.overload
1728
- def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1729
- """
1730
- Specifies the flow(s) that this flow depends on.
1731
-
1732
- ```
1733
- @trigger_on_finish(flow='FooFlow')
1734
- ```
1735
- or
1736
- ```
1737
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1738
- ```
1739
- This decorator respects the @project decorator and triggers the flow
1740
- when upstream runs within the same namespace complete successfully
1741
-
1742
- Additionally, you can specify project aware upstream flow dependencies
1743
- by specifying the fully qualified project_flow_name.
1744
- ```
1745
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1746
- ```
1747
- or
1748
- ```
1749
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1750
- ```
1751
-
1752
- You can also specify just the project or project branch (other values will be
1753
- inferred from the current project or project branch):
1754
- ```
1755
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1756
- ```
1757
-
1758
- Note that `branch` is typically one of:
1759
- - `prod`
1760
- - `user.bob`
1761
- - `test.my_experiment`
1762
- - `prod.staging`
1763
-
1764
- Parameters
1765
- ----------
1766
- flow : Union[str, Dict[str, str]], optional, default None
1767
- Upstream flow dependency for this flow.
1768
- flows : List[Union[str, Dict[str, str]]], default []
1769
- Upstream flow dependencies for this flow.
1770
- options : Dict[str, Any], default {}
1771
- Backend-specific configuration for tuning eventing behavior.
1772
-
1773
-
1774
- """
1775
- ...
1776
-
1777
- @typing.overload
1778
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1779
- ...
1780
-
1781
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1782
- """
1783
- Specifies the flow(s) that this flow depends on.
1784
-
1785
- ```
1786
- @trigger_on_finish(flow='FooFlow')
1787
- ```
1788
- or
1789
- ```
1790
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1791
- ```
1792
- This decorator respects the @project decorator and triggers the flow
1793
- when upstream runs within the same namespace complete successfully
1794
-
1795
- Additionally, you can specify project aware upstream flow dependencies
1796
- by specifying the fully qualified project_flow_name.
1797
- ```
1798
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1799
- ```
1800
- or
1801
- ```
1802
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1803
- ```
1804
-
1805
- You can also specify just the project or project branch (other values will be
1806
- inferred from the current project or project branch):
1807
- ```
1808
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1809
- ```
1810
-
1811
- Note that `branch` is typically one of:
1812
- - `prod`
1813
- - `user.bob`
1814
- - `test.my_experiment`
1815
- - `prod.staging`
1816
-
1817
- Parameters
1818
- ----------
1819
- flow : Union[str, Dict[str, str]], optional, default None
1820
- Upstream flow dependency for this flow.
1821
- flows : List[Union[str, Dict[str, str]]], default []
1822
- Upstream flow dependencies for this flow.
1823
- options : Dict[str, Any], default {}
1824
- Backend-specific configuration for tuning eventing behavior.
1825
-
1826
-
1827
- """
1828
- ...
1829
-
1830
1830
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1831
1831
  """
1832
1832
  Switch namespace to the one provided.