ob-metaflow-stubs 2.11.10.3__py2.py3-none-any.whl → 3.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. metaflow-stubs/__init__.pyi +412 -409
  2. metaflow-stubs/cards.pyi +6 -6
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +4 -4
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +4 -2
  16. metaflow-stubs/metaflow_current.pyi +3 -3
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +3 -3
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +4 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +3 -3
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  37. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  45. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  47. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  56. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  60. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  61. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  62. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  64. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  65. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  68. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  70. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  77. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  80. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  81. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  82. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  83. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  84. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  85. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  86. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  87. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  90. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  91. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  92. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  93. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  94. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  96. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  98. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  99. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  102. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +4 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  105. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +4 -31
  106. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  107. metaflow-stubs/plugins/package_cli.pyi +2 -2
  108. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  109. metaflow-stubs/plugins/perimeters.pyi +24 -0
  110. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  112. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  114. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  117. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  119. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  120. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  121. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  123. metaflow-stubs/plugins/tag_cli.pyi +3 -3
  124. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  126. metaflow-stubs/procpoll.pyi +2 -2
  127. metaflow-stubs/profilers/__init__.pyi +16 -0
  128. metaflow-stubs/pylint_wrapper.pyi +2 -2
  129. metaflow-stubs/tagging_util.pyi +2 -2
  130. {ob_metaflow_stubs-2.11.10.3.dist-info → ob_metaflow_stubs-3.1.dist-info}/METADATA +1 -2
  131. ob_metaflow_stubs-3.1.dist-info/RECORD +134 -0
  132. {ob_metaflow_stubs-2.11.10.3.dist-info → ob_metaflow_stubs-3.1.dist-info}/WHEEL +1 -1
  133. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +0 -100
  134. ob_metaflow_stubs-2.11.10.3.dist-info/RECORD +0 -133
  135. {ob_metaflow_stubs-2.11.10.3.dist-info → ob_metaflow_stubs-3.1.dist-info}/top_level.txt +0 -0
@@ -1,23 +1,23 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.11.10.3 #
4
- # Generated on 2024-04-12T18:07:07.339961 #
3
+ # MF version: 2.11.10.3+ob(v1) #
4
+ # Generated on 2024-05-06T19:56:35.236170 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import io
11
+ import metaflow.client.core
12
+ import metaflow.parameters
12
13
  import metaflow.metaflow_current
14
+ import datetime
13
15
  import metaflow._vendor.click.types
16
+ import metaflow.events
14
17
  import typing
18
+ import io
15
19
  import metaflow.datastore.inputs
16
- import metaflow.client.core
17
- import metaflow.parameters
18
20
  import metaflow.plugins.datatools.s3.s3
19
- import metaflow.events
20
- import datetime
21
21
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
22
22
  StepFlag = typing.NewType("StepFlag", bool)
23
23
 
@@ -725,55 +725,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
725
725
  """
726
726
  ...
727
727
 
728
- @typing.overload
729
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
730
- """
731
- Specifies that the step will success under all circumstances.
732
-
733
- The decorator will create an optional artifact, specified by `var`, which
734
- contains the exception raised. You can use it to detect the presence
735
- of errors, indicating that all happy-path artifacts produced by the step
736
- are missing.
737
-
738
- Parameters
739
- ----------
740
- var : str, optional, default None
741
- Name of the artifact in which to store the caught exception.
742
- If not specified, the exception is not stored.
743
- print_exception : bool, default True
744
- Determines whether or not the exception is printed to
745
- stdout when caught.
746
- """
747
- ...
748
-
749
- @typing.overload
750
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
751
- ...
752
-
753
- @typing.overload
754
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
755
- ...
756
-
757
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
758
- """
759
- Specifies that the step will success under all circumstances.
760
-
761
- The decorator will create an optional artifact, specified by `var`, which
762
- contains the exception raised. You can use it to detect the presence
763
- of errors, indicating that all happy-path artifacts produced by the step
764
- are missing.
765
-
766
- Parameters
767
- ----------
768
- var : str, optional, default None
769
- Name of the artifact in which to store the caught exception.
770
- If not specified, the exception is not stored.
771
- print_exception : bool, default True
772
- Determines whether or not the exception is printed to
773
- stdout when caught.
774
- """
775
- ...
776
-
777
728
  @typing.overload
778
729
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
779
730
  """
@@ -831,198 +782,6 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
831
782
  """
832
783
  ...
833
784
 
834
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
835
- """
836
- Specifies that this step should execute on Kubernetes.
837
-
838
- Parameters
839
- ----------
840
- cpu : int, default 1
841
- Number of CPUs required for this step. If `@resources` is
842
- also present, the maximum value from all decorators is used.
843
- memory : int, default 4096
844
- Memory size (in MB) required for this step. If
845
- `@resources` is also present, the maximum value from all decorators is
846
- used.
847
- disk : int, default 10240
848
- Disk size (in MB) required for this step. If
849
- `@resources` is also present, the maximum value from all decorators is
850
- used.
851
- image : str, optional, default None
852
- Docker image to use when launching on Kubernetes. If not specified, and
853
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
854
- not, a default Docker image mapping to the current version of Python is used.
855
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
856
- If given, the imagePullPolicy to be applied to the Docker image of the step.
857
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
858
- Kubernetes service account to use when launching pod in Kubernetes.
859
- secrets : List[str], optional, default None
860
- Kubernetes secrets to use when launching pod in Kubernetes. These
861
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
862
- in Metaflow configuration.
863
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
864
- Kubernetes namespace to use when launching pod in Kubernetes.
865
- gpu : int, optional, default None
866
- Number of GPUs required for this step. A value of zero implies that
867
- the scheduled node should not have GPUs.
868
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
869
- The vendor of the GPUs to be used for this step.
870
- tolerations : List[str], default []
871
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
872
- Kubernetes tolerations to use when launching pod in Kubernetes.
873
- use_tmpfs : bool, default False
874
- This enables an explicit tmpfs mount for this step.
875
- tmpfs_tempdir : bool, default True
876
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
877
- tmpfs_size : int, optional, default: None
878
- The value for the size (in MiB) of the tmpfs mount for this step.
879
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
880
- memory allocated for this step.
881
- tmpfs_path : str, optional, default /metaflow_temp
882
- Path to tmpfs mount for this step.
883
- persistent_volume_claims : Dict[str, str], optional, default None
884
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
885
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
886
- shared_memory: int, optional
887
- Shared memory size (in MiB) required for this step
888
- port: int, optional
889
- Port number to specify in the Kubernetes job object
890
- """
891
- ...
892
-
893
- @typing.overload
894
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
895
- """
896
- Creates a human-readable report, a Metaflow Card, after this step completes.
897
-
898
- Note that you may add multiple `@card` decorators in a step with different parameters.
899
-
900
- Parameters
901
- ----------
902
- type : str, default 'default'
903
- Card type.
904
- id : str, optional, default None
905
- If multiple cards are present, use this id to identify this card.
906
- options : Dict[str, Any], default {}
907
- Options passed to the card. The contents depend on the card type.
908
- timeout : int, default 45
909
- Interrupt reporting if it takes more than this many seconds.
910
-
911
-
912
- """
913
- ...
914
-
915
- @typing.overload
916
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
917
- ...
918
-
919
- @typing.overload
920
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
921
- ...
922
-
923
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
924
- """
925
- Creates a human-readable report, a Metaflow Card, after this step completes.
926
-
927
- Note that you may add multiple `@card` decorators in a step with different parameters.
928
-
929
- Parameters
930
- ----------
931
- type : str, default 'default'
932
- Card type.
933
- id : str, optional, default None
934
- If multiple cards are present, use this id to identify this card.
935
- options : Dict[str, Any], default {}
936
- Options passed to the card. The contents depend on the card type.
937
- timeout : int, default 45
938
- Interrupt reporting if it takes more than this many seconds.
939
-
940
-
941
- """
942
- ...
943
-
944
- @typing.overload
945
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
946
- """
947
- Specifies secrets to be retrieved and injected as environment variables prior to
948
- the execution of a step.
949
-
950
- Parameters
951
- ----------
952
- sources : List[Union[str, Dict[str, Any]]], default: []
953
- List of secret specs, defining how the secrets are to be retrieved
954
- """
955
- ...
956
-
957
- @typing.overload
958
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
959
- ...
960
-
961
- @typing.overload
962
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
963
- ...
964
-
965
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
966
- """
967
- Specifies secrets to be retrieved and injected as environment variables prior to
968
- the execution of a step.
969
-
970
- Parameters
971
- ----------
972
- sources : List[Union[str, Dict[str, Any]]], default: []
973
- List of secret specs, defining how the secrets are to be retrieved
974
- """
975
- ...
976
-
977
- @typing.overload
978
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
979
- """
980
- Specifies the PyPI packages for the step.
981
-
982
- Information in this decorator will augment any
983
- attributes set in the `@pyi_base` flow-level decorator. Hence,
984
- you can use `@pypi_base` to set packages required by all
985
- steps and use `@pypi` to specify step-specific overrides.
986
-
987
- Parameters
988
- ----------
989
- packages : Dict[str, str], default: {}
990
- Packages to use for this step. The key is the name of the package
991
- and the value is the version to use.
992
- python : str, optional, default: None
993
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
994
- that the version used will correspond to the version of the Python interpreter used to start the run.
995
- """
996
- ...
997
-
998
- @typing.overload
999
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1000
- ...
1001
-
1002
- @typing.overload
1003
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1004
- ...
1005
-
1006
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1007
- """
1008
- Specifies the PyPI packages for the step.
1009
-
1010
- Information in this decorator will augment any
1011
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1012
- you can use `@pypi_base` to set packages required by all
1013
- steps and use `@pypi` to specify step-specific overrides.
1014
-
1015
- Parameters
1016
- ----------
1017
- packages : Dict[str, str], default: {}
1018
- Packages to use for this step. The key is the name of the package
1019
- and the value is the version to use.
1020
- python : str, optional, default: None
1021
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1022
- that the version used will correspond to the version of the Python interpreter used to start the run.
1023
- """
1024
- ...
1025
-
1026
785
  @typing.overload
1027
786
  def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1028
787
  """
@@ -1211,11 +970,60 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1211
970
  ...
1212
971
 
1213
972
  @typing.overload
1214
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
973
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1215
974
  """
1216
- Specifies environment variables to be set prior to the execution of a step.
975
+ Specifies the PyPI packages for the step.
1217
976
 
1218
- Parameters
977
+ Information in this decorator will augment any
978
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
979
+ you can use `@pypi_base` to set packages required by all
980
+ steps and use `@pypi` to specify step-specific overrides.
981
+
982
+ Parameters
983
+ ----------
984
+ packages : Dict[str, str], default: {}
985
+ Packages to use for this step. The key is the name of the package
986
+ and the value is the version to use.
987
+ python : str, optional, default: None
988
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
989
+ that the version used will correspond to the version of the Python interpreter used to start the run.
990
+ """
991
+ ...
992
+
993
+ @typing.overload
994
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
995
+ ...
996
+
997
+ @typing.overload
998
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
999
+ ...
1000
+
1001
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1002
+ """
1003
+ Specifies the PyPI packages for the step.
1004
+
1005
+ Information in this decorator will augment any
1006
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1007
+ you can use `@pypi_base` to set packages required by all
1008
+ steps and use `@pypi` to specify step-specific overrides.
1009
+
1010
+ Parameters
1011
+ ----------
1012
+ packages : Dict[str, str], default: {}
1013
+ Packages to use for this step. The key is the name of the package
1014
+ and the value is the version to use.
1015
+ python : str, optional, default: None
1016
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1017
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1018
+ """
1019
+ ...
1020
+
1021
+ @typing.overload
1022
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1023
+ """
1024
+ Specifies environment variables to be set prior to the execution of a step.
1025
+
1026
+ Parameters
1219
1027
  ----------
1220
1028
  vars : Dict[str, str], default {}
1221
1029
  Dictionary of environment variables to set.
@@ -1241,6 +1049,65 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
1241
1049
  """
1242
1050
  ...
1243
1051
 
1052
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1053
+ """
1054
+ Specifies that this step should execute on Kubernetes.
1055
+
1056
+ Parameters
1057
+ ----------
1058
+ cpu : int, default 1
1059
+ Number of CPUs required for this step. If `@resources` is
1060
+ also present, the maximum value from all decorators is used.
1061
+ memory : int, default 4096
1062
+ Memory size (in MB) required for this step. If
1063
+ `@resources` is also present, the maximum value from all decorators is
1064
+ used.
1065
+ disk : int, default 10240
1066
+ Disk size (in MB) required for this step. If
1067
+ `@resources` is also present, the maximum value from all decorators is
1068
+ used.
1069
+ image : str, optional, default None
1070
+ Docker image to use when launching on Kubernetes. If not specified, and
1071
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1072
+ not, a default Docker image mapping to the current version of Python is used.
1073
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1074
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1075
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1076
+ Kubernetes service account to use when launching pod in Kubernetes.
1077
+ secrets : List[str], optional, default None
1078
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1079
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1080
+ in Metaflow configuration.
1081
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1082
+ Kubernetes namespace to use when launching pod in Kubernetes.
1083
+ gpu : int, optional, default None
1084
+ Number of GPUs required for this step. A value of zero implies that
1085
+ the scheduled node should not have GPUs.
1086
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1087
+ The vendor of the GPUs to be used for this step.
1088
+ tolerations : List[str], default []
1089
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1090
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1091
+ use_tmpfs : bool, default False
1092
+ This enables an explicit tmpfs mount for this step.
1093
+ tmpfs_tempdir : bool, default True
1094
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1095
+ tmpfs_size : int, optional, default: None
1096
+ The value for the size (in MiB) of the tmpfs mount for this step.
1097
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1098
+ memory allocated for this step.
1099
+ tmpfs_path : str, optional, default /metaflow_temp
1100
+ Path to tmpfs mount for this step.
1101
+ persistent_volume_claims : Dict[str, str], optional, default None
1102
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1103
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1104
+ shared_memory: int, optional
1105
+ Shared memory size (in MiB) required for this step
1106
+ port: int, optional
1107
+ Port number to specify in the Kubernetes job object
1108
+ """
1109
+ ...
1110
+
1244
1111
  @typing.overload
1245
1112
  def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1246
1113
  """
@@ -1388,143 +1255,220 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1388
1255
  """
1389
1256
  ...
1390
1257
 
1391
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1258
+ @typing.overload
1259
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1392
1260
  """
1393
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1394
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1395
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1396
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1397
- starts only after all sensors finish.
1261
+ Specifies secrets to be retrieved and injected as environment variables prior to
1262
+ the execution of a step.
1398
1263
 
1399
1264
  Parameters
1400
1265
  ----------
1401
- timeout : int
1402
- Time, in seconds before the task times out and fails. (Default: 3600)
1403
- poke_interval : int
1404
- Time in seconds that the job should wait in between each try. (Default: 60)
1405
- mode : str
1406
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1407
- exponential_backoff : bool
1408
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1409
- pool : str
1410
- the slot pool this task should run in,
1411
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1412
- soft_fail : bool
1413
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1414
- name : str
1415
- Name of the sensor on Airflow
1416
- description : str
1417
- Description of sensor in the Airflow UI
1418
- bucket_key : Union[str, List[str]]
1419
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1420
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1421
- bucket_name : str
1422
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1423
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1424
- wildcard_match : bool
1425
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1426
- aws_conn_id : str
1427
- a reference to the s3 connection on Airflow. (Default: None)
1428
- verify : bool
1429
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1266
+ sources : List[Union[str, Dict[str, Any]]], default: []
1267
+ List of secret specs, defining how the secrets are to be retrieved
1430
1268
  """
1431
1269
  ...
1432
1270
 
1433
1271
  @typing.overload
1434
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1272
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1273
+ ...
1274
+
1275
+ @typing.overload
1276
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1277
+ ...
1278
+
1279
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1435
1280
  """
1436
- Specifies the Conda environment for all steps of the flow.
1281
+ Specifies secrets to be retrieved and injected as environment variables prior to
1282
+ the execution of a step.
1437
1283
 
1438
- Use `@conda_base` to set common libraries required by all
1439
- steps and use `@conda` to specify step-specific additions.
1284
+ Parameters
1285
+ ----------
1286
+ sources : List[Union[str, Dict[str, Any]]], default: []
1287
+ List of secret specs, defining how the secrets are to be retrieved
1288
+ """
1289
+ ...
1290
+
1291
+ @typing.overload
1292
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1293
+ """
1294
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1295
+
1296
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1440
1297
 
1441
1298
  Parameters
1442
1299
  ----------
1443
- packages : Dict[str, str], default {}
1444
- Packages to use for this flow. The key is the name of the package
1445
- and the value is the version to use.
1446
- libraries : Dict[str, str], default {}
1447
- Supported for backward compatibility. When used with packages, packages will take precedence.
1448
- python : str, optional, default None
1449
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1450
- that the version used will correspond to the version of the Python interpreter used to start the run.
1451
- disabled : bool, default False
1452
- If set to True, disables Conda.
1300
+ type : str, default 'default'
1301
+ Card type.
1302
+ id : str, optional, default None
1303
+ If multiple cards are present, use this id to identify this card.
1304
+ options : Dict[str, Any], default {}
1305
+ Options passed to the card. The contents depend on the card type.
1306
+ timeout : int, default 45
1307
+ Interrupt reporting if it takes more than this many seconds.
1308
+
1309
+
1453
1310
  """
1454
1311
  ...
1455
1312
 
1456
1313
  @typing.overload
1457
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1314
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1458
1315
  ...
1459
1316
 
1460
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1317
+ @typing.overload
1318
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1319
+ ...
1320
+
1321
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1461
1322
  """
1462
- Specifies the Conda environment for all steps of the flow.
1323
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1463
1324
 
1464
- Use `@conda_base` to set common libraries required by all
1465
- steps and use `@conda` to specify step-specific additions.
1325
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1466
1326
 
1467
1327
  Parameters
1468
1328
  ----------
1469
- packages : Dict[str, str], default {}
1470
- Packages to use for this flow. The key is the name of the package
1471
- and the value is the version to use.
1472
- libraries : Dict[str, str], default {}
1473
- Supported for backward compatibility. When used with packages, packages will take precedence.
1474
- python : str, optional, default None
1475
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1476
- that the version used will correspond to the version of the Python interpreter used to start the run.
1477
- disabled : bool, default False
1478
- If set to True, disables Conda.
1329
+ type : str, default 'default'
1330
+ Card type.
1331
+ id : str, optional, default None
1332
+ If multiple cards are present, use this id to identify this card.
1333
+ options : Dict[str, Any], default {}
1334
+ Options passed to the card. The contents depend on the card type.
1335
+ timeout : int, default 45
1336
+ Interrupt reporting if it takes more than this many seconds.
1337
+
1338
+
1479
1339
  """
1480
1340
  ...
1481
1341
 
1482
1342
  @typing.overload
1483
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1343
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1484
1344
  """
1485
- Specifies the times when the flow should be run when running on a
1486
- production scheduler.
1345
+ Specifies that the step will success under all circumstances.
1346
+
1347
+ The decorator will create an optional artifact, specified by `var`, which
1348
+ contains the exception raised. You can use it to detect the presence
1349
+ of errors, indicating that all happy-path artifacts produced by the step
1350
+ are missing.
1487
1351
 
1488
1352
  Parameters
1489
1353
  ----------
1490
- hourly : bool, default False
1491
- Run the workflow hourly.
1492
- daily : bool, default True
1493
- Run the workflow daily.
1494
- weekly : bool, default False
1495
- Run the workflow weekly.
1496
- cron : str, optional, default None
1497
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1498
- specified by this expression.
1499
- timezone : str, optional, default None
1500
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1501
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1354
+ var : str, optional, default None
1355
+ Name of the artifact in which to store the caught exception.
1356
+ If not specified, the exception is not stored.
1357
+ print_exception : bool, default True
1358
+ Determines whether or not the exception is printed to
1359
+ stdout when caught.
1360
+ """
1361
+ ...
1362
+
1363
+ @typing.overload
1364
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1365
+ ...
1366
+
1367
+ @typing.overload
1368
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1369
+ ...
1370
+
1371
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1372
+ """
1373
+ Specifies that the step will success under all circumstances.
1374
+
1375
+ The decorator will create an optional artifact, specified by `var`, which
1376
+ contains the exception raised. You can use it to detect the presence
1377
+ of errors, indicating that all happy-path artifacts produced by the step
1378
+ are missing.
1379
+
1380
+ Parameters
1381
+ ----------
1382
+ var : str, optional, default None
1383
+ Name of the artifact in which to store the caught exception.
1384
+ If not specified, the exception is not stored.
1385
+ print_exception : bool, default True
1386
+ Determines whether or not the exception is printed to
1387
+ stdout when caught.
1388
+ """
1389
+ ...
1390
+
1391
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1392
+ """
1393
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1394
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1395
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1396
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1397
+ starts only after all sensors finish.
1398
+
1399
+ Parameters
1400
+ ----------
1401
+ timeout : int
1402
+ Time, in seconds before the task times out and fails. (Default: 3600)
1403
+ poke_interval : int
1404
+ Time in seconds that the job should wait in between each try. (Default: 60)
1405
+ mode : str
1406
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1407
+ exponential_backoff : bool
1408
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1409
+ pool : str
1410
+ the slot pool this task should run in,
1411
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1412
+ soft_fail : bool
1413
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1414
+ name : str
1415
+ Name of the sensor on Airflow
1416
+ description : str
1417
+ Description of sensor in the Airflow UI
1418
+ bucket_key : Union[str, List[str]]
1419
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1420
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1421
+ bucket_name : str
1422
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1423
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1424
+ wildcard_match : bool
1425
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1426
+ aws_conn_id : str
1427
+ a reference to the s3 connection on Airflow. (Default: None)
1428
+ verify : bool
1429
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1502
1430
  """
1503
1431
  ...
1504
1432
 
1505
- @typing.overload
1506
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1507
- ...
1508
-
1509
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1433
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1510
1434
  """
1511
- Specifies the times when the flow should be run when running on a
1512
- production scheduler.
1435
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1436
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1513
1437
 
1514
1438
  Parameters
1515
1439
  ----------
1516
- hourly : bool, default False
1517
- Run the workflow hourly.
1518
- daily : bool, default True
1519
- Run the workflow daily.
1520
- weekly : bool, default False
1521
- Run the workflow weekly.
1522
- cron : str, optional, default None
1523
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1524
- specified by this expression.
1525
- timezone : str, optional, default None
1526
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1527
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1440
+ timeout : int
1441
+ Time, in seconds before the task times out and fails. (Default: 3600)
1442
+ poke_interval : int
1443
+ Time in seconds that the job should wait in between each try. (Default: 60)
1444
+ mode : str
1445
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1446
+ exponential_backoff : bool
1447
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1448
+ pool : str
1449
+ the slot pool this task should run in,
1450
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1451
+ soft_fail : bool
1452
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1453
+ name : str
1454
+ Name of the sensor on Airflow
1455
+ description : str
1456
+ Description of sensor in the Airflow UI
1457
+ external_dag_id : str
1458
+ The dag_id that contains the task you want to wait for.
1459
+ external_task_ids : List[str]
1460
+ The list of task_ids that you want to wait for.
1461
+ If None (default value) the sensor waits for the DAG. (Default: None)
1462
+ allowed_states : List[str]
1463
+ Iterable of allowed states, (Default: ['success'])
1464
+ failed_states : List[str]
1465
+ Iterable of failed or dis-allowed states. (Default: None)
1466
+ execution_delta : datetime.timedelta
1467
+ time difference with the previous execution to look at,
1468
+ the default is the same logical date as the current task or DAG. (Default: None)
1469
+ check_existence: bool
1470
+ Set to True to check if the external task exists or check if
1471
+ the DAG to wait for exists. (Default: True)
1528
1472
  """
1529
1473
  ...
1530
1474
 
@@ -1631,6 +1575,73 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1631
1575
  """
1632
1576
  ...
1633
1577
 
1578
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1579
+ """
1580
+ Specifies what flows belong to the same project.
1581
+
1582
+ A project-specific namespace is created for all flows that
1583
+ use the same `@project(name)`.
1584
+
1585
+ Parameters
1586
+ ----------
1587
+ name : str
1588
+ Project name. Make sure that the name is unique amongst all
1589
+ projects that use the same production scheduler. The name may
1590
+ contain only lowercase alphanumeric characters and underscores.
1591
+
1592
+
1593
+ """
1594
+ ...
1595
+
1596
+ @typing.overload
1597
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1598
+ """
1599
+ Specifies the times when the flow should be run when running on a
1600
+ production scheduler.
1601
+
1602
+ Parameters
1603
+ ----------
1604
+ hourly : bool, default False
1605
+ Run the workflow hourly.
1606
+ daily : bool, default True
1607
+ Run the workflow daily.
1608
+ weekly : bool, default False
1609
+ Run the workflow weekly.
1610
+ cron : str, optional, default None
1611
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1612
+ specified by this expression.
1613
+ timezone : str, optional, default None
1614
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1615
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1616
+ """
1617
+ ...
1618
+
1619
+ @typing.overload
1620
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1621
+ ...
1622
+
1623
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1624
+ """
1625
+ Specifies the times when the flow should be run when running on a
1626
+ production scheduler.
1627
+
1628
+ Parameters
1629
+ ----------
1630
+ hourly : bool, default False
1631
+ Run the workflow hourly.
1632
+ daily : bool, default True
1633
+ Run the workflow daily.
1634
+ weekly : bool, default False
1635
+ Run the workflow weekly.
1636
+ cron : str, optional, default None
1637
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1638
+ specified by this expression.
1639
+ timezone : str, optional, default None
1640
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1641
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1642
+ """
1643
+ ...
1644
+
1634
1645
  @typing.overload
1635
1646
  def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1636
1647
  """
@@ -1670,66 +1681,6 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
1670
1681
  """
1671
1682
  ...
1672
1683
 
1673
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1674
- """
1675
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1676
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1677
-
1678
- Parameters
1679
- ----------
1680
- timeout : int
1681
- Time, in seconds before the task times out and fails. (Default: 3600)
1682
- poke_interval : int
1683
- Time in seconds that the job should wait in between each try. (Default: 60)
1684
- mode : str
1685
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1686
- exponential_backoff : bool
1687
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1688
- pool : str
1689
- the slot pool this task should run in,
1690
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1691
- soft_fail : bool
1692
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1693
- name : str
1694
- Name of the sensor on Airflow
1695
- description : str
1696
- Description of sensor in the Airflow UI
1697
- external_dag_id : str
1698
- The dag_id that contains the task you want to wait for.
1699
- external_task_ids : List[str]
1700
- The list of task_ids that you want to wait for.
1701
- If None (default value) the sensor waits for the DAG. (Default: None)
1702
- allowed_states : List[str]
1703
- Iterable of allowed states, (Default: ['success'])
1704
- failed_states : List[str]
1705
- Iterable of failed or dis-allowed states. (Default: None)
1706
- execution_delta : datetime.timedelta
1707
- time difference with the previous execution to look at,
1708
- the default is the same logical date as the current task or DAG. (Default: None)
1709
- check_existence: bool
1710
- Set to True to check if the external task exists or check if
1711
- the DAG to wait for exists. (Default: True)
1712
- """
1713
- ...
1714
-
1715
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1716
- """
1717
- Specifies what flows belong to the same project.
1718
-
1719
- A project-specific namespace is created for all flows that
1720
- use the same `@project(name)`.
1721
-
1722
- Parameters
1723
- ----------
1724
- name : str
1725
- Project name. Make sure that the name is unique amongst all
1726
- projects that use the same production scheduler. The name may
1727
- contain only lowercase alphanumeric characters and underscores.
1728
-
1729
-
1730
- """
1731
- ...
1732
-
1733
1684
  @typing.overload
1734
1685
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1735
1686
  """
@@ -1825,6 +1776,55 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1825
1776
  """
1826
1777
  ...
1827
1778
 
1779
+ @typing.overload
1780
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1781
+ """
1782
+ Specifies the Conda environment for all steps of the flow.
1783
+
1784
+ Use `@conda_base` to set common libraries required by all
1785
+ steps and use `@conda` to specify step-specific additions.
1786
+
1787
+ Parameters
1788
+ ----------
1789
+ packages : Dict[str, str], default {}
1790
+ Packages to use for this flow. The key is the name of the package
1791
+ and the value is the version to use.
1792
+ libraries : Dict[str, str], default {}
1793
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1794
+ python : str, optional, default None
1795
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1796
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1797
+ disabled : bool, default False
1798
+ If set to True, disables Conda.
1799
+ """
1800
+ ...
1801
+
1802
+ @typing.overload
1803
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1804
+ ...
1805
+
1806
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1807
+ """
1808
+ Specifies the Conda environment for all steps of the flow.
1809
+
1810
+ Use `@conda_base` to set common libraries required by all
1811
+ steps and use `@conda` to specify step-specific additions.
1812
+
1813
+ Parameters
1814
+ ----------
1815
+ packages : Dict[str, str], default {}
1816
+ Packages to use for this flow. The key is the name of the package
1817
+ and the value is the version to use.
1818
+ libraries : Dict[str, str], default {}
1819
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1820
+ python : str, optional, default None
1821
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1822
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1823
+ disabled : bool, default False
1824
+ If set to True, disables Conda.
1825
+ """
1826
+ ...
1827
+
1828
1828
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1829
1829
  """
1830
1830
  Switch namespace to the one provided.
@@ -2775,3 +2775,6 @@ class DataArtifact(metaflow.client.core.MetaflowObject, metaclass=type):
2775
2775
  ...
2776
2776
  ...
2777
2777
 
2778
+ def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
2779
+ ...
2780
+