ob-metaflow-stubs 5.4__py2.py3-none-any.whl → 5.6__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. metaflow-stubs/__init__.pyi +366 -366
  2. metaflow-stubs/cards.pyi +6 -6
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +3 -3
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +5 -5
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  28. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +7 -7
  33. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  35. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  38. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  42. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  44. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  51. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  52. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  56. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  57. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  58. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  60. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  61. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  62. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  64. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  66. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  68. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  71. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  72. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  73. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  74. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  75. metaflow-stubs/plugins/datatools/__init__.pyi +4 -4
  76. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  77. metaflow-stubs/plugins/datatools/s3/__init__.pyi +4 -4
  78. metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
  79. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  80. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  81. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  82. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  83. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  84. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  87. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  93. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  98. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  99. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  100. metaflow-stubs/plugins/package_cli.pyi +2 -2
  101. metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
  102. metaflow-stubs/plugins/perimeters.pyi +2 -2
  103. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  104. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  105. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  107. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  110. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  111. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  113. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  114. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  116. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  117. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  118. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  119. metaflow-stubs/procpoll.pyi +2 -2
  120. metaflow-stubs/profilers/__init__.pyi +2 -2
  121. metaflow-stubs/pylint_wrapper.pyi +2 -2
  122. metaflow-stubs/runner/__init__.pyi +2 -2
  123. metaflow-stubs/runner/deployer.pyi +3 -3
  124. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  125. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  126. metaflow-stubs/runner/nbrun.pyi +2 -2
  127. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  128. metaflow-stubs/runner/utils.pyi +2 -2
  129. metaflow-stubs/system/__init__.pyi +3 -3
  130. metaflow-stubs/system/system_logger.pyi +3 -3
  131. metaflow-stubs/system/system_monitor.pyi +2 -2
  132. metaflow-stubs/tagging_util.pyi +2 -2
  133. metaflow-stubs/tuple_util.pyi +2 -2
  134. {ob_metaflow_stubs-5.4.dist-info → ob_metaflow_stubs-5.6.dist-info}/METADATA +1 -1
  135. ob_metaflow_stubs-5.6.dist-info/RECORD +138 -0
  136. ob_metaflow_stubs-5.4.dist-info/RECORD +0 -138
  137. {ob_metaflow_stubs-5.4.dist-info → ob_metaflow_stubs-5.6.dist-info}/WHEEL +0 -0
  138. {ob_metaflow_stubs-5.4.dist-info → ob_metaflow_stubs-5.6.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.16.1+ob(v1) #
4
- # Generated on 2024-08-26T23:03:56.776768 #
3
+ # MF version: 2.12.18.1+ob(v1) #
4
+ # Generated on 2024-08-28T20:35:42.371631 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
+ import metaflow.parameters
12
+ import metaflow.datastore.inputs
13
+ import metaflow.flowspec
11
14
  import io
12
15
  import metaflow.client.core
13
- import metaflow._vendor.click.types
14
- import metaflow.parameters
15
16
  import metaflow.plugins.datatools.s3.s3
17
+ import typing
18
+ import metaflow.metaflow_current
19
+ import metaflow._vendor.click.types
16
20
  import metaflow.events
17
21
  import datetime
18
- import metaflow.metaflow_current
19
22
  import metaflow.runner.metaflow_runner
20
- import metaflow.datastore.inputs
21
- import metaflow.flowspec
22
- import typing
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -727,6 +727,136 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
727
727
  """
728
728
  ...
729
729
 
730
+ @typing.overload
731
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
732
+ """
733
+ Decorator prototype for all step decorators. This function gets specialized
734
+ and imported for all decorators types by _import_plugin_decorators().
735
+ """
736
+ ...
737
+
738
+ @typing.overload
739
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
740
+ ...
741
+
742
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
743
+ """
744
+ Decorator prototype for all step decorators. This function gets specialized
745
+ and imported for all decorators types by _import_plugin_decorators().
746
+ """
747
+ ...
748
+
749
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
750
+ """
751
+ Specifies that this step should execute on Kubernetes.
752
+
753
+ Parameters
754
+ ----------
755
+ cpu : int, default 1
756
+ Number of CPUs required for this step. If `@resources` is
757
+ also present, the maximum value from all decorators is used.
758
+ memory : int, default 4096
759
+ Memory size (in MB) required for this step. If
760
+ `@resources` is also present, the maximum value from all decorators is
761
+ used.
762
+ disk : int, default 10240
763
+ Disk size (in MB) required for this step. If
764
+ `@resources` is also present, the maximum value from all decorators is
765
+ used.
766
+ image : str, optional, default None
767
+ Docker image to use when launching on Kubernetes. If not specified, and
768
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
769
+ not, a default Docker image mapping to the current version of Python is used.
770
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
771
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
772
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
773
+ Kubernetes service account to use when launching pod in Kubernetes.
774
+ secrets : List[str], optional, default None
775
+ Kubernetes secrets to use when launching pod in Kubernetes. These
776
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
777
+ in Metaflow configuration.
778
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
779
+ Kubernetes namespace to use when launching pod in Kubernetes.
780
+ gpu : int, optional, default None
781
+ Number of GPUs required for this step. A value of zero implies that
782
+ the scheduled node should not have GPUs.
783
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
784
+ The vendor of the GPUs to be used for this step.
785
+ tolerations : List[str], default []
786
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
787
+ Kubernetes tolerations to use when launching pod in Kubernetes.
788
+ use_tmpfs : bool, default False
789
+ This enables an explicit tmpfs mount for this step.
790
+ tmpfs_tempdir : bool, default True
791
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
792
+ tmpfs_size : int, optional, default: None
793
+ The value for the size (in MiB) of the tmpfs mount for this step.
794
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
795
+ memory allocated for this step.
796
+ tmpfs_path : str, optional, default /metaflow_temp
797
+ Path to tmpfs mount for this step.
798
+ persistent_volume_claims : Dict[str, str], optional, default None
799
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
800
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
801
+ shared_memory: int, optional
802
+ Shared memory size (in MiB) required for this step
803
+ port: int, optional
804
+ Port number to specify in the Kubernetes job object
805
+ compute_pool : str, optional, default None
806
+ Compute pool to be used for for this step.
807
+ If not specified, any accessible compute pool within the perimeter is used.
808
+ """
809
+ ...
810
+
811
+ @typing.overload
812
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
813
+ """
814
+ Specifies the PyPI packages for the step.
815
+
816
+ Information in this decorator will augment any
817
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
818
+ you can use `@pypi_base` to set packages required by all
819
+ steps and use `@pypi` to specify step-specific overrides.
820
+
821
+ Parameters
822
+ ----------
823
+ packages : Dict[str, str], default: {}
824
+ Packages to use for this step. The key is the name of the package
825
+ and the value is the version to use.
826
+ python : str, optional, default: None
827
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
828
+ that the version used will correspond to the version of the Python interpreter used to start the run.
829
+ """
830
+ ...
831
+
832
+ @typing.overload
833
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
834
+ ...
835
+
836
+ @typing.overload
837
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
838
+ ...
839
+
840
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
841
+ """
842
+ Specifies the PyPI packages for the step.
843
+
844
+ Information in this decorator will augment any
845
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
846
+ you can use `@pypi_base` to set packages required by all
847
+ steps and use `@pypi` to specify step-specific overrides.
848
+
849
+ Parameters
850
+ ----------
851
+ packages : Dict[str, str], default: {}
852
+ Packages to use for this step. The key is the name of the package
853
+ and the value is the version to use.
854
+ python : str, optional, default: None
855
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
856
+ that the version used will correspond to the version of the Python interpreter used to start the run.
857
+ """
858
+ ...
859
+
730
860
  @typing.overload
731
861
  def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
862
  """
@@ -758,6 +888,59 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
758
888
  """
759
889
  ...
760
890
 
891
+ @typing.overload
892
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
893
+ """
894
+ Specifies the number of times the task corresponding
895
+ to a step needs to be retried.
896
+
897
+ This decorator is useful for handling transient errors, such as networking issues.
898
+ If your task contains operations that can't be retried safely, e.g. database updates,
899
+ it is advisable to annotate it with `@retry(times=0)`.
900
+
901
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
902
+ decorator will execute a no-op task after all retries have been exhausted,
903
+ ensuring that the flow execution can continue.
904
+
905
+ Parameters
906
+ ----------
907
+ times : int, default 3
908
+ Number of times to retry this task.
909
+ minutes_between_retries : int, default 2
910
+ Number of minutes between retries.
911
+ """
912
+ ...
913
+
914
+ @typing.overload
915
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
916
+ ...
917
+
918
+ @typing.overload
919
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
920
+ ...
921
+
922
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
923
+ """
924
+ Specifies the number of times the task corresponding
925
+ to a step needs to be retried.
926
+
927
+ This decorator is useful for handling transient errors, such as networking issues.
928
+ If your task contains operations that can't be retried safely, e.g. database updates,
929
+ it is advisable to annotate it with `@retry(times=0)`.
930
+
931
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
932
+ decorator will execute a no-op task after all retries have been exhausted,
933
+ ensuring that the flow execution can continue.
934
+
935
+ Parameters
936
+ ----------
937
+ times : int, default 3
938
+ Number of times to retry this task.
939
+ minutes_between_retries : int, default 2
940
+ Number of minutes between retries.
941
+ """
942
+ ...
943
+
761
944
  @typing.overload
762
945
  def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
763
946
  """
@@ -807,55 +990,6 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
807
990
  """
808
991
  ...
809
992
 
810
- @typing.overload
811
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
812
- """
813
- Specifies the PyPI packages for the step.
814
-
815
- Information in this decorator will augment any
816
- attributes set in the `@pyi_base` flow-level decorator. Hence,
817
- you can use `@pypi_base` to set packages required by all
818
- steps and use `@pypi` to specify step-specific overrides.
819
-
820
- Parameters
821
- ----------
822
- packages : Dict[str, str], default: {}
823
- Packages to use for this step. The key is the name of the package
824
- and the value is the version to use.
825
- python : str, optional, default: None
826
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
827
- that the version used will correspond to the version of the Python interpreter used to start the run.
828
- """
829
- ...
830
-
831
- @typing.overload
832
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
833
- ...
834
-
835
- @typing.overload
836
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
837
- ...
838
-
839
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
840
- """
841
- Specifies the PyPI packages for the step.
842
-
843
- Information in this decorator will augment any
844
- attributes set in the `@pyi_base` flow-level decorator. Hence,
845
- you can use `@pypi_base` to set packages required by all
846
- steps and use `@pypi` to specify step-specific overrides.
847
-
848
- Parameters
849
- ----------
850
- packages : Dict[str, str], default: {}
851
- Packages to use for this step. The key is the name of the package
852
- and the value is the version to use.
853
- python : str, optional, default: None
854
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
855
- that the version used will correspond to the version of the Python interpreter used to start the run.
856
- """
857
- ...
858
-
859
993
  @typing.overload
860
994
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
861
995
  """
@@ -947,55 +1081,53 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
947
1081
  ...
948
1082
 
949
1083
  @typing.overload
950
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1084
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
951
1085
  """
952
- Specifies the number of times the task corresponding
953
- to a step needs to be retried.
1086
+ Creates a human-readable report, a Metaflow Card, after this step completes.
954
1087
 
955
- This decorator is useful for handling transient errors, such as networking issues.
956
- If your task contains operations that can't be retried safely, e.g. database updates,
957
- it is advisable to annotate it with `@retry(times=0)`.
958
-
959
- This can be used in conjunction with the `@catch` decorator. The `@catch`
960
- decorator will execute a no-op task after all retries have been exhausted,
961
- ensuring that the flow execution can continue.
1088
+ Note that you may add multiple `@card` decorators in a step with different parameters.
962
1089
 
963
1090
  Parameters
964
1091
  ----------
965
- times : int, default 3
966
- Number of times to retry this task.
967
- minutes_between_retries : int, default 2
968
- Number of minutes between retries.
1092
+ type : str, default 'default'
1093
+ Card type.
1094
+ id : str, optional, default None
1095
+ If multiple cards are present, use this id to identify this card.
1096
+ options : Dict[str, Any], default {}
1097
+ Options passed to the card. The contents depend on the card type.
1098
+ timeout : int, default 45
1099
+ Interrupt reporting if it takes more than this many seconds.
1100
+
1101
+
969
1102
  """
970
1103
  ...
971
1104
 
972
1105
  @typing.overload
973
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1106
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
974
1107
  ...
975
1108
 
976
1109
  @typing.overload
977
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1110
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
978
1111
  ...
979
1112
 
980
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1113
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
981
1114
  """
982
- Specifies the number of times the task corresponding
983
- to a step needs to be retried.
984
-
985
- This decorator is useful for handling transient errors, such as networking issues.
986
- If your task contains operations that can't be retried safely, e.g. database updates,
987
- it is advisable to annotate it with `@retry(times=0)`.
1115
+ Creates a human-readable report, a Metaflow Card, after this step completes.
988
1116
 
989
- This can be used in conjunction with the `@catch` decorator. The `@catch`
990
- decorator will execute a no-op task after all retries have been exhausted,
991
- ensuring that the flow execution can continue.
1117
+ Note that you may add multiple `@card` decorators in a step with different parameters.
992
1118
 
993
1119
  Parameters
994
1120
  ----------
995
- times : int, default 3
996
- Number of times to retry this task.
997
- minutes_between_retries : int, default 2
998
- Number of minutes between retries.
1121
+ type : str, default 'default'
1122
+ Card type.
1123
+ id : str, optional, default None
1124
+ If multiple cards are present, use this id to identify this card.
1125
+ options : Dict[str, Any], default {}
1126
+ Options passed to the card. The contents depend on the card type.
1127
+ timeout : int, default 45
1128
+ Interrupt reporting if it takes more than this many seconds.
1129
+
1130
+
999
1131
  """
1000
1132
  ...
1001
1133
 
@@ -1056,6 +1188,23 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1056
1188
  """
1057
1189
  ...
1058
1190
 
1191
+ @typing.overload
1192
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1193
+ """
1194
+ Internal decorator to support Fast bakery
1195
+ """
1196
+ ...
1197
+
1198
+ @typing.overload
1199
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1200
+ ...
1201
+
1202
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1203
+ """
1204
+ Internal decorator to support Fast bakery
1205
+ """
1206
+ ...
1207
+
1059
1208
  @typing.overload
1060
1209
  def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1061
1210
  """
@@ -1133,155 +1282,6 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1133
1282
  """
1134
1283
  ...
1135
1284
 
1136
- @typing.overload
1137
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1138
- """
1139
- Internal decorator to support Fast bakery
1140
- """
1141
- ...
1142
-
1143
- @typing.overload
1144
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1145
- ...
1146
-
1147
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1148
- """
1149
- Internal decorator to support Fast bakery
1150
- """
1151
- ...
1152
-
1153
- @typing.overload
1154
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1155
- """
1156
- Decorator prototype for all step decorators. This function gets specialized
1157
- and imported for all decorators types by _import_plugin_decorators().
1158
- """
1159
- ...
1160
-
1161
- @typing.overload
1162
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1163
- ...
1164
-
1165
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1166
- """
1167
- Decorator prototype for all step decorators. This function gets specialized
1168
- and imported for all decorators types by _import_plugin_decorators().
1169
- """
1170
- ...
1171
-
1172
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1173
- """
1174
- Specifies that this step should execute on Kubernetes.
1175
-
1176
- Parameters
1177
- ----------
1178
- cpu : int, default 1
1179
- Number of CPUs required for this step. If `@resources` is
1180
- also present, the maximum value from all decorators is used.
1181
- memory : int, default 4096
1182
- Memory size (in MB) required for this step. If
1183
- `@resources` is also present, the maximum value from all decorators is
1184
- used.
1185
- disk : int, default 10240
1186
- Disk size (in MB) required for this step. If
1187
- `@resources` is also present, the maximum value from all decorators is
1188
- used.
1189
- image : str, optional, default None
1190
- Docker image to use when launching on Kubernetes. If not specified, and
1191
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1192
- not, a default Docker image mapping to the current version of Python is used.
1193
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1194
- If given, the imagePullPolicy to be applied to the Docker image of the step.
1195
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1196
- Kubernetes service account to use when launching pod in Kubernetes.
1197
- secrets : List[str], optional, default None
1198
- Kubernetes secrets to use when launching pod in Kubernetes. These
1199
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1200
- in Metaflow configuration.
1201
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1202
- Kubernetes namespace to use when launching pod in Kubernetes.
1203
- gpu : int, optional, default None
1204
- Number of GPUs required for this step. A value of zero implies that
1205
- the scheduled node should not have GPUs.
1206
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1207
- The vendor of the GPUs to be used for this step.
1208
- tolerations : List[str], default []
1209
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1210
- Kubernetes tolerations to use when launching pod in Kubernetes.
1211
- use_tmpfs : bool, default False
1212
- This enables an explicit tmpfs mount for this step.
1213
- tmpfs_tempdir : bool, default True
1214
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1215
- tmpfs_size : int, optional, default: None
1216
- The value for the size (in MiB) of the tmpfs mount for this step.
1217
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1218
- memory allocated for this step.
1219
- tmpfs_path : str, optional, default /metaflow_temp
1220
- Path to tmpfs mount for this step.
1221
- persistent_volume_claims : Dict[str, str], optional, default None
1222
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1223
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1224
- shared_memory: int, optional
1225
- Shared memory size (in MiB) required for this step
1226
- port: int, optional
1227
- Port number to specify in the Kubernetes job object
1228
- compute_pool : str, optional, default None
1229
- Compute pool to be used for for this step.
1230
- If not specified, any accessible compute pool within the perimeter is used.
1231
- """
1232
- ...
1233
-
1234
- @typing.overload
1235
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1236
- """
1237
- Creates a human-readable report, a Metaflow Card, after this step completes.
1238
-
1239
- Note that you may add multiple `@card` decorators in a step with different parameters.
1240
-
1241
- Parameters
1242
- ----------
1243
- type : str, default 'default'
1244
- Card type.
1245
- id : str, optional, default None
1246
- If multiple cards are present, use this id to identify this card.
1247
- options : Dict[str, Any], default {}
1248
- Options passed to the card. The contents depend on the card type.
1249
- timeout : int, default 45
1250
- Interrupt reporting if it takes more than this many seconds.
1251
-
1252
-
1253
- """
1254
- ...
1255
-
1256
- @typing.overload
1257
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1258
- ...
1259
-
1260
- @typing.overload
1261
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1262
- ...
1263
-
1264
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1265
- """
1266
- Creates a human-readable report, a Metaflow Card, after this step completes.
1267
-
1268
- Note that you may add multiple `@card` decorators in a step with different parameters.
1269
-
1270
- Parameters
1271
- ----------
1272
- type : str, default 'default'
1273
- Card type.
1274
- id : str, optional, default None
1275
- If multiple cards are present, use this id to identify this card.
1276
- options : Dict[str, Any], default {}
1277
- Options passed to the card. The contents depend on the card type.
1278
- timeout : int, default 45
1279
- Interrupt reporting if it takes more than this many seconds.
1280
-
1281
-
1282
- """
1283
- ...
1284
-
1285
1285
  @typing.overload
1286
1286
  def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1287
1287
  """
@@ -1424,6 +1424,48 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1424
1424
  """
1425
1425
  ...
1426
1426
 
1427
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1428
+ """
1429
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1430
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1431
+
1432
+ Parameters
1433
+ ----------
1434
+ timeout : int
1435
+ Time, in seconds before the task times out and fails. (Default: 3600)
1436
+ poke_interval : int
1437
+ Time in seconds that the job should wait in between each try. (Default: 60)
1438
+ mode : str
1439
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1440
+ exponential_backoff : bool
1441
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1442
+ pool : str
1443
+ the slot pool this task should run in,
1444
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1445
+ soft_fail : bool
1446
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1447
+ name : str
1448
+ Name of the sensor on Airflow
1449
+ description : str
1450
+ Description of sensor in the Airflow UI
1451
+ external_dag_id : str
1452
+ The dag_id that contains the task you want to wait for.
1453
+ external_task_ids : List[str]
1454
+ The list of task_ids that you want to wait for.
1455
+ If None (default value) the sensor waits for the DAG. (Default: None)
1456
+ allowed_states : List[str]
1457
+ Iterable of allowed states, (Default: ['success'])
1458
+ failed_states : List[str]
1459
+ Iterable of failed or dis-allowed states. (Default: None)
1460
+ execution_delta : datetime.timedelta
1461
+ time difference with the previous execution to look at,
1462
+ the default is the same logical date as the current task or DAG. (Default: None)
1463
+ check_existence: bool
1464
+ Set to True to check if the external task exists or check if
1465
+ the DAG to wait for exists. (Default: True)
1466
+ """
1467
+ ...
1468
+
1427
1469
  def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1428
1470
  """
1429
1471
  The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
@@ -1466,6 +1508,86 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1466
1508
  """
1467
1509
  ...
1468
1510
 
1511
+ @typing.overload
1512
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1513
+ """
1514
+ Specifies the Conda environment for all steps of the flow.
1515
+
1516
+ Use `@conda_base` to set common libraries required by all
1517
+ steps and use `@conda` to specify step-specific additions.
1518
+
1519
+ Parameters
1520
+ ----------
1521
+ packages : Dict[str, str], default {}
1522
+ Packages to use for this flow. The key is the name of the package
1523
+ and the value is the version to use.
1524
+ libraries : Dict[str, str], default {}
1525
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1526
+ python : str, optional, default None
1527
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1528
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1529
+ disabled : bool, default False
1530
+ If set to True, disables Conda.
1531
+ """
1532
+ ...
1533
+
1534
+ @typing.overload
1535
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1536
+ ...
1537
+
1538
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1539
+ """
1540
+ Specifies the Conda environment for all steps of the flow.
1541
+
1542
+ Use `@conda_base` to set common libraries required by all
1543
+ steps and use `@conda` to specify step-specific additions.
1544
+
1545
+ Parameters
1546
+ ----------
1547
+ packages : Dict[str, str], default {}
1548
+ Packages to use for this flow. The key is the name of the package
1549
+ and the value is the version to use.
1550
+ libraries : Dict[str, str], default {}
1551
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1552
+ python : str, optional, default None
1553
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1554
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1555
+ disabled : bool, default False
1556
+ If set to True, disables Conda.
1557
+ """
1558
+ ...
1559
+
1560
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1561
+ """
1562
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1563
+
1564
+ User code call
1565
+ -----------
1566
+ @nim(
1567
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1568
+ backend='managed'
1569
+ )
1570
+
1571
+ Valid backend options
1572
+ ---------------------
1573
+ - 'managed': Outerbounds selects a compute provider based on the model.
1574
+ - 🚧 'dataplane': Run in your account.
1575
+
1576
+ Valid model options
1577
+ ----------------
1578
+ - 'meta/llama3-8b-instruct': 8B parameter model
1579
+ - 'meta/llama3-70b-instruct': 70B parameter model
1580
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1581
+
1582
+ Parameters
1583
+ ----------
1584
+ models: list[NIM]
1585
+ List of NIM containers running models in sidecars.
1586
+ backend: str
1587
+ Compute provider to run the NIM container.
1588
+ """
1589
+ ...
1590
+
1469
1591
  @typing.overload
1470
1592
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1471
1593
  """
@@ -1561,97 +1683,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1561
1683
  """
1562
1684
  ...
1563
1685
 
1564
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1565
- """
1566
- Specifies what flows belong to the same project.
1567
-
1568
- A project-specific namespace is created for all flows that
1569
- use the same `@project(name)`.
1570
-
1571
- Parameters
1572
- ----------
1573
- name : str
1574
- Project name. Make sure that the name is unique amongst all
1575
- projects that use the same production scheduler. The name may
1576
- contain only lowercase alphanumeric characters and underscores.
1577
-
1578
-
1579
- """
1580
- ...
1581
-
1582
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1583
- """
1584
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1585
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1586
-
1587
- Parameters
1588
- ----------
1589
- timeout : int
1590
- Time, in seconds before the task times out and fails. (Default: 3600)
1591
- poke_interval : int
1592
- Time in seconds that the job should wait in between each try. (Default: 60)
1593
- mode : str
1594
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1595
- exponential_backoff : bool
1596
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1597
- pool : str
1598
- the slot pool this task should run in,
1599
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1600
- soft_fail : bool
1601
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1602
- name : str
1603
- Name of the sensor on Airflow
1604
- description : str
1605
- Description of sensor in the Airflow UI
1606
- external_dag_id : str
1607
- The dag_id that contains the task you want to wait for.
1608
- external_task_ids : List[str]
1609
- The list of task_ids that you want to wait for.
1610
- If None (default value) the sensor waits for the DAG. (Default: None)
1611
- allowed_states : List[str]
1612
- Iterable of allowed states, (Default: ['success'])
1613
- failed_states : List[str]
1614
- Iterable of failed or dis-allowed states. (Default: None)
1615
- execution_delta : datetime.timedelta
1616
- time difference with the previous execution to look at,
1617
- the default is the same logical date as the current task or DAG. (Default: None)
1618
- check_existence: bool
1619
- Set to True to check if the external task exists or check if
1620
- the DAG to wait for exists. (Default: True)
1621
- """
1622
- ...
1623
-
1624
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1625
- """
1626
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1627
-
1628
- User code call
1629
- -----------
1630
- @nim(
1631
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1632
- backend='managed'
1633
- )
1634
-
1635
- Valid backend options
1636
- ---------------------
1637
- - 'managed': Outerbounds selects a compute provider based on the model.
1638
- - 🚧 'dataplane': Run in your account.
1639
-
1640
- Valid model options
1641
- ----------------
1642
- - 'meta/llama3-8b-instruct': 8B parameter model
1643
- - 'meta/llama3-70b-instruct': 70B parameter model
1644
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1645
-
1646
- Parameters
1647
- ----------
1648
- models: list[NIM]
1649
- List of NIM containers running models in sidecars.
1650
- backend: str
1651
- Compute provider to run the NIM container.
1652
- """
1653
- ...
1654
-
1655
1686
  @typing.overload
1656
1687
  def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1657
1688
  """
@@ -1701,52 +1732,21 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1701
1732
  """
1702
1733
  ...
1703
1734
 
1704
- @typing.overload
1705
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1735
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1706
1736
  """
1707
- Specifies the Conda environment for all steps of the flow.
1737
+ Specifies what flows belong to the same project.
1708
1738
 
1709
- Use `@conda_base` to set common libraries required by all
1710
- steps and use `@conda` to specify step-specific additions.
1739
+ A project-specific namespace is created for all flows that
1740
+ use the same `@project(name)`.
1711
1741
 
1712
1742
  Parameters
1713
1743
  ----------
1714
- packages : Dict[str, str], default {}
1715
- Packages to use for this flow. The key is the name of the package
1716
- and the value is the version to use.
1717
- libraries : Dict[str, str], default {}
1718
- Supported for backward compatibility. When used with packages, packages will take precedence.
1719
- python : str, optional, default None
1720
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1721
- that the version used will correspond to the version of the Python interpreter used to start the run.
1722
- disabled : bool, default False
1723
- If set to True, disables Conda.
1724
- """
1725
- ...
1726
-
1727
- @typing.overload
1728
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1729
- ...
1730
-
1731
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1732
- """
1733
- Specifies the Conda environment for all steps of the flow.
1744
+ name : str
1745
+ Project name. Make sure that the name is unique amongst all
1746
+ projects that use the same production scheduler. The name may
1747
+ contain only lowercase alphanumeric characters and underscores.
1734
1748
 
1735
- Use `@conda_base` to set common libraries required by all
1736
- steps and use `@conda` to specify step-specific additions.
1737
1749
 
1738
- Parameters
1739
- ----------
1740
- packages : Dict[str, str], default {}
1741
- Packages to use for this flow. The key is the name of the package
1742
- and the value is the version to use.
1743
- libraries : Dict[str, str], default {}
1744
- Supported for backward compatibility. When used with packages, packages will take precedence.
1745
- python : str, optional, default None
1746
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1747
- that the version used will correspond to the version of the Python interpreter used to start the run.
1748
- disabled : bool, default False
1749
- If set to True, disables Conda.
1750
1750
  """
1751
1751
  ...
1752
1752