metaflow-stubs 2.12.13__py2.py3-none-any.whl → 2.12.15__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. metaflow-stubs/__init__.pyi +358 -358
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/metadata/metadata.pyi +3 -3
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +23 -23
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +8 -6
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
  37. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +20 -5
  38. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  40. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  48. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +19 -4
  58. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  61. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  62. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  63. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  64. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  65. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
  67. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  68. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  73. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  80. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  81. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  84. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  85. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  86. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  90. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  91. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  92. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  93. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  94. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  95. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  96. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  99. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  101. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  104. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  106. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +6 -2
  107. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  108. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  112. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  113. metaflow-stubs/plugins/package_cli.pyi +2 -2
  114. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  117. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  119. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +6 -2
  120. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  121. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  122. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  124. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  125. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  126. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  128. metaflow-stubs/plugins/tag_cli.pyi +5 -5
  129. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  130. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  131. metaflow-stubs/procpoll.pyi +2 -2
  132. metaflow-stubs/pylint_wrapper.pyi +2 -2
  133. metaflow-stubs/runner/__init__.pyi +2 -2
  134. metaflow-stubs/runner/deployer.pyi +21 -6
  135. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  136. metaflow-stubs/runner/nbdeploy.pyi +4 -4
  137. metaflow-stubs/runner/nbrun.pyi +4 -4
  138. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  139. metaflow-stubs/runner/utils.pyi +2 -2
  140. metaflow-stubs/system/__init__.pyi +4 -4
  141. metaflow-stubs/system/system_logger.pyi +3 -3
  142. metaflow-stubs/system/system_monitor.pyi +3 -3
  143. metaflow-stubs/tagging_util.pyi +2 -2
  144. metaflow-stubs/tuple_util.pyi +2 -2
  145. metaflow-stubs/version.pyi +2 -2
  146. {metaflow_stubs-2.12.13.dist-info → metaflow_stubs-2.12.15.dist-info}/METADATA +2 -2
  147. metaflow_stubs-2.12.15.dist-info/RECORD +150 -0
  148. metaflow_stubs-2.12.13.dist-info/RECORD +0 -150
  149. {metaflow_stubs-2.12.13.dist-info → metaflow_stubs-2.12.15.dist-info}/WHEEL +0 -0
  150. {metaflow_stubs-2.12.13.dist-info → metaflow_stubs-2.12.15.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.13 #
4
- # Generated on 2024-08-21T05:55:53.002542 #
3
+ # MF version: 2.12.15 #
4
+ # Generated on 2024-08-22T20:18:55.258441 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
- import metaflow.client.core
11
+ import metaflow.runner.metaflow_runner
13
12
  import metaflow.parameters
14
13
  import io
15
- import metaflow.metaflow_current
16
14
  import metaflow.flowspec
17
- import datetime
15
+ import metaflow.client.core
18
16
  import metaflow.events
17
+ import datetime
19
18
  import metaflow.datastore.inputs
19
+ import typing
20
20
  import metaflow.plugins.datatools.s3.s3
21
- import metaflow.runner.metaflow_runner
22
21
  import metaflow._vendor.click.types
22
+ import metaflow.metaflow_current
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -727,112 +727,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
727
727
  """
728
728
  ...
729
729
 
730
- @typing.overload
731
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
- """
733
- Specifies the PyPI packages for the step.
734
-
735
- Information in this decorator will augment any
736
- attributes set in the `@pyi_base` flow-level decorator. Hence,
737
- you can use `@pypi_base` to set packages required by all
738
- steps and use `@pypi` to specify step-specific overrides.
739
-
740
- Parameters
741
- ----------
742
- packages : Dict[str, str], default: {}
743
- Packages to use for this step. The key is the name of the package
744
- and the value is the version to use.
745
- python : str, optional, default: None
746
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
747
- that the version used will correspond to the version of the Python interpreter used to start the run.
748
- """
749
- ...
750
-
751
- @typing.overload
752
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
753
- ...
754
-
755
- @typing.overload
756
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
757
- ...
758
-
759
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
760
- """
761
- Specifies the PyPI packages for the step.
762
-
763
- Information in this decorator will augment any
764
- attributes set in the `@pyi_base` flow-level decorator. Hence,
765
- you can use `@pypi_base` to set packages required by all
766
- steps and use `@pypi` to specify step-specific overrides.
767
-
768
- Parameters
769
- ----------
770
- packages : Dict[str, str], default: {}
771
- Packages to use for this step. The key is the name of the package
772
- and the value is the version to use.
773
- python : str, optional, default: None
774
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
775
- that the version used will correspond to the version of the Python interpreter used to start the run.
776
- """
777
- ...
778
-
779
- @typing.overload
780
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
781
- """
782
- Specifies the Conda environment for the step.
783
-
784
- Information in this decorator will augment any
785
- attributes set in the `@conda_base` flow-level decorator. Hence,
786
- you can use `@conda_base` to set packages required by all
787
- steps and use `@conda` to specify step-specific overrides.
788
-
789
- Parameters
790
- ----------
791
- packages : Dict[str, str], default {}
792
- Packages to use for this step. The key is the name of the package
793
- and the value is the version to use.
794
- libraries : Dict[str, str], default {}
795
- Supported for backward compatibility. When used with packages, packages will take precedence.
796
- python : str, optional, default None
797
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
798
- that the version used will correspond to the version of the Python interpreter used to start the run.
799
- disabled : bool, default False
800
- If set to True, disables @conda.
801
- """
802
- ...
803
-
804
- @typing.overload
805
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
806
- ...
807
-
808
- @typing.overload
809
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
810
- ...
811
-
812
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
813
- """
814
- Specifies the Conda environment for the step.
815
-
816
- Information in this decorator will augment any
817
- attributes set in the `@conda_base` flow-level decorator. Hence,
818
- you can use `@conda_base` to set packages required by all
819
- steps and use `@conda` to specify step-specific overrides.
820
-
821
- Parameters
822
- ----------
823
- packages : Dict[str, str], default {}
824
- Packages to use for this step. The key is the name of the package
825
- and the value is the version to use.
826
- libraries : Dict[str, str], default {}
827
- Supported for backward compatibility. When used with packages, packages will take precedence.
828
- python : str, optional, default None
829
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
830
- that the version used will correspond to the version of the Python interpreter used to start the run.
831
- disabled : bool, default False
832
- If set to True, disables @conda.
833
- """
834
- ...
835
-
836
730
  def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
837
731
  """
838
732
  Specifies that this step should execute on Kubernetes.
@@ -949,59 +843,51 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
949
843
  ...
950
844
 
951
845
  @typing.overload
952
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
846
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
953
847
  """
954
- Specifies a timeout for your step.
955
-
956
- This decorator is useful if this step may hang indefinitely.
957
-
958
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
959
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
960
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
848
+ Specifies that the step will success under all circumstances.
961
849
 
962
- Note that all the values specified in parameters are added together so if you specify
963
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
850
+ The decorator will create an optional artifact, specified by `var`, which
851
+ contains the exception raised. You can use it to detect the presence
852
+ of errors, indicating that all happy-path artifacts produced by the step
853
+ are missing.
964
854
 
965
855
  Parameters
966
856
  ----------
967
- seconds : int, default 0
968
- Number of seconds to wait prior to timing out.
969
- minutes : int, default 0
970
- Number of minutes to wait prior to timing out.
971
- hours : int, default 0
972
- Number of hours to wait prior to timing out.
857
+ var : str, optional, default None
858
+ Name of the artifact in which to store the caught exception.
859
+ If not specified, the exception is not stored.
860
+ print_exception : bool, default True
861
+ Determines whether or not the exception is printed to
862
+ stdout when caught.
973
863
  """
974
864
  ...
975
865
 
976
866
  @typing.overload
977
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
867
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
978
868
  ...
979
869
 
980
870
  @typing.overload
981
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
871
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
982
872
  ...
983
873
 
984
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
874
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
985
875
  """
986
- Specifies a timeout for your step.
987
-
988
- This decorator is useful if this step may hang indefinitely.
989
-
990
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
991
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
992
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
876
+ Specifies that the step will success under all circumstances.
993
877
 
994
- Note that all the values specified in parameters are added together so if you specify
995
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
878
+ The decorator will create an optional artifact, specified by `var`, which
879
+ contains the exception raised. You can use it to detect the presence
880
+ of errors, indicating that all happy-path artifacts produced by the step
881
+ are missing.
996
882
 
997
883
  Parameters
998
884
  ----------
999
- seconds : int, default 0
1000
- Number of seconds to wait prior to timing out.
1001
- minutes : int, default 0
1002
- Number of minutes to wait prior to timing out.
1003
- hours : int, default 0
1004
- Number of hours to wait prior to timing out.
885
+ var : str, optional, default None
886
+ Name of the artifact in which to store the caught exception.
887
+ If not specified, the exception is not stored.
888
+ print_exception : bool, default True
889
+ Determines whether or not the exception is printed to
890
+ stdout when caught.
1005
891
  """
1006
892
  ...
1007
893
 
@@ -1083,73 +969,118 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1083
969
  ...
1084
970
 
1085
971
  @typing.overload
1086
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
972
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1087
973
  """
1088
- Specifies secrets to be retrieved and injected as environment variables prior to
1089
- the execution of a step.
974
+ Specifies environment variables to be set prior to the execution of a step.
1090
975
 
1091
976
  Parameters
1092
977
  ----------
1093
- sources : List[Union[str, Dict[str, Any]]], default: []
1094
- List of secret specs, defining how the secrets are to be retrieved
978
+ vars : Dict[str, str], default {}
979
+ Dictionary of environment variables to set.
1095
980
  """
1096
981
  ...
1097
982
 
1098
983
  @typing.overload
1099
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
984
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1100
985
  ...
1101
986
 
1102
987
  @typing.overload
1103
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
988
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1104
989
  ...
1105
990
 
1106
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
991
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1107
992
  """
1108
- Specifies secrets to be retrieved and injected as environment variables prior to
1109
- the execution of a step.
993
+ Specifies environment variables to be set prior to the execution of a step.
1110
994
 
1111
995
  Parameters
1112
996
  ----------
1113
- sources : List[Union[str, Dict[str, Any]]], default: []
1114
- List of secret specs, defining how the secrets are to be retrieved
997
+ vars : Dict[str, str], default {}
998
+ Dictionary of environment variables to set.
1115
999
  """
1116
1000
  ...
1117
1001
 
1118
1002
  @typing.overload
1119
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1003
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1120
1004
  """
1121
- Creates a human-readable report, a Metaflow Card, after this step completes.
1005
+ Decorator prototype for all step decorators. This function gets specialized
1006
+ and imported for all decorators types by _import_plugin_decorators().
1007
+ """
1008
+ ...
1009
+
1010
+ @typing.overload
1011
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1012
+ ...
1013
+
1014
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1015
+ """
1016
+ Decorator prototype for all step decorators. This function gets specialized
1017
+ and imported for all decorators types by _import_plugin_decorators().
1018
+ """
1019
+ ...
1020
+
1021
+ @typing.overload
1022
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1023
+ """
1024
+ Specifies a timeout for your step.
1122
1025
 
1123
- Note that you may add multiple `@card` decorators in a step with different parameters.
1026
+ This decorator is useful if this step may hang indefinitely.
1124
1027
 
1125
- Parameters
1126
- ----------
1127
- type : str, default 'default'
1128
- Card type.
1129
- id : str, optional, default None
1130
- If multiple cards are present, use this id to identify this card.
1131
- options : Dict[str, Any], default {}
1132
- Options passed to the card. The contents depend on the card type.
1133
- timeout : int, default 45
1134
- Interrupt reporting if it takes more than this many seconds.
1028
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1029
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1030
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1135
1031
 
1032
+ Note that all the values specified in parameters are added together so if you specify
1033
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1136
1034
 
1035
+ Parameters
1036
+ ----------
1037
+ seconds : int, default 0
1038
+ Number of seconds to wait prior to timing out.
1039
+ minutes : int, default 0
1040
+ Number of minutes to wait prior to timing out.
1041
+ hours : int, default 0
1042
+ Number of hours to wait prior to timing out.
1137
1043
  """
1138
1044
  ...
1139
1045
 
1140
1046
  @typing.overload
1141
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1047
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1142
1048
  ...
1143
1049
 
1144
1050
  @typing.overload
1145
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1051
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1146
1052
  ...
1147
1053
 
1148
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1054
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1149
1055
  """
1150
- Creates a human-readable report, a Metaflow Card, after this step completes.
1056
+ Specifies a timeout for your step.
1151
1057
 
1152
- Note that you may add multiple `@card` decorators in a step with different parameters.
1058
+ This decorator is useful if this step may hang indefinitely.
1059
+
1060
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1061
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1062
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1063
+
1064
+ Note that all the values specified in parameters are added together so if you specify
1065
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1066
+
1067
+ Parameters
1068
+ ----------
1069
+ seconds : int, default 0
1070
+ Number of seconds to wait prior to timing out.
1071
+ minutes : int, default 0
1072
+ Number of minutes to wait prior to timing out.
1073
+ hours : int, default 0
1074
+ Number of hours to wait prior to timing out.
1075
+ """
1076
+ ...
1077
+
1078
+ @typing.overload
1079
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1080
+ """
1081
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1082
+
1083
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1153
1084
 
1154
1085
  Parameters
1155
1086
  ----------
@@ -1167,33 +1098,31 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1167
1098
  ...
1168
1099
 
1169
1100
  @typing.overload
1170
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1171
- """
1172
- Specifies environment variables to be set prior to the execution of a step.
1173
-
1174
- Parameters
1175
- ----------
1176
- vars : Dict[str, str], default {}
1177
- Dictionary of environment variables to set.
1178
- """
1179
- ...
1180
-
1181
- @typing.overload
1182
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1101
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1183
1102
  ...
1184
1103
 
1185
1104
  @typing.overload
1186
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1105
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1187
1106
  ...
1188
1107
 
1189
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1108
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1190
1109
  """
1191
- Specifies environment variables to be set prior to the execution of a step.
1110
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1111
+
1112
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1192
1113
 
1193
1114
  Parameters
1194
1115
  ----------
1195
- vars : Dict[str, str], default {}
1196
- Dictionary of environment variables to set.
1116
+ type : str, default 'default'
1117
+ Card type.
1118
+ id : str, optional, default None
1119
+ If multiple cards are present, use this id to identify this card.
1120
+ options : Dict[str, Any], default {}
1121
+ Options passed to the card. The contents depend on the card type.
1122
+ timeout : int, default 45
1123
+ Interrupt reporting if it takes more than this many seconds.
1124
+
1125
+
1197
1126
  """
1198
1127
  ...
1199
1128
 
@@ -1345,109 +1274,190 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1345
1274
  ...
1346
1275
 
1347
1276
  @typing.overload
1348
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1277
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1349
1278
  """
1350
- Decorator prototype for all step decorators. This function gets specialized
1351
- and imported for all decorators types by _import_plugin_decorators().
1279
+ Specifies the PyPI packages for the step.
1280
+
1281
+ Information in this decorator will augment any
1282
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1283
+ you can use `@pypi_base` to set packages required by all
1284
+ steps and use `@pypi` to specify step-specific overrides.
1285
+
1286
+ Parameters
1287
+ ----------
1288
+ packages : Dict[str, str], default: {}
1289
+ Packages to use for this step. The key is the name of the package
1290
+ and the value is the version to use.
1291
+ python : str, optional, default: None
1292
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1293
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1352
1294
  """
1353
1295
  ...
1354
1296
 
1355
1297
  @typing.overload
1356
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1298
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1357
1299
  ...
1358
1300
 
1359
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1301
+ @typing.overload
1302
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1303
+ ...
1304
+
1305
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1360
1306
  """
1361
- Decorator prototype for all step decorators. This function gets specialized
1362
- and imported for all decorators types by _import_plugin_decorators().
1307
+ Specifies the PyPI packages for the step.
1308
+
1309
+ Information in this decorator will augment any
1310
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1311
+ you can use `@pypi_base` to set packages required by all
1312
+ steps and use `@pypi` to specify step-specific overrides.
1313
+
1314
+ Parameters
1315
+ ----------
1316
+ packages : Dict[str, str], default: {}
1317
+ Packages to use for this step. The key is the name of the package
1318
+ and the value is the version to use.
1319
+ python : str, optional, default: None
1320
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1321
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1363
1322
  """
1364
1323
  ...
1365
1324
 
1366
1325
  @typing.overload
1367
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1326
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1368
1327
  """
1369
- Specifies that the step will success under all circumstances.
1370
-
1371
- The decorator will create an optional artifact, specified by `var`, which
1372
- contains the exception raised. You can use it to detect the presence
1373
- of errors, indicating that all happy-path artifacts produced by the step
1374
- are missing.
1328
+ Specifies secrets to be retrieved and injected as environment variables prior to
1329
+ the execution of a step.
1375
1330
 
1376
1331
  Parameters
1377
1332
  ----------
1378
- var : str, optional, default None
1379
- Name of the artifact in which to store the caught exception.
1380
- If not specified, the exception is not stored.
1381
- print_exception : bool, default True
1382
- Determines whether or not the exception is printed to
1383
- stdout when caught.
1333
+ sources : List[Union[str, Dict[str, Any]]], default: []
1334
+ List of secret specs, defining how the secrets are to be retrieved
1384
1335
  """
1385
1336
  ...
1386
1337
 
1387
1338
  @typing.overload
1388
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1339
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1389
1340
  ...
1390
1341
 
1391
1342
  @typing.overload
1392
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1343
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1393
1344
  ...
1394
1345
 
1395
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1346
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1396
1347
  """
1397
- Specifies that the step will success under all circumstances.
1398
-
1399
- The decorator will create an optional artifact, specified by `var`, which
1400
- contains the exception raised. You can use it to detect the presence
1401
- of errors, indicating that all happy-path artifacts produced by the step
1402
- are missing.
1348
+ Specifies secrets to be retrieved and injected as environment variables prior to
1349
+ the execution of a step.
1403
1350
 
1404
1351
  Parameters
1405
1352
  ----------
1406
- var : str, optional, default None
1407
- Name of the artifact in which to store the caught exception.
1408
- If not specified, the exception is not stored.
1409
- print_exception : bool, default True
1410
- Determines whether or not the exception is printed to
1411
- stdout when caught.
1353
+ sources : List[Union[str, Dict[str, Any]]], default: []
1354
+ List of secret specs, defining how the secrets are to be retrieved
1412
1355
  """
1413
1356
  ...
1414
1357
 
1415
1358
  @typing.overload
1416
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1359
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1417
1360
  """
1418
- Specifies the PyPI packages for all steps of the flow.
1361
+ Specifies the Conda environment for the step.
1362
+
1363
+ Information in this decorator will augment any
1364
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1365
+ you can use `@conda_base` to set packages required by all
1366
+ steps and use `@conda` to specify step-specific overrides.
1419
1367
 
1420
- Use `@pypi_base` to set common packages required by all
1421
- steps and use `@pypi` to specify step-specific overrides.
1422
1368
  Parameters
1423
1369
  ----------
1424
- packages : Dict[str, str], default: {}
1425
- Packages to use for this flow. The key is the name of the package
1370
+ packages : Dict[str, str], default {}
1371
+ Packages to use for this step. The key is the name of the package
1426
1372
  and the value is the version to use.
1427
- python : str, optional, default: None
1373
+ libraries : Dict[str, str], default {}
1374
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1375
+ python : str, optional, default None
1428
1376
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1429
1377
  that the version used will correspond to the version of the Python interpreter used to start the run.
1378
+ disabled : bool, default False
1379
+ If set to True, disables @conda.
1430
1380
  """
1431
1381
  ...
1432
1382
 
1433
1383
  @typing.overload
1434
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1384
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1435
1385
  ...
1436
1386
 
1437
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1387
+ @typing.overload
1388
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1389
+ ...
1390
+
1391
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1438
1392
  """
1439
- Specifies the PyPI packages for all steps of the flow.
1393
+ Specifies the Conda environment for the step.
1394
+
1395
+ Information in this decorator will augment any
1396
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1397
+ you can use `@conda_base` to set packages required by all
1398
+ steps and use `@conda` to specify step-specific overrides.
1440
1399
 
1441
- Use `@pypi_base` to set common packages required by all
1442
- steps and use `@pypi` to specify step-specific overrides.
1443
1400
  Parameters
1444
1401
  ----------
1445
- packages : Dict[str, str], default: {}
1446
- Packages to use for this flow. The key is the name of the package
1402
+ packages : Dict[str, str], default {}
1403
+ Packages to use for this step. The key is the name of the package
1447
1404
  and the value is the version to use.
1448
- python : str, optional, default: None
1405
+ libraries : Dict[str, str], default {}
1406
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1407
+ python : str, optional, default None
1449
1408
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1450
1409
  that the version used will correspond to the version of the Python interpreter used to start the run.
1410
+ disabled : bool, default False
1411
+ If set to True, disables @conda.
1412
+ """
1413
+ ...
1414
+
1415
+ @typing.overload
1416
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1417
+ """
1418
+ Specifies the times when the flow should be run when running on a
1419
+ production scheduler.
1420
+
1421
+ Parameters
1422
+ ----------
1423
+ hourly : bool, default False
1424
+ Run the workflow hourly.
1425
+ daily : bool, default True
1426
+ Run the workflow daily.
1427
+ weekly : bool, default False
1428
+ Run the workflow weekly.
1429
+ cron : str, optional, default None
1430
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1431
+ specified by this expression.
1432
+ timezone : str, optional, default None
1433
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1434
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1435
+ """
1436
+ ...
1437
+
1438
+ @typing.overload
1439
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1440
+ ...
1441
+
1442
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1443
+ """
1444
+ Specifies the times when the flow should be run when running on a
1445
+ production scheduler.
1446
+
1447
+ Parameters
1448
+ ----------
1449
+ hourly : bool, default False
1450
+ Run the workflow hourly.
1451
+ daily : bool, default True
1452
+ Run the workflow daily.
1453
+ weekly : bool, default False
1454
+ Run the workflow weekly.
1455
+ cron : str, optional, default None
1456
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1457
+ specified by this expression.
1458
+ timezone : str, optional, default None
1459
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1460
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1451
1461
  """
1452
1462
  ...
1453
1463
 
@@ -1554,115 +1564,6 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1554
1564
  """
1555
1565
  ...
1556
1566
 
1557
- @typing.overload
1558
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1559
- """
1560
- Specifies the times when the flow should be run when running on a
1561
- production scheduler.
1562
-
1563
- Parameters
1564
- ----------
1565
- hourly : bool, default False
1566
- Run the workflow hourly.
1567
- daily : bool, default True
1568
- Run the workflow daily.
1569
- weekly : bool, default False
1570
- Run the workflow weekly.
1571
- cron : str, optional, default None
1572
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1573
- specified by this expression.
1574
- timezone : str, optional, default None
1575
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1576
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1577
- """
1578
- ...
1579
-
1580
- @typing.overload
1581
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1582
- ...
1583
-
1584
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1585
- """
1586
- Specifies the times when the flow should be run when running on a
1587
- production scheduler.
1588
-
1589
- Parameters
1590
- ----------
1591
- hourly : bool, default False
1592
- Run the workflow hourly.
1593
- daily : bool, default True
1594
- Run the workflow daily.
1595
- weekly : bool, default False
1596
- Run the workflow weekly.
1597
- cron : str, optional, default None
1598
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1599
- specified by this expression.
1600
- timezone : str, optional, default None
1601
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1602
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1603
- """
1604
- ...
1605
-
1606
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1607
- """
1608
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1609
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1610
-
1611
- Parameters
1612
- ----------
1613
- timeout : int
1614
- Time, in seconds before the task times out and fails. (Default: 3600)
1615
- poke_interval : int
1616
- Time in seconds that the job should wait in between each try. (Default: 60)
1617
- mode : str
1618
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1619
- exponential_backoff : bool
1620
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1621
- pool : str
1622
- the slot pool this task should run in,
1623
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1624
- soft_fail : bool
1625
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1626
- name : str
1627
- Name of the sensor on Airflow
1628
- description : str
1629
- Description of sensor in the Airflow UI
1630
- external_dag_id : str
1631
- The dag_id that contains the task you want to wait for.
1632
- external_task_ids : List[str]
1633
- The list of task_ids that you want to wait for.
1634
- If None (default value) the sensor waits for the DAG. (Default: None)
1635
- allowed_states : List[str]
1636
- Iterable of allowed states, (Default: ['success'])
1637
- failed_states : List[str]
1638
- Iterable of failed or dis-allowed states. (Default: None)
1639
- execution_delta : datetime.timedelta
1640
- time difference with the previous execution to look at,
1641
- the default is the same logical date as the current task or DAG. (Default: None)
1642
- check_existence: bool
1643
- Set to True to check if the external task exists or check if
1644
- the DAG to wait for exists. (Default: True)
1645
- """
1646
- ...
1647
-
1648
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1649
- """
1650
- Specifies what flows belong to the same project.
1651
-
1652
- A project-specific namespace is created for all flows that
1653
- use the same `@project(name)`.
1654
-
1655
- Parameters
1656
- ----------
1657
- name : str
1658
- Project name. Make sure that the name is unique amongst all
1659
- projects that use the same production scheduler. The name may
1660
- contain only lowercase alphanumeric characters and underscores.
1661
-
1662
-
1663
- """
1664
- ...
1665
-
1666
1567
  @typing.overload
1667
1568
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1668
1569
  """
@@ -1807,6 +1708,66 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1807
1708
  """
1808
1709
  ...
1809
1710
 
1711
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1712
+ """
1713
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1714
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1715
+
1716
+ Parameters
1717
+ ----------
1718
+ timeout : int
1719
+ Time, in seconds before the task times out and fails. (Default: 3600)
1720
+ poke_interval : int
1721
+ Time in seconds that the job should wait in between each try. (Default: 60)
1722
+ mode : str
1723
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1724
+ exponential_backoff : bool
1725
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1726
+ pool : str
1727
+ the slot pool this task should run in,
1728
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1729
+ soft_fail : bool
1730
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1731
+ name : str
1732
+ Name of the sensor on Airflow
1733
+ description : str
1734
+ Description of sensor in the Airflow UI
1735
+ external_dag_id : str
1736
+ The dag_id that contains the task you want to wait for.
1737
+ external_task_ids : List[str]
1738
+ The list of task_ids that you want to wait for.
1739
+ If None (default value) the sensor waits for the DAG. (Default: None)
1740
+ allowed_states : List[str]
1741
+ Iterable of allowed states, (Default: ['success'])
1742
+ failed_states : List[str]
1743
+ Iterable of failed or dis-allowed states. (Default: None)
1744
+ execution_delta : datetime.timedelta
1745
+ time difference with the previous execution to look at,
1746
+ the default is the same logical date as the current task or DAG. (Default: None)
1747
+ check_existence: bool
1748
+ Set to True to check if the external task exists or check if
1749
+ the DAG to wait for exists. (Default: True)
1750
+ """
1751
+ ...
1752
+
1753
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1754
+ """
1755
+ Specifies what flows belong to the same project.
1756
+
1757
+ A project-specific namespace is created for all flows that
1758
+ use the same `@project(name)`.
1759
+
1760
+ Parameters
1761
+ ----------
1762
+ name : str
1763
+ Project name. Make sure that the name is unique amongst all
1764
+ projects that use the same production scheduler. The name may
1765
+ contain only lowercase alphanumeric characters and underscores.
1766
+
1767
+
1768
+ """
1769
+ ...
1770
+
1810
1771
  def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1811
1772
  """
1812
1773
  The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
@@ -1849,6 +1810,45 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1849
1810
  """
1850
1811
  ...
1851
1812
 
1813
+ @typing.overload
1814
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1815
+ """
1816
+ Specifies the PyPI packages for all steps of the flow.
1817
+
1818
+ Use `@pypi_base` to set common packages required by all
1819
+ steps and use `@pypi` to specify step-specific overrides.
1820
+ Parameters
1821
+ ----------
1822
+ packages : Dict[str, str], default: {}
1823
+ Packages to use for this flow. The key is the name of the package
1824
+ and the value is the version to use.
1825
+ python : str, optional, default: None
1826
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1827
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1828
+ """
1829
+ ...
1830
+
1831
+ @typing.overload
1832
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1833
+ ...
1834
+
1835
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1836
+ """
1837
+ Specifies the PyPI packages for all steps of the flow.
1838
+
1839
+ Use `@pypi_base` to set common packages required by all
1840
+ steps and use `@pypi` to specify step-specific overrides.
1841
+ Parameters
1842
+ ----------
1843
+ packages : Dict[str, str], default: {}
1844
+ Packages to use for this flow. The key is the name of the package
1845
+ and the value is the version to use.
1846
+ python : str, optional, default: None
1847
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1848
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1849
+ """
1850
+ ...
1851
+
1852
1852
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1853
1853
  """
1854
1854
  Switch namespace to the one provided.
@@ -2800,7 +2800,7 @@ class DataArtifact(metaflow.client.core.MetaflowObject, metaclass=type):
2800
2800
  ...
2801
2801
 
2802
2802
  class Runner(object, metaclass=type):
2803
- def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, **kwargs):
2803
+ def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
2804
2804
  ...
2805
2805
  def __enter__(self) -> metaflow.runner.metaflow_runner.Runner:
2806
2806
  ...
@@ -2892,7 +2892,7 @@ class Runner(object, metaclass=type):
2892
2892
  ...
2893
2893
 
2894
2894
  class NBRunner(object, metaclass=type):
2895
- def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", **kwargs):
2895
+ def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", file_read_timeout: int = 3600, **kwargs):
2896
2896
  ...
2897
2897
  def nbrun(self, **kwargs):
2898
2898
  """
@@ -2998,7 +2998,7 @@ class NBRunner(object, metaclass=type):
2998
2998
  ...
2999
2999
 
3000
3000
  class Deployer(object, metaclass=type):
3001
- def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, **kwargs):
3001
+ def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
3002
3002
  ...
3003
3003
  def _Deployer__make_function(self, deployer_class):
3004
3004
  """
@@ -3018,7 +3018,7 @@ class Deployer(object, metaclass=type):
3018
3018
  ...
3019
3019
 
3020
3020
  class NBDeployer(object, metaclass=type):
3021
- def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", **kwargs):
3021
+ def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", file_read_timeout: int = 3600, **kwargs):
3022
3022
  ...
3023
3023
  def cleanup(self):
3024
3024
  """