ob-metaflow-stubs 6.0.3.114__py2.py3-none-any.whl → 6.0.3.115__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +498 -498
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +3 -3
  4. metaflow-stubs/client/__init__.pyi +5 -5
  5. metaflow-stubs/client/core.pyi +11 -9
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +5 -5
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +6 -3
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +68 -68
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +3 -3
  21. metaflow-stubs/plugins/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +5 -5
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +5 -5
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  61. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  74. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +4 -4
  76. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +7 -7
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +5 -5
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +3 -3
  113. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +4 -4
  126. metaflow-stubs/runner/metaflow_runner.pyi +5 -33
  127. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  128. metaflow-stubs/runner/nbrun.pyi +2 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  130. metaflow-stubs/runner/utils.pyi +2 -2
  131. metaflow-stubs/system/__init__.pyi +4 -4
  132. metaflow-stubs/system/system_logger.pyi +3 -3
  133. metaflow-stubs/system/system_monitor.pyi +3 -3
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.114.dist-info → ob_metaflow_stubs-6.0.3.115.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.115.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.114.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.114.dist-info → ob_metaflow_stubs-6.0.3.115.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.114.dist-info → ob_metaflow_stubs-6.0.3.115.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.26.1+obcheckpoint(0.1.1);ob(v1) #
4
- # Generated on 2024-10-23T20:57:01.122821 #
3
+ # MF version: 2.12.27.1+obcheckpoint(0.1.1);ob(v1) #
4
+ # Generated on 2024-10-25T21:39:54.730102 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.flowspec
12
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
- import metaflow.datastore.inputs
14
- import metaflow.parameters
15
- import metaflow.events
16
- import metaflow.runner.metaflow_runner
17
- import metaflow.client.core
11
+ import typing
18
12
  import metaflow.metaflow_current
19
13
  import datetime
20
14
  import metaflow._vendor.click.types
21
- import typing
15
+ import metaflow.runner.metaflow_runner
16
+ import metaflow.parameters
17
+ import metaflow.events
18
+ import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
19
+ import metaflow.client.core
20
+ import metaflow.datastore.inputs
21
+ import metaflow.flowspec
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -512,6 +512,56 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
512
512
  """
513
513
  ...
514
514
 
515
+ @typing.overload
516
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
517
+ """
518
+ Internal decorator to support Fast bakery
519
+ """
520
+ ...
521
+
522
+ @typing.overload
523
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
524
+ ...
525
+
526
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
527
+ """
528
+ Internal decorator to support Fast bakery
529
+ """
530
+ ...
531
+
532
+ @typing.overload
533
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
534
+ """
535
+ Specifies secrets to be retrieved and injected as environment variables prior to
536
+ the execution of a step.
537
+
538
+ Parameters
539
+ ----------
540
+ sources : List[Union[str, Dict[str, Any]]], default: []
541
+ List of secret specs, defining how the secrets are to be retrieved
542
+ """
543
+ ...
544
+
545
+ @typing.overload
546
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
547
+ ...
548
+
549
+ @typing.overload
550
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
551
+ ...
552
+
553
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
554
+ """
555
+ Specifies secrets to be retrieved and injected as environment variables prior to
556
+ the execution of a step.
557
+
558
+ Parameters
559
+ ----------
560
+ sources : List[Union[str, Dict[str, Any]]], default: []
561
+ List of secret specs, defining how the secrets are to be retrieved
562
+ """
563
+ ...
564
+
515
565
  @typing.overload
516
566
  def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
517
567
  """
@@ -574,228 +624,124 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
574
624
  ...
575
625
 
576
626
  @typing.overload
577
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
627
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
578
628
  """
579
- Specifies secrets to be retrieved and injected as environment variables prior to
580
- the execution of a step.
629
+ Enables loading / saving of models within a step.
630
+
581
631
 
582
632
  Parameters
583
633
  ----------
584
- sources : List[Union[str, Dict[str, Any]]], default: []
585
- List of secret specs, defining how the secrets are to be retrieved
634
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
635
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
636
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
637
+ - `current.checkpoint`
638
+ - `current.model`
639
+ - `current.huggingface_hub`
640
+
641
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
642
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
643
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
644
+
645
+ temp_dir_root : str, default: None
646
+ The root directory under which `current.model.loaded` will store loaded models
647
+
648
+
649
+
586
650
  """
587
651
  ...
588
652
 
589
653
  @typing.overload
590
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
654
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
591
655
  ...
592
656
 
593
657
  @typing.overload
594
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
658
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
595
659
  ...
596
660
 
597
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
661
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
598
662
  """
599
- Specifies secrets to be retrieved and injected as environment variables prior to
600
- the execution of a step.
663
+ Enables loading / saving of models within a step.
601
664
 
602
- Parameters
603
- ----------
604
- sources : List[Union[str, Dict[str, Any]]], default: []
605
- List of secret specs, defining how the secrets are to be retrieved
606
- """
607
- ...
608
-
609
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
610
- """
611
- Specifies that this step should execute on Kubernetes.
612
665
 
613
666
  Parameters
614
667
  ----------
615
- cpu : int, default 1
616
- Number of CPUs required for this step. If `@resources` is
617
- also present, the maximum value from all decorators is used.
618
- memory : int, default 4096
619
- Memory size (in MB) required for this step. If
620
- `@resources` is also present, the maximum value from all decorators is
621
- used.
622
- disk : int, default 10240
623
- Disk size (in MB) required for this step. If
624
- `@resources` is also present, the maximum value from all decorators is
625
- used.
626
- image : str, optional, default None
627
- Docker image to use when launching on Kubernetes. If not specified, and
628
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
629
- not, a default Docker image mapping to the current version of Python is used.
630
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
631
- If given, the imagePullPolicy to be applied to the Docker image of the step.
632
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
633
- Kubernetes service account to use when launching pod in Kubernetes.
634
- secrets : List[str], optional, default None
635
- Kubernetes secrets to use when launching pod in Kubernetes. These
636
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
637
- in Metaflow configuration.
638
- node_selector: Union[Dict[str,str], str], optional, default None
639
- Kubernetes node selector(s) to apply to the pod running the task.
640
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
641
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
642
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
643
- Kubernetes namespace to use when launching pod in Kubernetes.
644
- gpu : int, optional, default None
645
- Number of GPUs required for this step. A value of zero implies that
646
- the scheduled node should not have GPUs.
647
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
648
- The vendor of the GPUs to be used for this step.
649
- tolerations : List[str], default []
650
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
651
- Kubernetes tolerations to use when launching pod in Kubernetes.
652
- use_tmpfs : bool, default False
653
- This enables an explicit tmpfs mount for this step.
654
- tmpfs_tempdir : bool, default True
655
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
656
- tmpfs_size : int, optional, default: None
657
- The value for the size (in MiB) of the tmpfs mount for this step.
658
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
659
- memory allocated for this step.
660
- tmpfs_path : str, optional, default /metaflow_temp
661
- Path to tmpfs mount for this step.
662
- persistent_volume_claims : Dict[str, str], optional, default None
663
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
664
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
665
- shared_memory: int, optional
666
- Shared memory size (in MiB) required for this step
667
- port: int, optional
668
- Port number to specify in the Kubernetes job object
669
- compute_pool : str, optional, default None
670
- Compute pool to be used for for this step.
671
- If not specified, any accessible compute pool within the perimeter is used.
672
- hostname_resolution_timeout: int, default 10 * 60
673
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
674
- Only applicable when @parallel is used.
668
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
669
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
670
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
671
+ - `current.checkpoint`
672
+ - `current.model`
673
+ - `current.huggingface_hub`
674
+
675
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
676
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
677
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
678
+
679
+ temp_dir_root : str, default: None
680
+ The root directory under which `current.model.loaded` will store loaded models
681
+
682
+
683
+
675
684
  """
676
685
  ...
677
686
 
678
687
  @typing.overload
679
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
688
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
680
689
  """
681
- Specifies the Conda environment for the step.
690
+ Creates a human-readable report, a Metaflow Card, after this step completes.
682
691
 
683
- Information in this decorator will augment any
684
- attributes set in the `@conda_base` flow-level decorator. Hence,
685
- you can use `@conda_base` to set packages required by all
686
- steps and use `@conda` to specify step-specific overrides.
692
+ Note that you may add multiple `@card` decorators in a step with different parameters.
687
693
 
688
694
  Parameters
689
695
  ----------
690
- packages : Dict[str, str], default {}
691
- Packages to use for this step. The key is the name of the package
692
- and the value is the version to use.
693
- libraries : Dict[str, str], default {}
694
- Supported for backward compatibility. When used with packages, packages will take precedence.
695
- python : str, optional, default None
696
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
697
- that the version used will correspond to the version of the Python interpreter used to start the run.
698
- disabled : bool, default False
699
- If set to True, disables @conda.
696
+ type : str, default 'default'
697
+ Card type.
698
+ id : str, optional, default None
699
+ If multiple cards are present, use this id to identify this card.
700
+ options : Dict[str, Any], default {}
701
+ Options passed to the card. The contents depend on the card type.
702
+ timeout : int, default 45
703
+ Interrupt reporting if it takes more than this many seconds.
704
+
705
+
700
706
  """
701
707
  ...
702
708
 
703
709
  @typing.overload
704
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
710
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
705
711
  ...
706
712
 
707
713
  @typing.overload
708
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
714
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
709
715
  ...
710
716
 
711
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
717
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
712
718
  """
713
- Specifies the Conda environment for the step.
719
+ Creates a human-readable report, a Metaflow Card, after this step completes.
714
720
 
715
- Information in this decorator will augment any
716
- attributes set in the `@conda_base` flow-level decorator. Hence,
717
- you can use `@conda_base` to set packages required by all
718
- steps and use `@conda` to specify step-specific overrides.
721
+ Note that you may add multiple `@card` decorators in a step with different parameters.
719
722
 
720
723
  Parameters
721
724
  ----------
722
- packages : Dict[str, str], default {}
723
- Packages to use for this step. The key is the name of the package
724
- and the value is the version to use.
725
- libraries : Dict[str, str], default {}
726
- Supported for backward compatibility. When used with packages, packages will take precedence.
727
- python : str, optional, default None
728
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
729
- that the version used will correspond to the version of the Python interpreter used to start the run.
730
- disabled : bool, default False
731
- If set to True, disables @conda.
725
+ type : str, default 'default'
726
+ Card type.
727
+ id : str, optional, default None
728
+ If multiple cards are present, use this id to identify this card.
729
+ options : Dict[str, Any], default {}
730
+ Options passed to the card. The contents depend on the card type.
731
+ timeout : int, default 45
732
+ Interrupt reporting if it takes more than this many seconds.
733
+
734
+
732
735
  """
733
736
  ...
734
737
 
735
738
  @typing.overload
736
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
739
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
737
740
  """
738
- Specifies a timeout for your step.
741
+ Specifies the resources needed when executing this step.
739
742
 
740
- This decorator is useful if this step may hang indefinitely.
741
-
742
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
743
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
744
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
745
-
746
- Note that all the values specified in parameters are added together so if you specify
747
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
748
-
749
- Parameters
750
- ----------
751
- seconds : int, default 0
752
- Number of seconds to wait prior to timing out.
753
- minutes : int, default 0
754
- Number of minutes to wait prior to timing out.
755
- hours : int, default 0
756
- Number of hours to wait prior to timing out.
757
- """
758
- ...
759
-
760
- @typing.overload
761
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
762
- ...
763
-
764
- @typing.overload
765
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
766
- ...
767
-
768
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
769
- """
770
- Specifies a timeout for your step.
771
-
772
- This decorator is useful if this step may hang indefinitely.
773
-
774
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
775
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
776
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
777
-
778
- Note that all the values specified in parameters are added together so if you specify
779
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
780
-
781
- Parameters
782
- ----------
783
- seconds : int, default 0
784
- Number of seconds to wait prior to timing out.
785
- minutes : int, default 0
786
- Number of minutes to wait prior to timing out.
787
- hours : int, default 0
788
- Number of hours to wait prior to timing out.
789
- """
790
- ...
791
-
792
- @typing.overload
793
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
794
- """
795
- Specifies the resources needed when executing this step.
796
-
797
- Use `@resources` to specify the resource requirements
798
- independently of the specific compute layer (`@batch`, `@kubernetes`).
743
+ Use `@resources` to specify the resource requirements
744
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
799
745
 
800
746
  You can choose the compute layer on the command line by executing e.g.
801
747
  ```
@@ -812,7 +758,7 @@ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None,
812
758
  ----------
813
759
  cpu : int, default 1
814
760
  Number of CPUs required for this step.
815
- gpu : int, default 0
761
+ gpu : int, optional, default None
816
762
  Number of GPUs required for this step.
817
763
  disk : int, optional, default None
818
764
  Disk size (in MB) required for this step. Only applies on Kubernetes.
@@ -832,7 +778,7 @@ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.C
832
778
  def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
833
779
  ...
834
780
 
835
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
781
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
836
782
  """
837
783
  Specifies the resources needed when executing this step.
838
784
 
@@ -854,7 +800,7 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
854
800
  ----------
855
801
  cpu : int, default 1
856
802
  Number of CPUs required for this step.
857
- gpu : int, default 0
803
+ gpu : int, optional, default None
858
804
  Number of GPUs required for this step.
859
805
  disk : int, optional, default None
860
806
  Disk size (in MB) required for this step. Only applies on Kubernetes.
@@ -867,19 +813,90 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
867
813
  ...
868
814
 
869
815
  @typing.overload
870
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
816
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
871
817
  """
872
- Internal decorator to support Fast bakery
818
+ Specifies the Conda environment for the step.
819
+
820
+ Information in this decorator will augment any
821
+ attributes set in the `@conda_base` flow-level decorator. Hence,
822
+ you can use `@conda_base` to set packages required by all
823
+ steps and use `@conda` to specify step-specific overrides.
824
+
825
+ Parameters
826
+ ----------
827
+ packages : Dict[str, str], default {}
828
+ Packages to use for this step. The key is the name of the package
829
+ and the value is the version to use.
830
+ libraries : Dict[str, str], default {}
831
+ Supported for backward compatibility. When used with packages, packages will take precedence.
832
+ python : str, optional, default None
833
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
834
+ that the version used will correspond to the version of the Python interpreter used to start the run.
835
+ disabled : bool, default False
836
+ If set to True, disables @conda.
873
837
  """
874
838
  ...
875
839
 
876
840
  @typing.overload
877
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
841
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
878
842
  ...
879
843
 
880
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
844
+ @typing.overload
845
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
846
+ ...
847
+
848
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
881
849
  """
882
- Internal decorator to support Fast bakery
850
+ Specifies the Conda environment for the step.
851
+
852
+ Information in this decorator will augment any
853
+ attributes set in the `@conda_base` flow-level decorator. Hence,
854
+ you can use `@conda_base` to set packages required by all
855
+ steps and use `@conda` to specify step-specific overrides.
856
+
857
+ Parameters
858
+ ----------
859
+ packages : Dict[str, str], default {}
860
+ Packages to use for this step. The key is the name of the package
861
+ and the value is the version to use.
862
+ libraries : Dict[str, str], default {}
863
+ Supported for backward compatibility. When used with packages, packages will take precedence.
864
+ python : str, optional, default None
865
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
866
+ that the version used will correspond to the version of the Python interpreter used to start the run.
867
+ disabled : bool, default False
868
+ If set to True, disables @conda.
869
+ """
870
+ ...
871
+
872
+ @typing.overload
873
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
874
+ """
875
+ Specifies environment variables to be set prior to the execution of a step.
876
+
877
+ Parameters
878
+ ----------
879
+ vars : Dict[str, str], default {}
880
+ Dictionary of environment variables to set.
881
+ """
882
+ ...
883
+
884
+ @typing.overload
885
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
886
+ ...
887
+
888
+ @typing.overload
889
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
890
+ ...
891
+
892
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
893
+ """
894
+ Specifies environment variables to be set prior to the execution of a step.
895
+
896
+ Parameters
897
+ ----------
898
+ vars : Dict[str, str], default {}
899
+ Dictionary of environment variables to set.
883
900
  """
884
901
  ...
885
902
 
@@ -932,57 +949,6 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
932
949
  """
933
950
  ...
934
951
 
935
- @typing.overload
936
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
937
- """
938
- Creates a human-readable report, a Metaflow Card, after this step completes.
939
-
940
- Note that you may add multiple `@card` decorators in a step with different parameters.
941
-
942
- Parameters
943
- ----------
944
- type : str, default 'default'
945
- Card type.
946
- id : str, optional, default None
947
- If multiple cards are present, use this id to identify this card.
948
- options : Dict[str, Any], default {}
949
- Options passed to the card. The contents depend on the card type.
950
- timeout : int, default 45
951
- Interrupt reporting if it takes more than this many seconds.
952
-
953
-
954
- """
955
- ...
956
-
957
- @typing.overload
958
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
959
- ...
960
-
961
- @typing.overload
962
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
963
- ...
964
-
965
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
966
- """
967
- Creates a human-readable report, a Metaflow Card, after this step completes.
968
-
969
- Note that you may add multiple `@card` decorators in a step with different parameters.
970
-
971
- Parameters
972
- ----------
973
- type : str, default 'default'
974
- Card type.
975
- id : str, optional, default None
976
- If multiple cards are present, use this id to identify this card.
977
- options : Dict[str, Any], default {}
978
- Options passed to the card. The contents depend on the card type.
979
- timeout : int, default 45
980
- Interrupt reporting if it takes more than this many seconds.
981
-
982
-
983
- """
984
- ...
985
-
986
952
  @typing.overload
987
953
  def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
988
954
  """
@@ -1002,37 +968,6 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1002
968
  """
1003
969
  ...
1004
970
 
1005
- @typing.overload
1006
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1007
- """
1008
- Specifies environment variables to be set prior to the execution of a step.
1009
-
1010
- Parameters
1011
- ----------
1012
- vars : Dict[str, str], default {}
1013
- Dictionary of environment variables to set.
1014
- """
1015
- ...
1016
-
1017
- @typing.overload
1018
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1019
- ...
1020
-
1021
- @typing.overload
1022
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1023
- ...
1024
-
1025
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1026
- """
1027
- Specifies environment variables to be set prior to the execution of a step.
1028
-
1029
- Parameters
1030
- ----------
1031
- vars : Dict[str, str], default {}
1032
- Dictionary of environment variables to set.
1033
- """
1034
- ...
1035
-
1036
971
  @typing.overload
1037
972
  def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1038
973
  """
@@ -1082,6 +1017,132 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1082
1017
  """
1083
1018
  ...
1084
1019
 
1020
+ @typing.overload
1021
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1022
+ """
1023
+ Specifies a timeout for your step.
1024
+
1025
+ This decorator is useful if this step may hang indefinitely.
1026
+
1027
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1028
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1029
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1030
+
1031
+ Note that all the values specified in parameters are added together so if you specify
1032
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1033
+
1034
+ Parameters
1035
+ ----------
1036
+ seconds : int, default 0
1037
+ Number of seconds to wait prior to timing out.
1038
+ minutes : int, default 0
1039
+ Number of minutes to wait prior to timing out.
1040
+ hours : int, default 0
1041
+ Number of hours to wait prior to timing out.
1042
+ """
1043
+ ...
1044
+
1045
+ @typing.overload
1046
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1047
+ ...
1048
+
1049
+ @typing.overload
1050
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1051
+ ...
1052
+
1053
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1054
+ """
1055
+ Specifies a timeout for your step.
1056
+
1057
+ This decorator is useful if this step may hang indefinitely.
1058
+
1059
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1060
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1061
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1062
+
1063
+ Note that all the values specified in parameters are added together so if you specify
1064
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1065
+
1066
+ Parameters
1067
+ ----------
1068
+ seconds : int, default 0
1069
+ Number of seconds to wait prior to timing out.
1070
+ minutes : int, default 0
1071
+ Number of minutes to wait prior to timing out.
1072
+ hours : int, default 0
1073
+ Number of hours to wait prior to timing out.
1074
+ """
1075
+ ...
1076
+
1077
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1078
+ """
1079
+ Specifies that this step should execute on Kubernetes.
1080
+
1081
+ Parameters
1082
+ ----------
1083
+ cpu : int, default 1
1084
+ Number of CPUs required for this step. If `@resources` is
1085
+ also present, the maximum value from all decorators is used.
1086
+ memory : int, default 4096
1087
+ Memory size (in MB) required for this step. If
1088
+ `@resources` is also present, the maximum value from all decorators is
1089
+ used.
1090
+ disk : int, default 10240
1091
+ Disk size (in MB) required for this step. If
1092
+ `@resources` is also present, the maximum value from all decorators is
1093
+ used.
1094
+ image : str, optional, default None
1095
+ Docker image to use when launching on Kubernetes. If not specified, and
1096
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1097
+ not, a default Docker image mapping to the current version of Python is used.
1098
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1099
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1100
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1101
+ Kubernetes service account to use when launching pod in Kubernetes.
1102
+ secrets : List[str], optional, default None
1103
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1104
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1105
+ in Metaflow configuration.
1106
+ node_selector: Union[Dict[str,str], str], optional, default None
1107
+ Kubernetes node selector(s) to apply to the pod running the task.
1108
+ Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
1109
+ or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
1110
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1111
+ Kubernetes namespace to use when launching pod in Kubernetes.
1112
+ gpu : int, optional, default None
1113
+ Number of GPUs required for this step. A value of zero implies that
1114
+ the scheduled node should not have GPUs.
1115
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1116
+ The vendor of the GPUs to be used for this step.
1117
+ tolerations : List[str], default []
1118
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1119
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1120
+ use_tmpfs : bool, default False
1121
+ This enables an explicit tmpfs mount for this step.
1122
+ tmpfs_tempdir : bool, default True
1123
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1124
+ tmpfs_size : int, optional, default: None
1125
+ The value for the size (in MiB) of the tmpfs mount for this step.
1126
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1127
+ memory allocated for this step.
1128
+ tmpfs_path : str, optional, default /metaflow_temp
1129
+ Path to tmpfs mount for this step.
1130
+ persistent_volume_claims : Dict[str, str], optional, default None
1131
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1132
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1133
+ shared_memory: int, optional
1134
+ Shared memory size (in MiB) required for this step
1135
+ port: int, optional
1136
+ Port number to specify in the Kubernetes job object
1137
+ compute_pool : str, optional, default None
1138
+ Compute pool to be used for for this step.
1139
+ If not specified, any accessible compute pool within the perimeter is used.
1140
+ hostname_resolution_timeout: int, default 10 * 60
1141
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1142
+ Only applicable when @parallel is used.
1143
+ """
1144
+ ...
1145
+
1085
1146
  def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1086
1147
  """
1087
1148
  Decorator that helps cache, version and store models/datasets from huggingface hub.
@@ -1148,64 +1209,34 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1148
1209
  """
1149
1210
  ...
1150
1211
 
1151
- @typing.overload
1152
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1212
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1153
1213
  """
1154
- Enables loading / saving of models within a step.
1155
-
1156
-
1157
- Parameters
1158
- ----------
1159
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1160
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1161
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
1162
- - `current.checkpoint`
1163
- - `current.model`
1164
- - `current.huggingface_hub`
1165
-
1166
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1167
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1168
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1169
-
1170
- temp_dir_root : str, default: None
1171
- The root directory under which `current.model.loaded` will store loaded models
1172
-
1214
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1173
1215
 
1216
+ User code call
1217
+ -----------
1218
+ @nim(
1219
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1220
+ backend='managed'
1221
+ )
1174
1222
 
1175
- """
1176
- ...
1177
-
1178
- @typing.overload
1179
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1180
- ...
1181
-
1182
- @typing.overload
1183
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1184
- ...
1185
-
1186
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1187
- """
1188
- Enables loading / saving of models within a step.
1223
+ Valid backend options
1224
+ ---------------------
1225
+ - 'managed': Outerbounds selects a compute provider based on the model.
1226
+ - 🚧 'dataplane': Run in your account.
1189
1227
 
1228
+ Valid model options
1229
+ ----------------
1230
+ - 'meta/llama3-8b-instruct': 8B parameter model
1231
+ - 'meta/llama3-70b-instruct': 70B parameter model
1232
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1190
1233
 
1191
1234
  Parameters
1192
1235
  ----------
1193
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1194
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1195
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
1196
- - `current.checkpoint`
1197
- - `current.model`
1198
- - `current.huggingface_hub`
1199
-
1200
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1201
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1202
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1203
-
1204
- temp_dir_root : str, default: None
1205
- The root directory under which `current.model.loaded` will store loaded models
1206
-
1207
-
1208
-
1236
+ models: list[NIM]
1237
+ List of NIM containers running models in sidecars.
1238
+ backend: str
1239
+ Compute provider to run the NIM container.
1209
1240
  """
1210
1241
  ...
1211
1242
 
@@ -1278,84 +1309,35 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1278
1309
  when upstream runs within the same namespace complete successfully
1279
1310
 
1280
1311
  Additionally, you can specify project aware upstream flow dependencies
1281
- by specifying the fully qualified project_flow_name.
1282
- ```
1283
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1284
- ```
1285
- or
1286
- ```
1287
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1288
- ```
1289
-
1290
- You can also specify just the project or project branch (other values will be
1291
- inferred from the current project or project branch):
1292
- ```
1293
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1294
- ```
1295
-
1296
- Note that `branch` is typically one of:
1297
- - `prod`
1298
- - `user.bob`
1299
- - `test.my_experiment`
1300
- - `prod.staging`
1301
-
1302
- Parameters
1303
- ----------
1304
- flow : Union[str, Dict[str, str]], optional, default None
1305
- Upstream flow dependency for this flow.
1306
- flows : List[Union[str, Dict[str, str]]], default []
1307
- Upstream flow dependencies for this flow.
1308
- options : Dict[str, Any], default {}
1309
- Backend-specific configuration for tuning eventing behavior.
1310
-
1311
-
1312
- """
1313
- ...
1314
-
1315
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1316
- """
1317
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1318
-
1319
- User code call
1320
- -----------
1321
- @nim(
1322
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1323
- backend='managed'
1324
- )
1325
-
1326
- Valid backend options
1327
- ---------------------
1328
- - 'managed': Outerbounds selects a compute provider based on the model.
1329
- - 🚧 'dataplane': Run in your account.
1330
-
1331
- Valid model options
1332
- ----------------
1333
- - 'meta/llama3-8b-instruct': 8B parameter model
1334
- - 'meta/llama3-70b-instruct': 70B parameter model
1335
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1312
+ by specifying the fully qualified project_flow_name.
1313
+ ```
1314
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1315
+ ```
1316
+ or
1317
+ ```
1318
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1319
+ ```
1336
1320
 
1337
- Parameters
1338
- ----------
1339
- models: list[NIM]
1340
- List of NIM containers running models in sidecars.
1341
- backend: str
1342
- Compute provider to run the NIM container.
1343
- """
1344
- ...
1345
-
1346
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1347
- """
1348
- Specifies what flows belong to the same project.
1321
+ You can also specify just the project or project branch (other values will be
1322
+ inferred from the current project or project branch):
1323
+ ```
1324
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1325
+ ```
1349
1326
 
1350
- A project-specific namespace is created for all flows that
1351
- use the same `@project(name)`.
1327
+ Note that `branch` is typically one of:
1328
+ - `prod`
1329
+ - `user.bob`
1330
+ - `test.my_experiment`
1331
+ - `prod.staging`
1352
1332
 
1353
1333
  Parameters
1354
1334
  ----------
1355
- name : str
1356
- Project name. Make sure that the name is unique amongst all
1357
- projects that use the same production scheduler. The name may
1358
- contain only lowercase alphanumeric characters and underscores.
1335
+ flow : Union[str, Dict[str, str]], optional, default None
1336
+ Upstream flow dependency for this flow.
1337
+ flows : List[Union[str, Dict[str, str]]], default []
1338
+ Upstream flow dependencies for this flow.
1339
+ options : Dict[str, Any], default {}
1340
+ Backend-specific configuration for tuning eventing behavior.
1359
1341
 
1360
1342
 
1361
1343
  """
@@ -1403,104 +1385,6 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1403
1385
  """
1404
1386
  ...
1405
1387
 
1406
- @typing.overload
1407
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1408
- """
1409
- Specifies the Conda environment for all steps of the flow.
1410
-
1411
- Use `@conda_base` to set common libraries required by all
1412
- steps and use `@conda` to specify step-specific additions.
1413
-
1414
- Parameters
1415
- ----------
1416
- packages : Dict[str, str], default {}
1417
- Packages to use for this flow. The key is the name of the package
1418
- and the value is the version to use.
1419
- libraries : Dict[str, str], default {}
1420
- Supported for backward compatibility. When used with packages, packages will take precedence.
1421
- python : str, optional, default None
1422
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1423
- that the version used will correspond to the version of the Python interpreter used to start the run.
1424
- disabled : bool, default False
1425
- If set to True, disables Conda.
1426
- """
1427
- ...
1428
-
1429
- @typing.overload
1430
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1431
- ...
1432
-
1433
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1434
- """
1435
- Specifies the Conda environment for all steps of the flow.
1436
-
1437
- Use `@conda_base` to set common libraries required by all
1438
- steps and use `@conda` to specify step-specific additions.
1439
-
1440
- Parameters
1441
- ----------
1442
- packages : Dict[str, str], default {}
1443
- Packages to use for this flow. The key is the name of the package
1444
- and the value is the version to use.
1445
- libraries : Dict[str, str], default {}
1446
- Supported for backward compatibility. When used with packages, packages will take precedence.
1447
- python : str, optional, default None
1448
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1449
- that the version used will correspond to the version of the Python interpreter used to start the run.
1450
- disabled : bool, default False
1451
- If set to True, disables Conda.
1452
- """
1453
- ...
1454
-
1455
- @typing.overload
1456
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1457
- """
1458
- Specifies the times when the flow should be run when running on a
1459
- production scheduler.
1460
-
1461
- Parameters
1462
- ----------
1463
- hourly : bool, default False
1464
- Run the workflow hourly.
1465
- daily : bool, default True
1466
- Run the workflow daily.
1467
- weekly : bool, default False
1468
- Run the workflow weekly.
1469
- cron : str, optional, default None
1470
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1471
- specified by this expression.
1472
- timezone : str, optional, default None
1473
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1474
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1475
- """
1476
- ...
1477
-
1478
- @typing.overload
1479
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1480
- ...
1481
-
1482
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1483
- """
1484
- Specifies the times when the flow should be run when running on a
1485
- production scheduler.
1486
-
1487
- Parameters
1488
- ----------
1489
- hourly : bool, default False
1490
- Run the workflow hourly.
1491
- daily : bool, default True
1492
- Run the workflow daily.
1493
- weekly : bool, default False
1494
- Run the workflow weekly.
1495
- cron : str, optional, default None
1496
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1497
- specified by this expression.
1498
- timezone : str, optional, default None
1499
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1500
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1501
- """
1502
- ...
1503
-
1504
1388
  @typing.overload
1505
1389
  def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1506
1390
  """
@@ -1582,6 +1466,55 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1582
1466
  """
1583
1467
  ...
1584
1468
 
1469
+ @typing.overload
1470
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1471
+ """
1472
+ Specifies the Conda environment for all steps of the flow.
1473
+
1474
+ Use `@conda_base` to set common libraries required by all
1475
+ steps and use `@conda` to specify step-specific additions.
1476
+
1477
+ Parameters
1478
+ ----------
1479
+ packages : Dict[str, str], default {}
1480
+ Packages to use for this flow. The key is the name of the package
1481
+ and the value is the version to use.
1482
+ libraries : Dict[str, str], default {}
1483
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1484
+ python : str, optional, default None
1485
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1486
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1487
+ disabled : bool, default False
1488
+ If set to True, disables Conda.
1489
+ """
1490
+ ...
1491
+
1492
+ @typing.overload
1493
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1494
+ ...
1495
+
1496
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1497
+ """
1498
+ Specifies the Conda environment for all steps of the flow.
1499
+
1500
+ Use `@conda_base` to set common libraries required by all
1501
+ steps and use `@conda` to specify step-specific additions.
1502
+
1503
+ Parameters
1504
+ ----------
1505
+ packages : Dict[str, str], default {}
1506
+ Packages to use for this flow. The key is the name of the package
1507
+ and the value is the version to use.
1508
+ libraries : Dict[str, str], default {}
1509
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1510
+ python : str, optional, default None
1511
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1512
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1513
+ disabled : bool, default False
1514
+ If set to True, disables Conda.
1515
+ """
1516
+ ...
1517
+
1585
1518
  @typing.overload
1586
1519
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1587
1520
  """
@@ -1677,6 +1610,73 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1677
1610
  """
1678
1611
  ...
1679
1612
 
1613
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1614
+ """
1615
+ Specifies what flows belong to the same project.
1616
+
1617
+ A project-specific namespace is created for all flows that
1618
+ use the same `@project(name)`.
1619
+
1620
+ Parameters
1621
+ ----------
1622
+ name : str
1623
+ Project name. Make sure that the name is unique amongst all
1624
+ projects that use the same production scheduler. The name may
1625
+ contain only lowercase alphanumeric characters and underscores.
1626
+
1627
+
1628
+ """
1629
+ ...
1630
+
1631
+ @typing.overload
1632
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1633
+ """
1634
+ Specifies the times when the flow should be run when running on a
1635
+ production scheduler.
1636
+
1637
+ Parameters
1638
+ ----------
1639
+ hourly : bool, default False
1640
+ Run the workflow hourly.
1641
+ daily : bool, default True
1642
+ Run the workflow daily.
1643
+ weekly : bool, default False
1644
+ Run the workflow weekly.
1645
+ cron : str, optional, default None
1646
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1647
+ specified by this expression.
1648
+ timezone : str, optional, default None
1649
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1650
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1651
+ """
1652
+ ...
1653
+
1654
+ @typing.overload
1655
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1656
+ ...
1657
+
1658
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1659
+ """
1660
+ Specifies the times when the flow should be run when running on a
1661
+ production scheduler.
1662
+
1663
+ Parameters
1664
+ ----------
1665
+ hourly : bool, default False
1666
+ Run the workflow hourly.
1667
+ daily : bool, default True
1668
+ Run the workflow daily.
1669
+ weekly : bool, default False
1670
+ Run the workflow weekly.
1671
+ cron : str, optional, default None
1672
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1673
+ specified by this expression.
1674
+ timezone : str, optional, default None
1675
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1676
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1677
+ """
1678
+ ...
1679
+
1680
1680
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1681
1681
  """
1682
1682
  Switch namespace to the one provided.
@@ -1768,7 +1768,7 @@ class Metaflow(object, metaclass=type):
1768
1768
  flows present in the current namespace will be returned. A `Flow` is present in a namespace
1769
1769
  if it has at least one run in the namespace.
1770
1770
  """
1771
- def __init__(self):
1771
+ def __init__(self, _current_metadata: typing.Optional[str] = None):
1772
1772
  ...
1773
1773
  @property
1774
1774
  def flows(self) -> typing.List[metaflow.client.core.Flow]: