ob-metaflow-stubs 6.0.3.109rc0__py2.py3-none-any.whl → 6.0.3.110__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +660 -647
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +6 -6
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +3 -3
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +4 -2
  17. metaflow-stubs/metaflow_current.pyi +69 -35
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +4 -4
  21. metaflow-stubs/plugins/__init__.pyi +4 -2
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +5 -5
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +8 -8
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +132 -4
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +6 -3
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
  61. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +4 -4
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  74. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  76. metaflow-stubs/plugins/datatools/__init__.pyi +4 -4
  77. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +4 -4
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +4 -4
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +5 -5
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +20 -4
  126. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  127. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  128. metaflow-stubs/runner/nbrun.pyi +2 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  130. metaflow-stubs/runner/utils.pyi +2 -2
  131. metaflow-stubs/system/__init__.pyi +3 -3
  132. metaflow-stubs/system/system_logger.pyi +2 -2
  133. metaflow-stubs/system/system_monitor.pyi +3 -3
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.109rc0.dist-info → ob_metaflow_stubs-6.0.3.110.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.110.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.109rc0.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.109rc0.dist-info → ob_metaflow_stubs-6.0.3.110.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.109rc0.dist-info → ob_metaflow_stubs-6.0.3.110.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.25.1+obcheckpoint(0.0.12);ob(v1) #
4
- # Generated on 2024-10-10T01:08:08.920544 #
3
+ # MF version: 2.12.25.2+obcheckpoint(0.1.0);ob(v1) #
4
+ # Generated on 2024-10-16T01:27:45.836502 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.events
12
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
11
+ import metaflow.datastore.inputs
13
12
  import metaflow.runner.metaflow_runner
13
+ import metaflow.flowspec
14
+ import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
15
+ import metaflow.parameters
14
16
  import datetime
15
17
  import metaflow.metaflow_current
18
+ import metaflow.client.core
16
19
  import typing
20
+ import metaflow.events
17
21
  import metaflow._vendor.click.types
18
- import metaflow.parameters
19
- import metaflow.client.core
20
- import metaflow.datastore.inputs
21
- import metaflow.flowspec
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -512,126 +512,99 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
512
512
  """
513
513
  ...
514
514
 
515
- @typing.overload
516
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
517
- """
518
- Internal decorator to support Fast bakery
519
- """
520
- ...
521
-
522
- @typing.overload
523
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
524
- ...
525
-
526
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
527
- """
528
- Internal decorator to support Fast bakery
529
- """
530
- ...
531
-
532
- @typing.overload
533
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
515
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
534
516
  """
535
- Specifies that the step will success under all circumstances.
536
-
537
- The decorator will create an optional artifact, specified by `var`, which
538
- contains the exception raised. You can use it to detect the presence
539
- of errors, indicating that all happy-path artifacts produced by the step
540
- are missing.
517
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
541
518
 
542
519
  Parameters
543
520
  ----------
544
- var : str, optional, default None
545
- Name of the artifact in which to store the caught exception.
546
- If not specified, the exception is not stored.
547
- print_exception : bool, default True
548
- Determines whether or not the exception is printed to
549
- stdout when caught.
550
- """
551
- ...
552
-
553
- @typing.overload
554
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
555
- ...
556
-
557
- @typing.overload
558
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
559
- ...
560
-
561
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
562
- """
563
- Specifies that the step will success under all circumstances.
521
+ temp_dir_root : str, optional
522
+ The root directory that will hold the temporary directory where objects will be downloaded.
564
523
 
565
- The decorator will create an optional artifact, specified by `var`, which
566
- contains the exception raised. You can use it to detect the presence
567
- of errors, indicating that all happy-path artifacts produced by the step
568
- are missing.
569
524
 
570
- Parameters
571
- ----------
572
- var : str, optional, default None
573
- Name of the artifact in which to store the caught exception.
574
- If not specified, the exception is not stored.
575
- print_exception : bool, default True
576
- Determines whether or not the exception is printed to
577
- stdout when caught.
578
525
  """
579
526
  ...
580
527
 
581
- @typing.overload
582
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
528
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
583
529
  """
584
- Specifies a timeout for your step.
585
-
586
- This decorator is useful if this step may hang indefinitely.
587
-
588
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
589
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
590
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
591
-
592
- Note that all the values specified in parameters are added together so if you specify
593
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
530
+ Specifies that this step should execute on Kubernetes.
594
531
 
595
532
  Parameters
596
533
  ----------
597
- seconds : int, default 0
598
- Number of seconds to wait prior to timing out.
599
- minutes : int, default 0
600
- Number of minutes to wait prior to timing out.
601
- hours : int, default 0
602
- Number of hours to wait prior to timing out.
534
+ cpu : int, default 1
535
+ Number of CPUs required for this step. If `@resources` is
536
+ also present, the maximum value from all decorators is used.
537
+ memory : int, default 4096
538
+ Memory size (in MB) required for this step. If
539
+ `@resources` is also present, the maximum value from all decorators is
540
+ used.
541
+ disk : int, default 10240
542
+ Disk size (in MB) required for this step. If
543
+ `@resources` is also present, the maximum value from all decorators is
544
+ used.
545
+ image : str, optional, default None
546
+ Docker image to use when launching on Kubernetes. If not specified, and
547
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
548
+ not, a default Docker image mapping to the current version of Python is used.
549
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
550
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
551
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
552
+ Kubernetes service account to use when launching pod in Kubernetes.
553
+ secrets : List[str], optional, default None
554
+ Kubernetes secrets to use when launching pod in Kubernetes. These
555
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
556
+ in Metaflow configuration.
557
+ node_selector: Union[Dict[str,str], str], optional, default None
558
+ Kubernetes node selector(s) to apply to the pod running the task.
559
+ Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
560
+ or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
561
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
562
+ Kubernetes namespace to use when launching pod in Kubernetes.
563
+ gpu : int, optional, default None
564
+ Number of GPUs required for this step. A value of zero implies that
565
+ the scheduled node should not have GPUs.
566
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
567
+ The vendor of the GPUs to be used for this step.
568
+ tolerations : List[str], default []
569
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
570
+ Kubernetes tolerations to use when launching pod in Kubernetes.
571
+ use_tmpfs : bool, default False
572
+ This enables an explicit tmpfs mount for this step.
573
+ tmpfs_tempdir : bool, default True
574
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
575
+ tmpfs_size : int, optional, default: None
576
+ The value for the size (in MiB) of the tmpfs mount for this step.
577
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
578
+ memory allocated for this step.
579
+ tmpfs_path : str, optional, default /metaflow_temp
580
+ Path to tmpfs mount for this step.
581
+ persistent_volume_claims : Dict[str, str], optional, default None
582
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
583
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
584
+ shared_memory: int, optional
585
+ Shared memory size (in MiB) required for this step
586
+ port: int, optional
587
+ Port number to specify in the Kubernetes job object
588
+ compute_pool : str, optional, default None
589
+ Compute pool to be used for for this step.
590
+ If not specified, any accessible compute pool within the perimeter is used.
603
591
  """
604
592
  ...
605
593
 
606
594
  @typing.overload
607
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
595
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
596
+ """
597
+ Internal decorator to support Fast bakery
598
+ """
608
599
  ...
609
600
 
610
601
  @typing.overload
611
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
602
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
612
603
  ...
613
604
 
614
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
605
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
615
606
  """
616
- Specifies a timeout for your step.
617
-
618
- This decorator is useful if this step may hang indefinitely.
619
-
620
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
621
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
622
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
623
-
624
- Note that all the values specified in parameters are added together so if you specify
625
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
626
-
627
- Parameters
628
- ----------
629
- seconds : int, default 0
630
- Number of seconds to wait prior to timing out.
631
- minutes : int, default 0
632
- Number of minutes to wait prior to timing out.
633
- hours : int, default 0
634
- Number of hours to wait prior to timing out.
607
+ Internal decorator to support Fast bakery
635
608
  """
636
609
  ...
637
610
 
@@ -697,136 +670,51 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
697
670
  ...
698
671
 
699
672
  @typing.overload
700
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
673
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
701
674
  """
702
- Specifies the resources needed when executing this step.
703
-
704
- Use `@resources` to specify the resource requirements
705
- independently of the specific compute layer (`@batch`, `@kubernetes`).
675
+ Specifies the PyPI packages for the step.
706
676
 
707
- You can choose the compute layer on the command line by executing e.g.
708
- ```
709
- python myflow.py run --with batch
710
- ```
711
- or
712
- ```
713
- python myflow.py run --with kubernetes
714
- ```
715
- which executes the flow on the desired system using the
716
- requirements specified in `@resources`.
677
+ Information in this decorator will augment any
678
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
679
+ you can use `@pypi_base` to set packages required by all
680
+ steps and use `@pypi` to specify step-specific overrides.
717
681
 
718
682
  Parameters
719
683
  ----------
720
- cpu : int, default 1
721
- Number of CPUs required for this step.
722
- gpu : int, default 0
723
- Number of GPUs required for this step.
724
- disk : int, optional, default None
725
- Disk size (in MB) required for this step. Only applies on Kubernetes.
726
- memory : int, default 4096
727
- Memory size (in MB) required for this step.
728
- shared_memory : int, optional, default None
729
- The value for the size (in MiB) of the /dev/shm volume for this step.
730
- This parameter maps to the `--shm-size` option in Docker.
684
+ packages : Dict[str, str], default: {}
685
+ Packages to use for this step. The key is the name of the package
686
+ and the value is the version to use.
687
+ python : str, optional, default: None
688
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
689
+ that the version used will correspond to the version of the Python interpreter used to start the run.
731
690
  """
732
691
  ...
733
692
 
734
693
  @typing.overload
735
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
694
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
736
695
  ...
737
696
 
738
697
  @typing.overload
739
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
698
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
740
699
  ...
741
700
 
742
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
701
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
743
702
  """
744
- Specifies the resources needed when executing this step.
745
-
746
- Use `@resources` to specify the resource requirements
747
- independently of the specific compute layer (`@batch`, `@kubernetes`).
703
+ Specifies the PyPI packages for the step.
748
704
 
749
- You can choose the compute layer on the command line by executing e.g.
750
- ```
751
- python myflow.py run --with batch
752
- ```
753
- or
754
- ```
755
- python myflow.py run --with kubernetes
756
- ```
757
- which executes the flow on the desired system using the
758
- requirements specified in `@resources`.
705
+ Information in this decorator will augment any
706
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
707
+ you can use `@pypi_base` to set packages required by all
708
+ steps and use `@pypi` to specify step-specific overrides.
759
709
 
760
710
  Parameters
761
711
  ----------
762
- cpu : int, default 1
763
- Number of CPUs required for this step.
764
- gpu : int, default 0
765
- Number of GPUs required for this step.
766
- disk : int, optional, default None
767
- Disk size (in MB) required for this step. Only applies on Kubernetes.
768
- memory : int, default 4096
769
- Memory size (in MB) required for this step.
770
- shared_memory : int, optional, default None
771
- The value for the size (in MiB) of the /dev/shm volume for this step.
772
- This parameter maps to the `--shm-size` option in Docker.
773
- """
774
- ...
775
-
776
- @typing.overload
777
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
778
- """
779
- Specifies the Conda environment for the step.
780
-
781
- Information in this decorator will augment any
782
- attributes set in the `@conda_base` flow-level decorator. Hence,
783
- you can use `@conda_base` to set packages required by all
784
- steps and use `@conda` to specify step-specific overrides.
785
-
786
- Parameters
787
- ----------
788
- packages : Dict[str, str], default {}
789
- Packages to use for this step. The key is the name of the package
790
- and the value is the version to use.
791
- libraries : Dict[str, str], default {}
792
- Supported for backward compatibility. When used with packages, packages will take precedence.
793
- python : str, optional, default None
794
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
795
- that the version used will correspond to the version of the Python interpreter used to start the run.
796
- disabled : bool, default False
797
- If set to True, disables @conda.
798
- """
799
- ...
800
-
801
- @typing.overload
802
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
803
- ...
804
-
805
- @typing.overload
806
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
807
- ...
808
-
809
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
810
- """
811
- Specifies the Conda environment for the step.
812
-
813
- Information in this decorator will augment any
814
- attributes set in the `@conda_base` flow-level decorator. Hence,
815
- you can use `@conda_base` to set packages required by all
816
- steps and use `@conda` to specify step-specific overrides.
817
-
818
- Parameters
819
- ----------
820
- packages : Dict[str, str], default {}
821
- Packages to use for this step. The key is the name of the package
822
- and the value is the version to use.
823
- libraries : Dict[str, str], default {}
824
- Supported for backward compatibility. When used with packages, packages will take precedence.
825
- python : str, optional, default None
826
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
827
- that the version used will correspond to the version of the Python interpreter used to start the run.
828
- disabled : bool, default False
829
- If set to True, disables @conda.
712
+ packages : Dict[str, str], default: {}
713
+ Packages to use for this step. The key is the name of the package
714
+ and the value is the version to use.
715
+ python : str, optional, default: None
716
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
717
+ that the version used will correspond to the version of the Python interpreter used to start the run.
830
718
  """
831
719
  ...
832
720
 
@@ -864,167 +752,59 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
864
752
  ...
865
753
 
866
754
  @typing.overload
867
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
755
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
868
756
  """
869
- Specifies environment variables to be set prior to the execution of a step.
757
+ Specifies a timeout for your step.
870
758
 
871
- Parameters
872
- ----------
873
- vars : Dict[str, str], default {}
874
- Dictionary of environment variables to set.
875
- """
876
- ...
877
-
878
- @typing.overload
879
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
880
- ...
881
-
882
- @typing.overload
883
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
884
- ...
885
-
886
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
887
- """
888
- Specifies environment variables to be set prior to the execution of a step.
759
+ This decorator is useful if this step may hang indefinitely.
889
760
 
890
- Parameters
891
- ----------
892
- vars : Dict[str, str], default {}
893
- Dictionary of environment variables to set.
894
- """
895
- ...
896
-
897
- @typing.overload
898
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
899
- """
900
- Specifies the PyPI packages for the step.
761
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
762
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
763
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
901
764
 
902
- Information in this decorator will augment any
903
- attributes set in the `@pyi_base` flow-level decorator. Hence,
904
- you can use `@pypi_base` to set packages required by all
905
- steps and use `@pypi` to specify step-specific overrides.
765
+ Note that all the values specified in parameters are added together so if you specify
766
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
906
767
 
907
768
  Parameters
908
769
  ----------
909
- packages : Dict[str, str], default: {}
910
- Packages to use for this step. The key is the name of the package
911
- and the value is the version to use.
912
- python : str, optional, default: None
913
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
914
- that the version used will correspond to the version of the Python interpreter used to start the run.
770
+ seconds : int, default 0
771
+ Number of seconds to wait prior to timing out.
772
+ minutes : int, default 0
773
+ Number of minutes to wait prior to timing out.
774
+ hours : int, default 0
775
+ Number of hours to wait prior to timing out.
915
776
  """
916
777
  ...
917
778
 
918
779
  @typing.overload
919
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
780
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
920
781
  ...
921
782
 
922
783
  @typing.overload
923
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
784
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
924
785
  ...
925
786
 
926
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
787
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
927
788
  """
928
- Specifies the PyPI packages for the step.
789
+ Specifies a timeout for your step.
929
790
 
930
- Information in this decorator will augment any
931
- attributes set in the `@pyi_base` flow-level decorator. Hence,
932
- you can use `@pypi_base` to set packages required by all
933
- steps and use `@pypi` to specify step-specific overrides.
791
+ This decorator is useful if this step may hang indefinitely.
934
792
 
935
- Parameters
936
- ----------
937
- packages : Dict[str, str], default: {}
938
- Packages to use for this step. The key is the name of the package
939
- and the value is the version to use.
940
- python : str, optional, default: None
941
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
942
- that the version used will correspond to the version of the Python interpreter used to start the run.
943
- """
944
- ...
945
-
946
- @typing.overload
947
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
948
- """
949
- Decorator prototype for all step decorators. This function gets specialized
950
- and imported for all decorators types by _import_plugin_decorators().
951
- """
952
- ...
953
-
954
- @typing.overload
955
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
956
- ...
957
-
958
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
959
- """
960
- Decorator prototype for all step decorators. This function gets specialized
961
- and imported for all decorators types by _import_plugin_decorators().
962
- """
963
- ...
964
-
965
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
966
- """
967
- Specifies that this step should execute on Kubernetes.
793
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
794
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
795
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
796
+
797
+ Note that all the values specified in parameters are added together so if you specify
798
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
968
799
 
969
800
  Parameters
970
801
  ----------
971
- cpu : int, default 1
972
- Number of CPUs required for this step. If `@resources` is
973
- also present, the maximum value from all decorators is used.
974
- memory : int, default 4096
975
- Memory size (in MB) required for this step. If
976
- `@resources` is also present, the maximum value from all decorators is
977
- used.
978
- disk : int, default 10240
979
- Disk size (in MB) required for this step. If
980
- `@resources` is also present, the maximum value from all decorators is
981
- used.
982
- image : str, optional, default None
983
- Docker image to use when launching on Kubernetes. If not specified, and
984
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
985
- not, a default Docker image mapping to the current version of Python is used.
986
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
987
- If given, the imagePullPolicy to be applied to the Docker image of the step.
988
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
989
- Kubernetes service account to use when launching pod in Kubernetes.
990
- secrets : List[str], optional, default None
991
- Kubernetes secrets to use when launching pod in Kubernetes. These
992
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
993
- in Metaflow configuration.
994
- node_selector: Union[Dict[str,str], str], optional, default None
995
- Kubernetes node selector(s) to apply to the pod running the task.
996
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
997
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
998
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
999
- Kubernetes namespace to use when launching pod in Kubernetes.
1000
- gpu : int, optional, default None
1001
- Number of GPUs required for this step. A value of zero implies that
1002
- the scheduled node should not have GPUs.
1003
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1004
- The vendor of the GPUs to be used for this step.
1005
- tolerations : List[str], default []
1006
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1007
- Kubernetes tolerations to use when launching pod in Kubernetes.
1008
- use_tmpfs : bool, default False
1009
- This enables an explicit tmpfs mount for this step.
1010
- tmpfs_tempdir : bool, default True
1011
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1012
- tmpfs_size : int, optional, default: None
1013
- The value for the size (in MiB) of the tmpfs mount for this step.
1014
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1015
- memory allocated for this step.
1016
- tmpfs_path : str, optional, default /metaflow_temp
1017
- Path to tmpfs mount for this step.
1018
- persistent_volume_claims : Dict[str, str], optional, default None
1019
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1020
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1021
- shared_memory: int, optional
1022
- Shared memory size (in MiB) required for this step
1023
- port: int, optional
1024
- Port number to specify in the Kubernetes job object
1025
- compute_pool : str, optional, default None
1026
- Compute pool to be used for for this step.
1027
- If not specified, any accessible compute pool within the perimeter is used.
802
+ seconds : int, default 0
803
+ Number of seconds to wait prior to timing out.
804
+ minutes : int, default 0
805
+ Number of minutes to wait prior to timing out.
806
+ hours : int, default 0
807
+ Number of hours to wait prior to timing out.
1028
808
  """
1029
809
  ...
1030
810
 
@@ -1090,33 +870,319 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
1090
870
  ...
1091
871
 
1092
872
  @typing.overload
1093
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
873
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1094
874
  """
1095
- Creates a human-readable report, a Metaflow Card, after this step completes.
875
+ Specifies the number of times the task corresponding
876
+ to a step needs to be retried.
1096
877
 
1097
- Note that you may add multiple `@card` decorators in a step with different parameters.
878
+ This decorator is useful for handling transient errors, such as networking issues.
879
+ If your task contains operations that can't be retried safely, e.g. database updates,
880
+ it is advisable to annotate it with `@retry(times=0)`.
881
+
882
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
883
+ decorator will execute a no-op task after all retries have been exhausted,
884
+ ensuring that the flow execution can continue.
1098
885
 
1099
886
  Parameters
1100
887
  ----------
1101
- type : str, default 'default'
1102
- Card type.
1103
- id : str, optional, default None
1104
- If multiple cards are present, use this id to identify this card.
1105
- options : Dict[str, Any], default {}
1106
- Options passed to the card. The contents depend on the card type.
1107
- timeout : int, default 45
1108
- Interrupt reporting if it takes more than this many seconds.
1109
-
1110
-
888
+ times : int, default 3
889
+ Number of times to retry this task.
890
+ minutes_between_retries : int, default 2
891
+ Number of minutes between retries.
1111
892
  """
1112
893
  ...
1113
894
 
1114
895
  @typing.overload
1115
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
896
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1116
897
  ...
1117
898
 
1118
899
  @typing.overload
1119
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
900
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
901
+ ...
902
+
903
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
904
+ """
905
+ Specifies the number of times the task corresponding
906
+ to a step needs to be retried.
907
+
908
+ This decorator is useful for handling transient errors, such as networking issues.
909
+ If your task contains operations that can't be retried safely, e.g. database updates,
910
+ it is advisable to annotate it with `@retry(times=0)`.
911
+
912
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
913
+ decorator will execute a no-op task after all retries have been exhausted,
914
+ ensuring that the flow execution can continue.
915
+
916
+ Parameters
917
+ ----------
918
+ times : int, default 3
919
+ Number of times to retry this task.
920
+ minutes_between_retries : int, default 2
921
+ Number of minutes between retries.
922
+ """
923
+ ...
924
+
925
+ @typing.overload
926
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
927
+ """
928
+ Specifies that the step will success under all circumstances.
929
+
930
+ The decorator will create an optional artifact, specified by `var`, which
931
+ contains the exception raised. You can use it to detect the presence
932
+ of errors, indicating that all happy-path artifacts produced by the step
933
+ are missing.
934
+
935
+ Parameters
936
+ ----------
937
+ var : str, optional, default None
938
+ Name of the artifact in which to store the caught exception.
939
+ If not specified, the exception is not stored.
940
+ print_exception : bool, default True
941
+ Determines whether or not the exception is printed to
942
+ stdout when caught.
943
+ """
944
+ ...
945
+
946
+ @typing.overload
947
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
948
+ ...
949
+
950
+ @typing.overload
951
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
952
+ ...
953
+
954
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
955
+ """
956
+ Specifies that the step will success under all circumstances.
957
+
958
+ The decorator will create an optional artifact, specified by `var`, which
959
+ contains the exception raised. You can use it to detect the presence
960
+ of errors, indicating that all happy-path artifacts produced by the step
961
+ are missing.
962
+
963
+ Parameters
964
+ ----------
965
+ var : str, optional, default None
966
+ Name of the artifact in which to store the caught exception.
967
+ If not specified, the exception is not stored.
968
+ print_exception : bool, default True
969
+ Determines whether or not the exception is printed to
970
+ stdout when caught.
971
+ """
972
+ ...
973
+
974
+ @typing.overload
975
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
976
+ """
977
+ Decorator prototype for all step decorators. This function gets specialized
978
+ and imported for all decorators types by _import_plugin_decorators().
979
+ """
980
+ ...
981
+
982
+ @typing.overload
983
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
984
+ ...
985
+
986
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
987
+ """
988
+ Decorator prototype for all step decorators. This function gets specialized
989
+ and imported for all decorators types by _import_plugin_decorators().
990
+ """
991
+ ...
992
+
993
+ @typing.overload
994
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
995
+ """
996
+ Specifies environment variables to be set prior to the execution of a step.
997
+
998
+ Parameters
999
+ ----------
1000
+ vars : Dict[str, str], default {}
1001
+ Dictionary of environment variables to set.
1002
+ """
1003
+ ...
1004
+
1005
+ @typing.overload
1006
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1007
+ ...
1008
+
1009
+ @typing.overload
1010
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1011
+ ...
1012
+
1013
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1014
+ """
1015
+ Specifies environment variables to be set prior to the execution of a step.
1016
+
1017
+ Parameters
1018
+ ----------
1019
+ vars : Dict[str, str], default {}
1020
+ Dictionary of environment variables to set.
1021
+ """
1022
+ ...
1023
+
1024
+ @typing.overload
1025
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1026
+ """
1027
+ Specifies the resources needed when executing this step.
1028
+
1029
+ Use `@resources` to specify the resource requirements
1030
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1031
+
1032
+ You can choose the compute layer on the command line by executing e.g.
1033
+ ```
1034
+ python myflow.py run --with batch
1035
+ ```
1036
+ or
1037
+ ```
1038
+ python myflow.py run --with kubernetes
1039
+ ```
1040
+ which executes the flow on the desired system using the
1041
+ requirements specified in `@resources`.
1042
+
1043
+ Parameters
1044
+ ----------
1045
+ cpu : int, default 1
1046
+ Number of CPUs required for this step.
1047
+ gpu : int, default 0
1048
+ Number of GPUs required for this step.
1049
+ disk : int, optional, default None
1050
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1051
+ memory : int, default 4096
1052
+ Memory size (in MB) required for this step.
1053
+ shared_memory : int, optional, default None
1054
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1055
+ This parameter maps to the `--shm-size` option in Docker.
1056
+ """
1057
+ ...
1058
+
1059
+ @typing.overload
1060
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1061
+ ...
1062
+
1063
+ @typing.overload
1064
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1065
+ ...
1066
+
1067
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1068
+ """
1069
+ Specifies the resources needed when executing this step.
1070
+
1071
+ Use `@resources` to specify the resource requirements
1072
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1073
+
1074
+ You can choose the compute layer on the command line by executing e.g.
1075
+ ```
1076
+ python myflow.py run --with batch
1077
+ ```
1078
+ or
1079
+ ```
1080
+ python myflow.py run --with kubernetes
1081
+ ```
1082
+ which executes the flow on the desired system using the
1083
+ requirements specified in `@resources`.
1084
+
1085
+ Parameters
1086
+ ----------
1087
+ cpu : int, default 1
1088
+ Number of CPUs required for this step.
1089
+ gpu : int, default 0
1090
+ Number of GPUs required for this step.
1091
+ disk : int, optional, default None
1092
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1093
+ memory : int, default 4096
1094
+ Memory size (in MB) required for this step.
1095
+ shared_memory : int, optional, default None
1096
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1097
+ This parameter maps to the `--shm-size` option in Docker.
1098
+ """
1099
+ ...
1100
+
1101
+ @typing.overload
1102
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1103
+ """
1104
+ Specifies the Conda environment for the step.
1105
+
1106
+ Information in this decorator will augment any
1107
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1108
+ you can use `@conda_base` to set packages required by all
1109
+ steps and use `@conda` to specify step-specific overrides.
1110
+
1111
+ Parameters
1112
+ ----------
1113
+ packages : Dict[str, str], default {}
1114
+ Packages to use for this step. The key is the name of the package
1115
+ and the value is the version to use.
1116
+ libraries : Dict[str, str], default {}
1117
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1118
+ python : str, optional, default None
1119
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1120
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1121
+ disabled : bool, default False
1122
+ If set to True, disables @conda.
1123
+ """
1124
+ ...
1125
+
1126
+ @typing.overload
1127
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1128
+ ...
1129
+
1130
+ @typing.overload
1131
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1132
+ ...
1133
+
1134
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1135
+ """
1136
+ Specifies the Conda environment for the step.
1137
+
1138
+ Information in this decorator will augment any
1139
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1140
+ you can use `@conda_base` to set packages required by all
1141
+ steps and use `@conda` to specify step-specific overrides.
1142
+
1143
+ Parameters
1144
+ ----------
1145
+ packages : Dict[str, str], default {}
1146
+ Packages to use for this step. The key is the name of the package
1147
+ and the value is the version to use.
1148
+ libraries : Dict[str, str], default {}
1149
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1150
+ python : str, optional, default None
1151
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1152
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1153
+ disabled : bool, default False
1154
+ If set to True, disables @conda.
1155
+ """
1156
+ ...
1157
+
1158
+ @typing.overload
1159
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1160
+ """
1161
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1162
+
1163
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1164
+
1165
+ Parameters
1166
+ ----------
1167
+ type : str, default 'default'
1168
+ Card type.
1169
+ id : str, optional, default None
1170
+ If multiple cards are present, use this id to identify this card.
1171
+ options : Dict[str, Any], default {}
1172
+ Options passed to the card. The contents depend on the card type.
1173
+ timeout : int, default 45
1174
+ Interrupt reporting if it takes more than this many seconds.
1175
+
1176
+
1177
+ """
1178
+ ...
1179
+
1180
+ @typing.overload
1181
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1182
+ ...
1183
+
1184
+ @typing.overload
1185
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1120
1186
  ...
1121
1187
 
1122
1188
  def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
@@ -1141,154 +1207,163 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1141
1207
  ...
1142
1208
 
1143
1209
  @typing.overload
1144
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1210
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1145
1211
  """
1146
- Specifies the number of times the task corresponding
1147
- to a step needs to be retried.
1148
-
1149
- This decorator is useful for handling transient errors, such as networking issues.
1150
- If your task contains operations that can't be retried safely, e.g. database updates,
1151
- it is advisable to annotate it with `@retry(times=0)`.
1212
+ Specifies the Conda environment for all steps of the flow.
1152
1213
 
1153
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1154
- decorator will execute a no-op task after all retries have been exhausted,
1155
- ensuring that the flow execution can continue.
1214
+ Use `@conda_base` to set common libraries required by all
1215
+ steps and use `@conda` to specify step-specific additions.
1156
1216
 
1157
1217
  Parameters
1158
1218
  ----------
1159
- times : int, default 3
1160
- Number of times to retry this task.
1161
- minutes_between_retries : int, default 2
1162
- Number of minutes between retries.
1219
+ packages : Dict[str, str], default {}
1220
+ Packages to use for this flow. The key is the name of the package
1221
+ and the value is the version to use.
1222
+ libraries : Dict[str, str], default {}
1223
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1224
+ python : str, optional, default None
1225
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1226
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1227
+ disabled : bool, default False
1228
+ If set to True, disables Conda.
1163
1229
  """
1164
1230
  ...
1165
1231
 
1166
1232
  @typing.overload
1167
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1168
- ...
1169
-
1170
- @typing.overload
1171
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1233
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1172
1234
  ...
1173
1235
 
1174
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1236
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1175
1237
  """
1176
- Specifies the number of times the task corresponding
1177
- to a step needs to be retried.
1238
+ Specifies the Conda environment for all steps of the flow.
1178
1239
 
1179
- This decorator is useful for handling transient errors, such as networking issues.
1180
- If your task contains operations that can't be retried safely, e.g. database updates,
1181
- it is advisable to annotate it with `@retry(times=0)`.
1240
+ Use `@conda_base` to set common libraries required by all
1241
+ steps and use `@conda` to specify step-specific additions.
1182
1242
 
1183
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1184
- decorator will execute a no-op task after all retries have been exhausted,
1185
- ensuring that the flow execution can continue.
1243
+ Parameters
1244
+ ----------
1245
+ packages : Dict[str, str], default {}
1246
+ Packages to use for this flow. The key is the name of the package
1247
+ and the value is the version to use.
1248
+ libraries : Dict[str, str], default {}
1249
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1250
+ python : str, optional, default None
1251
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1252
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1253
+ disabled : bool, default False
1254
+ If set to True, disables Conda.
1255
+ """
1256
+ ...
1257
+
1258
+ @typing.overload
1259
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1260
+ """
1261
+ Specifies the PyPI packages for all steps of the flow.
1186
1262
 
1263
+ Use `@pypi_base` to set common packages required by all
1264
+ steps and use `@pypi` to specify step-specific overrides.
1187
1265
  Parameters
1188
1266
  ----------
1189
- times : int, default 3
1190
- Number of times to retry this task.
1191
- minutes_between_retries : int, default 2
1192
- Number of minutes between retries.
1267
+ packages : Dict[str, str], default: {}
1268
+ Packages to use for this flow. The key is the name of the package
1269
+ and the value is the version to use.
1270
+ python : str, optional, default: None
1271
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1272
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1193
1273
  """
1194
1274
  ...
1195
1275
 
1196
1276
  @typing.overload
1197
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1277
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1278
+ ...
1279
+
1280
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1198
1281
  """
1199
- Specifies the flow(s) that this flow depends on.
1282
+ Specifies the PyPI packages for all steps of the flow.
1200
1283
 
1201
- ```
1202
- @trigger_on_finish(flow='FooFlow')
1203
- ```
1204
- or
1205
- ```
1206
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1207
- ```
1208
- This decorator respects the @project decorator and triggers the flow
1209
- when upstream runs within the same namespace complete successfully
1284
+ Use `@pypi_base` to set common packages required by all
1285
+ steps and use `@pypi` to specify step-specific overrides.
1286
+ Parameters
1287
+ ----------
1288
+ packages : Dict[str, str], default: {}
1289
+ Packages to use for this flow. The key is the name of the package
1290
+ and the value is the version to use.
1291
+ python : str, optional, default: None
1292
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1293
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1294
+ """
1295
+ ...
1296
+
1297
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1298
+ """
1299
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1210
1300
 
1211
- Additionally, you can specify project aware upstream flow dependencies
1212
- by specifying the fully qualified project_flow_name.
1213
- ```
1214
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1215
- ```
1216
- or
1217
- ```
1218
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1219
- ```
1301
+ User code call
1302
+ -----------
1303
+ @nim(
1304
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1305
+ backend='managed'
1306
+ )
1220
1307
 
1221
- You can also specify just the project or project branch (other values will be
1222
- inferred from the current project or project branch):
1223
- ```
1224
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1225
- ```
1308
+ Valid backend options
1309
+ ---------------------
1310
+ - 'managed': Outerbounds selects a compute provider based on the model.
1311
+ - 🚧 'dataplane': Run in your account.
1226
1312
 
1227
- Note that `branch` is typically one of:
1228
- - `prod`
1229
- - `user.bob`
1230
- - `test.my_experiment`
1231
- - `prod.staging`
1313
+ Valid model options
1314
+ ----------------
1315
+ - 'meta/llama3-8b-instruct': 8B parameter model
1316
+ - 'meta/llama3-70b-instruct': 70B parameter model
1317
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1232
1318
 
1233
1319
  Parameters
1234
1320
  ----------
1235
- flow : Union[str, Dict[str, str]], optional, default None
1236
- Upstream flow dependency for this flow.
1237
- flows : List[Union[str, Dict[str, str]]], default []
1238
- Upstream flow dependencies for this flow.
1239
- options : Dict[str, Any], default {}
1240
- Backend-specific configuration for tuning eventing behavior.
1241
-
1242
-
1321
+ models: list[NIM]
1322
+ List of NIM containers running models in sidecars.
1323
+ backend: str
1324
+ Compute provider to run the NIM container.
1243
1325
  """
1244
1326
  ...
1245
1327
 
1246
1328
  @typing.overload
1247
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1248
- ...
1249
-
1250
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1329
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1251
1330
  """
1252
- Specifies the flow(s) that this flow depends on.
1331
+ Specifies the event(s) that this flow depends on.
1253
1332
 
1254
1333
  ```
1255
- @trigger_on_finish(flow='FooFlow')
1334
+ @trigger(event='foo')
1256
1335
  ```
1257
1336
  or
1258
1337
  ```
1259
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1338
+ @trigger(events=['foo', 'bar'])
1260
1339
  ```
1261
- This decorator respects the @project decorator and triggers the flow
1262
- when upstream runs within the same namespace complete successfully
1263
1340
 
1264
- Additionally, you can specify project aware upstream flow dependencies
1265
- by specifying the fully qualified project_flow_name.
1341
+ Additionally, you can specify the parameter mappings
1342
+ to map event payload to Metaflow parameters for the flow.
1266
1343
  ```
1267
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1344
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1268
1345
  ```
1269
1346
  or
1270
1347
  ```
1271
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1348
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1349
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1272
1350
  ```
1273
1351
 
1274
- You can also specify just the project or project branch (other values will be
1275
- inferred from the current project or project branch):
1352
+ 'parameters' can also be a list of strings and tuples like so:
1276
1353
  ```
1277
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1354
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1355
+ ```
1356
+ This is equivalent to:
1357
+ ```
1358
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1278
1359
  ```
1279
-
1280
- Note that `branch` is typically one of:
1281
- - `prod`
1282
- - `user.bob`
1283
- - `test.my_experiment`
1284
- - `prod.staging`
1285
1360
 
1286
1361
  Parameters
1287
1362
  ----------
1288
- flow : Union[str, Dict[str, str]], optional, default None
1289
- Upstream flow dependency for this flow.
1290
- flows : List[Union[str, Dict[str, str]]], default []
1291
- Upstream flow dependencies for this flow.
1363
+ event : Union[str, Dict[str, Any]], optional, default None
1364
+ Event dependency for this flow.
1365
+ events : List[Union[str, Dict[str, Any]]], default []
1366
+ Events dependency for this flow.
1292
1367
  options : Dict[str, Any], default {}
1293
1368
  Backend-specific configuration for tuning eventing behavior.
1294
1369
 
@@ -1296,76 +1371,52 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1296
1371
  """
1297
1372
  ...
1298
1373
 
1299
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1300
- """
1301
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1302
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1303
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1304
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1305
- starts only after all sensors finish.
1306
-
1307
- Parameters
1308
- ----------
1309
- timeout : int
1310
- Time, in seconds before the task times out and fails. (Default: 3600)
1311
- poke_interval : int
1312
- Time in seconds that the job should wait in between each try. (Default: 60)
1313
- mode : str
1314
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1315
- exponential_backoff : bool
1316
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1317
- pool : str
1318
- the slot pool this task should run in,
1319
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1320
- soft_fail : bool
1321
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1322
- name : str
1323
- Name of the sensor on Airflow
1324
- description : str
1325
- Description of sensor in the Airflow UI
1326
- bucket_key : Union[str, List[str]]
1327
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1328
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1329
- bucket_name : str
1330
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1331
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1332
- wildcard_match : bool
1333
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1334
- aws_conn_id : str
1335
- a reference to the s3 connection on Airflow. (Default: None)
1336
- verify : bool
1337
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1338
- """
1374
+ @typing.overload
1375
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1339
1376
  ...
1340
1377
 
1341
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1378
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1342
1379
  """
1343
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1380
+ Specifies the event(s) that this flow depends on.
1344
1381
 
1345
- User code call
1346
- -----------
1347
- @nim(
1348
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1349
- backend='managed'
1350
- )
1382
+ ```
1383
+ @trigger(event='foo')
1384
+ ```
1385
+ or
1386
+ ```
1387
+ @trigger(events=['foo', 'bar'])
1388
+ ```
1351
1389
 
1352
- Valid backend options
1353
- ---------------------
1354
- - 'managed': Outerbounds selects a compute provider based on the model.
1355
- - 🚧 'dataplane': Run in your account.
1390
+ Additionally, you can specify the parameter mappings
1391
+ to map event payload to Metaflow parameters for the flow.
1392
+ ```
1393
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1394
+ ```
1395
+ or
1396
+ ```
1397
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1398
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1399
+ ```
1356
1400
 
1357
- Valid model options
1358
- ----------------
1359
- - 'meta/llama3-8b-instruct': 8B parameter model
1360
- - 'meta/llama3-70b-instruct': 70B parameter model
1361
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1401
+ 'parameters' can also be a list of strings and tuples like so:
1402
+ ```
1403
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1404
+ ```
1405
+ This is equivalent to:
1406
+ ```
1407
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1408
+ ```
1362
1409
 
1363
1410
  Parameters
1364
1411
  ----------
1365
- models: list[NIM]
1366
- List of NIM containers running models in sidecars.
1367
- backend: str
1368
- Compute provider to run the NIM container.
1412
+ event : Union[str, Dict[str, Any]], optional, default None
1413
+ Event dependency for this flow.
1414
+ events : List[Union[str, Dict[str, Any]]], default []
1415
+ Events dependency for this flow.
1416
+ options : Dict[str, Any], default {}
1417
+ Backend-specific configuration for tuning eventing behavior.
1418
+
1419
+
1369
1420
  """
1370
1421
  ...
1371
1422
 
@@ -1436,42 +1487,45 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1436
1487
  """
1437
1488
  ...
1438
1489
 
1439
- @typing.overload
1440
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1441
- """
1442
- Specifies the PyPI packages for all steps of the flow.
1443
-
1444
- Use `@pypi_base` to set common packages required by all
1445
- steps and use `@pypi` to specify step-specific overrides.
1446
- Parameters
1447
- ----------
1448
- packages : Dict[str, str], default: {}
1449
- Packages to use for this flow. The key is the name of the package
1450
- and the value is the version to use.
1451
- python : str, optional, default: None
1452
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1453
- that the version used will correspond to the version of the Python interpreter used to start the run.
1454
- """
1455
- ...
1456
-
1457
- @typing.overload
1458
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1459
- ...
1460
-
1461
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1490
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1462
1491
  """
1463
- Specifies the PyPI packages for all steps of the flow.
1492
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1493
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1494
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1495
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1496
+ starts only after all sensors finish.
1464
1497
 
1465
- Use `@pypi_base` to set common packages required by all
1466
- steps and use `@pypi` to specify step-specific overrides.
1467
1498
  Parameters
1468
1499
  ----------
1469
- packages : Dict[str, str], default: {}
1470
- Packages to use for this flow. The key is the name of the package
1471
- and the value is the version to use.
1472
- python : str, optional, default: None
1473
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1474
- that the version used will correspond to the version of the Python interpreter used to start the run.
1500
+ timeout : int
1501
+ Time, in seconds before the task times out and fails. (Default: 3600)
1502
+ poke_interval : int
1503
+ Time in seconds that the job should wait in between each try. (Default: 60)
1504
+ mode : str
1505
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1506
+ exponential_backoff : bool
1507
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1508
+ pool : str
1509
+ the slot pool this task should run in,
1510
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1511
+ soft_fail : bool
1512
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1513
+ name : str
1514
+ Name of the sensor on Airflow
1515
+ description : str
1516
+ Description of sensor in the Airflow UI
1517
+ bucket_key : Union[str, List[str]]
1518
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1519
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1520
+ bucket_name : str
1521
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1522
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1523
+ wildcard_match : bool
1524
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1525
+ aws_conn_id : str
1526
+ a reference to the s3 connection on Airflow. (Default: None)
1527
+ verify : bool
1528
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1475
1529
  """
1476
1530
  ...
1477
1531
 
@@ -1518,93 +1572,48 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1518
1572
  ...
1519
1573
 
1520
1574
  @typing.overload
1521
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1522
- """
1523
- Specifies the Conda environment for all steps of the flow.
1524
-
1525
- Use `@conda_base` to set common libraries required by all
1526
- steps and use `@conda` to specify step-specific additions.
1527
-
1528
- Parameters
1529
- ----------
1530
- packages : Dict[str, str], default {}
1531
- Packages to use for this flow. The key is the name of the package
1532
- and the value is the version to use.
1533
- libraries : Dict[str, str], default {}
1534
- Supported for backward compatibility. When used with packages, packages will take precedence.
1535
- python : str, optional, default None
1536
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1537
- that the version used will correspond to the version of the Python interpreter used to start the run.
1538
- disabled : bool, default False
1539
- If set to True, disables Conda.
1540
- """
1541
- ...
1542
-
1543
- @typing.overload
1544
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1545
- ...
1546
-
1547
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1548
- """
1549
- Specifies the Conda environment for all steps of the flow.
1550
-
1551
- Use `@conda_base` to set common libraries required by all
1552
- steps and use `@conda` to specify step-specific additions.
1553
-
1554
- Parameters
1555
- ----------
1556
- packages : Dict[str, str], default {}
1557
- Packages to use for this flow. The key is the name of the package
1558
- and the value is the version to use.
1559
- libraries : Dict[str, str], default {}
1560
- Supported for backward compatibility. When used with packages, packages will take precedence.
1561
- python : str, optional, default None
1562
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1563
- that the version used will correspond to the version of the Python interpreter used to start the run.
1564
- disabled : bool, default False
1565
- If set to True, disables Conda.
1566
- """
1567
- ...
1568
-
1569
- @typing.overload
1570
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1575
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1571
1576
  """
1572
- Specifies the event(s) that this flow depends on.
1577
+ Specifies the flow(s) that this flow depends on.
1573
1578
 
1574
1579
  ```
1575
- @trigger(event='foo')
1580
+ @trigger_on_finish(flow='FooFlow')
1576
1581
  ```
1577
1582
  or
1578
1583
  ```
1579
- @trigger(events=['foo', 'bar'])
1584
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1580
1585
  ```
1586
+ This decorator respects the @project decorator and triggers the flow
1587
+ when upstream runs within the same namespace complete successfully
1581
1588
 
1582
- Additionally, you can specify the parameter mappings
1583
- to map event payload to Metaflow parameters for the flow.
1589
+ Additionally, you can specify project aware upstream flow dependencies
1590
+ by specifying the fully qualified project_flow_name.
1584
1591
  ```
1585
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1592
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1586
1593
  ```
1587
1594
  or
1588
1595
  ```
1589
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1590
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1596
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1591
1597
  ```
1592
1598
 
1593
- 'parameters' can also be a list of strings and tuples like so:
1594
- ```
1595
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1596
- ```
1597
- This is equivalent to:
1599
+ You can also specify just the project or project branch (other values will be
1600
+ inferred from the current project or project branch):
1598
1601
  ```
1599
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1602
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1600
1603
  ```
1601
1604
 
1605
+ Note that `branch` is typically one of:
1606
+ - `prod`
1607
+ - `user.bob`
1608
+ - `test.my_experiment`
1609
+ - `prod.staging`
1610
+
1602
1611
  Parameters
1603
1612
  ----------
1604
- event : Union[str, Dict[str, Any]], optional, default None
1605
- Event dependency for this flow.
1606
- events : List[Union[str, Dict[str, Any]]], default []
1607
- Events dependency for this flow.
1613
+ flow : Union[str, Dict[str, str]], optional, default None
1614
+ Upstream flow dependency for this flow.
1615
+ flows : List[Union[str, Dict[str, str]]], default []
1616
+ Upstream flow dependencies for this flow.
1608
1617
  options : Dict[str, Any], default {}
1609
1618
  Backend-specific configuration for tuning eventing behavior.
1610
1619
 
@@ -1613,47 +1622,51 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1613
1622
  ...
1614
1623
 
1615
1624
  @typing.overload
1616
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1625
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1617
1626
  ...
1618
1627
 
1619
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1628
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1620
1629
  """
1621
- Specifies the event(s) that this flow depends on.
1630
+ Specifies the flow(s) that this flow depends on.
1622
1631
 
1623
1632
  ```
1624
- @trigger(event='foo')
1633
+ @trigger_on_finish(flow='FooFlow')
1625
1634
  ```
1626
1635
  or
1627
1636
  ```
1628
- @trigger(events=['foo', 'bar'])
1637
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1629
1638
  ```
1639
+ This decorator respects the @project decorator and triggers the flow
1640
+ when upstream runs within the same namespace complete successfully
1630
1641
 
1631
- Additionally, you can specify the parameter mappings
1632
- to map event payload to Metaflow parameters for the flow.
1642
+ Additionally, you can specify project aware upstream flow dependencies
1643
+ by specifying the fully qualified project_flow_name.
1633
1644
  ```
1634
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1645
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1635
1646
  ```
1636
1647
  or
1637
1648
  ```
1638
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1639
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1649
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1640
1650
  ```
1641
1651
 
1642
- 'parameters' can also be a list of strings and tuples like so:
1643
- ```
1644
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1645
- ```
1646
- This is equivalent to:
1652
+ You can also specify just the project or project branch (other values will be
1653
+ inferred from the current project or project branch):
1647
1654
  ```
1648
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1655
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1649
1656
  ```
1650
1657
 
1658
+ Note that `branch` is typically one of:
1659
+ - `prod`
1660
+ - `user.bob`
1661
+ - `test.my_experiment`
1662
+ - `prod.staging`
1663
+
1651
1664
  Parameters
1652
1665
  ----------
1653
- event : Union[str, Dict[str, Any]], optional, default None
1654
- Event dependency for this flow.
1655
- events : List[Union[str, Dict[str, Any]]], default []
1656
- Events dependency for this flow.
1666
+ flow : Union[str, Dict[str, str]], optional, default None
1667
+ Upstream flow dependency for this flow.
1668
+ flows : List[Union[str, Dict[str, str]]], default []
1669
+ Upstream flow dependencies for this flow.
1657
1670
  options : Dict[str, Any], default {}
1658
1671
  Backend-specific configuration for tuning eventing behavior.
1659
1672