ob-metaflow-stubs 6.0.3.110__py2.py3-none-any.whl → 6.0.3.111__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +480 -480
  2. metaflow-stubs/cards.pyi +4 -4
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +1 -1
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +1 -1
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/info_file.pyi +1 -1
  14. metaflow-stubs/metadata/metadata.pyi +1 -1
  15. metaflow-stubs/metadata/util.pyi +1 -1
  16. metaflow-stubs/metaflow_config.pyi +1 -1
  17. metaflow-stubs/metaflow_current.pyi +118 -118
  18. metaflow-stubs/mflog/mflog.pyi +1 -1
  19. metaflow-stubs/multicore_utils.pyi +1 -1
  20. metaflow-stubs/parameters.pyi +3 -3
  21. metaflow-stubs/plugins/__init__.pyi +1 -1
  22. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  24. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  29. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  30. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  31. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +1 -1
  36. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  37. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +1 -1
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +1 -1
  52. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  59. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  60. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  61. metaflow-stubs/plugins/cards/card_client.pyi +1 -1
  62. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  74. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  75. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  76. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  82. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  83. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  84. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  85. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  101. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  102. metaflow-stubs/plugins/package_cli.pyi +1 -1
  103. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/perimeters.pyi +1 -1
  105. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  111. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  112. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  113. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  114. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  118. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  120. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  121. metaflow-stubs/procpoll.pyi +1 -1
  122. metaflow-stubs/profilers/__init__.pyi +1 -1
  123. metaflow-stubs/pylint_wrapper.pyi +1 -1
  124. metaflow-stubs/runner/__init__.pyi +1 -1
  125. metaflow-stubs/runner/deployer.pyi +2 -2
  126. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  127. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  128. metaflow-stubs/runner/nbrun.pyi +1 -1
  129. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  130. metaflow-stubs/runner/utils.pyi +1 -1
  131. metaflow-stubs/system/__init__.pyi +2 -2
  132. metaflow-stubs/system/system_logger.pyi +1 -1
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +1 -1
  135. metaflow-stubs/tuple_util.pyi +1 -1
  136. {ob_metaflow_stubs-6.0.3.110.dist-info → ob_metaflow_stubs-6.0.3.111.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.111.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.110.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.110.dist-info → ob_metaflow_stubs-6.0.3.111.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.110.dist-info → ob_metaflow_stubs-6.0.3.111.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.12.25.2+obcheckpoint(0.1.0);ob(v1) #
4
- # Generated on 2024-10-16T01:27:45.836502 #
4
+ # Generated on 2024-10-21T17:14:28.436023 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.datastore.inputs
12
- import metaflow.runner.metaflow_runner
13
11
  import metaflow.flowspec
14
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
15
12
  import metaflow.parameters
16
- import datetime
13
+ import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
14
+ import metaflow._vendor.click.types
15
+ import metaflow.events
16
+ import metaflow.datastore.inputs
17
+ import typing
17
18
  import metaflow.metaflow_current
18
19
  import metaflow.client.core
19
- import typing
20
- import metaflow.events
21
- import metaflow._vendor.click.types
20
+ import datetime
21
+ import metaflow.runner.metaflow_runner
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -512,99 +512,83 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
512
512
  """
513
513
  ...
514
514
 
515
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
515
+ @typing.overload
516
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
516
517
  """
517
- Decorator that helps cache, version and store models/datasets from huggingface hub.
518
-
519
- Parameters
520
- ----------
521
- temp_dir_root : str, optional
522
- The root directory that will hold the temporary directory where objects will be downloaded.
523
-
524
-
518
+ Decorator prototype for all step decorators. This function gets specialized
519
+ and imported for all decorators types by _import_plugin_decorators().
525
520
  """
526
521
  ...
527
522
 
528
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
523
+ @typing.overload
524
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
525
+ ...
526
+
527
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
529
528
  """
530
- Specifies that this step should execute on Kubernetes.
529
+ Decorator prototype for all step decorators. This function gets specialized
530
+ and imported for all decorators types by _import_plugin_decorators().
531
+ """
532
+ ...
533
+
534
+ @typing.overload
535
+ def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
536
+ """
537
+ Enables checkpointing for a step.
538
+
531
539
 
532
540
  Parameters
533
541
  ----------
534
- cpu : int, default 1
535
- Number of CPUs required for this step. If `@resources` is
536
- also present, the maximum value from all decorators is used.
537
- memory : int, default 4096
538
- Memory size (in MB) required for this step. If
539
- `@resources` is also present, the maximum value from all decorators is
540
- used.
541
- disk : int, default 10240
542
- Disk size (in MB) required for this step. If
543
- `@resources` is also present, the maximum value from all decorators is
544
- used.
545
- image : str, optional, default None
546
- Docker image to use when launching on Kubernetes. If not specified, and
547
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
548
- not, a default Docker image mapping to the current version of Python is used.
549
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
550
- If given, the imagePullPolicy to be applied to the Docker image of the step.
551
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
552
- Kubernetes service account to use when launching pod in Kubernetes.
553
- secrets : List[str], optional, default None
554
- Kubernetes secrets to use when launching pod in Kubernetes. These
555
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
556
- in Metaflow configuration.
557
- node_selector: Union[Dict[str,str], str], optional, default None
558
- Kubernetes node selector(s) to apply to the pod running the task.
559
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
560
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
561
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
562
- Kubernetes namespace to use when launching pod in Kubernetes.
563
- gpu : int, optional, default None
564
- Number of GPUs required for this step. A value of zero implies that
565
- the scheduled node should not have GPUs.
566
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
567
- The vendor of the GPUs to be used for this step.
568
- tolerations : List[str], default []
569
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
570
- Kubernetes tolerations to use when launching pod in Kubernetes.
571
- use_tmpfs : bool, default False
572
- This enables an explicit tmpfs mount for this step.
573
- tmpfs_tempdir : bool, default True
574
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
575
- tmpfs_size : int, optional, default: None
576
- The value for the size (in MiB) of the tmpfs mount for this step.
577
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
578
- memory allocated for this step.
579
- tmpfs_path : str, optional, default /metaflow_temp
580
- Path to tmpfs mount for this step.
581
- persistent_volume_claims : Dict[str, str], optional, default None
582
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
583
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
584
- shared_memory: int, optional
585
- Shared memory size (in MiB) required for this step
586
- port: int, optional
587
- Port number to specify in the Kubernetes job object
588
- compute_pool : str, optional, default None
589
- Compute pool to be used for for this step.
590
- If not specified, any accessible compute pool within the perimeter is used.
542
+ load_policy : str, default: "fresh"
543
+ The policy for loading the checkpoint. The following policies are supported:
544
+ - "eager": Loads the the latest available checkpoint within the namespace.
545
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
546
+ will be loaded at the start of the task.
547
+ - "none": Do not load any checkpoint
548
+ - "fresh": Loads the lastest checkpoint created within the running Task.
549
+ This mode helps loading checkpoints across various retry attempts of the same task.
550
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
551
+ created within the task will be loaded when the task is retries execution on failure.
552
+
553
+ temp_dir_root : str, default: None
554
+ The root directory under which `current.checkpoint.directory` will be created.
555
+
556
+
557
+
591
558
  """
592
559
  ...
593
560
 
594
561
  @typing.overload
595
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
596
- """
597
- Internal decorator to support Fast bakery
598
- """
562
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
599
563
  ...
600
564
 
601
565
  @typing.overload
602
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
566
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
603
567
  ...
604
568
 
605
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
569
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
606
570
  """
607
- Internal decorator to support Fast bakery
571
+ Enables checkpointing for a step.
572
+
573
+
574
+ Parameters
575
+ ----------
576
+ load_policy : str, default: "fresh"
577
+ The policy for loading the checkpoint. The following policies are supported:
578
+ - "eager": Loads the the latest available checkpoint within the namespace.
579
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
580
+ will be loaded at the start of the task.
581
+ - "none": Do not load any checkpoint
582
+ - "fresh": Loads the lastest checkpoint created within the running Task.
583
+ This mode helps loading checkpoints across various retry attempts of the same task.
584
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
585
+ created within the task will be loaded when the task is retries execution on failure.
586
+
587
+ temp_dir_root : str, default: None
588
+ The root directory under which `current.checkpoint.directory` will be created.
589
+
590
+
591
+
608
592
  """
609
593
  ...
610
594
 
@@ -666,55 +650,96 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
666
650
 
667
651
 
668
652
 
653
+ """
654
+ ...
655
+
656
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
657
+ """
658
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
659
+
660
+ Parameters
661
+ ----------
662
+ temp_dir_root : str, optional
663
+ The root directory that will hold the temporary directory where objects will be downloaded.
664
+
665
+
669
666
  """
670
667
  ...
671
668
 
672
669
  @typing.overload
673
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
670
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
674
671
  """
675
- Specifies the PyPI packages for the step.
672
+ Specifies the resources needed when executing this step.
676
673
 
677
- Information in this decorator will augment any
678
- attributes set in the `@pyi_base` flow-level decorator. Hence,
679
- you can use `@pypi_base` to set packages required by all
680
- steps and use `@pypi` to specify step-specific overrides.
674
+ Use `@resources` to specify the resource requirements
675
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
676
+
677
+ You can choose the compute layer on the command line by executing e.g.
678
+ ```
679
+ python myflow.py run --with batch
680
+ ```
681
+ or
682
+ ```
683
+ python myflow.py run --with kubernetes
684
+ ```
685
+ which executes the flow on the desired system using the
686
+ requirements specified in `@resources`.
681
687
 
682
688
  Parameters
683
689
  ----------
684
- packages : Dict[str, str], default: {}
685
- Packages to use for this step. The key is the name of the package
686
- and the value is the version to use.
687
- python : str, optional, default: None
688
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
689
- that the version used will correspond to the version of the Python interpreter used to start the run.
690
+ cpu : int, default 1
691
+ Number of CPUs required for this step.
692
+ gpu : int, default 0
693
+ Number of GPUs required for this step.
694
+ disk : int, optional, default None
695
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
696
+ memory : int, default 4096
697
+ Memory size (in MB) required for this step.
698
+ shared_memory : int, optional, default None
699
+ The value for the size (in MiB) of the /dev/shm volume for this step.
700
+ This parameter maps to the `--shm-size` option in Docker.
690
701
  """
691
702
  ...
692
703
 
693
704
  @typing.overload
694
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
705
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
695
706
  ...
696
707
 
697
708
  @typing.overload
698
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
709
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
699
710
  ...
700
711
 
701
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
712
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
702
713
  """
703
- Specifies the PyPI packages for the step.
714
+ Specifies the resources needed when executing this step.
704
715
 
705
- Information in this decorator will augment any
706
- attributes set in the `@pyi_base` flow-level decorator. Hence,
707
- you can use `@pypi_base` to set packages required by all
708
- steps and use `@pypi` to specify step-specific overrides.
716
+ Use `@resources` to specify the resource requirements
717
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
718
+
719
+ You can choose the compute layer on the command line by executing e.g.
720
+ ```
721
+ python myflow.py run --with batch
722
+ ```
723
+ or
724
+ ```
725
+ python myflow.py run --with kubernetes
726
+ ```
727
+ which executes the flow on the desired system using the
728
+ requirements specified in `@resources`.
709
729
 
710
730
  Parameters
711
731
  ----------
712
- packages : Dict[str, str], default: {}
713
- Packages to use for this step. The key is the name of the package
714
- and the value is the version to use.
715
- python : str, optional, default: None
716
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
717
- that the version used will correspond to the version of the Python interpreter used to start the run.
732
+ cpu : int, default 1
733
+ Number of CPUs required for this step.
734
+ gpu : int, default 0
735
+ Number of GPUs required for this step.
736
+ disk : int, optional, default None
737
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
738
+ memory : int, default 4096
739
+ Memory size (in MB) required for this step.
740
+ shared_memory : int, optional, default None
741
+ The value for the size (in MiB) of the /dev/shm volume for this step.
742
+ This parameter maps to the `--shm-size` option in Docker.
718
743
  """
719
744
  ...
720
745
 
@@ -809,63 +834,168 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
809
834
  ...
810
835
 
811
836
  @typing.overload
812
- def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
837
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
813
838
  """
814
- Enables checkpointing for a step.
839
+ Creates a human-readable report, a Metaflow Card, after this step completes.
815
840
 
841
+ Note that you may add multiple `@card` decorators in a step with different parameters.
816
842
 
817
843
  Parameters
818
844
  ----------
819
- load_policy : str, default: "fresh"
820
- The policy for loading the checkpoint. The following policies are supported:
821
- - "eager": Loads the the latest available checkpoint within the namespace.
822
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
823
- will be loaded at the start of the task.
824
- - "none": Do not load any checkpoint
825
- - "fresh": Loads the lastest checkpoint created within the running Task.
826
- This mode helps loading checkpoints across various retry attempts of the same task.
827
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
828
- created within the task will be loaded when the task is retries execution on failure.
829
-
830
- temp_dir_root : str, default: None
831
- The root directory under which `current.checkpoint.directory` will be created.
832
-
845
+ type : str, default 'default'
846
+ Card type.
847
+ id : str, optional, default None
848
+ If multiple cards are present, use this id to identify this card.
849
+ options : Dict[str, Any], default {}
850
+ Options passed to the card. The contents depend on the card type.
851
+ timeout : int, default 45
852
+ Interrupt reporting if it takes more than this many seconds.
833
853
 
834
854
 
835
855
  """
836
856
  ...
837
857
 
838
858
  @typing.overload
839
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
859
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
840
860
  ...
841
861
 
842
862
  @typing.overload
843
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
863
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
844
864
  ...
845
865
 
846
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
866
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
847
867
  """
848
- Enables checkpointing for a step.
868
+ Creates a human-readable report, a Metaflow Card, after this step completes.
849
869
 
870
+ Note that you may add multiple `@card` decorators in a step with different parameters.
850
871
 
851
872
  Parameters
852
873
  ----------
853
- load_policy : str, default: "fresh"
854
- The policy for loading the checkpoint. The following policies are supported:
855
- - "eager": Loads the the latest available checkpoint within the namespace.
856
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
857
- will be loaded at the start of the task.
858
- - "none": Do not load any checkpoint
859
- - "fresh": Loads the lastest checkpoint created within the running Task.
860
- This mode helps loading checkpoints across various retry attempts of the same task.
861
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
862
- created within the task will be loaded when the task is retries execution on failure.
874
+ type : str, default 'default'
875
+ Card type.
876
+ id : str, optional, default None
877
+ If multiple cards are present, use this id to identify this card.
878
+ options : Dict[str, Any], default {}
879
+ Options passed to the card. The contents depend on the card type.
880
+ timeout : int, default 45
881
+ Interrupt reporting if it takes more than this many seconds.
863
882
 
864
- temp_dir_root : str, default: None
865
- The root directory under which `current.checkpoint.directory` will be created.
866
883
 
884
+ """
885
+ ...
886
+
887
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
888
+ """
889
+ Specifies that this step should execute on Kubernetes.
890
+
891
+ Parameters
892
+ ----------
893
+ cpu : int, default 1
894
+ Number of CPUs required for this step. If `@resources` is
895
+ also present, the maximum value from all decorators is used.
896
+ memory : int, default 4096
897
+ Memory size (in MB) required for this step. If
898
+ `@resources` is also present, the maximum value from all decorators is
899
+ used.
900
+ disk : int, default 10240
901
+ Disk size (in MB) required for this step. If
902
+ `@resources` is also present, the maximum value from all decorators is
903
+ used.
904
+ image : str, optional, default None
905
+ Docker image to use when launching on Kubernetes. If not specified, and
906
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
907
+ not, a default Docker image mapping to the current version of Python is used.
908
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
909
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
910
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
911
+ Kubernetes service account to use when launching pod in Kubernetes.
912
+ secrets : List[str], optional, default None
913
+ Kubernetes secrets to use when launching pod in Kubernetes. These
914
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
915
+ in Metaflow configuration.
916
+ node_selector: Union[Dict[str,str], str], optional, default None
917
+ Kubernetes node selector(s) to apply to the pod running the task.
918
+ Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
919
+ or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
920
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
921
+ Kubernetes namespace to use when launching pod in Kubernetes.
922
+ gpu : int, optional, default None
923
+ Number of GPUs required for this step. A value of zero implies that
924
+ the scheduled node should not have GPUs.
925
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
926
+ The vendor of the GPUs to be used for this step.
927
+ tolerations : List[str], default []
928
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
929
+ Kubernetes tolerations to use when launching pod in Kubernetes.
930
+ use_tmpfs : bool, default False
931
+ This enables an explicit tmpfs mount for this step.
932
+ tmpfs_tempdir : bool, default True
933
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
934
+ tmpfs_size : int, optional, default: None
935
+ The value for the size (in MiB) of the tmpfs mount for this step.
936
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
937
+ memory allocated for this step.
938
+ tmpfs_path : str, optional, default /metaflow_temp
939
+ Path to tmpfs mount for this step.
940
+ persistent_volume_claims : Dict[str, str], optional, default None
941
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
942
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
943
+ shared_memory: int, optional
944
+ Shared memory size (in MiB) required for this step
945
+ port: int, optional
946
+ Port number to specify in the Kubernetes job object
947
+ compute_pool : str, optional, default None
948
+ Compute pool to be used for for this step.
949
+ If not specified, any accessible compute pool within the perimeter is used.
950
+ """
951
+ ...
952
+
953
+ @typing.overload
954
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
955
+ """
956
+ Specifies the PyPI packages for the step.
957
+
958
+ Information in this decorator will augment any
959
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
960
+ you can use `@pypi_base` to set packages required by all
961
+ steps and use `@pypi` to specify step-specific overrides.
962
+
963
+ Parameters
964
+ ----------
965
+ packages : Dict[str, str], default: {}
966
+ Packages to use for this step. The key is the name of the package
967
+ and the value is the version to use.
968
+ python : str, optional, default: None
969
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
970
+ that the version used will correspond to the version of the Python interpreter used to start the run.
971
+ """
972
+ ...
973
+
974
+ @typing.overload
975
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
976
+ ...
977
+
978
+ @typing.overload
979
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
980
+ ...
981
+
982
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
983
+ """
984
+ Specifies the PyPI packages for the step.
867
985
 
986
+ Information in this decorator will augment any
987
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
988
+ you can use `@pypi_base` to set packages required by all
989
+ steps and use `@pypi` to specify step-specific overrides.
868
990
 
991
+ Parameters
992
+ ----------
993
+ packages : Dict[str, str], default: {}
994
+ Packages to use for this step. The key is the name of the package
995
+ and the value is the version to use.
996
+ python : str, optional, default: None
997
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
998
+ that the version used will correspond to the version of the Python interpreter used to start the run.
869
999
  """
870
1000
  ...
871
1001
 
@@ -971,25 +1101,6 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
971
1101
  """
972
1102
  ...
973
1103
 
974
- @typing.overload
975
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
976
- """
977
- Decorator prototype for all step decorators. This function gets specialized
978
- and imported for all decorators types by _import_plugin_decorators().
979
- """
980
- ...
981
-
982
- @typing.overload
983
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
984
- ...
985
-
986
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
987
- """
988
- Decorator prototype for all step decorators. This function gets specialized
989
- and imported for all decorators types by _import_plugin_decorators().
990
- """
991
- ...
992
-
993
1104
  @typing.overload
994
1105
  def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
995
1106
  """
@@ -1022,79 +1133,19 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
1022
1133
  ...
1023
1134
 
1024
1135
  @typing.overload
1025
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1136
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1026
1137
  """
1027
- Specifies the resources needed when executing this step.
1028
-
1029
- Use `@resources` to specify the resource requirements
1030
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1031
-
1032
- You can choose the compute layer on the command line by executing e.g.
1033
- ```
1034
- python myflow.py run --with batch
1035
- ```
1036
- or
1037
- ```
1038
- python myflow.py run --with kubernetes
1039
- ```
1040
- which executes the flow on the desired system using the
1041
- requirements specified in `@resources`.
1042
-
1043
- Parameters
1044
- ----------
1045
- cpu : int, default 1
1046
- Number of CPUs required for this step.
1047
- gpu : int, default 0
1048
- Number of GPUs required for this step.
1049
- disk : int, optional, default None
1050
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1051
- memory : int, default 4096
1052
- Memory size (in MB) required for this step.
1053
- shared_memory : int, optional, default None
1054
- The value for the size (in MiB) of the /dev/shm volume for this step.
1055
- This parameter maps to the `--shm-size` option in Docker.
1138
+ Internal decorator to support Fast bakery
1056
1139
  """
1057
1140
  ...
1058
1141
 
1059
1142
  @typing.overload
1060
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1061
- ...
1062
-
1063
- @typing.overload
1064
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1143
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1065
1144
  ...
1066
1145
 
1067
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1146
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1068
1147
  """
1069
- Specifies the resources needed when executing this step.
1070
-
1071
- Use `@resources` to specify the resource requirements
1072
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1073
-
1074
- You can choose the compute layer on the command line by executing e.g.
1075
- ```
1076
- python myflow.py run --with batch
1077
- ```
1078
- or
1079
- ```
1080
- python myflow.py run --with kubernetes
1081
- ```
1082
- which executes the flow on the desired system using the
1083
- requirements specified in `@resources`.
1084
-
1085
- Parameters
1086
- ----------
1087
- cpu : int, default 1
1088
- Number of CPUs required for this step.
1089
- gpu : int, default 0
1090
- Number of GPUs required for this step.
1091
- disk : int, optional, default None
1092
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1093
- memory : int, default 4096
1094
- Memory size (in MB) required for this step.
1095
- shared_memory : int, optional, default None
1096
- The value for the size (in MiB) of the /dev/shm volume for this step.
1097
- This parameter maps to the `--shm-size` option in Docker.
1148
+ Internal decorator to support Fast bakery
1098
1149
  """
1099
1150
  ...
1100
1151
 
@@ -1156,53 +1207,90 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1156
1207
  ...
1157
1208
 
1158
1209
  @typing.overload
1159
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1210
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1160
1211
  """
1161
- Creates a human-readable report, a Metaflow Card, after this step completes.
1162
-
1163
- Note that you may add multiple `@card` decorators in a step with different parameters.
1212
+ Specifies the PyPI packages for all steps of the flow.
1164
1213
 
1214
+ Use `@pypi_base` to set common packages required by all
1215
+ steps and use `@pypi` to specify step-specific overrides.
1165
1216
  Parameters
1166
1217
  ----------
1167
- type : str, default 'default'
1168
- Card type.
1169
- id : str, optional, default None
1170
- If multiple cards are present, use this id to identify this card.
1171
- options : Dict[str, Any], default {}
1172
- Options passed to the card. The contents depend on the card type.
1173
- timeout : int, default 45
1174
- Interrupt reporting if it takes more than this many seconds.
1175
-
1218
+ packages : Dict[str, str], default: {}
1219
+ Packages to use for this flow. The key is the name of the package
1220
+ and the value is the version to use.
1221
+ python : str, optional, default: None
1222
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1223
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1224
+ """
1225
+ ...
1226
+
1227
+ @typing.overload
1228
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1229
+ ...
1230
+
1231
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1232
+ """
1233
+ Specifies the PyPI packages for all steps of the flow.
1176
1234
 
1235
+ Use `@pypi_base` to set common packages required by all
1236
+ steps and use `@pypi` to specify step-specific overrides.
1237
+ Parameters
1238
+ ----------
1239
+ packages : Dict[str, str], default: {}
1240
+ Packages to use for this flow. The key is the name of the package
1241
+ and the value is the version to use.
1242
+ python : str, optional, default: None
1243
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1244
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1177
1245
  """
1178
1246
  ...
1179
1247
 
1180
1248
  @typing.overload
1181
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1249
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1250
+ """
1251
+ Specifies the times when the flow should be run when running on a
1252
+ production scheduler.
1253
+
1254
+ Parameters
1255
+ ----------
1256
+ hourly : bool, default False
1257
+ Run the workflow hourly.
1258
+ daily : bool, default True
1259
+ Run the workflow daily.
1260
+ weekly : bool, default False
1261
+ Run the workflow weekly.
1262
+ cron : str, optional, default None
1263
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1264
+ specified by this expression.
1265
+ timezone : str, optional, default None
1266
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1267
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1268
+ """
1182
1269
  ...
1183
1270
 
1184
1271
  @typing.overload
1185
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1272
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1186
1273
  ...
1187
1274
 
1188
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1275
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1189
1276
  """
1190
- Creates a human-readable report, a Metaflow Card, after this step completes.
1191
-
1192
- Note that you may add multiple `@card` decorators in a step with different parameters.
1277
+ Specifies the times when the flow should be run when running on a
1278
+ production scheduler.
1193
1279
 
1194
1280
  Parameters
1195
1281
  ----------
1196
- type : str, default 'default'
1197
- Card type.
1198
- id : str, optional, default None
1199
- If multiple cards are present, use this id to identify this card.
1200
- options : Dict[str, Any], default {}
1201
- Options passed to the card. The contents depend on the card type.
1202
- timeout : int, default 45
1203
- Interrupt reporting if it takes more than this many seconds.
1204
-
1205
-
1282
+ hourly : bool, default False
1283
+ Run the workflow hourly.
1284
+ daily : bool, default True
1285
+ Run the workflow daily.
1286
+ weekly : bool, default False
1287
+ Run the workflow weekly.
1288
+ cron : str, optional, default None
1289
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1290
+ specified by this expression.
1291
+ timezone : str, optional, default None
1292
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1293
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1206
1294
  """
1207
1295
  ...
1208
1296
 
@@ -1255,45 +1343,6 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1255
1343
  """
1256
1344
  ...
1257
1345
 
1258
- @typing.overload
1259
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1260
- """
1261
- Specifies the PyPI packages for all steps of the flow.
1262
-
1263
- Use `@pypi_base` to set common packages required by all
1264
- steps and use `@pypi` to specify step-specific overrides.
1265
- Parameters
1266
- ----------
1267
- packages : Dict[str, str], default: {}
1268
- Packages to use for this flow. The key is the name of the package
1269
- and the value is the version to use.
1270
- python : str, optional, default: None
1271
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1272
- that the version used will correspond to the version of the Python interpreter used to start the run.
1273
- """
1274
- ...
1275
-
1276
- @typing.overload
1277
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1278
- ...
1279
-
1280
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1281
- """
1282
- Specifies the PyPI packages for all steps of the flow.
1283
-
1284
- Use `@pypi_base` to set common packages required by all
1285
- steps and use `@pypi` to specify step-specific overrides.
1286
- Parameters
1287
- ----------
1288
- packages : Dict[str, str], default: {}
1289
- Packages to use for this flow. The key is the name of the package
1290
- and the value is the version to use.
1291
- python : str, optional, default: None
1292
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1293
- that the version used will correspond to the version of the Python interpreter used to start the run.
1294
- """
1295
- ...
1296
-
1297
1346
  def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1298
1347
  """
1299
1348
  This decorator is used to run NIM containers in Metaflow tasks as sidecars.
@@ -1325,45 +1374,67 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
1325
1374
  """
1326
1375
  ...
1327
1376
 
1377
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1378
+ """
1379
+ Specifies what flows belong to the same project.
1380
+
1381
+ A project-specific namespace is created for all flows that
1382
+ use the same `@project(name)`.
1383
+
1384
+ Parameters
1385
+ ----------
1386
+ name : str
1387
+ Project name. Make sure that the name is unique amongst all
1388
+ projects that use the same production scheduler. The name may
1389
+ contain only lowercase alphanumeric characters and underscores.
1390
+
1391
+
1392
+ """
1393
+ ...
1394
+
1328
1395
  @typing.overload
1329
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1396
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1330
1397
  """
1331
- Specifies the event(s) that this flow depends on.
1398
+ Specifies the flow(s) that this flow depends on.
1332
1399
 
1333
1400
  ```
1334
- @trigger(event='foo')
1401
+ @trigger_on_finish(flow='FooFlow')
1335
1402
  ```
1336
1403
  or
1337
1404
  ```
1338
- @trigger(events=['foo', 'bar'])
1405
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1339
1406
  ```
1407
+ This decorator respects the @project decorator and triggers the flow
1408
+ when upstream runs within the same namespace complete successfully
1340
1409
 
1341
- Additionally, you can specify the parameter mappings
1342
- to map event payload to Metaflow parameters for the flow.
1410
+ Additionally, you can specify project aware upstream flow dependencies
1411
+ by specifying the fully qualified project_flow_name.
1343
1412
  ```
1344
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1413
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1345
1414
  ```
1346
1415
  or
1347
1416
  ```
1348
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1349
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1417
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1350
1418
  ```
1351
1419
 
1352
- 'parameters' can also be a list of strings and tuples like so:
1353
- ```
1354
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1355
- ```
1356
- This is equivalent to:
1420
+ You can also specify just the project or project branch (other values will be
1421
+ inferred from the current project or project branch):
1357
1422
  ```
1358
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1423
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1359
1424
  ```
1360
1425
 
1426
+ Note that `branch` is typically one of:
1427
+ - `prod`
1428
+ - `user.bob`
1429
+ - `test.my_experiment`
1430
+ - `prod.staging`
1431
+
1361
1432
  Parameters
1362
1433
  ----------
1363
- event : Union[str, Dict[str, Any]], optional, default None
1364
- Event dependency for this flow.
1365
- events : List[Union[str, Dict[str, Any]]], default []
1366
- Events dependency for this flow.
1434
+ flow : Union[str, Dict[str, str]], optional, default None
1435
+ Upstream flow dependency for this flow.
1436
+ flows : List[Union[str, Dict[str, str]]], default []
1437
+ Upstream flow dependencies for this flow.
1367
1438
  options : Dict[str, Any], default {}
1368
1439
  Backend-specific configuration for tuning eventing behavior.
1369
1440
 
@@ -1372,47 +1443,51 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1372
1443
  ...
1373
1444
 
1374
1445
  @typing.overload
1375
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1446
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1376
1447
  ...
1377
1448
 
1378
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1449
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1379
1450
  """
1380
- Specifies the event(s) that this flow depends on.
1451
+ Specifies the flow(s) that this flow depends on.
1381
1452
 
1382
1453
  ```
1383
- @trigger(event='foo')
1454
+ @trigger_on_finish(flow='FooFlow')
1384
1455
  ```
1385
1456
  or
1386
1457
  ```
1387
- @trigger(events=['foo', 'bar'])
1458
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1388
1459
  ```
1460
+ This decorator respects the @project decorator and triggers the flow
1461
+ when upstream runs within the same namespace complete successfully
1389
1462
 
1390
- Additionally, you can specify the parameter mappings
1391
- to map event payload to Metaflow parameters for the flow.
1463
+ Additionally, you can specify project aware upstream flow dependencies
1464
+ by specifying the fully qualified project_flow_name.
1392
1465
  ```
1393
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1466
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1394
1467
  ```
1395
1468
  or
1396
1469
  ```
1397
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1398
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1470
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1399
1471
  ```
1400
1472
 
1401
- 'parameters' can also be a list of strings and tuples like so:
1402
- ```
1403
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1404
- ```
1405
- This is equivalent to:
1473
+ You can also specify just the project or project branch (other values will be
1474
+ inferred from the current project or project branch):
1406
1475
  ```
1407
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1476
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1408
1477
  ```
1409
1478
 
1479
+ Note that `branch` is typically one of:
1480
+ - `prod`
1481
+ - `user.bob`
1482
+ - `test.my_experiment`
1483
+ - `prod.staging`
1484
+
1410
1485
  Parameters
1411
1486
  ----------
1412
- event : Union[str, Dict[str, Any]], optional, default None
1413
- Event dependency for this flow.
1414
- events : List[Union[str, Dict[str, Any]]], default []
1415
- Events dependency for this flow.
1487
+ flow : Union[str, Dict[str, str]], optional, default None
1488
+ Upstream flow dependency for this flow.
1489
+ flows : List[Union[str, Dict[str, str]]], default []
1490
+ Upstream flow dependencies for this flow.
1416
1491
  options : Dict[str, Any], default {}
1417
1492
  Backend-specific configuration for tuning eventing behavior.
1418
1493
 
@@ -1420,70 +1495,45 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1420
1495
  """
1421
1496
  ...
1422
1497
 
1423
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1498
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1424
1499
  """
1425
- Specifies what flows belong to the same project.
1426
-
1427
- A project-specific namespace is created for all flows that
1428
- use the same `@project(name)`.
1500
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1501
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1429
1502
 
1430
1503
  Parameters
1431
1504
  ----------
1505
+ timeout : int
1506
+ Time, in seconds before the task times out and fails. (Default: 3600)
1507
+ poke_interval : int
1508
+ Time in seconds that the job should wait in between each try. (Default: 60)
1509
+ mode : str
1510
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1511
+ exponential_backoff : bool
1512
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1513
+ pool : str
1514
+ the slot pool this task should run in,
1515
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1516
+ soft_fail : bool
1517
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1432
1518
  name : str
1433
- Project name. Make sure that the name is unique amongst all
1434
- projects that use the same production scheduler. The name may
1435
- contain only lowercase alphanumeric characters and underscores.
1436
-
1437
-
1438
- """
1439
- ...
1440
-
1441
- @typing.overload
1442
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1443
- """
1444
- Specifies the times when the flow should be run when running on a
1445
- production scheduler.
1446
-
1447
- Parameters
1448
- ----------
1449
- hourly : bool, default False
1450
- Run the workflow hourly.
1451
- daily : bool, default True
1452
- Run the workflow daily.
1453
- weekly : bool, default False
1454
- Run the workflow weekly.
1455
- cron : str, optional, default None
1456
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1457
- specified by this expression.
1458
- timezone : str, optional, default None
1459
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1460
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1461
- """
1462
- ...
1463
-
1464
- @typing.overload
1465
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1466
- ...
1467
-
1468
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1469
- """
1470
- Specifies the times when the flow should be run when running on a
1471
- production scheduler.
1472
-
1473
- Parameters
1474
- ----------
1475
- hourly : bool, default False
1476
- Run the workflow hourly.
1477
- daily : bool, default True
1478
- Run the workflow daily.
1479
- weekly : bool, default False
1480
- Run the workflow weekly.
1481
- cron : str, optional, default None
1482
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1483
- specified by this expression.
1484
- timezone : str, optional, default None
1485
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1486
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1519
+ Name of the sensor on Airflow
1520
+ description : str
1521
+ Description of sensor in the Airflow UI
1522
+ external_dag_id : str
1523
+ The dag_id that contains the task you want to wait for.
1524
+ external_task_ids : List[str]
1525
+ The list of task_ids that you want to wait for.
1526
+ If None (default value) the sensor waits for the DAG. (Default: None)
1527
+ allowed_states : List[str]
1528
+ Iterable of allowed states, (Default: ['success'])
1529
+ failed_states : List[str]
1530
+ Iterable of failed or dis-allowed states. (Default: None)
1531
+ execution_delta : datetime.timedelta
1532
+ time difference with the previous execution to look at,
1533
+ the default is the same logical date as the current task or DAG. (Default: None)
1534
+ check_existence: bool
1535
+ Set to True to check if the external task exists or check if
1536
+ the DAG to wait for exists. (Default: True)
1487
1537
  """
1488
1538
  ...
1489
1539
 
@@ -1529,91 +1579,45 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1529
1579
  """
1530
1580
  ...
1531
1581
 
1532
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1533
- """
1534
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1535
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1536
-
1537
- Parameters
1538
- ----------
1539
- timeout : int
1540
- Time, in seconds before the task times out and fails. (Default: 3600)
1541
- poke_interval : int
1542
- Time in seconds that the job should wait in between each try. (Default: 60)
1543
- mode : str
1544
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1545
- exponential_backoff : bool
1546
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1547
- pool : str
1548
- the slot pool this task should run in,
1549
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1550
- soft_fail : bool
1551
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1552
- name : str
1553
- Name of the sensor on Airflow
1554
- description : str
1555
- Description of sensor in the Airflow UI
1556
- external_dag_id : str
1557
- The dag_id that contains the task you want to wait for.
1558
- external_task_ids : List[str]
1559
- The list of task_ids that you want to wait for.
1560
- If None (default value) the sensor waits for the DAG. (Default: None)
1561
- allowed_states : List[str]
1562
- Iterable of allowed states, (Default: ['success'])
1563
- failed_states : List[str]
1564
- Iterable of failed or dis-allowed states. (Default: None)
1565
- execution_delta : datetime.timedelta
1566
- time difference with the previous execution to look at,
1567
- the default is the same logical date as the current task or DAG. (Default: None)
1568
- check_existence: bool
1569
- Set to True to check if the external task exists or check if
1570
- the DAG to wait for exists. (Default: True)
1571
- """
1572
- ...
1573
-
1574
1582
  @typing.overload
1575
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1583
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1576
1584
  """
1577
- Specifies the flow(s) that this flow depends on.
1585
+ Specifies the event(s) that this flow depends on.
1578
1586
 
1579
1587
  ```
1580
- @trigger_on_finish(flow='FooFlow')
1588
+ @trigger(event='foo')
1581
1589
  ```
1582
1590
  or
1583
1591
  ```
1584
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1592
+ @trigger(events=['foo', 'bar'])
1585
1593
  ```
1586
- This decorator respects the @project decorator and triggers the flow
1587
- when upstream runs within the same namespace complete successfully
1588
1594
 
1589
- Additionally, you can specify project aware upstream flow dependencies
1590
- by specifying the fully qualified project_flow_name.
1595
+ Additionally, you can specify the parameter mappings
1596
+ to map event payload to Metaflow parameters for the flow.
1591
1597
  ```
1592
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1598
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1593
1599
  ```
1594
1600
  or
1595
1601
  ```
1596
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1602
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1603
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1597
1604
  ```
1598
1605
 
1599
- You can also specify just the project or project branch (other values will be
1600
- inferred from the current project or project branch):
1606
+ 'parameters' can also be a list of strings and tuples like so:
1601
1607
  ```
1602
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1608
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1609
+ ```
1610
+ This is equivalent to:
1611
+ ```
1612
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1603
1613
  ```
1604
-
1605
- Note that `branch` is typically one of:
1606
- - `prod`
1607
- - `user.bob`
1608
- - `test.my_experiment`
1609
- - `prod.staging`
1610
1614
 
1611
1615
  Parameters
1612
1616
  ----------
1613
- flow : Union[str, Dict[str, str]], optional, default None
1614
- Upstream flow dependency for this flow.
1615
- flows : List[Union[str, Dict[str, str]]], default []
1616
- Upstream flow dependencies for this flow.
1617
+ event : Union[str, Dict[str, Any]], optional, default None
1618
+ Event dependency for this flow.
1619
+ events : List[Union[str, Dict[str, Any]]], default []
1620
+ Events dependency for this flow.
1617
1621
  options : Dict[str, Any], default {}
1618
1622
  Backend-specific configuration for tuning eventing behavior.
1619
1623
 
@@ -1622,51 +1626,47 @@ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] =
1622
1626
  ...
1623
1627
 
1624
1628
  @typing.overload
1625
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1629
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1626
1630
  ...
1627
1631
 
1628
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1632
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1629
1633
  """
1630
- Specifies the flow(s) that this flow depends on.
1634
+ Specifies the event(s) that this flow depends on.
1631
1635
 
1632
1636
  ```
1633
- @trigger_on_finish(flow='FooFlow')
1637
+ @trigger(event='foo')
1634
1638
  ```
1635
1639
  or
1636
1640
  ```
1637
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1641
+ @trigger(events=['foo', 'bar'])
1638
1642
  ```
1639
- This decorator respects the @project decorator and triggers the flow
1640
- when upstream runs within the same namespace complete successfully
1641
1643
 
1642
- Additionally, you can specify project aware upstream flow dependencies
1643
- by specifying the fully qualified project_flow_name.
1644
+ Additionally, you can specify the parameter mappings
1645
+ to map event payload to Metaflow parameters for the flow.
1644
1646
  ```
1645
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1647
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1646
1648
  ```
1647
1649
  or
1648
1650
  ```
1649
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1651
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1652
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1650
1653
  ```
1651
1654
 
1652
- You can also specify just the project or project branch (other values will be
1653
- inferred from the current project or project branch):
1655
+ 'parameters' can also be a list of strings and tuples like so:
1654
1656
  ```
1655
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1657
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1658
+ ```
1659
+ This is equivalent to:
1660
+ ```
1661
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1656
1662
  ```
1657
-
1658
- Note that `branch` is typically one of:
1659
- - `prod`
1660
- - `user.bob`
1661
- - `test.my_experiment`
1662
- - `prod.staging`
1663
1663
 
1664
1664
  Parameters
1665
1665
  ----------
1666
- flow : Union[str, Dict[str, str]], optional, default None
1667
- Upstream flow dependency for this flow.
1668
- flows : List[Union[str, Dict[str, str]]], default []
1669
- Upstream flow dependencies for this flow.
1666
+ event : Union[str, Dict[str, Any]], optional, default None
1667
+ Event dependency for this flow.
1668
+ events : List[Union[str, Dict[str, Any]]], default []
1669
+ Events dependency for this flow.
1670
1670
  options : Dict[str, Any], default {}
1671
1671
  Backend-specific configuration for tuning eventing behavior.
1672
1672