ob-metaflow-stubs 6.0.3.102rc0__py2.py3-none-any.whl → 6.0.3.103rc0__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +496 -496
  2. metaflow-stubs/cards.pyi +4 -4
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +1 -1
  7. metaflow-stubs/clone_util.pyi +1 -1
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +1 -1
  10. metaflow-stubs/flowspec.pyi +3 -3
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/info_file.pyi +1 -1
  14. metaflow-stubs/metadata/metadata.pyi +2 -2
  15. metaflow-stubs/metadata/util.pyi +1 -1
  16. metaflow-stubs/metaflow_config.pyi +1 -1
  17. metaflow-stubs/metaflow_current.pyi +65 -65
  18. metaflow-stubs/mflog/mflog.pyi +1 -1
  19. metaflow-stubs/multicore_utils.pyi +1 -1
  20. metaflow-stubs/parameters.pyi +3 -3
  21. metaflow-stubs/plugins/__init__.pyi +1 -1
  22. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  24. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  30. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  31. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  36. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  37. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +1 -1
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  52. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  59. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  60. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  61. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  62. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  74. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  75. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  76. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  82. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  83. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  84. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  85. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  88. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  101. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  102. metaflow-stubs/plugins/package_cli.pyi +1 -1
  103. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/perimeters.pyi +1 -1
  105. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  111. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  112. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  113. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  114. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  118. metaflow-stubs/plugins/tag_cli.pyi +3 -3
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  120. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  121. metaflow-stubs/procpoll.pyi +1 -1
  122. metaflow-stubs/profilers/__init__.pyi +1 -1
  123. metaflow-stubs/pylint_wrapper.pyi +1 -1
  124. metaflow-stubs/runner/__init__.pyi +1 -1
  125. metaflow-stubs/runner/deployer.pyi +2 -2
  126. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  127. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  128. metaflow-stubs/runner/nbrun.pyi +1 -1
  129. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  130. metaflow-stubs/runner/utils.pyi +1 -1
  131. metaflow-stubs/system/__init__.pyi +2 -2
  132. metaflow-stubs/system/system_logger.pyi +1 -1
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +1 -1
  135. metaflow-stubs/tuple_util.pyi +1 -1
  136. {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.103rc0.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.103rc0.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.102rc0.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.103rc0.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.103rc0.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.12.22.1+obcheckpoint(0.0.10);ob(v1) #
4
- # Generated on 2024-09-20T18:35:21.800373 #
4
+ # Generated on 2024-09-20T18:44:03.631910 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.events
12
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
11
+ import datetime
13
12
  import metaflow.datastore.inputs
14
- import metaflow.metaflow_current
15
- import metaflow._vendor.click.types
16
- import metaflow.runner.metaflow_runner
17
- import typing
18
- import metaflow.flowspec
19
13
  import metaflow.client.core
14
+ import metaflow.events
20
15
  import metaflow.parameters
21
- import datetime
16
+ import metaflow.runner.metaflow_runner
17
+ import metaflow._vendor.click.types
18
+ import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
19
+ import metaflow.flowspec
20
+ import typing
21
+ import metaflow.metaflow_current
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -484,112 +484,59 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
484
484
  ...
485
485
 
486
486
  @typing.overload
487
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
487
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
488
488
  """
489
- Specifies the number of times the task corresponding
490
- to a step needs to be retried.
489
+ Specifies a timeout for your step.
491
490
 
492
- This decorator is useful for handling transient errors, such as networking issues.
493
- If your task contains operations that can't be retried safely, e.g. database updates,
494
- it is advisable to annotate it with `@retry(times=0)`.
491
+ This decorator is useful if this step may hang indefinitely.
495
492
 
496
- This can be used in conjunction with the `@catch` decorator. The `@catch`
497
- decorator will execute a no-op task after all retries have been exhausted,
498
- ensuring that the flow execution can continue.
493
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
494
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
495
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
496
+
497
+ Note that all the values specified in parameters are added together so if you specify
498
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
499
499
 
500
500
  Parameters
501
501
  ----------
502
- times : int, default 3
503
- Number of times to retry this task.
504
- minutes_between_retries : int, default 2
505
- Number of minutes between retries.
502
+ seconds : int, default 0
503
+ Number of seconds to wait prior to timing out.
504
+ minutes : int, default 0
505
+ Number of minutes to wait prior to timing out.
506
+ hours : int, default 0
507
+ Number of hours to wait prior to timing out.
506
508
  """
507
509
  ...
508
510
 
509
511
  @typing.overload
510
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
512
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
511
513
  ...
512
514
 
513
515
  @typing.overload
514
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
515
- ...
516
-
517
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
518
- """
519
- Specifies the number of times the task corresponding
520
- to a step needs to be retried.
521
-
522
- This decorator is useful for handling transient errors, such as networking issues.
523
- If your task contains operations that can't be retried safely, e.g. database updates,
524
- it is advisable to annotate it with `@retry(times=0)`.
525
-
526
- This can be used in conjunction with the `@catch` decorator. The `@catch`
527
- decorator will execute a no-op task after all retries have been exhausted,
528
- ensuring that the flow execution can continue.
529
-
530
- Parameters
531
- ----------
532
- times : int, default 3
533
- Number of times to retry this task.
534
- minutes_between_retries : int, default 2
535
- Number of minutes between retries.
536
- """
516
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
537
517
  ...
538
518
 
539
- @typing.overload
540
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
519
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
541
520
  """
542
- Specifies the Conda environment for the step.
521
+ Specifies a timeout for your step.
543
522
 
544
- Information in this decorator will augment any
545
- attributes set in the `@conda_base` flow-level decorator. Hence,
546
- you can use `@conda_base` to set packages required by all
547
- steps and use `@conda` to specify step-specific overrides.
523
+ This decorator is useful if this step may hang indefinitely.
548
524
 
549
- Parameters
550
- ----------
551
- packages : Dict[str, str], default {}
552
- Packages to use for this step. The key is the name of the package
553
- and the value is the version to use.
554
- libraries : Dict[str, str], default {}
555
- Supported for backward compatibility. When used with packages, packages will take precedence.
556
- python : str, optional, default None
557
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
558
- that the version used will correspond to the version of the Python interpreter used to start the run.
559
- disabled : bool, default False
560
- If set to True, disables @conda.
561
- """
562
- ...
563
-
564
- @typing.overload
565
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
566
- ...
567
-
568
- @typing.overload
569
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
570
- ...
571
-
572
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
573
- """
574
- Specifies the Conda environment for the step.
525
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
526
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
527
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
575
528
 
576
- Information in this decorator will augment any
577
- attributes set in the `@conda_base` flow-level decorator. Hence,
578
- you can use `@conda_base` to set packages required by all
579
- steps and use `@conda` to specify step-specific overrides.
529
+ Note that all the values specified in parameters are added together so if you specify
530
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
580
531
 
581
532
  Parameters
582
533
  ----------
583
- packages : Dict[str, str], default {}
584
- Packages to use for this step. The key is the name of the package
585
- and the value is the version to use.
586
- libraries : Dict[str, str], default {}
587
- Supported for backward compatibility. When used with packages, packages will take precedence.
588
- python : str, optional, default None
589
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
590
- that the version used will correspond to the version of the Python interpreter used to start the run.
591
- disabled : bool, default False
592
- If set to True, disables @conda.
534
+ seconds : int, default 0
535
+ Number of seconds to wait prior to timing out.
536
+ minutes : int, default 0
537
+ Number of minutes to wait prior to timing out.
538
+ hours : int, default 0
539
+ Number of hours to wait prior to timing out.
593
540
  """
594
541
  ...
595
542
 
@@ -627,167 +574,247 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
627
574
  ...
628
575
 
629
576
  @typing.overload
630
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
577
+ def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
631
578
  """
632
- Specifies a timeout for your step.
633
-
634
- This decorator is useful if this step may hang indefinitely.
635
-
636
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
637
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
638
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
579
+ Enables checkpointing for a step.
639
580
 
640
- Note that all the values specified in parameters are added together so if you specify
641
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
642
581
 
643
582
  Parameters
644
583
  ----------
645
- seconds : int, default 0
646
- Number of seconds to wait prior to timing out.
647
- minutes : int, default 0
648
- Number of minutes to wait prior to timing out.
649
- hours : int, default 0
650
- Number of hours to wait prior to timing out.
584
+ load_policy : str, default: "fresh"
585
+ The policy for loading the checkpoint. The following policies are supported:
586
+ - "eager": Loads the the latest available checkpoint within the namespace.
587
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
588
+ will be loaded at the start of the task.
589
+ - "none": Do not load any checkpoint
590
+ - "fresh": Loads the lastest checkpoint created within the running Task.
591
+ This mode helps loading checkpoints across various retry attempts of the same task.
592
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
593
+ created within the task will be loaded when the task is retries execution on failure.
594
+
595
+ temp_dir_root : str, default: None
596
+ The root directory under which `current.checkpoint.directory` will be created.
597
+
598
+
599
+
651
600
  """
652
601
  ...
653
602
 
654
603
  @typing.overload
655
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
604
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
656
605
  ...
657
606
 
658
607
  @typing.overload
659
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
608
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
660
609
  ...
661
610
 
662
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
611
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
663
612
  """
664
- Specifies a timeout for your step.
665
-
666
- This decorator is useful if this step may hang indefinitely.
667
-
668
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
669
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
670
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
613
+ Enables checkpointing for a step.
671
614
 
672
- Note that all the values specified in parameters are added together so if you specify
673
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
674
615
 
675
616
  Parameters
676
617
  ----------
677
- seconds : int, default 0
678
- Number of seconds to wait prior to timing out.
679
- minutes : int, default 0
680
- Number of minutes to wait prior to timing out.
681
- hours : int, default 0
682
- Number of hours to wait prior to timing out.
618
+ load_policy : str, default: "fresh"
619
+ The policy for loading the checkpoint. The following policies are supported:
620
+ - "eager": Loads the the latest available checkpoint within the namespace.
621
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
622
+ will be loaded at the start of the task.
623
+ - "none": Do not load any checkpoint
624
+ - "fresh": Loads the lastest checkpoint created within the running Task.
625
+ This mode helps loading checkpoints across various retry attempts of the same task.
626
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
627
+ created within the task will be loaded when the task is retries execution on failure.
628
+
629
+ temp_dir_root : str, default: None
630
+ The root directory under which `current.checkpoint.directory` will be created.
631
+
632
+
633
+
683
634
  """
684
635
  ...
685
636
 
686
637
  @typing.overload
687
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
638
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
688
639
  """
689
- Specifies environment variables to be set prior to the execution of a step.
640
+ Creates a human-readable report, a Metaflow Card, after this step completes.
641
+
642
+ Note that you may add multiple `@card` decorators in a step with different parameters.
690
643
 
691
644
  Parameters
692
645
  ----------
693
- vars : Dict[str, str], default {}
694
- Dictionary of environment variables to set.
646
+ type : str, default 'default'
647
+ Card type.
648
+ id : str, optional, default None
649
+ If multiple cards are present, use this id to identify this card.
650
+ options : Dict[str, Any], default {}
651
+ Options passed to the card. The contents depend on the card type.
652
+ timeout : int, default 45
653
+ Interrupt reporting if it takes more than this many seconds.
654
+
655
+
695
656
  """
696
657
  ...
697
658
 
698
659
  @typing.overload
699
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
660
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
700
661
  ...
701
662
 
702
663
  @typing.overload
703
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
664
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
704
665
  ...
705
666
 
706
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
667
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
707
668
  """
708
- Specifies environment variables to be set prior to the execution of a step.
669
+ Creates a human-readable report, a Metaflow Card, after this step completes.
670
+
671
+ Note that you may add multiple `@card` decorators in a step with different parameters.
709
672
 
710
673
  Parameters
711
674
  ----------
712
- vars : Dict[str, str], default {}
713
- Dictionary of environment variables to set.
675
+ type : str, default 'default'
676
+ Card type.
677
+ id : str, optional, default None
678
+ If multiple cards are present, use this id to identify this card.
679
+ options : Dict[str, Any], default {}
680
+ Options passed to the card. The contents depend on the card type.
681
+ timeout : int, default 45
682
+ Interrupt reporting if it takes more than this many seconds.
683
+
684
+
714
685
  """
715
686
  ...
716
687
 
717
688
  @typing.overload
718
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
689
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
719
690
  """
720
- Specifies the resources needed when executing this step.
691
+ Specifies the number of times the task corresponding
692
+ to a step needs to be retried.
721
693
 
722
- Use `@resources` to specify the resource requirements
723
- independently of the specific compute layer (`@batch`, `@kubernetes`).
694
+ This decorator is useful for handling transient errors, such as networking issues.
695
+ If your task contains operations that can't be retried safely, e.g. database updates,
696
+ it is advisable to annotate it with `@retry(times=0)`.
724
697
 
725
- You can choose the compute layer on the command line by executing e.g.
726
- ```
727
- python myflow.py run --with batch
728
- ```
729
- or
730
- ```
731
- python myflow.py run --with kubernetes
732
- ```
733
- which executes the flow on the desired system using the
734
- requirements specified in `@resources`.
698
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
699
+ decorator will execute a no-op task after all retries have been exhausted,
700
+ ensuring that the flow execution can continue.
735
701
 
736
702
  Parameters
737
703
  ----------
738
- cpu : int, default 1
739
- Number of CPUs required for this step.
740
- gpu : int, default 0
741
- Number of GPUs required for this step.
742
- disk : int, optional, default None
743
- Disk size (in MB) required for this step. Only applies on Kubernetes.
744
- memory : int, default 4096
745
- Memory size (in MB) required for this step.
746
- shared_memory : int, optional, default None
747
- The value for the size (in MiB) of the /dev/shm volume for this step.
748
- This parameter maps to the `--shm-size` option in Docker.
704
+ times : int, default 3
705
+ Number of times to retry this task.
706
+ minutes_between_retries : int, default 2
707
+ Number of minutes between retries.
749
708
  """
750
709
  ...
751
710
 
752
711
  @typing.overload
753
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
712
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
754
713
  ...
755
714
 
756
715
  @typing.overload
757
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
716
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
758
717
  ...
759
718
 
760
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
719
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
761
720
  """
762
- Specifies the resources needed when executing this step.
721
+ Specifies the number of times the task corresponding
722
+ to a step needs to be retried.
763
723
 
764
- Use `@resources` to specify the resource requirements
765
- independently of the specific compute layer (`@batch`, `@kubernetes`).
724
+ This decorator is useful for handling transient errors, such as networking issues.
725
+ If your task contains operations that can't be retried safely, e.g. database updates,
726
+ it is advisable to annotate it with `@retry(times=0)`.
766
727
 
767
- You can choose the compute layer on the command line by executing e.g.
768
- ```
769
- python myflow.py run --with batch
770
- ```
771
- or
772
- ```
773
- python myflow.py run --with kubernetes
774
- ```
775
- which executes the flow on the desired system using the
776
- requirements specified in `@resources`.
728
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
729
+ decorator will execute a no-op task after all retries have been exhausted,
730
+ ensuring that the flow execution can continue.
777
731
 
778
732
  Parameters
779
733
  ----------
780
- cpu : int, default 1
781
- Number of CPUs required for this step.
782
- gpu : int, default 0
783
- Number of GPUs required for this step.
784
- disk : int, optional, default None
785
- Disk size (in MB) required for this step. Only applies on Kubernetes.
786
- memory : int, default 4096
787
- Memory size (in MB) required for this step.
788
- shared_memory : int, optional, default None
789
- The value for the size (in MiB) of the /dev/shm volume for this step.
790
- This parameter maps to the `--shm-size` option in Docker.
734
+ times : int, default 3
735
+ Number of times to retry this task.
736
+ minutes_between_retries : int, default 2
737
+ Number of minutes between retries.
738
+ """
739
+ ...
740
+
741
+ @typing.overload
742
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
743
+ """
744
+ Specifies environment variables to be set prior to the execution of a step.
745
+
746
+ Parameters
747
+ ----------
748
+ vars : Dict[str, str], default {}
749
+ Dictionary of environment variables to set.
750
+ """
751
+ ...
752
+
753
+ @typing.overload
754
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
755
+ ...
756
+
757
+ @typing.overload
758
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
759
+ ...
760
+
761
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
762
+ """
763
+ Specifies environment variables to be set prior to the execution of a step.
764
+
765
+ Parameters
766
+ ----------
767
+ vars : Dict[str, str], default {}
768
+ Dictionary of environment variables to set.
769
+ """
770
+ ...
771
+
772
+ @typing.overload
773
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
774
+ """
775
+ Specifies the PyPI packages for the step.
776
+
777
+ Information in this decorator will augment any
778
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
779
+ you can use `@pypi_base` to set packages required by all
780
+ steps and use `@pypi` to specify step-specific overrides.
781
+
782
+ Parameters
783
+ ----------
784
+ packages : Dict[str, str], default: {}
785
+ Packages to use for this step. The key is the name of the package
786
+ and the value is the version to use.
787
+ python : str, optional, default: None
788
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
789
+ that the version used will correspond to the version of the Python interpreter used to start the run.
790
+ """
791
+ ...
792
+
793
+ @typing.overload
794
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
795
+ ...
796
+
797
+ @typing.overload
798
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
799
+ ...
800
+
801
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
802
+ """
803
+ Specifies the PyPI packages for the step.
804
+
805
+ Information in this decorator will augment any
806
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
807
+ you can use `@pypi_base` to set packages required by all
808
+ steps and use `@pypi` to specify step-specific overrides.
809
+
810
+ Parameters
811
+ ----------
812
+ packages : Dict[str, str], default: {}
813
+ Packages to use for this step. The key is the name of the package
814
+ and the value is the version to use.
815
+ python : str, optional, default: None
816
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
817
+ that the version used will correspond to the version of the Python interpreter used to start the run.
791
818
  """
792
819
  ...
793
820
 
@@ -877,163 +904,136 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
877
904
  ...
878
905
 
879
906
  @typing.overload
880
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
907
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
881
908
  """
882
- Specifies the PyPI packages for the step.
909
+ Specifies the Conda environment for the step.
883
910
 
884
911
  Information in this decorator will augment any
885
- attributes set in the `@pyi_base` flow-level decorator. Hence,
886
- you can use `@pypi_base` to set packages required by all
887
- steps and use `@pypi` to specify step-specific overrides.
912
+ attributes set in the `@conda_base` flow-level decorator. Hence,
913
+ you can use `@conda_base` to set packages required by all
914
+ steps and use `@conda` to specify step-specific overrides.
888
915
 
889
916
  Parameters
890
917
  ----------
891
- packages : Dict[str, str], default: {}
918
+ packages : Dict[str, str], default {}
892
919
  Packages to use for this step. The key is the name of the package
893
920
  and the value is the version to use.
894
- python : str, optional, default: None
921
+ libraries : Dict[str, str], default {}
922
+ Supported for backward compatibility. When used with packages, packages will take precedence.
923
+ python : str, optional, default None
895
924
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
896
925
  that the version used will correspond to the version of the Python interpreter used to start the run.
926
+ disabled : bool, default False
927
+ If set to True, disables @conda.
897
928
  """
898
929
  ...
899
930
 
900
931
  @typing.overload
901
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
932
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
902
933
  ...
903
934
 
904
935
  @typing.overload
905
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
936
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
906
937
  ...
907
938
 
908
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
939
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
909
940
  """
910
- Specifies the PyPI packages for the step.
941
+ Specifies the Conda environment for the step.
911
942
 
912
943
  Information in this decorator will augment any
913
- attributes set in the `@pyi_base` flow-level decorator. Hence,
914
- you can use `@pypi_base` to set packages required by all
915
- steps and use `@pypi` to specify step-specific overrides.
944
+ attributes set in the `@conda_base` flow-level decorator. Hence,
945
+ you can use `@conda_base` to set packages required by all
946
+ steps and use `@conda` to specify step-specific overrides.
916
947
 
917
948
  Parameters
918
949
  ----------
919
- packages : Dict[str, str], default: {}
950
+ packages : Dict[str, str], default {}
920
951
  Packages to use for this step. The key is the name of the package
921
952
  and the value is the version to use.
922
- python : str, optional, default: None
953
+ libraries : Dict[str, str], default {}
954
+ Supported for backward compatibility. When used with packages, packages will take precedence.
955
+ python : str, optional, default None
923
956
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
924
957
  that the version used will correspond to the version of the Python interpreter used to start the run.
958
+ disabled : bool, default False
959
+ If set to True, disables @conda.
925
960
  """
926
961
  ...
927
962
 
928
963
  @typing.overload
929
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
964
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
930
965
  """
931
- Creates a human-readable report, a Metaflow Card, after this step completes.
966
+ Specifies the resources needed when executing this step.
932
967
 
933
- Note that you may add multiple `@card` decorators in a step with different parameters.
968
+ Use `@resources` to specify the resource requirements
969
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
970
+
971
+ You can choose the compute layer on the command line by executing e.g.
972
+ ```
973
+ python myflow.py run --with batch
974
+ ```
975
+ or
976
+ ```
977
+ python myflow.py run --with kubernetes
978
+ ```
979
+ which executes the flow on the desired system using the
980
+ requirements specified in `@resources`.
934
981
 
935
982
  Parameters
936
983
  ----------
937
- type : str, default 'default'
938
- Card type.
939
- id : str, optional, default None
940
- If multiple cards are present, use this id to identify this card.
941
- options : Dict[str, Any], default {}
942
- Options passed to the card. The contents depend on the card type.
943
- timeout : int, default 45
944
- Interrupt reporting if it takes more than this many seconds.
945
-
946
-
984
+ cpu : int, default 1
985
+ Number of CPUs required for this step.
986
+ gpu : int, default 0
987
+ Number of GPUs required for this step.
988
+ disk : int, optional, default None
989
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
990
+ memory : int, default 4096
991
+ Memory size (in MB) required for this step.
992
+ shared_memory : int, optional, default None
993
+ The value for the size (in MiB) of the /dev/shm volume for this step.
994
+ This parameter maps to the `--shm-size` option in Docker.
947
995
  """
948
996
  ...
949
997
 
950
998
  @typing.overload
951
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
999
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
952
1000
  ...
953
1001
 
954
1002
  @typing.overload
955
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1003
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
956
1004
  ...
957
1005
 
958
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1006
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
959
1007
  """
960
- Creates a human-readable report, a Metaflow Card, after this step completes.
1008
+ Specifies the resources needed when executing this step.
961
1009
 
962
- Note that you may add multiple `@card` decorators in a step with different parameters.
1010
+ Use `@resources` to specify the resource requirements
1011
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1012
+
1013
+ You can choose the compute layer on the command line by executing e.g.
1014
+ ```
1015
+ python myflow.py run --with batch
1016
+ ```
1017
+ or
1018
+ ```
1019
+ python myflow.py run --with kubernetes
1020
+ ```
1021
+ which executes the flow on the desired system using the
1022
+ requirements specified in `@resources`.
963
1023
 
964
1024
  Parameters
965
1025
  ----------
966
- type : str, default 'default'
967
- Card type.
968
- id : str, optional, default None
969
- If multiple cards are present, use this id to identify this card.
970
- options : Dict[str, Any], default {}
971
- Options passed to the card. The contents depend on the card type.
972
- timeout : int, default 45
973
- Interrupt reporting if it takes more than this many seconds.
974
-
975
-
976
- """
977
- ...
978
-
979
- @typing.overload
980
- def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
981
- """
982
- Enables checkpointing for a step.
983
-
984
-
985
- Parameters
986
- ----------
987
- load_policy : str, default: "fresh"
988
- The policy for loading the checkpoint. The following policies are supported:
989
- - "eager": Loads the the latest available checkpoint within the namespace.
990
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
991
- will be loaded at the start of the task.
992
- - "none": Do not load any checkpoint
993
- - "fresh": Loads the lastest checkpoint created within the running Task.
994
- This mode helps loading checkpoints across various retry attempts of the same task.
995
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
996
- created within the task will be loaded when the task is retries execution on failure.
997
-
998
- temp_dir_root : str, default: None
999
- The root directory under which `current.checkpoint.directory` will be created.
1000
-
1001
-
1002
-
1003
- """
1004
- ...
1005
-
1006
- @typing.overload
1007
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1008
- ...
1009
-
1010
- @typing.overload
1011
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1012
- ...
1013
-
1014
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
1015
- """
1016
- Enables checkpointing for a step.
1017
-
1018
-
1019
- Parameters
1020
- ----------
1021
- load_policy : str, default: "fresh"
1022
- The policy for loading the checkpoint. The following policies are supported:
1023
- - "eager": Loads the the latest available checkpoint within the namespace.
1024
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1025
- will be loaded at the start of the task.
1026
- - "none": Do not load any checkpoint
1027
- - "fresh": Loads the lastest checkpoint created within the running Task.
1028
- This mode helps loading checkpoints across various retry attempts of the same task.
1029
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1030
- created within the task will be loaded when the task is retries execution on failure.
1031
-
1032
- temp_dir_root : str, default: None
1033
- The root directory under which `current.checkpoint.directory` will be created.
1034
-
1035
-
1036
-
1026
+ cpu : int, default 1
1027
+ Number of CPUs required for this step.
1028
+ gpu : int, default 0
1029
+ Number of GPUs required for this step.
1030
+ disk : int, optional, default None
1031
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1032
+ memory : int, default 4096
1033
+ Memory size (in MB) required for this step.
1034
+ shared_memory : int, optional, default None
1035
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1036
+ This parameter maps to the `--shm-size` option in Docker.
1037
1037
  """
1038
1038
  ...
1039
1039
 
@@ -1115,6 +1115,24 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
1115
1115
  """
1116
1116
  ...
1117
1117
 
1118
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1119
+ """
1120
+ Specifies what flows belong to the same project.
1121
+
1122
+ A project-specific namespace is created for all flows that
1123
+ use the same `@project(name)`.
1124
+
1125
+ Parameters
1126
+ ----------
1127
+ name : str
1128
+ Project name. Make sure that the name is unique amongst all
1129
+ projects that use the same production scheduler. The name may
1130
+ contain only lowercase alphanumeric characters and underscores.
1131
+
1132
+
1133
+ """
1134
+ ...
1135
+
1118
1136
  @typing.overload
1119
1137
  def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1120
1138
  """
@@ -1165,44 +1183,48 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1165
1183
  ...
1166
1184
 
1167
1185
  @typing.overload
1168
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1186
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1169
1187
  """
1170
- Specifies the event(s) that this flow depends on.
1188
+ Specifies the flow(s) that this flow depends on.
1171
1189
 
1172
1190
  ```
1173
- @trigger(event='foo')
1191
+ @trigger_on_finish(flow='FooFlow')
1174
1192
  ```
1175
1193
  or
1176
1194
  ```
1177
- @trigger(events=['foo', 'bar'])
1195
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1178
1196
  ```
1197
+ This decorator respects the @project decorator and triggers the flow
1198
+ when upstream runs within the same namespace complete successfully
1179
1199
 
1180
- Additionally, you can specify the parameter mappings
1181
- to map event payload to Metaflow parameters for the flow.
1200
+ Additionally, you can specify project aware upstream flow dependencies
1201
+ by specifying the fully qualified project_flow_name.
1182
1202
  ```
1183
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1203
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1184
1204
  ```
1185
1205
  or
1186
1206
  ```
1187
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1188
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1207
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1189
1208
  ```
1190
1209
 
1191
- 'parameters' can also be a list of strings and tuples like so:
1192
- ```
1193
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1194
- ```
1195
- This is equivalent to:
1210
+ You can also specify just the project or project branch (other values will be
1211
+ inferred from the current project or project branch):
1196
1212
  ```
1197
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1213
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1198
1214
  ```
1199
1215
 
1216
+ Note that `branch` is typically one of:
1217
+ - `prod`
1218
+ - `user.bob`
1219
+ - `test.my_experiment`
1220
+ - `prod.staging`
1221
+
1200
1222
  Parameters
1201
1223
  ----------
1202
- event : Union[str, Dict[str, Any]], optional, default None
1203
- Event dependency for this flow.
1204
- events : List[Union[str, Dict[str, Any]]], default []
1205
- Events dependency for this flow.
1224
+ flow : Union[str, Dict[str, str]], optional, default None
1225
+ Upstream flow dependency for this flow.
1226
+ flows : List[Union[str, Dict[str, str]]], default []
1227
+ Upstream flow dependencies for this flow.
1206
1228
  options : Dict[str, Any], default {}
1207
1229
  Backend-specific configuration for tuning eventing behavior.
1208
1230
 
@@ -1211,47 +1233,51 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1211
1233
  ...
1212
1234
 
1213
1235
  @typing.overload
1214
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1236
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1215
1237
  ...
1216
1238
 
1217
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1239
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1218
1240
  """
1219
- Specifies the event(s) that this flow depends on.
1241
+ Specifies the flow(s) that this flow depends on.
1220
1242
 
1221
1243
  ```
1222
- @trigger(event='foo')
1244
+ @trigger_on_finish(flow='FooFlow')
1223
1245
  ```
1224
1246
  or
1225
1247
  ```
1226
- @trigger(events=['foo', 'bar'])
1248
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1227
1249
  ```
1250
+ This decorator respects the @project decorator and triggers the flow
1251
+ when upstream runs within the same namespace complete successfully
1228
1252
 
1229
- Additionally, you can specify the parameter mappings
1230
- to map event payload to Metaflow parameters for the flow.
1253
+ Additionally, you can specify project aware upstream flow dependencies
1254
+ by specifying the fully qualified project_flow_name.
1231
1255
  ```
1232
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1256
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1233
1257
  ```
1234
1258
  or
1235
1259
  ```
1236
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1237
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1260
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1238
1261
  ```
1239
1262
 
1240
- 'parameters' can also be a list of strings and tuples like so:
1241
- ```
1242
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1243
- ```
1244
- This is equivalent to:
1263
+ You can also specify just the project or project branch (other values will be
1264
+ inferred from the current project or project branch):
1245
1265
  ```
1246
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1266
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1247
1267
  ```
1248
1268
 
1269
+ Note that `branch` is typically one of:
1270
+ - `prod`
1271
+ - `user.bob`
1272
+ - `test.my_experiment`
1273
+ - `prod.staging`
1274
+
1249
1275
  Parameters
1250
1276
  ----------
1251
- event : Union[str, Dict[str, Any]], optional, default None
1252
- Event dependency for this flow.
1253
- events : List[Union[str, Dict[str, Any]]], default []
1254
- Events dependency for this flow.
1277
+ flow : Union[str, Dict[str, str]], optional, default None
1278
+ Upstream flow dependency for this flow.
1279
+ flows : List[Union[str, Dict[str, str]]], default []
1280
+ Upstream flow dependencies for this flow.
1255
1281
  options : Dict[str, Any], default {}
1256
1282
  Backend-specific configuration for tuning eventing behavior.
1257
1283
 
@@ -1302,41 +1328,97 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1302
1328
  ...
1303
1329
 
1304
1330
  @typing.overload
1305
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1331
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1306
1332
  """
1307
- Specifies the PyPI packages for all steps of the flow.
1333
+ Specifies the event(s) that this flow depends on.
1334
+
1335
+ ```
1336
+ @trigger(event='foo')
1337
+ ```
1338
+ or
1339
+ ```
1340
+ @trigger(events=['foo', 'bar'])
1341
+ ```
1342
+
1343
+ Additionally, you can specify the parameter mappings
1344
+ to map event payload to Metaflow parameters for the flow.
1345
+ ```
1346
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1347
+ ```
1348
+ or
1349
+ ```
1350
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1351
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1352
+ ```
1353
+
1354
+ 'parameters' can also be a list of strings and tuples like so:
1355
+ ```
1356
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1357
+ ```
1358
+ This is equivalent to:
1359
+ ```
1360
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1361
+ ```
1308
1362
 
1309
- Use `@pypi_base` to set common packages required by all
1310
- steps and use `@pypi` to specify step-specific overrides.
1311
1363
  Parameters
1312
1364
  ----------
1313
- packages : Dict[str, str], default: {}
1314
- Packages to use for this flow. The key is the name of the package
1315
- and the value is the version to use.
1316
- python : str, optional, default: None
1317
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1318
- that the version used will correspond to the version of the Python interpreter used to start the run.
1365
+ event : Union[str, Dict[str, Any]], optional, default None
1366
+ Event dependency for this flow.
1367
+ events : List[Union[str, Dict[str, Any]]], default []
1368
+ Events dependency for this flow.
1369
+ options : Dict[str, Any], default {}
1370
+ Backend-specific configuration for tuning eventing behavior.
1371
+
1372
+
1319
1373
  """
1320
1374
  ...
1321
1375
 
1322
1376
  @typing.overload
1323
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1377
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1324
1378
  ...
1325
1379
 
1326
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1380
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1327
1381
  """
1328
- Specifies the PyPI packages for all steps of the flow.
1382
+ Specifies the event(s) that this flow depends on.
1383
+
1384
+ ```
1385
+ @trigger(event='foo')
1386
+ ```
1387
+ or
1388
+ ```
1389
+ @trigger(events=['foo', 'bar'])
1390
+ ```
1391
+
1392
+ Additionally, you can specify the parameter mappings
1393
+ to map event payload to Metaflow parameters for the flow.
1394
+ ```
1395
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1396
+ ```
1397
+ or
1398
+ ```
1399
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1400
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1401
+ ```
1402
+
1403
+ 'parameters' can also be a list of strings and tuples like so:
1404
+ ```
1405
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1406
+ ```
1407
+ This is equivalent to:
1408
+ ```
1409
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1410
+ ```
1329
1411
 
1330
- Use `@pypi_base` to set common packages required by all
1331
- steps and use `@pypi` to specify step-specific overrides.
1332
1412
  Parameters
1333
1413
  ----------
1334
- packages : Dict[str, str], default: {}
1335
- Packages to use for this flow. The key is the name of the package
1336
- and the value is the version to use.
1337
- python : str, optional, default: None
1338
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1339
- that the version used will correspond to the version of the Python interpreter used to start the run.
1414
+ event : Union[str, Dict[str, Any]], optional, default None
1415
+ Event dependency for this flow.
1416
+ events : List[Union[str, Dict[str, Any]]], default []
1417
+ Events dependency for this flow.
1418
+ options : Dict[str, Any], default {}
1419
+ Backend-specific configuration for tuning eventing behavior.
1420
+
1421
+
1340
1422
  """
1341
1423
  ...
1342
1424
 
@@ -1389,106 +1471,45 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1389
1471
  """
1390
1472
  ...
1391
1473
 
1392
- @typing.overload
1393
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1394
- """
1395
- Specifies the flow(s) that this flow depends on.
1396
-
1397
- ```
1398
- @trigger_on_finish(flow='FooFlow')
1399
- ```
1400
- or
1401
- ```
1402
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1403
- ```
1404
- This decorator respects the @project decorator and triggers the flow
1405
- when upstream runs within the same namespace complete successfully
1406
-
1407
- Additionally, you can specify project aware upstream flow dependencies
1408
- by specifying the fully qualified project_flow_name.
1409
- ```
1410
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1411
- ```
1412
- or
1413
- ```
1414
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1415
- ```
1416
-
1417
- You can also specify just the project or project branch (other values will be
1418
- inferred from the current project or project branch):
1419
- ```
1420
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1421
- ```
1422
-
1423
- Note that `branch` is typically one of:
1424
- - `prod`
1425
- - `user.bob`
1426
- - `test.my_experiment`
1427
- - `prod.staging`
1428
-
1429
- Parameters
1430
- ----------
1431
- flow : Union[str, Dict[str, str]], optional, default None
1432
- Upstream flow dependency for this flow.
1433
- flows : List[Union[str, Dict[str, str]]], default []
1434
- Upstream flow dependencies for this flow.
1435
- options : Dict[str, Any], default {}
1436
- Backend-specific configuration for tuning eventing behavior.
1437
-
1438
-
1439
- """
1440
- ...
1441
-
1442
- @typing.overload
1443
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1444
- ...
1445
-
1446
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1474
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1447
1475
  """
1448
- Specifies the flow(s) that this flow depends on.
1449
-
1450
- ```
1451
- @trigger_on_finish(flow='FooFlow')
1452
- ```
1453
- or
1454
- ```
1455
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1456
- ```
1457
- This decorator respects the @project decorator and triggers the flow
1458
- when upstream runs within the same namespace complete successfully
1459
-
1460
- Additionally, you can specify project aware upstream flow dependencies
1461
- by specifying the fully qualified project_flow_name.
1462
- ```
1463
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1464
- ```
1465
- or
1466
- ```
1467
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1468
- ```
1469
-
1470
- You can also specify just the project or project branch (other values will be
1471
- inferred from the current project or project branch):
1472
- ```
1473
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1474
- ```
1475
-
1476
- Note that `branch` is typically one of:
1477
- - `prod`
1478
- - `user.bob`
1479
- - `test.my_experiment`
1480
- - `prod.staging`
1476
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1477
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1481
1478
 
1482
1479
  Parameters
1483
1480
  ----------
1484
- flow : Union[str, Dict[str, str]], optional, default None
1485
- Upstream flow dependency for this flow.
1486
- flows : List[Union[str, Dict[str, str]]], default []
1487
- Upstream flow dependencies for this flow.
1488
- options : Dict[str, Any], default {}
1489
- Backend-specific configuration for tuning eventing behavior.
1490
-
1491
-
1481
+ timeout : int
1482
+ Time, in seconds before the task times out and fails. (Default: 3600)
1483
+ poke_interval : int
1484
+ Time in seconds that the job should wait in between each try. (Default: 60)
1485
+ mode : str
1486
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1487
+ exponential_backoff : bool
1488
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1489
+ pool : str
1490
+ the slot pool this task should run in,
1491
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1492
+ soft_fail : bool
1493
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1494
+ name : str
1495
+ Name of the sensor on Airflow
1496
+ description : str
1497
+ Description of sensor in the Airflow UI
1498
+ external_dag_id : str
1499
+ The dag_id that contains the task you want to wait for.
1500
+ external_task_ids : List[str]
1501
+ The list of task_ids that you want to wait for.
1502
+ If None (default value) the sensor waits for the DAG. (Default: None)
1503
+ allowed_states : List[str]
1504
+ Iterable of allowed states, (Default: ['success'])
1505
+ failed_states : List[str]
1506
+ Iterable of failed or dis-allowed states. (Default: None)
1507
+ execution_delta : datetime.timedelta
1508
+ time difference with the previous execution to look at,
1509
+ the default is the same logical date as the current task or DAG. (Default: None)
1510
+ check_existence: bool
1511
+ Set to True to check if the external task exists or check if
1512
+ the DAG to wait for exists. (Default: True)
1492
1513
  """
1493
1514
  ...
1494
1515
 
@@ -1523,63 +1544,42 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
1523
1544
  """
1524
1545
  ...
1525
1546
 
1526
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1547
+ @typing.overload
1548
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1527
1549
  """
1528
- Specifies what flows belong to the same project.
1529
-
1530
- A project-specific namespace is created for all flows that
1531
- use the same `@project(name)`.
1550
+ Specifies the PyPI packages for all steps of the flow.
1532
1551
 
1552
+ Use `@pypi_base` to set common packages required by all
1553
+ steps and use `@pypi` to specify step-specific overrides.
1533
1554
  Parameters
1534
1555
  ----------
1535
- name : str
1536
- Project name. Make sure that the name is unique amongst all
1537
- projects that use the same production scheduler. The name may
1538
- contain only lowercase alphanumeric characters and underscores.
1539
-
1540
-
1556
+ packages : Dict[str, str], default: {}
1557
+ Packages to use for this flow. The key is the name of the package
1558
+ and the value is the version to use.
1559
+ python : str, optional, default: None
1560
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1561
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1541
1562
  """
1542
1563
  ...
1543
1564
 
1544
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1565
+ @typing.overload
1566
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1567
+ ...
1568
+
1569
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1545
1570
  """
1546
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1547
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1571
+ Specifies the PyPI packages for all steps of the flow.
1548
1572
 
1573
+ Use `@pypi_base` to set common packages required by all
1574
+ steps and use `@pypi` to specify step-specific overrides.
1549
1575
  Parameters
1550
1576
  ----------
1551
- timeout : int
1552
- Time, in seconds before the task times out and fails. (Default: 3600)
1553
- poke_interval : int
1554
- Time in seconds that the job should wait in between each try. (Default: 60)
1555
- mode : str
1556
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1557
- exponential_backoff : bool
1558
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1559
- pool : str
1560
- the slot pool this task should run in,
1561
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1562
- soft_fail : bool
1563
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1564
- name : str
1565
- Name of the sensor on Airflow
1566
- description : str
1567
- Description of sensor in the Airflow UI
1568
- external_dag_id : str
1569
- The dag_id that contains the task you want to wait for.
1570
- external_task_ids : List[str]
1571
- The list of task_ids that you want to wait for.
1572
- If None (default value) the sensor waits for the DAG. (Default: None)
1573
- allowed_states : List[str]
1574
- Iterable of allowed states, (Default: ['success'])
1575
- failed_states : List[str]
1576
- Iterable of failed or dis-allowed states. (Default: None)
1577
- execution_delta : datetime.timedelta
1578
- time difference with the previous execution to look at,
1579
- the default is the same logical date as the current task or DAG. (Default: None)
1580
- check_existence: bool
1581
- Set to True to check if the external task exists or check if
1582
- the DAG to wait for exists. (Default: True)
1577
+ packages : Dict[str, str], default: {}
1578
+ Packages to use for this flow. The key is the name of the package
1579
+ and the value is the version to use.
1580
+ python : str, optional, default: None
1581
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1582
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1583
1583
  """
1584
1584
  ...
1585
1585