ob-metaflow-stubs 6.0.3.103rc0__py2.py3-none-any.whl → 6.0.3.103rc3__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +466 -466
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +3 -3
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +3 -3
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +64 -64
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +3 -3
  21. metaflow-stubs/plugins/__init__.pyi +3 -3
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +4 -4
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
  61. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +4 -4
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  74. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  76. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -4
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +4 -4
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +4 -4
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  120. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +3 -3
  126. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  127. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  128. metaflow-stubs/runner/nbrun.pyi +2 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  130. metaflow-stubs/runner/utils.pyi +2 -2
  131. metaflow-stubs/system/__init__.pyi +2 -2
  132. metaflow-stubs/system/system_logger.pyi +2 -2
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.103rc0.dist-info → ob_metaflow_stubs-6.0.3.103rc3.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.103rc3.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.103rc0.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.103rc0.dist-info → ob_metaflow_stubs-6.0.3.103rc3.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.103rc0.dist-info → ob_metaflow_stubs-6.0.3.103rc3.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.10);ob(v1) #
4
- # Generated on 2024-09-20T18:44:03.631910 #
3
+ # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-09-20T21:42:44.532164 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
- import metaflow.datastore.inputs
13
- import metaflow.client.core
14
- import metaflow.events
11
+ import metaflow.flowspec
15
12
  import metaflow.parameters
16
- import metaflow.runner.metaflow_runner
13
+ import typing
14
+ import metaflow.datastore.inputs
17
15
  import metaflow._vendor.click.types
16
+ import metaflow.runner.metaflow_runner
18
17
  import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
19
- import metaflow.flowspec
20
- import typing
18
+ import datetime
19
+ import metaflow.events
21
20
  import metaflow.metaflow_current
21
+ import metaflow.client.core
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -483,6 +483,37 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
483
483
  """
484
484
  ...
485
485
 
486
+ @typing.overload
487
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
488
+ """
489
+ Specifies environment variables to be set prior to the execution of a step.
490
+
491
+ Parameters
492
+ ----------
493
+ vars : Dict[str, str], default {}
494
+ Dictionary of environment variables to set.
495
+ """
496
+ ...
497
+
498
+ @typing.overload
499
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
500
+ ...
501
+
502
+ @typing.overload
503
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
504
+ ...
505
+
506
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
507
+ """
508
+ Specifies environment variables to be set prior to the execution of a step.
509
+
510
+ Parameters
511
+ ----------
512
+ vars : Dict[str, str], default {}
513
+ Dictionary of environment variables to set.
514
+ """
515
+ ...
516
+
486
517
  @typing.overload
487
518
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
488
519
  """
@@ -540,151 +571,6 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
540
571
  """
541
572
  ...
542
573
 
543
- @typing.overload
544
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
545
- """
546
- Specifies secrets to be retrieved and injected as environment variables prior to
547
- the execution of a step.
548
-
549
- Parameters
550
- ----------
551
- sources : List[Union[str, Dict[str, Any]]], default: []
552
- List of secret specs, defining how the secrets are to be retrieved
553
- """
554
- ...
555
-
556
- @typing.overload
557
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
558
- ...
559
-
560
- @typing.overload
561
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
562
- ...
563
-
564
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
565
- """
566
- Specifies secrets to be retrieved and injected as environment variables prior to
567
- the execution of a step.
568
-
569
- Parameters
570
- ----------
571
- sources : List[Union[str, Dict[str, Any]]], default: []
572
- List of secret specs, defining how the secrets are to be retrieved
573
- """
574
- ...
575
-
576
- @typing.overload
577
- def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
578
- """
579
- Enables checkpointing for a step.
580
-
581
-
582
- Parameters
583
- ----------
584
- load_policy : str, default: "fresh"
585
- The policy for loading the checkpoint. The following policies are supported:
586
- - "eager": Loads the the latest available checkpoint within the namespace.
587
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
588
- will be loaded at the start of the task.
589
- - "none": Do not load any checkpoint
590
- - "fresh": Loads the lastest checkpoint created within the running Task.
591
- This mode helps loading checkpoints across various retry attempts of the same task.
592
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
593
- created within the task will be loaded when the task is retries execution on failure.
594
-
595
- temp_dir_root : str, default: None
596
- The root directory under which `current.checkpoint.directory` will be created.
597
-
598
-
599
-
600
- """
601
- ...
602
-
603
- @typing.overload
604
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
605
- ...
606
-
607
- @typing.overload
608
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
609
- ...
610
-
611
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
612
- """
613
- Enables checkpointing for a step.
614
-
615
-
616
- Parameters
617
- ----------
618
- load_policy : str, default: "fresh"
619
- The policy for loading the checkpoint. The following policies are supported:
620
- - "eager": Loads the the latest available checkpoint within the namespace.
621
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
622
- will be loaded at the start of the task.
623
- - "none": Do not load any checkpoint
624
- - "fresh": Loads the lastest checkpoint created within the running Task.
625
- This mode helps loading checkpoints across various retry attempts of the same task.
626
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
627
- created within the task will be loaded when the task is retries execution on failure.
628
-
629
- temp_dir_root : str, default: None
630
- The root directory under which `current.checkpoint.directory` will be created.
631
-
632
-
633
-
634
- """
635
- ...
636
-
637
- @typing.overload
638
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
639
- """
640
- Creates a human-readable report, a Metaflow Card, after this step completes.
641
-
642
- Note that you may add multiple `@card` decorators in a step with different parameters.
643
-
644
- Parameters
645
- ----------
646
- type : str, default 'default'
647
- Card type.
648
- id : str, optional, default None
649
- If multiple cards are present, use this id to identify this card.
650
- options : Dict[str, Any], default {}
651
- Options passed to the card. The contents depend on the card type.
652
- timeout : int, default 45
653
- Interrupt reporting if it takes more than this many seconds.
654
-
655
-
656
- """
657
- ...
658
-
659
- @typing.overload
660
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
661
- ...
662
-
663
- @typing.overload
664
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
665
- ...
666
-
667
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
668
- """
669
- Creates a human-readable report, a Metaflow Card, after this step completes.
670
-
671
- Note that you may add multiple `@card` decorators in a step with different parameters.
672
-
673
- Parameters
674
- ----------
675
- type : str, default 'default'
676
- Card type.
677
- id : str, optional, default None
678
- If multiple cards are present, use this id to identify this card.
679
- options : Dict[str, Any], default {}
680
- Options passed to the card. The contents depend on the card type.
681
- timeout : int, default 45
682
- Interrupt reporting if it takes more than this many seconds.
683
-
684
-
685
- """
686
- ...
687
-
688
574
  @typing.overload
689
575
  def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
690
576
  """
@@ -739,101 +625,148 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
739
625
  ...
740
626
 
741
627
  @typing.overload
742
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
628
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
743
629
  """
744
- Specifies environment variables to be set prior to the execution of a step.
745
-
746
- Parameters
747
- ----------
748
- vars : Dict[str, str], default {}
749
- Dictionary of environment variables to set.
630
+ Internal decorator to support Fast bakery
750
631
  """
751
632
  ...
752
633
 
753
634
  @typing.overload
754
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
755
- ...
756
-
757
- @typing.overload
758
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
635
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
759
636
  ...
760
637
 
761
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
638
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
762
639
  """
763
- Specifies environment variables to be set prior to the execution of a step.
764
-
765
- Parameters
766
- ----------
767
- vars : Dict[str, str], default {}
768
- Dictionary of environment variables to set.
640
+ Internal decorator to support Fast bakery
769
641
  """
770
642
  ...
771
643
 
772
644
  @typing.overload
773
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
645
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
774
646
  """
775
- Specifies the PyPI packages for the step.
776
-
777
- Information in this decorator will augment any
778
- attributes set in the `@pyi_base` flow-level decorator. Hence,
779
- you can use `@pypi_base` to set packages required by all
780
- steps and use `@pypi` to specify step-specific overrides.
781
-
782
- Parameters
783
- ----------
784
- packages : Dict[str, str], default: {}
785
- Packages to use for this step. The key is the name of the package
786
- and the value is the version to use.
787
- python : str, optional, default: None
788
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
789
- that the version used will correspond to the version of the Python interpreter used to start the run.
647
+ Decorator prototype for all step decorators. This function gets specialized
648
+ and imported for all decorators types by _import_plugin_decorators().
790
649
  """
791
650
  ...
792
651
 
793
652
  @typing.overload
794
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
653
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
795
654
  ...
796
655
 
797
- @typing.overload
798
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
656
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
657
+ """
658
+ Decorator prototype for all step decorators. This function gets specialized
659
+ and imported for all decorators types by _import_plugin_decorators().
660
+ """
799
661
  ...
800
662
 
801
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
663
+ @typing.overload
664
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
802
665
  """
803
- Specifies the PyPI packages for the step.
666
+ Specifies the resources needed when executing this step.
804
667
 
805
- Information in this decorator will augment any
806
- attributes set in the `@pyi_base` flow-level decorator. Hence,
807
- you can use `@pypi_base` to set packages required by all
808
- steps and use `@pypi` to specify step-specific overrides.
668
+ Use `@resources` to specify the resource requirements
669
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
809
670
 
810
- Parameters
811
- ----------
812
- packages : Dict[str, str], default: {}
813
- Packages to use for this step. The key is the name of the package
814
- and the value is the version to use.
815
- python : str, optional, default: None
816
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
817
- that the version used will correspond to the version of the Python interpreter used to start the run.
671
+ You can choose the compute layer on the command line by executing e.g.
672
+ ```
673
+ python myflow.py run --with batch
674
+ ```
675
+ or
676
+ ```
677
+ python myflow.py run --with kubernetes
678
+ ```
679
+ which executes the flow on the desired system using the
680
+ requirements specified in `@resources`.
681
+
682
+ Parameters
683
+ ----------
684
+ cpu : int, default 1
685
+ Number of CPUs required for this step.
686
+ gpu : int, default 0
687
+ Number of GPUs required for this step.
688
+ disk : int, optional, default None
689
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
690
+ memory : int, default 4096
691
+ Memory size (in MB) required for this step.
692
+ shared_memory : int, optional, default None
693
+ The value for the size (in MiB) of the /dev/shm volume for this step.
694
+ This parameter maps to the `--shm-size` option in Docker.
818
695
  """
819
696
  ...
820
697
 
821
698
  @typing.overload
822
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
699
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
700
+ ...
701
+
702
+ @typing.overload
703
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
704
+ ...
705
+
706
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
823
707
  """
824
- Decorator prototype for all step decorators. This function gets specialized
825
- and imported for all decorators types by _import_plugin_decorators().
708
+ Specifies the resources needed when executing this step.
709
+
710
+ Use `@resources` to specify the resource requirements
711
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
712
+
713
+ You can choose the compute layer on the command line by executing e.g.
714
+ ```
715
+ python myflow.py run --with batch
716
+ ```
717
+ or
718
+ ```
719
+ python myflow.py run --with kubernetes
720
+ ```
721
+ which executes the flow on the desired system using the
722
+ requirements specified in `@resources`.
723
+
724
+ Parameters
725
+ ----------
726
+ cpu : int, default 1
727
+ Number of CPUs required for this step.
728
+ gpu : int, default 0
729
+ Number of GPUs required for this step.
730
+ disk : int, optional, default None
731
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
732
+ memory : int, default 4096
733
+ Memory size (in MB) required for this step.
734
+ shared_memory : int, optional, default None
735
+ The value for the size (in MiB) of the /dev/shm volume for this step.
736
+ This parameter maps to the `--shm-size` option in Docker.
826
737
  """
827
738
  ...
828
739
 
829
740
  @typing.overload
830
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
741
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
742
+ """
743
+ Specifies secrets to be retrieved and injected as environment variables prior to
744
+ the execution of a step.
745
+
746
+ Parameters
747
+ ----------
748
+ sources : List[Union[str, Dict[str, Any]]], default: []
749
+ List of secret specs, defining how the secrets are to be retrieved
750
+ """
831
751
  ...
832
752
 
833
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
753
+ @typing.overload
754
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
755
+ ...
756
+
757
+ @typing.overload
758
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
759
+ ...
760
+
761
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
834
762
  """
835
- Decorator prototype for all step decorators. This function gets specialized
836
- and imported for all decorators types by _import_plugin_decorators().
763
+ Specifies secrets to be retrieved and injected as environment variables prior to
764
+ the execution of a step.
765
+
766
+ Parameters
767
+ ----------
768
+ sources : List[Union[str, Dict[str, Any]]], default: []
769
+ List of secret specs, defining how the secrets are to be retrieved
837
770
  """
838
771
  ...
839
772
 
@@ -961,79 +894,63 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
961
894
  ...
962
895
 
963
896
  @typing.overload
964
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
897
+ def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
965
898
  """
966
- Specifies the resources needed when executing this step.
967
-
968
- Use `@resources` to specify the resource requirements
969
- independently of the specific compute layer (`@batch`, `@kubernetes`).
899
+ Enables checkpointing for a step.
970
900
 
971
- You can choose the compute layer on the command line by executing e.g.
972
- ```
973
- python myflow.py run --with batch
974
- ```
975
- or
976
- ```
977
- python myflow.py run --with kubernetes
978
- ```
979
- which executes the flow on the desired system using the
980
- requirements specified in `@resources`.
981
901
 
982
902
  Parameters
983
903
  ----------
984
- cpu : int, default 1
985
- Number of CPUs required for this step.
986
- gpu : int, default 0
987
- Number of GPUs required for this step.
988
- disk : int, optional, default None
989
- Disk size (in MB) required for this step. Only applies on Kubernetes.
990
- memory : int, default 4096
991
- Memory size (in MB) required for this step.
992
- shared_memory : int, optional, default None
993
- The value for the size (in MiB) of the /dev/shm volume for this step.
994
- This parameter maps to the `--shm-size` option in Docker.
904
+ load_policy : str, default: "fresh"
905
+ The policy for loading the checkpoint. The following policies are supported:
906
+ - "eager": Loads the the latest available checkpoint within the namespace.
907
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
908
+ will be loaded at the start of the task.
909
+ - "none": Do not load any checkpoint
910
+ - "fresh": Loads the lastest checkpoint created within the running Task.
911
+ This mode helps loading checkpoints across various retry attempts of the same task.
912
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
913
+ created within the task will be loaded when the task is retries execution on failure.
914
+
915
+ temp_dir_root : str, default: None
916
+ The root directory under which `current.checkpoint.directory` will be created.
917
+
918
+
919
+
995
920
  """
996
921
  ...
997
922
 
998
923
  @typing.overload
999
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
924
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1000
925
  ...
1001
926
 
1002
927
  @typing.overload
1003
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
928
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1004
929
  ...
1005
930
 
1006
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
931
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
1007
932
  """
1008
- Specifies the resources needed when executing this step.
1009
-
1010
- Use `@resources` to specify the resource requirements
1011
- independently of the specific compute layer (`@batch`, `@kubernetes`).
933
+ Enables checkpointing for a step.
1012
934
 
1013
- You can choose the compute layer on the command line by executing e.g.
1014
- ```
1015
- python myflow.py run --with batch
1016
- ```
1017
- or
1018
- ```
1019
- python myflow.py run --with kubernetes
1020
- ```
1021
- which executes the flow on the desired system using the
1022
- requirements specified in `@resources`.
1023
935
 
1024
936
  Parameters
1025
937
  ----------
1026
- cpu : int, default 1
1027
- Number of CPUs required for this step.
1028
- gpu : int, default 0
1029
- Number of GPUs required for this step.
1030
- disk : int, optional, default None
1031
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1032
- memory : int, default 4096
1033
- Memory size (in MB) required for this step.
1034
- shared_memory : int, optional, default None
1035
- The value for the size (in MiB) of the /dev/shm volume for this step.
1036
- This parameter maps to the `--shm-size` option in Docker.
938
+ load_policy : str, default: "fresh"
939
+ The policy for loading the checkpoint. The following policies are supported:
940
+ - "eager": Loads the the latest available checkpoint within the namespace.
941
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
942
+ will be loaded at the start of the task.
943
+ - "none": Do not load any checkpoint
944
+ - "fresh": Loads the lastest checkpoint created within the running Task.
945
+ This mode helps loading checkpoints across various retry attempts of the same task.
946
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
947
+ created within the task will be loaded when the task is retries execution on failure.
948
+
949
+ temp_dir_root : str, default: None
950
+ The root directory under which `current.checkpoint.directory` will be created.
951
+
952
+
953
+
1037
954
  """
1038
955
  ...
1039
956
 
@@ -1099,189 +1016,141 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1099
1016
  ...
1100
1017
 
1101
1018
  @typing.overload
1102
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1019
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1103
1020
  """
1104
- Internal decorator to support Fast bakery
1021
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1022
+
1023
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1024
+
1025
+ Parameters
1026
+ ----------
1027
+ type : str, default 'default'
1028
+ Card type.
1029
+ id : str, optional, default None
1030
+ If multiple cards are present, use this id to identify this card.
1031
+ options : Dict[str, Any], default {}
1032
+ Options passed to the card. The contents depend on the card type.
1033
+ timeout : int, default 45
1034
+ Interrupt reporting if it takes more than this many seconds.
1035
+
1036
+
1105
1037
  """
1106
1038
  ...
1107
1039
 
1108
1040
  @typing.overload
1109
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1041
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1110
1042
  ...
1111
1043
 
1112
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1113
- """
1114
- Internal decorator to support Fast bakery
1115
- """
1044
+ @typing.overload
1045
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1116
1046
  ...
1117
1047
 
1118
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1048
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1119
1049
  """
1120
- Specifies what flows belong to the same project.
1050
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1121
1051
 
1122
- A project-specific namespace is created for all flows that
1123
- use the same `@project(name)`.
1052
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1124
1053
 
1125
1054
  Parameters
1126
1055
  ----------
1127
- name : str
1128
- Project name. Make sure that the name is unique amongst all
1129
- projects that use the same production scheduler. The name may
1130
- contain only lowercase alphanumeric characters and underscores.
1056
+ type : str, default 'default'
1057
+ Card type.
1058
+ id : str, optional, default None
1059
+ If multiple cards are present, use this id to identify this card.
1060
+ options : Dict[str, Any], default {}
1061
+ Options passed to the card. The contents depend on the card type.
1062
+ timeout : int, default 45
1063
+ Interrupt reporting if it takes more than this many seconds.
1131
1064
 
1132
1065
 
1133
1066
  """
1134
1067
  ...
1135
1068
 
1136
1069
  @typing.overload
1137
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1070
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1138
1071
  """
1139
- Specifies the times when the flow should be run when running on a
1140
- production scheduler.
1072
+ Specifies the PyPI packages for the step.
1073
+
1074
+ Information in this decorator will augment any
1075
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1076
+ you can use `@pypi_base` to set packages required by all
1077
+ steps and use `@pypi` to specify step-specific overrides.
1141
1078
 
1142
1079
  Parameters
1143
1080
  ----------
1144
- hourly : bool, default False
1145
- Run the workflow hourly.
1146
- daily : bool, default True
1147
- Run the workflow daily.
1148
- weekly : bool, default False
1149
- Run the workflow weekly.
1150
- cron : str, optional, default None
1151
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1152
- specified by this expression.
1153
- timezone : str, optional, default None
1154
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1155
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1081
+ packages : Dict[str, str], default: {}
1082
+ Packages to use for this step. The key is the name of the package
1083
+ and the value is the version to use.
1084
+ python : str, optional, default: None
1085
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1086
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1156
1087
  """
1157
1088
  ...
1158
1089
 
1159
1090
  @typing.overload
1160
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1091
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1161
1092
  ...
1162
1093
 
1163
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1094
+ @typing.overload
1095
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1096
+ ...
1097
+
1098
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1164
1099
  """
1165
- Specifies the times when the flow should be run when running on a
1166
- production scheduler.
1100
+ Specifies the PyPI packages for the step.
1101
+
1102
+ Information in this decorator will augment any
1103
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1104
+ you can use `@pypi_base` to set packages required by all
1105
+ steps and use `@pypi` to specify step-specific overrides.
1167
1106
 
1168
1107
  Parameters
1169
1108
  ----------
1170
- hourly : bool, default False
1171
- Run the workflow hourly.
1172
- daily : bool, default True
1173
- Run the workflow daily.
1174
- weekly : bool, default False
1175
- Run the workflow weekly.
1176
- cron : str, optional, default None
1177
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1178
- specified by this expression.
1179
- timezone : str, optional, default None
1180
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1181
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1109
+ packages : Dict[str, str], default: {}
1110
+ Packages to use for this step. The key is the name of the package
1111
+ and the value is the version to use.
1112
+ python : str, optional, default: None
1113
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1114
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1182
1115
  """
1183
1116
  ...
1184
1117
 
1185
1118
  @typing.overload
1186
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1119
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1187
1120
  """
1188
- Specifies the flow(s) that this flow depends on.
1189
-
1190
- ```
1191
- @trigger_on_finish(flow='FooFlow')
1192
- ```
1193
- or
1194
- ```
1195
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1196
- ```
1197
- This decorator respects the @project decorator and triggers the flow
1198
- when upstream runs within the same namespace complete successfully
1199
-
1200
- Additionally, you can specify project aware upstream flow dependencies
1201
- by specifying the fully qualified project_flow_name.
1202
- ```
1203
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1204
- ```
1205
- or
1206
- ```
1207
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1208
- ```
1209
-
1210
- You can also specify just the project or project branch (other values will be
1211
- inferred from the current project or project branch):
1212
- ```
1213
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1214
- ```
1215
-
1216
- Note that `branch` is typically one of:
1217
- - `prod`
1218
- - `user.bob`
1219
- - `test.my_experiment`
1220
- - `prod.staging`
1121
+ Specifies the PyPI packages for all steps of the flow.
1221
1122
 
1123
+ Use `@pypi_base` to set common packages required by all
1124
+ steps and use `@pypi` to specify step-specific overrides.
1222
1125
  Parameters
1223
1126
  ----------
1224
- flow : Union[str, Dict[str, str]], optional, default None
1225
- Upstream flow dependency for this flow.
1226
- flows : List[Union[str, Dict[str, str]]], default []
1227
- Upstream flow dependencies for this flow.
1228
- options : Dict[str, Any], default {}
1229
- Backend-specific configuration for tuning eventing behavior.
1230
-
1231
-
1127
+ packages : Dict[str, str], default: {}
1128
+ Packages to use for this flow. The key is the name of the package
1129
+ and the value is the version to use.
1130
+ python : str, optional, default: None
1131
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1132
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1232
1133
  """
1233
1134
  ...
1234
1135
 
1235
1136
  @typing.overload
1236
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1137
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1237
1138
  ...
1238
1139
 
1239
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1140
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1240
1141
  """
1241
- Specifies the flow(s) that this flow depends on.
1242
-
1243
- ```
1244
- @trigger_on_finish(flow='FooFlow')
1245
- ```
1246
- or
1247
- ```
1248
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1249
- ```
1250
- This decorator respects the @project decorator and triggers the flow
1251
- when upstream runs within the same namespace complete successfully
1252
-
1253
- Additionally, you can specify project aware upstream flow dependencies
1254
- by specifying the fully qualified project_flow_name.
1255
- ```
1256
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1257
- ```
1258
- or
1259
- ```
1260
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1261
- ```
1262
-
1263
- You can also specify just the project or project branch (other values will be
1264
- inferred from the current project or project branch):
1265
- ```
1266
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1267
- ```
1268
-
1269
- Note that `branch` is typically one of:
1270
- - `prod`
1271
- - `user.bob`
1272
- - `test.my_experiment`
1273
- - `prod.staging`
1142
+ Specifies the PyPI packages for all steps of the flow.
1274
1143
 
1144
+ Use `@pypi_base` to set common packages required by all
1145
+ steps and use `@pypi` to specify step-specific overrides.
1275
1146
  Parameters
1276
1147
  ----------
1277
- flow : Union[str, Dict[str, str]], optional, default None
1278
- Upstream flow dependency for this flow.
1279
- flows : List[Union[str, Dict[str, str]]], default []
1280
- Upstream flow dependencies for this flow.
1281
- options : Dict[str, Any], default {}
1282
- Backend-specific configuration for tuning eventing behavior.
1283
-
1284
-
1148
+ packages : Dict[str, str], default: {}
1149
+ Packages to use for this flow. The key is the name of the package
1150
+ and the value is the version to use.
1151
+ python : str, optional, default: None
1152
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1153
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1285
1154
  """
1286
1155
  ...
1287
1156
 
@@ -1327,6 +1196,55 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1327
1196
  """
1328
1197
  ...
1329
1198
 
1199
+ @typing.overload
1200
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1201
+ """
1202
+ Specifies the Conda environment for all steps of the flow.
1203
+
1204
+ Use `@conda_base` to set common libraries required by all
1205
+ steps and use `@conda` to specify step-specific additions.
1206
+
1207
+ Parameters
1208
+ ----------
1209
+ packages : Dict[str, str], default {}
1210
+ Packages to use for this flow. The key is the name of the package
1211
+ and the value is the version to use.
1212
+ libraries : Dict[str, str], default {}
1213
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1214
+ python : str, optional, default None
1215
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1216
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1217
+ disabled : bool, default False
1218
+ If set to True, disables Conda.
1219
+ """
1220
+ ...
1221
+
1222
+ @typing.overload
1223
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1224
+ ...
1225
+
1226
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1227
+ """
1228
+ Specifies the Conda environment for all steps of the flow.
1229
+
1230
+ Use `@conda_base` to set common libraries required by all
1231
+ steps and use `@conda` to specify step-specific additions.
1232
+
1233
+ Parameters
1234
+ ----------
1235
+ packages : Dict[str, str], default {}
1236
+ Packages to use for this flow. The key is the name of the package
1237
+ and the value is the version to use.
1238
+ libraries : Dict[str, str], default {}
1239
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1240
+ python : str, optional, default None
1241
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1242
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1243
+ disabled : bool, default False
1244
+ If set to True, disables Conda.
1245
+ """
1246
+ ...
1247
+
1330
1248
  @typing.overload
1331
1249
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1332
1250
  """
@@ -1423,51 +1341,172 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1423
1341
  ...
1424
1342
 
1425
1343
  @typing.overload
1426
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1344
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1427
1345
  """
1428
- Specifies the Conda environment for all steps of the flow.
1346
+ Specifies the times when the flow should be run when running on a
1347
+ production scheduler.
1429
1348
 
1430
- Use `@conda_base` to set common libraries required by all
1431
- steps and use `@conda` to specify step-specific additions.
1349
+ Parameters
1350
+ ----------
1351
+ hourly : bool, default False
1352
+ Run the workflow hourly.
1353
+ daily : bool, default True
1354
+ Run the workflow daily.
1355
+ weekly : bool, default False
1356
+ Run the workflow weekly.
1357
+ cron : str, optional, default None
1358
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1359
+ specified by this expression.
1360
+ timezone : str, optional, default None
1361
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1362
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1363
+ """
1364
+ ...
1365
+
1366
+ @typing.overload
1367
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1368
+ ...
1369
+
1370
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1371
+ """
1372
+ Specifies the times when the flow should be run when running on a
1373
+ production scheduler.
1432
1374
 
1433
1375
  Parameters
1434
1376
  ----------
1435
- packages : Dict[str, str], default {}
1436
- Packages to use for this flow. The key is the name of the package
1437
- and the value is the version to use.
1438
- libraries : Dict[str, str], default {}
1439
- Supported for backward compatibility. When used with packages, packages will take precedence.
1440
- python : str, optional, default None
1441
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1442
- that the version used will correspond to the version of the Python interpreter used to start the run.
1443
- disabled : bool, default False
1444
- If set to True, disables Conda.
1377
+ hourly : bool, default False
1378
+ Run the workflow hourly.
1379
+ daily : bool, default True
1380
+ Run the workflow daily.
1381
+ weekly : bool, default False
1382
+ Run the workflow weekly.
1383
+ cron : str, optional, default None
1384
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1385
+ specified by this expression.
1386
+ timezone : str, optional, default None
1387
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1388
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1445
1389
  """
1446
1390
  ...
1447
1391
 
1448
1392
  @typing.overload
1449
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1393
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1394
+ """
1395
+ Specifies the flow(s) that this flow depends on.
1396
+
1397
+ ```
1398
+ @trigger_on_finish(flow='FooFlow')
1399
+ ```
1400
+ or
1401
+ ```
1402
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1403
+ ```
1404
+ This decorator respects the @project decorator and triggers the flow
1405
+ when upstream runs within the same namespace complete successfully
1406
+
1407
+ Additionally, you can specify project aware upstream flow dependencies
1408
+ by specifying the fully qualified project_flow_name.
1409
+ ```
1410
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1411
+ ```
1412
+ or
1413
+ ```
1414
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1415
+ ```
1416
+
1417
+ You can also specify just the project or project branch (other values will be
1418
+ inferred from the current project or project branch):
1419
+ ```
1420
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1421
+ ```
1422
+
1423
+ Note that `branch` is typically one of:
1424
+ - `prod`
1425
+ - `user.bob`
1426
+ - `test.my_experiment`
1427
+ - `prod.staging`
1428
+
1429
+ Parameters
1430
+ ----------
1431
+ flow : Union[str, Dict[str, str]], optional, default None
1432
+ Upstream flow dependency for this flow.
1433
+ flows : List[Union[str, Dict[str, str]]], default []
1434
+ Upstream flow dependencies for this flow.
1435
+ options : Dict[str, Any], default {}
1436
+ Backend-specific configuration for tuning eventing behavior.
1437
+
1438
+
1439
+ """
1450
1440
  ...
1451
1441
 
1452
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1442
+ @typing.overload
1443
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1444
+ ...
1445
+
1446
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1453
1447
  """
1454
- Specifies the Conda environment for all steps of the flow.
1448
+ Specifies the flow(s) that this flow depends on.
1455
1449
 
1456
- Use `@conda_base` to set common libraries required by all
1457
- steps and use `@conda` to specify step-specific additions.
1450
+ ```
1451
+ @trigger_on_finish(flow='FooFlow')
1452
+ ```
1453
+ or
1454
+ ```
1455
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1456
+ ```
1457
+ This decorator respects the @project decorator and triggers the flow
1458
+ when upstream runs within the same namespace complete successfully
1459
+
1460
+ Additionally, you can specify project aware upstream flow dependencies
1461
+ by specifying the fully qualified project_flow_name.
1462
+ ```
1463
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1464
+ ```
1465
+ or
1466
+ ```
1467
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1468
+ ```
1469
+
1470
+ You can also specify just the project or project branch (other values will be
1471
+ inferred from the current project or project branch):
1472
+ ```
1473
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1474
+ ```
1475
+
1476
+ Note that `branch` is typically one of:
1477
+ - `prod`
1478
+ - `user.bob`
1479
+ - `test.my_experiment`
1480
+ - `prod.staging`
1458
1481
 
1459
1482
  Parameters
1460
1483
  ----------
1461
- packages : Dict[str, str], default {}
1462
- Packages to use for this flow. The key is the name of the package
1463
- and the value is the version to use.
1464
- libraries : Dict[str, str], default {}
1465
- Supported for backward compatibility. When used with packages, packages will take precedence.
1466
- python : str, optional, default None
1467
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1468
- that the version used will correspond to the version of the Python interpreter used to start the run.
1469
- disabled : bool, default False
1470
- If set to True, disables Conda.
1484
+ flow : Union[str, Dict[str, str]], optional, default None
1485
+ Upstream flow dependency for this flow.
1486
+ flows : List[Union[str, Dict[str, str]]], default []
1487
+ Upstream flow dependencies for this flow.
1488
+ options : Dict[str, Any], default {}
1489
+ Backend-specific configuration for tuning eventing behavior.
1490
+
1491
+
1492
+ """
1493
+ ...
1494
+
1495
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1496
+ """
1497
+ Specifies what flows belong to the same project.
1498
+
1499
+ A project-specific namespace is created for all flows that
1500
+ use the same `@project(name)`.
1501
+
1502
+ Parameters
1503
+ ----------
1504
+ name : str
1505
+ Project name. Make sure that the name is unique amongst all
1506
+ projects that use the same production scheduler. The name may
1507
+ contain only lowercase alphanumeric characters and underscores.
1508
+
1509
+
1471
1510
  """
1472
1511
  ...
1473
1512
 
@@ -1544,45 +1583,6 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
1544
1583
  """
1545
1584
  ...
1546
1585
 
1547
- @typing.overload
1548
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1549
- """
1550
- Specifies the PyPI packages for all steps of the flow.
1551
-
1552
- Use `@pypi_base` to set common packages required by all
1553
- steps and use `@pypi` to specify step-specific overrides.
1554
- Parameters
1555
- ----------
1556
- packages : Dict[str, str], default: {}
1557
- Packages to use for this flow. The key is the name of the package
1558
- and the value is the version to use.
1559
- python : str, optional, default: None
1560
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1561
- that the version used will correspond to the version of the Python interpreter used to start the run.
1562
- """
1563
- ...
1564
-
1565
- @typing.overload
1566
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1567
- ...
1568
-
1569
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1570
- """
1571
- Specifies the PyPI packages for all steps of the flow.
1572
-
1573
- Use `@pypi_base` to set common packages required by all
1574
- steps and use `@pypi` to specify step-specific overrides.
1575
- Parameters
1576
- ----------
1577
- packages : Dict[str, str], default: {}
1578
- Packages to use for this flow. The key is the name of the package
1579
- and the value is the version to use.
1580
- python : str, optional, default: None
1581
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1582
- that the version used will correspond to the version of the Python interpreter used to start the run.
1583
- """
1584
- ...
1585
-
1586
1586
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1587
1587
  """
1588
1588
  Switch namespace to the one provided.