ob-metaflow-stubs 5.9.1__py2.py3-none-any.whl → 5.10__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +496 -498
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +3 -3
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/info_file.pyi +16 -0
  14. metaflow-stubs/metadata/metadata.pyi +3 -3
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +17 -17
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +4 -4
  21. metaflow-stubs/plugins/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +3 -3
  61. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +4 -4
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +5 -5
  74. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  76. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +4 -4
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +4 -4
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +4 -4
  126. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  127. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  128. metaflow-stubs/runner/nbrun.pyi +2 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  130. metaflow-stubs/runner/utils.pyi +2 -2
  131. metaflow-stubs/system/__init__.pyi +4 -4
  132. metaflow-stubs/system/system_logger.pyi +3 -3
  133. metaflow-stubs/system/system_monitor.pyi +3 -3
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-5.9.1.dist-info → ob_metaflow_stubs-5.10.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-5.10.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-5.9.1.dist-info/RECORD +0 -139
  139. {ob_metaflow_stubs-5.9.1.dist-info → ob_metaflow_stubs-5.10.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-5.9.1.dist-info → ob_metaflow_stubs-5.10.dist-info}/top_level.txt +0 -0
@@ -1,30 +1,26 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.20.1+ob(v1) #
4
- # Generated on 2024-09-16T18:41:26.221357 #
3
+ # MF version: 2.12.22.1+ob(v1) #
4
+ # Generated on 2024-09-20T00:12:02.818960 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow._vendor.click.types
12
- import metaflow.client.core
13
- import metaflow.runner.metaflow_runner
14
- import datetime
15
11
  import metaflow.events
12
+ import metaflow.datastore.inputs
13
+ import metaflow.runner.metaflow_runner
16
14
  import metaflow.metaflow_current
17
- import typing
18
15
  import metaflow.flowspec
16
+ import datetime
17
+ import metaflow.client.core
18
+ import metaflow._vendor.click.types
19
+ import typing
19
20
  import metaflow.parameters
20
- import metaflow.datastore.inputs
21
21
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
22
22
  StepFlag = typing.NewType("StepFlag", bool)
23
23
 
24
- CURRENT_DIRECTORY: str
25
-
26
- INFO_FILE: str
27
-
28
24
  EXT_PKG: str
29
25
 
30
26
  def parallel_imap_unordered(func: typing.Callable[[typing.Any], typing.Any], iterable: typing.Iterable[typing.Any], max_parallel: typing.Optional[int] = None, dir: typing.Optional[str] = None) -> typing.Iterator[typing.Any]:
@@ -437,6 +433,172 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
437
433
  """
438
434
  ...
439
435
 
436
+ @typing.overload
437
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
438
+ """
439
+ Creates a human-readable report, a Metaflow Card, after this step completes.
440
+
441
+ Note that you may add multiple `@card` decorators in a step with different parameters.
442
+
443
+ Parameters
444
+ ----------
445
+ type : str, default 'default'
446
+ Card type.
447
+ id : str, optional, default None
448
+ If multiple cards are present, use this id to identify this card.
449
+ options : Dict[str, Any], default {}
450
+ Options passed to the card. The contents depend on the card type.
451
+ timeout : int, default 45
452
+ Interrupt reporting if it takes more than this many seconds.
453
+
454
+
455
+ """
456
+ ...
457
+
458
+ @typing.overload
459
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
460
+ ...
461
+
462
+ @typing.overload
463
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
464
+ ...
465
+
466
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
467
+ """
468
+ Creates a human-readable report, a Metaflow Card, after this step completes.
469
+
470
+ Note that you may add multiple `@card` decorators in a step with different parameters.
471
+
472
+ Parameters
473
+ ----------
474
+ type : str, default 'default'
475
+ Card type.
476
+ id : str, optional, default None
477
+ If multiple cards are present, use this id to identify this card.
478
+ options : Dict[str, Any], default {}
479
+ Options passed to the card. The contents depend on the card type.
480
+ timeout : int, default 45
481
+ Interrupt reporting if it takes more than this many seconds.
482
+
483
+
484
+ """
485
+ ...
486
+
487
+ @typing.overload
488
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
489
+ """
490
+ Specifies that the step will success under all circumstances.
491
+
492
+ The decorator will create an optional artifact, specified by `var`, which
493
+ contains the exception raised. You can use it to detect the presence
494
+ of errors, indicating that all happy-path artifacts produced by the step
495
+ are missing.
496
+
497
+ Parameters
498
+ ----------
499
+ var : str, optional, default None
500
+ Name of the artifact in which to store the caught exception.
501
+ If not specified, the exception is not stored.
502
+ print_exception : bool, default True
503
+ Determines whether or not the exception is printed to
504
+ stdout when caught.
505
+ """
506
+ ...
507
+
508
+ @typing.overload
509
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
510
+ ...
511
+
512
+ @typing.overload
513
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
514
+ ...
515
+
516
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
517
+ """
518
+ Specifies that the step will success under all circumstances.
519
+
520
+ The decorator will create an optional artifact, specified by `var`, which
521
+ contains the exception raised. You can use it to detect the presence
522
+ of errors, indicating that all happy-path artifacts produced by the step
523
+ are missing.
524
+
525
+ Parameters
526
+ ----------
527
+ var : str, optional, default None
528
+ Name of the artifact in which to store the caught exception.
529
+ If not specified, the exception is not stored.
530
+ print_exception : bool, default True
531
+ Determines whether or not the exception is printed to
532
+ stdout when caught.
533
+ """
534
+ ...
535
+
536
+ @typing.overload
537
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
538
+ """
539
+ Specifies the PyPI packages for the step.
540
+
541
+ Information in this decorator will augment any
542
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
543
+ you can use `@pypi_base` to set packages required by all
544
+ steps and use `@pypi` to specify step-specific overrides.
545
+
546
+ Parameters
547
+ ----------
548
+ packages : Dict[str, str], default: {}
549
+ Packages to use for this step. The key is the name of the package
550
+ and the value is the version to use.
551
+ python : str, optional, default: None
552
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
553
+ that the version used will correspond to the version of the Python interpreter used to start the run.
554
+ """
555
+ ...
556
+
557
+ @typing.overload
558
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
559
+ ...
560
+
561
+ @typing.overload
562
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
563
+ ...
564
+
565
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
566
+ """
567
+ Specifies the PyPI packages for the step.
568
+
569
+ Information in this decorator will augment any
570
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
571
+ you can use `@pypi_base` to set packages required by all
572
+ steps and use `@pypi` to specify step-specific overrides.
573
+
574
+ Parameters
575
+ ----------
576
+ packages : Dict[str, str], default: {}
577
+ Packages to use for this step. The key is the name of the package
578
+ and the value is the version to use.
579
+ python : str, optional, default: None
580
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
581
+ that the version used will correspond to the version of the Python interpreter used to start the run.
582
+ """
583
+ ...
584
+
585
+ @typing.overload
586
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
587
+ """
588
+ Internal decorator to support Fast bakery
589
+ """
590
+ ...
591
+
592
+ @typing.overload
593
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
594
+ ...
595
+
596
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
597
+ """
598
+ Internal decorator to support Fast bakery
599
+ """
600
+ ...
601
+
440
602
  @typing.overload
441
603
  def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
442
604
  """
@@ -515,342 +677,252 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
515
677
  ...
516
678
 
517
679
  @typing.overload
518
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
680
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
519
681
  """
520
- Specifies environment variables to be set prior to the execution of a step.
682
+ Specifies secrets to be retrieved and injected as environment variables prior to
683
+ the execution of a step.
521
684
 
522
685
  Parameters
523
686
  ----------
524
- vars : Dict[str, str], default {}
525
- Dictionary of environment variables to set.
687
+ sources : List[Union[str, Dict[str, Any]]], default: []
688
+ List of secret specs, defining how the secrets are to be retrieved
526
689
  """
527
690
  ...
528
691
 
529
692
  @typing.overload
530
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
693
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
531
694
  ...
532
695
 
533
696
  @typing.overload
534
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
697
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
535
698
  ...
536
699
 
537
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
700
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
538
701
  """
539
- Specifies environment variables to be set prior to the execution of a step.
702
+ Specifies secrets to be retrieved and injected as environment variables prior to
703
+ the execution of a step.
540
704
 
541
705
  Parameters
542
706
  ----------
543
- vars : Dict[str, str], default {}
544
- Dictionary of environment variables to set.
707
+ sources : List[Union[str, Dict[str, Any]]], default: []
708
+ List of secret specs, defining how the secrets are to be retrieved
545
709
  """
546
710
  ...
547
711
 
548
712
  @typing.overload
549
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
713
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
550
714
  """
551
- Specifies secrets to be retrieved and injected as environment variables prior to
552
- the execution of a step.
715
+ Specifies environment variables to be set prior to the execution of a step.
553
716
 
554
717
  Parameters
555
718
  ----------
556
- sources : List[Union[str, Dict[str, Any]]], default: []
557
- List of secret specs, defining how the secrets are to be retrieved
719
+ vars : Dict[str, str], default {}
720
+ Dictionary of environment variables to set.
558
721
  """
559
722
  ...
560
723
 
561
724
  @typing.overload
562
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
725
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
563
726
  ...
564
727
 
565
728
  @typing.overload
566
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
729
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
567
730
  ...
568
731
 
569
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
732
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
570
733
  """
571
- Specifies secrets to be retrieved and injected as environment variables prior to
572
- the execution of a step.
734
+ Specifies environment variables to be set prior to the execution of a step.
573
735
 
574
736
  Parameters
575
737
  ----------
576
- sources : List[Union[str, Dict[str, Any]]], default: []
577
- List of secret specs, defining how the secrets are to be retrieved
738
+ vars : Dict[str, str], default {}
739
+ Dictionary of environment variables to set.
578
740
  """
579
741
  ...
580
742
 
581
743
  @typing.overload
582
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
744
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
583
745
  """
584
- Specifies the Conda environment for the step.
746
+ Specifies the number of times the task corresponding
747
+ to a step needs to be retried.
585
748
 
586
- Information in this decorator will augment any
587
- attributes set in the `@conda_base` flow-level decorator. Hence,
588
- you can use `@conda_base` to set packages required by all
589
- steps and use `@conda` to specify step-specific overrides.
749
+ This decorator is useful for handling transient errors, such as networking issues.
750
+ If your task contains operations that can't be retried safely, e.g. database updates,
751
+ it is advisable to annotate it with `@retry(times=0)`.
752
+
753
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
754
+ decorator will execute a no-op task after all retries have been exhausted,
755
+ ensuring that the flow execution can continue.
590
756
 
591
757
  Parameters
592
758
  ----------
593
- packages : Dict[str, str], default {}
594
- Packages to use for this step. The key is the name of the package
595
- and the value is the version to use.
596
- libraries : Dict[str, str], default {}
597
- Supported for backward compatibility. When used with packages, packages will take precedence.
598
- python : str, optional, default None
599
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
600
- that the version used will correspond to the version of the Python interpreter used to start the run.
601
- disabled : bool, default False
602
- If set to True, disables @conda.
759
+ times : int, default 3
760
+ Number of times to retry this task.
761
+ minutes_between_retries : int, default 2
762
+ Number of minutes between retries.
603
763
  """
604
764
  ...
605
765
 
606
766
  @typing.overload
607
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
767
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
608
768
  ...
609
769
 
610
770
  @typing.overload
611
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
612
- ...
613
-
614
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
615
- """
616
- Specifies the Conda environment for the step.
617
-
618
- Information in this decorator will augment any
619
- attributes set in the `@conda_base` flow-level decorator. Hence,
620
- you can use `@conda_base` to set packages required by all
621
- steps and use `@conda` to specify step-specific overrides.
622
-
623
- Parameters
624
- ----------
625
- packages : Dict[str, str], default {}
626
- Packages to use for this step. The key is the name of the package
627
- and the value is the version to use.
628
- libraries : Dict[str, str], default {}
629
- Supported for backward compatibility. When used with packages, packages will take precedence.
630
- python : str, optional, default None
631
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
632
- that the version used will correspond to the version of the Python interpreter used to start the run.
633
- disabled : bool, default False
634
- If set to True, disables @conda.
635
- """
771
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
636
772
  ...
637
773
 
638
- @typing.overload
639
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
774
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
640
775
  """
641
- Specifies that the step will success under all circumstances.
642
-
643
- The decorator will create an optional artifact, specified by `var`, which
644
- contains the exception raised. You can use it to detect the presence
645
- of errors, indicating that all happy-path artifacts produced by the step
646
- are missing.
776
+ Specifies the number of times the task corresponding
777
+ to a step needs to be retried.
647
778
 
648
- Parameters
649
- ----------
650
- var : str, optional, default None
651
- Name of the artifact in which to store the caught exception.
652
- If not specified, the exception is not stored.
653
- print_exception : bool, default True
654
- Determines whether or not the exception is printed to
655
- stdout when caught.
656
- """
657
- ...
658
-
659
- @typing.overload
660
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
661
- ...
662
-
663
- @typing.overload
664
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
665
- ...
666
-
667
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
668
- """
669
- Specifies that the step will success under all circumstances.
779
+ This decorator is useful for handling transient errors, such as networking issues.
780
+ If your task contains operations that can't be retried safely, e.g. database updates,
781
+ it is advisable to annotate it with `@retry(times=0)`.
670
782
 
671
- The decorator will create an optional artifact, specified by `var`, which
672
- contains the exception raised. You can use it to detect the presence
673
- of errors, indicating that all happy-path artifacts produced by the step
674
- are missing.
783
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
784
+ decorator will execute a no-op task after all retries have been exhausted,
785
+ ensuring that the flow execution can continue.
675
786
 
676
787
  Parameters
677
788
  ----------
678
- var : str, optional, default None
679
- Name of the artifact in which to store the caught exception.
680
- If not specified, the exception is not stored.
681
- print_exception : bool, default True
682
- Determines whether or not the exception is printed to
683
- stdout when caught.
684
- """
685
- ...
686
-
687
- @typing.overload
688
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
689
- """
690
- Internal decorator to support Fast bakery
691
- """
692
- ...
693
-
694
- @typing.overload
695
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
696
- ...
697
-
698
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
699
- """
700
- Internal decorator to support Fast bakery
789
+ times : int, default 3
790
+ Number of times to retry this task.
791
+ minutes_between_retries : int, default 2
792
+ Number of minutes between retries.
701
793
  """
702
794
  ...
703
795
 
704
796
  @typing.overload
705
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
797
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
706
798
  """
707
- Creates a human-readable report, a Metaflow Card, after this step completes.
708
-
709
- Note that you may add multiple `@card` decorators in a step with different parameters.
710
-
711
- Parameters
712
- ----------
713
- type : str, default 'default'
714
- Card type.
715
- id : str, optional, default None
716
- If multiple cards are present, use this id to identify this card.
717
- options : Dict[str, Any], default {}
718
- Options passed to the card. The contents depend on the card type.
719
- timeout : int, default 45
720
- Interrupt reporting if it takes more than this many seconds.
721
-
722
-
799
+ Decorator prototype for all step decorators. This function gets specialized
800
+ and imported for all decorators types by _import_plugin_decorators().
723
801
  """
724
802
  ...
725
803
 
726
804
  @typing.overload
727
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
728
- ...
729
-
730
- @typing.overload
731
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
805
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
732
806
  ...
733
807
 
734
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
808
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
735
809
  """
736
- Creates a human-readable report, a Metaflow Card, after this step completes.
737
-
738
- Note that you may add multiple `@card` decorators in a step with different parameters.
739
-
740
- Parameters
741
- ----------
742
- type : str, default 'default'
743
- Card type.
744
- id : str, optional, default None
745
- If multiple cards are present, use this id to identify this card.
746
- options : Dict[str, Any], default {}
747
- Options passed to the card. The contents depend on the card type.
748
- timeout : int, default 45
749
- Interrupt reporting if it takes more than this many seconds.
750
-
751
-
810
+ Decorator prototype for all step decorators. This function gets specialized
811
+ and imported for all decorators types by _import_plugin_decorators().
752
812
  """
753
813
  ...
754
814
 
755
815
  @typing.overload
756
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
816
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
757
817
  """
758
- Specifies the PyPI packages for the step.
818
+ Specifies the Conda environment for the step.
759
819
 
760
820
  Information in this decorator will augment any
761
- attributes set in the `@pyi_base` flow-level decorator. Hence,
762
- you can use `@pypi_base` to set packages required by all
763
- steps and use `@pypi` to specify step-specific overrides.
821
+ attributes set in the `@conda_base` flow-level decorator. Hence,
822
+ you can use `@conda_base` to set packages required by all
823
+ steps and use `@conda` to specify step-specific overrides.
764
824
 
765
825
  Parameters
766
826
  ----------
767
- packages : Dict[str, str], default: {}
827
+ packages : Dict[str, str], default {}
768
828
  Packages to use for this step. The key is the name of the package
769
829
  and the value is the version to use.
770
- python : str, optional, default: None
830
+ libraries : Dict[str, str], default {}
831
+ Supported for backward compatibility. When used with packages, packages will take precedence.
832
+ python : str, optional, default None
771
833
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
772
834
  that the version used will correspond to the version of the Python interpreter used to start the run.
835
+ disabled : bool, default False
836
+ If set to True, disables @conda.
773
837
  """
774
838
  ...
775
839
 
776
840
  @typing.overload
777
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
841
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
778
842
  ...
779
843
 
780
844
  @typing.overload
781
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
845
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
782
846
  ...
783
847
 
784
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
848
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
785
849
  """
786
- Specifies the PyPI packages for the step.
850
+ Specifies the Conda environment for the step.
787
851
 
788
852
  Information in this decorator will augment any
789
- attributes set in the `@pyi_base` flow-level decorator. Hence,
790
- you can use `@pypi_base` to set packages required by all
791
- steps and use `@pypi` to specify step-specific overrides.
853
+ attributes set in the `@conda_base` flow-level decorator. Hence,
854
+ you can use `@conda_base` to set packages required by all
855
+ steps and use `@conda` to specify step-specific overrides.
792
856
 
793
857
  Parameters
794
858
  ----------
795
- packages : Dict[str, str], default: {}
859
+ packages : Dict[str, str], default {}
796
860
  Packages to use for this step. The key is the name of the package
797
861
  and the value is the version to use.
798
- python : str, optional, default: None
862
+ libraries : Dict[str, str], default {}
863
+ Supported for backward compatibility. When used with packages, packages will take precedence.
864
+ python : str, optional, default None
799
865
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
800
866
  that the version used will correspond to the version of the Python interpreter used to start the run.
867
+ disabled : bool, default False
868
+ If set to True, disables @conda.
801
869
  """
802
870
  ...
803
871
 
804
872
  @typing.overload
805
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
873
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
806
874
  """
807
- Specifies the number of times the task corresponding
808
- to a step needs to be retried.
875
+ Specifies a timeout for your step.
809
876
 
810
- This decorator is useful for handling transient errors, such as networking issues.
811
- If your task contains operations that can't be retried safely, e.g. database updates,
812
- it is advisable to annotate it with `@retry(times=0)`.
877
+ This decorator is useful if this step may hang indefinitely.
813
878
 
814
- This can be used in conjunction with the `@catch` decorator. The `@catch`
815
- decorator will execute a no-op task after all retries have been exhausted,
816
- ensuring that the flow execution can continue.
879
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
880
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
881
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
882
+
883
+ Note that all the values specified in parameters are added together so if you specify
884
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
817
885
 
818
886
  Parameters
819
887
  ----------
820
- times : int, default 3
821
- Number of times to retry this task.
822
- minutes_between_retries : int, default 2
823
- Number of minutes between retries.
888
+ seconds : int, default 0
889
+ Number of seconds to wait prior to timing out.
890
+ minutes : int, default 0
891
+ Number of minutes to wait prior to timing out.
892
+ hours : int, default 0
893
+ Number of hours to wait prior to timing out.
824
894
  """
825
895
  ...
826
896
 
827
897
  @typing.overload
828
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
898
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
829
899
  ...
830
900
 
831
901
  @typing.overload
832
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
902
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
833
903
  ...
834
904
 
835
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
905
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
836
906
  """
837
- Specifies the number of times the task corresponding
838
- to a step needs to be retried.
907
+ Specifies a timeout for your step.
839
908
 
840
- This decorator is useful for handling transient errors, such as networking issues.
841
- If your task contains operations that can't be retried safely, e.g. database updates,
842
- it is advisable to annotate it with `@retry(times=0)`.
909
+ This decorator is useful if this step may hang indefinitely.
843
910
 
844
- This can be used in conjunction with the `@catch` decorator. The `@catch`
845
- decorator will execute a no-op task after all retries have been exhausted,
846
- ensuring that the flow execution can continue.
911
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
912
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
913
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
914
+
915
+ Note that all the values specified in parameters are added together so if you specify
916
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
847
917
 
848
918
  Parameters
849
919
  ----------
850
- times : int, default 3
851
- Number of times to retry this task.
852
- minutes_between_retries : int, default 2
853
- Number of minutes between retries.
920
+ seconds : int, default 0
921
+ Number of seconds to wait prior to timing out.
922
+ minutes : int, default 0
923
+ Number of minutes to wait prior to timing out.
924
+ hours : int, default 0
925
+ Number of hours to wait prior to timing out.
854
926
  """
855
927
  ...
856
928
 
@@ -921,78 +993,41 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
921
993
  ...
922
994
 
923
995
  @typing.overload
924
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
925
- """
926
- Decorator prototype for all step decorators. This function gets specialized
927
- and imported for all decorators types by _import_plugin_decorators().
928
- """
929
- ...
930
-
931
- @typing.overload
932
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
933
- ...
934
-
935
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
936
- """
937
- Decorator prototype for all step decorators. This function gets specialized
938
- and imported for all decorators types by _import_plugin_decorators().
939
- """
940
- ...
941
-
942
- @typing.overload
943
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
996
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
944
997
  """
945
- Specifies a timeout for your step.
946
-
947
- This decorator is useful if this step may hang indefinitely.
948
-
949
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
950
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
951
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
952
-
953
- Note that all the values specified in parameters are added together so if you specify
954
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
998
+ Specifies the PyPI packages for all steps of the flow.
955
999
 
1000
+ Use `@pypi_base` to set common packages required by all
1001
+ steps and use `@pypi` to specify step-specific overrides.
956
1002
  Parameters
957
1003
  ----------
958
- seconds : int, default 0
959
- Number of seconds to wait prior to timing out.
960
- minutes : int, default 0
961
- Number of minutes to wait prior to timing out.
962
- hours : int, default 0
963
- Number of hours to wait prior to timing out.
1004
+ packages : Dict[str, str], default: {}
1005
+ Packages to use for this flow. The key is the name of the package
1006
+ and the value is the version to use.
1007
+ python : str, optional, default: None
1008
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1009
+ that the version used will correspond to the version of the Python interpreter used to start the run.
964
1010
  """
965
1011
  ...
966
1012
 
967
1013
  @typing.overload
968
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
969
- ...
970
-
971
- @typing.overload
972
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1014
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
973
1015
  ...
974
1016
 
975
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1017
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
976
1018
  """
977
- Specifies a timeout for your step.
978
-
979
- This decorator is useful if this step may hang indefinitely.
980
-
981
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
982
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
983
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
984
-
985
- Note that all the values specified in parameters are added together so if you specify
986
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1019
+ Specifies the PyPI packages for all steps of the flow.
987
1020
 
1021
+ Use `@pypi_base` to set common packages required by all
1022
+ steps and use `@pypi` to specify step-specific overrides.
988
1023
  Parameters
989
1024
  ----------
990
- seconds : int, default 0
991
- Number of seconds to wait prior to timing out.
992
- minutes : int, default 0
993
- Number of minutes to wait prior to timing out.
994
- hours : int, default 0
995
- Number of hours to wait prior to timing out.
1025
+ packages : Dict[str, str], default: {}
1026
+ Packages to use for this flow. The key is the name of the package
1027
+ and the value is the version to use.
1028
+ python : str, optional, default: None
1029
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1030
+ that the version used will correspond to the version of the Python interpreter used to start the run.
996
1031
  """
997
1032
  ...
998
1033
 
@@ -1045,45 +1080,52 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1045
1080
  """
1046
1081
  ...
1047
1082
 
1048
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1083
+ @typing.overload
1084
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1049
1085
  """
1050
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1051
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1052
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1053
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1054
- starts only after all sensors finish.
1086
+ Specifies the Conda environment for all steps of the flow.
1087
+
1088
+ Use `@conda_base` to set common libraries required by all
1089
+ steps and use `@conda` to specify step-specific additions.
1055
1090
 
1056
1091
  Parameters
1057
1092
  ----------
1058
- timeout : int
1059
- Time, in seconds before the task times out and fails. (Default: 3600)
1060
- poke_interval : int
1061
- Time in seconds that the job should wait in between each try. (Default: 60)
1062
- mode : str
1063
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1064
- exponential_backoff : bool
1065
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1066
- pool : str
1067
- the slot pool this task should run in,
1068
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1069
- soft_fail : bool
1070
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1071
- name : str
1072
- Name of the sensor on Airflow
1073
- description : str
1074
- Description of sensor in the Airflow UI
1075
- bucket_key : Union[str, List[str]]
1076
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1077
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1078
- bucket_name : str
1079
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1080
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1081
- wildcard_match : bool
1082
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1083
- aws_conn_id : str
1084
- a reference to the s3 connection on Airflow. (Default: None)
1085
- verify : bool
1086
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1093
+ packages : Dict[str, str], default {}
1094
+ Packages to use for this flow. The key is the name of the package
1095
+ and the value is the version to use.
1096
+ libraries : Dict[str, str], default {}
1097
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1098
+ python : str, optional, default None
1099
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1100
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1101
+ disabled : bool, default False
1102
+ If set to True, disables Conda.
1103
+ """
1104
+ ...
1105
+
1106
+ @typing.overload
1107
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1108
+ ...
1109
+
1110
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1111
+ """
1112
+ Specifies the Conda environment for all steps of the flow.
1113
+
1114
+ Use `@conda_base` to set common libraries required by all
1115
+ steps and use `@conda` to specify step-specific additions.
1116
+
1117
+ Parameters
1118
+ ----------
1119
+ packages : Dict[str, str], default {}
1120
+ Packages to use for this flow. The key is the name of the package
1121
+ and the value is the version to use.
1122
+ libraries : Dict[str, str], default {}
1123
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1124
+ python : str, optional, default None
1125
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1126
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1127
+ disabled : bool, default False
1128
+ If set to True, disables Conda.
1087
1129
  """
1088
1130
  ...
1089
1131
 
@@ -1129,140 +1171,189 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1129
1171
  """
1130
1172
  ...
1131
1173
 
1132
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1174
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1133
1175
  """
1134
- Specifies what flows belong to the same project.
1135
-
1136
- A project-specific namespace is created for all flows that
1137
- use the same `@project(name)`.
1176
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1138
1177
 
1139
- Parameters
1140
- ----------
1141
- name : str
1142
- Project name. Make sure that the name is unique amongst all
1143
- projects that use the same production scheduler. The name may
1144
- contain only lowercase alphanumeric characters and underscores.
1178
+ User code call
1179
+ -----------
1180
+ @nim(
1181
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1182
+ backend='managed'
1183
+ )
1145
1184
 
1185
+ Valid backend options
1186
+ ---------------------
1187
+ - 'managed': Outerbounds selects a compute provider based on the model.
1188
+ - 🚧 'dataplane': Run in your account.
1146
1189
 
1147
- """
1148
- ...
1149
-
1150
- @typing.overload
1151
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1152
- """
1153
- Specifies the PyPI packages for all steps of the flow.
1190
+ Valid model options
1191
+ ----------------
1192
+ - 'meta/llama3-8b-instruct': 8B parameter model
1193
+ - 'meta/llama3-70b-instruct': 70B parameter model
1194
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1154
1195
 
1155
- Use `@pypi_base` to set common packages required by all
1156
- steps and use `@pypi` to specify step-specific overrides.
1157
1196
  Parameters
1158
1197
  ----------
1159
- packages : Dict[str, str], default: {}
1160
- Packages to use for this flow. The key is the name of the package
1161
- and the value is the version to use.
1162
- python : str, optional, default: None
1163
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1164
- that the version used will correspond to the version of the Python interpreter used to start the run.
1198
+ models: list[NIM]
1199
+ List of NIM containers running models in sidecars.
1200
+ backend: str
1201
+ Compute provider to run the NIM container.
1165
1202
  """
1166
1203
  ...
1167
1204
 
1168
1205
  @typing.overload
1169
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1170
- ...
1171
-
1172
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1206
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1173
1207
  """
1174
- Specifies the PyPI packages for all steps of the flow.
1208
+ Specifies the event(s) that this flow depends on.
1209
+
1210
+ ```
1211
+ @trigger(event='foo')
1212
+ ```
1213
+ or
1214
+ ```
1215
+ @trigger(events=['foo', 'bar'])
1216
+ ```
1217
+
1218
+ Additionally, you can specify the parameter mappings
1219
+ to map event payload to Metaflow parameters for the flow.
1220
+ ```
1221
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1222
+ ```
1223
+ or
1224
+ ```
1225
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1226
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1227
+ ```
1228
+
1229
+ 'parameters' can also be a list of strings and tuples like so:
1230
+ ```
1231
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1232
+ ```
1233
+ This is equivalent to:
1234
+ ```
1235
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1236
+ ```
1175
1237
 
1176
- Use `@pypi_base` to set common packages required by all
1177
- steps and use `@pypi` to specify step-specific overrides.
1178
1238
  Parameters
1179
1239
  ----------
1180
- packages : Dict[str, str], default: {}
1181
- Packages to use for this flow. The key is the name of the package
1182
- and the value is the version to use.
1183
- python : str, optional, default: None
1184
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1185
- that the version used will correspond to the version of the Python interpreter used to start the run.
1240
+ event : Union[str, Dict[str, Any]], optional, default None
1241
+ Event dependency for this flow.
1242
+ events : List[Union[str, Dict[str, Any]]], default []
1243
+ Events dependency for this flow.
1244
+ options : Dict[str, Any], default {}
1245
+ Backend-specific configuration for tuning eventing behavior.
1246
+
1247
+
1186
1248
  """
1187
1249
  ...
1188
1250
 
1189
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1251
+ @typing.overload
1252
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1253
+ ...
1254
+
1255
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1190
1256
  """
1191
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1257
+ Specifies the event(s) that this flow depends on.
1192
1258
 
1193
- User code call
1194
- -----------
1195
- @nim(
1196
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1197
- backend='managed'
1198
- )
1259
+ ```
1260
+ @trigger(event='foo')
1261
+ ```
1262
+ or
1263
+ ```
1264
+ @trigger(events=['foo', 'bar'])
1265
+ ```
1199
1266
 
1200
- Valid backend options
1201
- ---------------------
1202
- - 'managed': Outerbounds selects a compute provider based on the model.
1203
- - 🚧 'dataplane': Run in your account.
1267
+ Additionally, you can specify the parameter mappings
1268
+ to map event payload to Metaflow parameters for the flow.
1269
+ ```
1270
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1271
+ ```
1272
+ or
1273
+ ```
1274
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1275
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1276
+ ```
1204
1277
 
1205
- Valid model options
1206
- ----------------
1207
- - 'meta/llama3-8b-instruct': 8B parameter model
1208
- - 'meta/llama3-70b-instruct': 70B parameter model
1209
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1278
+ 'parameters' can also be a list of strings and tuples like so:
1279
+ ```
1280
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1281
+ ```
1282
+ This is equivalent to:
1283
+ ```
1284
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1285
+ ```
1210
1286
 
1211
1287
  Parameters
1212
1288
  ----------
1213
- models: list[NIM]
1214
- List of NIM containers running models in sidecars.
1215
- backend: str
1216
- Compute provider to run the NIM container.
1289
+ event : Union[str, Dict[str, Any]], optional, default None
1290
+ Event dependency for this flow.
1291
+ events : List[Union[str, Dict[str, Any]]], default []
1292
+ Events dependency for this flow.
1293
+ options : Dict[str, Any], default {}
1294
+ Backend-specific configuration for tuning eventing behavior.
1295
+
1296
+
1217
1297
  """
1218
1298
  ...
1219
1299
 
1220
- @typing.overload
1221
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1300
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1222
1301
  """
1223
- Specifies the Conda environment for all steps of the flow.
1302
+ Specifies what flows belong to the same project.
1224
1303
 
1225
- Use `@conda_base` to set common libraries required by all
1226
- steps and use `@conda` to specify step-specific additions.
1304
+ A project-specific namespace is created for all flows that
1305
+ use the same `@project(name)`.
1227
1306
 
1228
1307
  Parameters
1229
1308
  ----------
1230
- packages : Dict[str, str], default {}
1231
- Packages to use for this flow. The key is the name of the package
1232
- and the value is the version to use.
1233
- libraries : Dict[str, str], default {}
1234
- Supported for backward compatibility. When used with packages, packages will take precedence.
1235
- python : str, optional, default None
1236
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1237
- that the version used will correspond to the version of the Python interpreter used to start the run.
1238
- disabled : bool, default False
1239
- If set to True, disables Conda.
1309
+ name : str
1310
+ Project name. Make sure that the name is unique amongst all
1311
+ projects that use the same production scheduler. The name may
1312
+ contain only lowercase alphanumeric characters and underscores.
1313
+
1314
+
1240
1315
  """
1241
1316
  ...
1242
1317
 
1243
- @typing.overload
1244
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1245
- ...
1246
-
1247
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1318
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1248
1319
  """
1249
- Specifies the Conda environment for all steps of the flow.
1250
-
1251
- Use `@conda_base` to set common libraries required by all
1252
- steps and use `@conda` to specify step-specific additions.
1320
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1321
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1322
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1323
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1324
+ starts only after all sensors finish.
1253
1325
 
1254
1326
  Parameters
1255
1327
  ----------
1256
- packages : Dict[str, str], default {}
1257
- Packages to use for this flow. The key is the name of the package
1258
- and the value is the version to use.
1259
- libraries : Dict[str, str], default {}
1260
- Supported for backward compatibility. When used with packages, packages will take precedence.
1261
- python : str, optional, default None
1262
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1263
- that the version used will correspond to the version of the Python interpreter used to start the run.
1264
- disabled : bool, default False
1265
- If set to True, disables Conda.
1328
+ timeout : int
1329
+ Time, in seconds before the task times out and fails. (Default: 3600)
1330
+ poke_interval : int
1331
+ Time in seconds that the job should wait in between each try. (Default: 60)
1332
+ mode : str
1333
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1334
+ exponential_backoff : bool
1335
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1336
+ pool : str
1337
+ the slot pool this task should run in,
1338
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1339
+ soft_fail : bool
1340
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1341
+ name : str
1342
+ Name of the sensor on Airflow
1343
+ description : str
1344
+ Description of sensor in the Airflow UI
1345
+ bucket_key : Union[str, List[str]]
1346
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1347
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1348
+ bucket_name : str
1349
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1350
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1351
+ wildcard_match : bool
1352
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1353
+ aws_conn_id : str
1354
+ a reference to the s3 connection on Airflow. (Default: None)
1355
+ verify : bool
1356
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1266
1357
  """
1267
1358
  ...
1268
1359
 
@@ -1369,101 +1460,6 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1369
1460
  """
1370
1461
  ...
1371
1462
 
1372
- @typing.overload
1373
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1374
- """
1375
- Specifies the event(s) that this flow depends on.
1376
-
1377
- ```
1378
- @trigger(event='foo')
1379
- ```
1380
- or
1381
- ```
1382
- @trigger(events=['foo', 'bar'])
1383
- ```
1384
-
1385
- Additionally, you can specify the parameter mappings
1386
- to map event payload to Metaflow parameters for the flow.
1387
- ```
1388
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1389
- ```
1390
- or
1391
- ```
1392
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1393
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1394
- ```
1395
-
1396
- 'parameters' can also be a list of strings and tuples like so:
1397
- ```
1398
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1399
- ```
1400
- This is equivalent to:
1401
- ```
1402
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1403
- ```
1404
-
1405
- Parameters
1406
- ----------
1407
- event : Union[str, Dict[str, Any]], optional, default None
1408
- Event dependency for this flow.
1409
- events : List[Union[str, Dict[str, Any]]], default []
1410
- Events dependency for this flow.
1411
- options : Dict[str, Any], default {}
1412
- Backend-specific configuration for tuning eventing behavior.
1413
-
1414
-
1415
- """
1416
- ...
1417
-
1418
- @typing.overload
1419
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1420
- ...
1421
-
1422
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1423
- """
1424
- Specifies the event(s) that this flow depends on.
1425
-
1426
- ```
1427
- @trigger(event='foo')
1428
- ```
1429
- or
1430
- ```
1431
- @trigger(events=['foo', 'bar'])
1432
- ```
1433
-
1434
- Additionally, you can specify the parameter mappings
1435
- to map event payload to Metaflow parameters for the flow.
1436
- ```
1437
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1438
- ```
1439
- or
1440
- ```
1441
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1442
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1443
- ```
1444
-
1445
- 'parameters' can also be a list of strings and tuples like so:
1446
- ```
1447
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1448
- ```
1449
- This is equivalent to:
1450
- ```
1451
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1452
- ```
1453
-
1454
- Parameters
1455
- ----------
1456
- event : Union[str, Dict[str, Any]], optional, default None
1457
- Event dependency for this flow.
1458
- events : List[Union[str, Dict[str, Any]]], default []
1459
- Events dependency for this flow.
1460
- options : Dict[str, Any], default {}
1461
- Backend-specific configuration for tuning eventing behavior.
1462
-
1463
-
1464
- """
1465
- ...
1466
-
1467
1463
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1468
1464
  """
1469
1465
  Switch namespace to the one provided.
@@ -2642,6 +2638,8 @@ class NBDeployer(object, metaclass=type):
2642
2638
  ...
2643
2639
  ...
2644
2640
 
2641
+ pkg_name: str
2642
+
2645
2643
  def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
2646
2644
  ...
2647
2645