ob-metaflow-stubs 6.0.3.102rc7__py2.py3-none-any.whl → 6.0.3.103__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +594 -594
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +7 -7
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +6 -6
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +3 -3
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +89 -89
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +4 -4
  21. metaflow-stubs/plugins/__init__.pyi +3 -3
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +7 -7
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  61. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  74. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  76. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +4 -4
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +4 -4
  126. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  127. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  128. metaflow-stubs/runner/nbrun.pyi +2 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  130. metaflow-stubs/runner/utils.pyi +2 -2
  131. metaflow-stubs/system/__init__.pyi +4 -4
  132. metaflow-stubs/system/system_logger.pyi +3 -3
  133. metaflow-stubs/system/system_monitor.pyi +3 -3
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.102rc7.dist-info → ob_metaflow_stubs-6.0.3.103.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.103.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.102rc7.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.102rc7.dist-info → ob_metaflow_stubs-6.0.3.103.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.102rc7.dist-info → ob_metaflow_stubs-6.0.3.103.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.10);ob(v1) #
4
- # Generated on 2024-09-20T19:26:01.310999 #
3
+ # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-09-20T20:42:33.459900 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
- import metaflow.flowspec
13
11
  import metaflow._vendor.click.types
14
- import metaflow.parameters
12
+ import metaflow.datastore.inputs
13
+ import metaflow.runner.metaflow_runner
15
14
  import datetime
15
+ import metaflow.flowspec
16
16
  import metaflow.events
17
- import metaflow.runner.metaflow_runner
18
- import metaflow.client.core
19
17
  import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
20
18
  import metaflow.metaflow_current
21
- import metaflow.datastore.inputs
19
+ import metaflow.parameters
20
+ import metaflow.client.core
21
+ import typing
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -435,193 +435,249 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
435
435
  ...
436
436
 
437
437
  @typing.overload
438
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
438
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
439
439
  """
440
- Specifies secrets to be retrieved and injected as environment variables prior to
441
- the execution of a step.
442
-
443
- Parameters
444
- ----------
445
- sources : List[Union[str, Dict[str, Any]]], default: []
446
- List of secret specs, defining how the secrets are to be retrieved
440
+ Decorator prototype for all step decorators. This function gets specialized
441
+ and imported for all decorators types by _import_plugin_decorators().
447
442
  """
448
443
  ...
449
444
 
450
445
  @typing.overload
451
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
452
- ...
453
-
454
- @typing.overload
455
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
446
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
456
447
  ...
457
448
 
458
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
449
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
459
450
  """
460
- Specifies secrets to be retrieved and injected as environment variables prior to
461
- the execution of a step.
462
-
463
- Parameters
464
- ----------
465
- sources : List[Union[str, Dict[str, Any]]], default: []
466
- List of secret specs, defining how the secrets are to be retrieved
451
+ Decorator prototype for all step decorators. This function gets specialized
452
+ and imported for all decorators types by _import_plugin_decorators().
467
453
  """
468
454
  ...
469
455
 
470
456
  @typing.overload
471
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
457
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
472
458
  """
473
- Creates a human-readable report, a Metaflow Card, after this step completes.
459
+ Specifies the resources needed when executing this step.
474
460
 
475
- Note that you may add multiple `@card` decorators in a step with different parameters.
461
+ Use `@resources` to specify the resource requirements
462
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
463
+
464
+ You can choose the compute layer on the command line by executing e.g.
465
+ ```
466
+ python myflow.py run --with batch
467
+ ```
468
+ or
469
+ ```
470
+ python myflow.py run --with kubernetes
471
+ ```
472
+ which executes the flow on the desired system using the
473
+ requirements specified in `@resources`.
476
474
 
477
475
  Parameters
478
476
  ----------
479
- type : str, default 'default'
480
- Card type.
481
- id : str, optional, default None
482
- If multiple cards are present, use this id to identify this card.
483
- options : Dict[str, Any], default {}
484
- Options passed to the card. The contents depend on the card type.
485
- timeout : int, default 45
486
- Interrupt reporting if it takes more than this many seconds.
487
-
488
-
477
+ cpu : int, default 1
478
+ Number of CPUs required for this step.
479
+ gpu : int, default 0
480
+ Number of GPUs required for this step.
481
+ disk : int, optional, default None
482
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
483
+ memory : int, default 4096
484
+ Memory size (in MB) required for this step.
485
+ shared_memory : int, optional, default None
486
+ The value for the size (in MiB) of the /dev/shm volume for this step.
487
+ This parameter maps to the `--shm-size` option in Docker.
489
488
  """
490
489
  ...
491
490
 
492
491
  @typing.overload
493
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
492
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
494
493
  ...
495
494
 
496
495
  @typing.overload
497
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
496
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
498
497
  ...
499
498
 
500
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
499
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
501
500
  """
502
- Creates a human-readable report, a Metaflow Card, after this step completes.
501
+ Specifies the resources needed when executing this step.
503
502
 
504
- Note that you may add multiple `@card` decorators in a step with different parameters.
503
+ Use `@resources` to specify the resource requirements
504
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
505
+
506
+ You can choose the compute layer on the command line by executing e.g.
507
+ ```
508
+ python myflow.py run --with batch
509
+ ```
510
+ or
511
+ ```
512
+ python myflow.py run --with kubernetes
513
+ ```
514
+ which executes the flow on the desired system using the
515
+ requirements specified in `@resources`.
505
516
 
506
517
  Parameters
507
518
  ----------
508
- type : str, default 'default'
509
- Card type.
510
- id : str, optional, default None
511
- If multiple cards are present, use this id to identify this card.
512
- options : Dict[str, Any], default {}
513
- Options passed to the card. The contents depend on the card type.
514
- timeout : int, default 45
515
- Interrupt reporting if it takes more than this many seconds.
516
-
517
-
519
+ cpu : int, default 1
520
+ Number of CPUs required for this step.
521
+ gpu : int, default 0
522
+ Number of GPUs required for this step.
523
+ disk : int, optional, default None
524
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
525
+ memory : int, default 4096
526
+ Memory size (in MB) required for this step.
527
+ shared_memory : int, optional, default None
528
+ The value for the size (in MiB) of the /dev/shm volume for this step.
529
+ This parameter maps to the `--shm-size` option in Docker.
518
530
  """
519
531
  ...
520
532
 
521
533
  @typing.overload
522
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
534
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
523
535
  """
524
- Specifies environment variables to be set prior to the execution of a step.
536
+ Specifies that the step will success under all circumstances.
537
+
538
+ The decorator will create an optional artifact, specified by `var`, which
539
+ contains the exception raised. You can use it to detect the presence
540
+ of errors, indicating that all happy-path artifacts produced by the step
541
+ are missing.
525
542
 
526
543
  Parameters
527
544
  ----------
528
- vars : Dict[str, str], default {}
529
- Dictionary of environment variables to set.
545
+ var : str, optional, default None
546
+ Name of the artifact in which to store the caught exception.
547
+ If not specified, the exception is not stored.
548
+ print_exception : bool, default True
549
+ Determines whether or not the exception is printed to
550
+ stdout when caught.
530
551
  """
531
552
  ...
532
553
 
533
554
  @typing.overload
534
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
555
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
535
556
  ...
536
557
 
537
558
  @typing.overload
538
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
559
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
539
560
  ...
540
561
 
541
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
562
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
542
563
  """
543
- Specifies environment variables to be set prior to the execution of a step.
564
+ Specifies that the step will success under all circumstances.
565
+
566
+ The decorator will create an optional artifact, specified by `var`, which
567
+ contains the exception raised. You can use it to detect the presence
568
+ of errors, indicating that all happy-path artifacts produced by the step
569
+ are missing.
544
570
 
545
571
  Parameters
546
572
  ----------
547
- vars : Dict[str, str], default {}
548
- Dictionary of environment variables to set.
573
+ var : str, optional, default None
574
+ Name of the artifact in which to store the caught exception.
575
+ If not specified, the exception is not stored.
576
+ print_exception : bool, default True
577
+ Determines whether or not the exception is printed to
578
+ stdout when caught.
549
579
  """
550
580
  ...
551
581
 
552
582
  @typing.overload
553
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
583
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
554
584
  """
555
- Decorator prototype for all step decorators. This function gets specialized
556
- and imported for all decorators types by _import_plugin_decorators().
585
+ Specifies the number of times the task corresponding
586
+ to a step needs to be retried.
587
+
588
+ This decorator is useful for handling transient errors, such as networking issues.
589
+ If your task contains operations that can't be retried safely, e.g. database updates,
590
+ it is advisable to annotate it with `@retry(times=0)`.
591
+
592
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
593
+ decorator will execute a no-op task after all retries have been exhausted,
594
+ ensuring that the flow execution can continue.
595
+
596
+ Parameters
597
+ ----------
598
+ times : int, default 3
599
+ Number of times to retry this task.
600
+ minutes_between_retries : int, default 2
601
+ Number of minutes between retries.
557
602
  """
558
603
  ...
559
604
 
560
605
  @typing.overload
561
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
606
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
562
607
  ...
563
608
 
564
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
609
+ @typing.overload
610
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
611
+ ...
612
+
613
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
565
614
  """
566
- Decorator prototype for all step decorators. This function gets specialized
567
- and imported for all decorators types by _import_plugin_decorators().
615
+ Specifies the number of times the task corresponding
616
+ to a step needs to be retried.
617
+
618
+ This decorator is useful for handling transient errors, such as networking issues.
619
+ If your task contains operations that can't be retried safely, e.g. database updates,
620
+ it is advisable to annotate it with `@retry(times=0)`.
621
+
622
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
623
+ decorator will execute a no-op task after all retries have been exhausted,
624
+ ensuring that the flow execution can continue.
625
+
626
+ Parameters
627
+ ----------
628
+ times : int, default 3
629
+ Number of times to retry this task.
630
+ minutes_between_retries : int, default 2
631
+ Number of minutes between retries.
568
632
  """
569
633
  ...
570
634
 
571
635
  @typing.overload
572
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
636
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
573
637
  """
574
- Specifies a timeout for your step.
575
-
576
- This decorator is useful if this step may hang indefinitely.
577
-
578
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
579
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
580
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
638
+ Specifies the PyPI packages for the step.
581
639
 
582
- Note that all the values specified in parameters are added together so if you specify
583
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
640
+ Information in this decorator will augment any
641
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
642
+ you can use `@pypi_base` to set packages required by all
643
+ steps and use `@pypi` to specify step-specific overrides.
584
644
 
585
645
  Parameters
586
646
  ----------
587
- seconds : int, default 0
588
- Number of seconds to wait prior to timing out.
589
- minutes : int, default 0
590
- Number of minutes to wait prior to timing out.
591
- hours : int, default 0
592
- Number of hours to wait prior to timing out.
647
+ packages : Dict[str, str], default: {}
648
+ Packages to use for this step. The key is the name of the package
649
+ and the value is the version to use.
650
+ python : str, optional, default: None
651
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
652
+ that the version used will correspond to the version of the Python interpreter used to start the run.
593
653
  """
594
654
  ...
595
655
 
596
656
  @typing.overload
597
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
657
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
598
658
  ...
599
659
 
600
660
  @typing.overload
601
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
661
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
602
662
  ...
603
663
 
604
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
664
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
605
665
  """
606
- Specifies a timeout for your step.
607
-
608
- This decorator is useful if this step may hang indefinitely.
609
-
610
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
611
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
612
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
666
+ Specifies the PyPI packages for the step.
613
667
 
614
- Note that all the values specified in parameters are added together so if you specify
615
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
668
+ Information in this decorator will augment any
669
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
670
+ you can use `@pypi_base` to set packages required by all
671
+ steps and use `@pypi` to specify step-specific overrides.
616
672
 
617
673
  Parameters
618
674
  ----------
619
- seconds : int, default 0
620
- Number of seconds to wait prior to timing out.
621
- minutes : int, default 0
622
- Number of minutes to wait prior to timing out.
623
- hours : int, default 0
624
- Number of hours to wait prior to timing out.
675
+ packages : Dict[str, str], default: {}
676
+ Packages to use for this step. The key is the name of the package
677
+ and the value is the version to use.
678
+ python : str, optional, default: None
679
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
680
+ that the version used will correspond to the version of the Python interpreter used to start the run.
625
681
  """
626
682
  ...
627
683
 
@@ -692,177 +748,140 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
692
748
  ...
693
749
 
694
750
  @typing.overload
695
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
751
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
696
752
  """
697
- Enables loading / saving of models within a step.
698
-
699
-
700
- Parameters
701
- ----------
702
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
703
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
704
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
705
- - `current.checkpoint`
706
- - `current.model`
707
- - `current.huggingface_hub`
708
-
709
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
710
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
711
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
753
+ Specifies a timeout for your step.
712
754
 
713
- temp_dir_root : str, default: None
714
- The root directory under which `current.model.loaded` will store loaded models
755
+ This decorator is useful if this step may hang indefinitely.
715
756
 
757
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
758
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
759
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
716
760
 
761
+ Note that all the values specified in parameters are added together so if you specify
762
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
717
763
 
764
+ Parameters
765
+ ----------
766
+ seconds : int, default 0
767
+ Number of seconds to wait prior to timing out.
768
+ minutes : int, default 0
769
+ Number of minutes to wait prior to timing out.
770
+ hours : int, default 0
771
+ Number of hours to wait prior to timing out.
718
772
  """
719
773
  ...
720
774
 
721
775
  @typing.overload
722
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
776
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
723
777
  ...
724
778
 
725
779
  @typing.overload
726
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
780
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
727
781
  ...
728
782
 
729
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
783
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
730
784
  """
731
- Enables loading / saving of models within a step.
732
-
733
-
734
- Parameters
735
- ----------
736
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
737
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
738
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
739
- - `current.checkpoint`
740
- - `current.model`
741
- - `current.huggingface_hub`
742
-
743
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
744
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
745
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
785
+ Specifies a timeout for your step.
746
786
 
747
- temp_dir_root : str, default: None
748
- The root directory under which `current.model.loaded` will store loaded models
787
+ This decorator is useful if this step may hang indefinitely.
749
788
 
789
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
790
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
791
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
750
792
 
793
+ Note that all the values specified in parameters are added together so if you specify
794
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
751
795
 
796
+ Parameters
797
+ ----------
798
+ seconds : int, default 0
799
+ Number of seconds to wait prior to timing out.
800
+ minutes : int, default 0
801
+ Number of minutes to wait prior to timing out.
802
+ hours : int, default 0
803
+ Number of hours to wait prior to timing out.
752
804
  """
753
805
  ...
754
806
 
755
807
  @typing.overload
756
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
808
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
757
809
  """
758
- Specifies the number of times the task corresponding
759
- to a step needs to be retried.
760
-
761
- This decorator is useful for handling transient errors, such as networking issues.
762
- If your task contains operations that can't be retried safely, e.g. database updates,
763
- it is advisable to annotate it with `@retry(times=0)`.
764
-
765
- This can be used in conjunction with the `@catch` decorator. The `@catch`
766
- decorator will execute a no-op task after all retries have been exhausted,
767
- ensuring that the flow execution can continue.
810
+ Internal decorator to support Fast bakery
811
+ """
812
+ ...
813
+
814
+ @typing.overload
815
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
816
+ ...
817
+
818
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
819
+ """
820
+ Internal decorator to support Fast bakery
821
+ """
822
+ ...
823
+
824
+ @typing.overload
825
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
826
+ """
827
+ Specifies secrets to be retrieved and injected as environment variables prior to
828
+ the execution of a step.
768
829
 
769
830
  Parameters
770
831
  ----------
771
- times : int, default 3
772
- Number of times to retry this task.
773
- minutes_between_retries : int, default 2
774
- Number of minutes between retries.
832
+ sources : List[Union[str, Dict[str, Any]]], default: []
833
+ List of secret specs, defining how the secrets are to be retrieved
775
834
  """
776
835
  ...
777
836
 
778
837
  @typing.overload
779
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
838
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
780
839
  ...
781
840
 
782
841
  @typing.overload
783
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
842
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
784
843
  ...
785
844
 
786
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
845
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
787
846
  """
788
- Specifies the number of times the task corresponding
789
- to a step needs to be retried.
790
-
791
- This decorator is useful for handling transient errors, such as networking issues.
792
- If your task contains operations that can't be retried safely, e.g. database updates,
793
- it is advisable to annotate it with `@retry(times=0)`.
794
-
795
- This can be used in conjunction with the `@catch` decorator. The `@catch`
796
- decorator will execute a no-op task after all retries have been exhausted,
797
- ensuring that the flow execution can continue.
847
+ Specifies secrets to be retrieved and injected as environment variables prior to
848
+ the execution of a step.
798
849
 
799
850
  Parameters
800
851
  ----------
801
- times : int, default 3
802
- Number of times to retry this task.
803
- minutes_between_retries : int, default 2
804
- Number of minutes between retries.
852
+ sources : List[Union[str, Dict[str, Any]]], default: []
853
+ List of secret specs, defining how the secrets are to be retrieved
805
854
  """
806
855
  ...
807
856
 
808
857
  @typing.overload
809
- def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
858
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
810
859
  """
811
- Enables checkpointing for a step.
812
-
860
+ Specifies environment variables to be set prior to the execution of a step.
813
861
 
814
862
  Parameters
815
863
  ----------
816
- load_policy : str, default: "fresh"
817
- The policy for loading the checkpoint. The following policies are supported:
818
- - "eager": Loads the the latest available checkpoint within the namespace.
819
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
820
- will be loaded at the start of the task.
821
- - "none": Do not load any checkpoint
822
- - "fresh": Loads the lastest checkpoint created within the running Task.
823
- This mode helps loading checkpoints across various retry attempts of the same task.
824
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
825
- created within the task will be loaded when the task is retries execution on failure.
826
-
827
- temp_dir_root : str, default: None
828
- The root directory under which `current.checkpoint.directory` will be created.
829
-
830
-
831
-
864
+ vars : Dict[str, str], default {}
865
+ Dictionary of environment variables to set.
832
866
  """
833
867
  ...
834
868
 
835
869
  @typing.overload
836
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
870
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
837
871
  ...
838
872
 
839
873
  @typing.overload
840
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
874
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
841
875
  ...
842
876
 
843
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
877
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
844
878
  """
845
- Enables checkpointing for a step.
846
-
879
+ Specifies environment variables to be set prior to the execution of a step.
847
880
 
848
881
  Parameters
849
882
  ----------
850
- load_policy : str, default: "fresh"
851
- The policy for loading the checkpoint. The following policies are supported:
852
- - "eager": Loads the the latest available checkpoint within the namespace.
853
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
854
- will be loaded at the start of the task.
855
- - "none": Do not load any checkpoint
856
- - "fresh": Loads the lastest checkpoint created within the running Task.
857
- This mode helps loading checkpoints across various retry attempts of the same task.
858
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
859
- created within the task will be loaded when the task is retries execution on failure.
860
-
861
- temp_dir_root : str, default: None
862
- The root directory under which `current.checkpoint.directory` will be created.
863
-
864
-
865
-
883
+ vars : Dict[str, str], default {}
884
+ Dictionary of environment variables to set.
866
885
  """
867
886
  ...
868
887
 
@@ -924,331 +943,387 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
924
943
  ...
925
944
 
926
945
  @typing.overload
927
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
946
+ def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
928
947
  """
929
- Specifies the PyPI packages for the step.
948
+ Enables checkpointing for a step.
930
949
 
931
- Information in this decorator will augment any
932
- attributes set in the `@pyi_base` flow-level decorator. Hence,
933
- you can use `@pypi_base` to set packages required by all
934
- steps and use `@pypi` to specify step-specific overrides.
935
950
 
936
951
  Parameters
937
952
  ----------
938
- packages : Dict[str, str], default: {}
939
- Packages to use for this step. The key is the name of the package
940
- and the value is the version to use.
941
- python : str, optional, default: None
942
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
943
- that the version used will correspond to the version of the Python interpreter used to start the run.
953
+ load_policy : str, default: "fresh"
954
+ The policy for loading the checkpoint. The following policies are supported:
955
+ - "eager": Loads the the latest available checkpoint within the namespace.
956
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
957
+ will be loaded at the start of the task.
958
+ - "none": Do not load any checkpoint
959
+ - "fresh": Loads the lastest checkpoint created within the running Task.
960
+ This mode helps loading checkpoints across various retry attempts of the same task.
961
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
962
+ created within the task will be loaded when the task is retries execution on failure.
963
+
964
+ temp_dir_root : str, default: None
965
+ The root directory under which `current.checkpoint.directory` will be created.
966
+
967
+
968
+
944
969
  """
945
970
  ...
946
971
 
947
972
  @typing.overload
948
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
973
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
949
974
  ...
950
975
 
951
976
  @typing.overload
952
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
977
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
953
978
  ...
954
979
 
955
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
980
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
956
981
  """
957
- Specifies the PyPI packages for the step.
982
+ Enables checkpointing for a step.
958
983
 
959
- Information in this decorator will augment any
960
- attributes set in the `@pyi_base` flow-level decorator. Hence,
961
- you can use `@pypi_base` to set packages required by all
962
- steps and use `@pypi` to specify step-specific overrides.
963
984
 
964
985
  Parameters
965
986
  ----------
966
- packages : Dict[str, str], default: {}
967
- Packages to use for this step. The key is the name of the package
968
- and the value is the version to use.
969
- python : str, optional, default: None
970
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
971
- that the version used will correspond to the version of the Python interpreter used to start the run.
972
- """
973
- ...
974
-
975
- @typing.overload
976
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
977
- """
978
- Specifies that the step will success under all circumstances.
979
-
980
- The decorator will create an optional artifact, specified by `var`, which
981
- contains the exception raised. You can use it to detect the presence
982
- of errors, indicating that all happy-path artifacts produced by the step
983
- are missing.
984
-
987
+ load_policy : str, default: "fresh"
988
+ The policy for loading the checkpoint. The following policies are supported:
989
+ - "eager": Loads the the latest available checkpoint within the namespace.
990
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
991
+ will be loaded at the start of the task.
992
+ - "none": Do not load any checkpoint
993
+ - "fresh": Loads the lastest checkpoint created within the running Task.
994
+ This mode helps loading checkpoints across various retry attempts of the same task.
995
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
996
+ created within the task will be loaded when the task is retries execution on failure.
997
+
998
+ temp_dir_root : str, default: None
999
+ The root directory under which `current.checkpoint.directory` will be created.
1000
+
1001
+
1002
+
1003
+ """
1004
+ ...
1005
+
1006
+ @typing.overload
1007
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1008
+ """
1009
+ Enables loading / saving of models within a step.
1010
+
1011
+
985
1012
  Parameters
986
1013
  ----------
987
- var : str, optional, default None
988
- Name of the artifact in which to store the caught exception.
989
- If not specified, the exception is not stored.
990
- print_exception : bool, default True
991
- Determines whether or not the exception is printed to
992
- stdout when caught.
1014
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1015
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1016
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
1017
+ - `current.checkpoint`
1018
+ - `current.model`
1019
+ - `current.huggingface_hub`
1020
+
1021
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1022
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1023
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1024
+
1025
+ temp_dir_root : str, default: None
1026
+ The root directory under which `current.model.loaded` will store loaded models
1027
+
1028
+
1029
+
993
1030
  """
994
1031
  ...
995
1032
 
996
1033
  @typing.overload
997
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1034
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
998
1035
  ...
999
1036
 
1000
1037
  @typing.overload
1001
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1038
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1002
1039
  ...
1003
1040
 
1004
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1041
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1005
1042
  """
1006
- Specifies that the step will success under all circumstances.
1043
+ Enables loading / saving of models within a step.
1007
1044
 
1008
- The decorator will create an optional artifact, specified by `var`, which
1009
- contains the exception raised. You can use it to detect the presence
1010
- of errors, indicating that all happy-path artifacts produced by the step
1011
- are missing.
1012
1045
 
1013
1046
  Parameters
1014
1047
  ----------
1015
- var : str, optional, default None
1016
- Name of the artifact in which to store the caught exception.
1017
- If not specified, the exception is not stored.
1018
- print_exception : bool, default True
1019
- Determines whether or not the exception is printed to
1020
- stdout when caught.
1048
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1049
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1050
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
1051
+ - `current.checkpoint`
1052
+ - `current.model`
1053
+ - `current.huggingface_hub`
1054
+
1055
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1056
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1057
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1058
+
1059
+ temp_dir_root : str, default: None
1060
+ The root directory under which `current.model.loaded` will store loaded models
1061
+
1062
+
1063
+
1021
1064
  """
1022
1065
  ...
1023
1066
 
1024
1067
  @typing.overload
1025
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1068
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1026
1069
  """
1027
- Specifies the resources needed when executing this step.
1028
-
1029
- Use `@resources` to specify the resource requirements
1030
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1070
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1031
1071
 
1032
- You can choose the compute layer on the command line by executing e.g.
1033
- ```
1034
- python myflow.py run --with batch
1035
- ```
1036
- or
1037
- ```
1038
- python myflow.py run --with kubernetes
1039
- ```
1040
- which executes the flow on the desired system using the
1041
- requirements specified in `@resources`.
1072
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1042
1073
 
1043
1074
  Parameters
1044
1075
  ----------
1045
- cpu : int, default 1
1046
- Number of CPUs required for this step.
1047
- gpu : int, default 0
1048
- Number of GPUs required for this step.
1049
- disk : int, optional, default None
1050
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1051
- memory : int, default 4096
1052
- Memory size (in MB) required for this step.
1053
- shared_memory : int, optional, default None
1054
- The value for the size (in MiB) of the /dev/shm volume for this step.
1055
- This parameter maps to the `--shm-size` option in Docker.
1076
+ type : str, default 'default'
1077
+ Card type.
1078
+ id : str, optional, default None
1079
+ If multiple cards are present, use this id to identify this card.
1080
+ options : Dict[str, Any], default {}
1081
+ Options passed to the card. The contents depend on the card type.
1082
+ timeout : int, default 45
1083
+ Interrupt reporting if it takes more than this many seconds.
1084
+
1085
+
1056
1086
  """
1057
1087
  ...
1058
1088
 
1059
1089
  @typing.overload
1060
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1090
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1061
1091
  ...
1062
1092
 
1063
1093
  @typing.overload
1064
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1094
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1065
1095
  ...
1066
1096
 
1067
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1097
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1068
1098
  """
1069
- Specifies the resources needed when executing this step.
1070
-
1071
- Use `@resources` to specify the resource requirements
1072
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1099
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1073
1100
 
1074
- You can choose the compute layer on the command line by executing e.g.
1075
- ```
1076
- python myflow.py run --with batch
1077
- ```
1078
- or
1079
- ```
1080
- python myflow.py run --with kubernetes
1081
- ```
1082
- which executes the flow on the desired system using the
1083
- requirements specified in `@resources`.
1101
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1084
1102
 
1085
1103
  Parameters
1086
1104
  ----------
1087
- cpu : int, default 1
1088
- Number of CPUs required for this step.
1089
- gpu : int, default 0
1090
- Number of GPUs required for this step.
1091
- disk : int, optional, default None
1092
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1093
- memory : int, default 4096
1094
- Memory size (in MB) required for this step.
1095
- shared_memory : int, optional, default None
1096
- The value for the size (in MiB) of the /dev/shm volume for this step.
1097
- This parameter maps to the `--shm-size` option in Docker.
1098
- """
1099
- ...
1100
-
1101
- @typing.overload
1102
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1103
- """
1104
- Internal decorator to support Fast bakery
1105
- """
1106
- ...
1107
-
1108
- @typing.overload
1109
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1110
- ...
1111
-
1112
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1113
- """
1114
- Internal decorator to support Fast bakery
1105
+ type : str, default 'default'
1106
+ Card type.
1107
+ id : str, optional, default None
1108
+ If multiple cards are present, use this id to identify this card.
1109
+ options : Dict[str, Any], default {}
1110
+ Options passed to the card. The contents depend on the card type.
1111
+ timeout : int, default 45
1112
+ Interrupt reporting if it takes more than this many seconds.
1113
+
1114
+
1115
1115
  """
1116
1116
  ...
1117
1117
 
1118
- @typing.overload
1119
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1118
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1120
1119
  """
1121
- Specifies the times when the flow should be run when running on a
1122
- production scheduler.
1120
+ Specifies what flows belong to the same project.
1121
+
1122
+ A project-specific namespace is created for all flows that
1123
+ use the same `@project(name)`.
1123
1124
 
1124
1125
  Parameters
1125
1126
  ----------
1126
- hourly : bool, default False
1127
- Run the workflow hourly.
1128
- daily : bool, default True
1129
- Run the workflow daily.
1130
- weekly : bool, default False
1131
- Run the workflow weekly.
1132
- cron : str, optional, default None
1133
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1134
- specified by this expression.
1135
- timezone : str, optional, default None
1136
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1137
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1127
+ name : str
1128
+ Project name. Make sure that the name is unique amongst all
1129
+ projects that use the same production scheduler. The name may
1130
+ contain only lowercase alphanumeric characters and underscores.
1131
+
1132
+
1138
1133
  """
1139
1134
  ...
1140
1135
 
1141
- @typing.overload
1142
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1143
- ...
1144
-
1145
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1136
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1146
1137
  """
1147
- Specifies the times when the flow should be run when running on a
1148
- production scheduler.
1138
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1139
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1140
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1141
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1142
+ starts only after all sensors finish.
1149
1143
 
1150
1144
  Parameters
1151
1145
  ----------
1152
- hourly : bool, default False
1153
- Run the workflow hourly.
1154
- daily : bool, default True
1155
- Run the workflow daily.
1156
- weekly : bool, default False
1157
- Run the workflow weekly.
1158
- cron : str, optional, default None
1159
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1160
- specified by this expression.
1161
- timezone : str, optional, default None
1162
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1163
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1146
+ timeout : int
1147
+ Time, in seconds before the task times out and fails. (Default: 3600)
1148
+ poke_interval : int
1149
+ Time in seconds that the job should wait in between each try. (Default: 60)
1150
+ mode : str
1151
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1152
+ exponential_backoff : bool
1153
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1154
+ pool : str
1155
+ the slot pool this task should run in,
1156
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1157
+ soft_fail : bool
1158
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1159
+ name : str
1160
+ Name of the sensor on Airflow
1161
+ description : str
1162
+ Description of sensor in the Airflow UI
1163
+ bucket_key : Union[str, List[str]]
1164
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1165
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1166
+ bucket_name : str
1167
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1168
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1169
+ wildcard_match : bool
1170
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1171
+ aws_conn_id : str
1172
+ a reference to the s3 connection on Airflow. (Default: None)
1173
+ verify : bool
1174
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1164
1175
  """
1165
1176
  ...
1166
1177
 
1167
1178
  @typing.overload
1168
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1179
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1169
1180
  """
1170
- Specifies the Conda environment for all steps of the flow.
1181
+ Specifies the flow(s) that this flow depends on.
1171
1182
 
1172
- Use `@conda_base` to set common libraries required by all
1173
- steps and use `@conda` to specify step-specific additions.
1183
+ ```
1184
+ @trigger_on_finish(flow='FooFlow')
1185
+ ```
1186
+ or
1187
+ ```
1188
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1189
+ ```
1190
+ This decorator respects the @project decorator and triggers the flow
1191
+ when upstream runs within the same namespace complete successfully
1192
+
1193
+ Additionally, you can specify project aware upstream flow dependencies
1194
+ by specifying the fully qualified project_flow_name.
1195
+ ```
1196
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1197
+ ```
1198
+ or
1199
+ ```
1200
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1201
+ ```
1202
+
1203
+ You can also specify just the project or project branch (other values will be
1204
+ inferred from the current project or project branch):
1205
+ ```
1206
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1207
+ ```
1208
+
1209
+ Note that `branch` is typically one of:
1210
+ - `prod`
1211
+ - `user.bob`
1212
+ - `test.my_experiment`
1213
+ - `prod.staging`
1174
1214
 
1175
1215
  Parameters
1176
1216
  ----------
1177
- packages : Dict[str, str], default {}
1178
- Packages to use for this flow. The key is the name of the package
1179
- and the value is the version to use.
1180
- libraries : Dict[str, str], default {}
1181
- Supported for backward compatibility. When used with packages, packages will take precedence.
1182
- python : str, optional, default None
1183
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1184
- that the version used will correspond to the version of the Python interpreter used to start the run.
1185
- disabled : bool, default False
1186
- If set to True, disables Conda.
1217
+ flow : Union[str, Dict[str, str]], optional, default None
1218
+ Upstream flow dependency for this flow.
1219
+ flows : List[Union[str, Dict[str, str]]], default []
1220
+ Upstream flow dependencies for this flow.
1221
+ options : Dict[str, Any], default {}
1222
+ Backend-specific configuration for tuning eventing behavior.
1223
+
1224
+
1187
1225
  """
1188
1226
  ...
1189
1227
 
1190
1228
  @typing.overload
1191
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1229
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1192
1230
  ...
1193
1231
 
1194
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1232
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1195
1233
  """
1196
- Specifies the Conda environment for all steps of the flow.
1234
+ Specifies the flow(s) that this flow depends on.
1197
1235
 
1198
- Use `@conda_base` to set common libraries required by all
1199
- steps and use `@conda` to specify step-specific additions.
1236
+ ```
1237
+ @trigger_on_finish(flow='FooFlow')
1238
+ ```
1239
+ or
1240
+ ```
1241
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1242
+ ```
1243
+ This decorator respects the @project decorator and triggers the flow
1244
+ when upstream runs within the same namespace complete successfully
1245
+
1246
+ Additionally, you can specify project aware upstream flow dependencies
1247
+ by specifying the fully qualified project_flow_name.
1248
+ ```
1249
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1250
+ ```
1251
+ or
1252
+ ```
1253
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1254
+ ```
1255
+
1256
+ You can also specify just the project or project branch (other values will be
1257
+ inferred from the current project or project branch):
1258
+ ```
1259
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1260
+ ```
1261
+
1262
+ Note that `branch` is typically one of:
1263
+ - `prod`
1264
+ - `user.bob`
1265
+ - `test.my_experiment`
1266
+ - `prod.staging`
1200
1267
 
1201
1268
  Parameters
1202
1269
  ----------
1203
- packages : Dict[str, str], default {}
1204
- Packages to use for this flow. The key is the name of the package
1205
- and the value is the version to use.
1206
- libraries : Dict[str, str], default {}
1207
- Supported for backward compatibility. When used with packages, packages will take precedence.
1208
- python : str, optional, default None
1209
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1210
- that the version used will correspond to the version of the Python interpreter used to start the run.
1211
- disabled : bool, default False
1212
- If set to True, disables Conda.
1270
+ flow : Union[str, Dict[str, str]], optional, default None
1271
+ Upstream flow dependency for this flow.
1272
+ flows : List[Union[str, Dict[str, str]]], default []
1273
+ Upstream flow dependencies for this flow.
1274
+ options : Dict[str, Any], default {}
1275
+ Backend-specific configuration for tuning eventing behavior.
1276
+
1277
+
1213
1278
  """
1214
1279
  ...
1215
1280
 
1216
1281
  @typing.overload
1217
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1282
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1218
1283
  """
1219
- Specifies the PyPI packages for all steps of the flow.
1284
+ Specifies the times when the flow should be run when running on a
1285
+ production scheduler.
1220
1286
 
1221
- Use `@pypi_base` to set common packages required by all
1222
- steps and use `@pypi` to specify step-specific overrides.
1223
1287
  Parameters
1224
1288
  ----------
1225
- packages : Dict[str, str], default: {}
1226
- Packages to use for this flow. The key is the name of the package
1227
- and the value is the version to use.
1228
- python : str, optional, default: None
1229
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1230
- that the version used will correspond to the version of the Python interpreter used to start the run.
1289
+ hourly : bool, default False
1290
+ Run the workflow hourly.
1291
+ daily : bool, default True
1292
+ Run the workflow daily.
1293
+ weekly : bool, default False
1294
+ Run the workflow weekly.
1295
+ cron : str, optional, default None
1296
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1297
+ specified by this expression.
1298
+ timezone : str, optional, default None
1299
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1300
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1231
1301
  """
1232
1302
  ...
1233
1303
 
1234
1304
  @typing.overload
1235
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1305
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1236
1306
  ...
1237
1307
 
1238
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1308
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1239
1309
  """
1240
- Specifies the PyPI packages for all steps of the flow.
1241
-
1242
- Use `@pypi_base` to set common packages required by all
1243
- steps and use `@pypi` to specify step-specific overrides.
1244
- Parameters
1245
- ----------
1246
- packages : Dict[str, str], default: {}
1247
- Packages to use for this flow. The key is the name of the package
1248
- and the value is the version to use.
1249
- python : str, optional, default: None
1250
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1251
- that the version used will correspond to the version of the Python interpreter used to start the run.
1310
+ Specifies the times when the flow should be run when running on a
1311
+ production scheduler.
1312
+
1313
+ Parameters
1314
+ ----------
1315
+ hourly : bool, default False
1316
+ Run the workflow hourly.
1317
+ daily : bool, default True
1318
+ Run the workflow daily.
1319
+ weekly : bool, default False
1320
+ Run the workflow weekly.
1321
+ cron : str, optional, default None
1322
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1323
+ specified by this expression.
1324
+ timezone : str, optional, default None
1325
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1326
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1252
1327
  """
1253
1328
  ...
1254
1329
 
@@ -1347,166 +1422,73 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1347
1422
  """
1348
1423
  ...
1349
1424
 
1350
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1351
- """
1352
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1353
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1354
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1355
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1356
- starts only after all sensors finish.
1357
-
1358
- Parameters
1359
- ----------
1360
- timeout : int
1361
- Time, in seconds before the task times out and fails. (Default: 3600)
1362
- poke_interval : int
1363
- Time in seconds that the job should wait in between each try. (Default: 60)
1364
- mode : str
1365
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1366
- exponential_backoff : bool
1367
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1368
- pool : str
1369
- the slot pool this task should run in,
1370
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1371
- soft_fail : bool
1372
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1373
- name : str
1374
- Name of the sensor on Airflow
1375
- description : str
1376
- Description of sensor in the Airflow UI
1377
- bucket_key : Union[str, List[str]]
1378
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1379
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1380
- bucket_name : str
1381
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1382
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1383
- wildcard_match : bool
1384
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1385
- aws_conn_id : str
1386
- a reference to the s3 connection on Airflow. (Default: None)
1387
- verify : bool
1388
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1389
- """
1390
- ...
1391
-
1392
1425
  @typing.overload
1393
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1426
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1394
1427
  """
1395
- Specifies the flow(s) that this flow depends on.
1396
-
1397
- ```
1398
- @trigger_on_finish(flow='FooFlow')
1399
- ```
1400
- or
1401
- ```
1402
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1403
- ```
1404
- This decorator respects the @project decorator and triggers the flow
1405
- when upstream runs within the same namespace complete successfully
1406
-
1407
- Additionally, you can specify project aware upstream flow dependencies
1408
- by specifying the fully qualified project_flow_name.
1409
- ```
1410
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1411
- ```
1412
- or
1413
- ```
1414
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1415
- ```
1416
-
1417
- You can also specify just the project or project branch (other values will be
1418
- inferred from the current project or project branch):
1419
- ```
1420
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1421
- ```
1422
-
1423
- Note that `branch` is typically one of:
1424
- - `prod`
1425
- - `user.bob`
1426
- - `test.my_experiment`
1427
- - `prod.staging`
1428
+ Specifies the PyPI packages for all steps of the flow.
1428
1429
 
1430
+ Use `@pypi_base` to set common packages required by all
1431
+ steps and use `@pypi` to specify step-specific overrides.
1429
1432
  Parameters
1430
1433
  ----------
1431
- flow : Union[str, Dict[str, str]], optional, default None
1432
- Upstream flow dependency for this flow.
1433
- flows : List[Union[str, Dict[str, str]]], default []
1434
- Upstream flow dependencies for this flow.
1435
- options : Dict[str, Any], default {}
1436
- Backend-specific configuration for tuning eventing behavior.
1437
-
1438
-
1434
+ packages : Dict[str, str], default: {}
1435
+ Packages to use for this flow. The key is the name of the package
1436
+ and the value is the version to use.
1437
+ python : str, optional, default: None
1438
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1439
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1439
1440
  """
1440
1441
  ...
1441
1442
 
1442
1443
  @typing.overload
1443
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1444
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1444
1445
  ...
1445
1446
 
1446
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1447
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1447
1448
  """
1448
- Specifies the flow(s) that this flow depends on.
1449
-
1450
- ```
1451
- @trigger_on_finish(flow='FooFlow')
1452
- ```
1453
- or
1454
- ```
1455
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1456
- ```
1457
- This decorator respects the @project decorator and triggers the flow
1458
- when upstream runs within the same namespace complete successfully
1459
-
1460
- Additionally, you can specify project aware upstream flow dependencies
1461
- by specifying the fully qualified project_flow_name.
1462
- ```
1463
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1464
- ```
1465
- or
1466
- ```
1467
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1468
- ```
1469
-
1470
- You can also specify just the project or project branch (other values will be
1471
- inferred from the current project or project branch):
1472
- ```
1473
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1474
- ```
1475
-
1476
- Note that `branch` is typically one of:
1477
- - `prod`
1478
- - `user.bob`
1479
- - `test.my_experiment`
1480
- - `prod.staging`
1449
+ Specifies the PyPI packages for all steps of the flow.
1481
1450
 
1451
+ Use `@pypi_base` to set common packages required by all
1452
+ steps and use `@pypi` to specify step-specific overrides.
1482
1453
  Parameters
1483
1454
  ----------
1484
- flow : Union[str, Dict[str, str]], optional, default None
1485
- Upstream flow dependency for this flow.
1486
- flows : List[Union[str, Dict[str, str]]], default []
1487
- Upstream flow dependencies for this flow.
1488
- options : Dict[str, Any], default {}
1489
- Backend-specific configuration for tuning eventing behavior.
1490
-
1491
-
1455
+ packages : Dict[str, str], default: {}
1456
+ Packages to use for this flow. The key is the name of the package
1457
+ and the value is the version to use.
1458
+ python : str, optional, default: None
1459
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1460
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1492
1461
  """
1493
1462
  ...
1494
1463
 
1495
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1464
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1496
1465
  """
1497
- Specifies what flows belong to the same project.
1466
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1498
1467
 
1499
- A project-specific namespace is created for all flows that
1500
- use the same `@project(name)`.
1468
+ User code call
1469
+ -----------
1470
+ @nim(
1471
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1472
+ backend='managed'
1473
+ )
1501
1474
 
1502
- Parameters
1503
- ----------
1504
- name : str
1505
- Project name. Make sure that the name is unique amongst all
1506
- projects that use the same production scheduler. The name may
1507
- contain only lowercase alphanumeric characters and underscores.
1475
+ Valid backend options
1476
+ ---------------------
1477
+ - 'managed': Outerbounds selects a compute provider based on the model.
1478
+ - 🚧 'dataplane': Run in your account.
1508
1479
 
1480
+ Valid model options
1481
+ ----------------
1482
+ - 'meta/llama3-8b-instruct': 8B parameter model
1483
+ - 'meta/llama3-70b-instruct': 70B parameter model
1484
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1509
1485
 
1486
+ Parameters
1487
+ ----------
1488
+ models: list[NIM]
1489
+ List of NIM containers running models in sidecars.
1490
+ backend: str
1491
+ Compute provider to run the NIM container.
1510
1492
  """
1511
1493
  ...
1512
1494
 
@@ -1552,34 +1534,52 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1552
1534
  """
1553
1535
  ...
1554
1536
 
1555
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1537
+ @typing.overload
1538
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1556
1539
  """
1557
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1540
+ Specifies the Conda environment for all steps of the flow.
1558
1541
 
1559
- User code call
1560
- -----------
1561
- @nim(
1562
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1563
- backend='managed'
1564
- )
1542
+ Use `@conda_base` to set common libraries required by all
1543
+ steps and use `@conda` to specify step-specific additions.
1565
1544
 
1566
- Valid backend options
1567
- ---------------------
1568
- - 'managed': Outerbounds selects a compute provider based on the model.
1569
- - 🚧 'dataplane': Run in your account.
1545
+ Parameters
1546
+ ----------
1547
+ packages : Dict[str, str], default {}
1548
+ Packages to use for this flow. The key is the name of the package
1549
+ and the value is the version to use.
1550
+ libraries : Dict[str, str], default {}
1551
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1552
+ python : str, optional, default None
1553
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1554
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1555
+ disabled : bool, default False
1556
+ If set to True, disables Conda.
1557
+ """
1558
+ ...
1559
+
1560
+ @typing.overload
1561
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1562
+ ...
1563
+
1564
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1565
+ """
1566
+ Specifies the Conda environment for all steps of the flow.
1570
1567
 
1571
- Valid model options
1572
- ----------------
1573
- - 'meta/llama3-8b-instruct': 8B parameter model
1574
- - 'meta/llama3-70b-instruct': 70B parameter model
1575
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1568
+ Use `@conda_base` to set common libraries required by all
1569
+ steps and use `@conda` to specify step-specific additions.
1576
1570
 
1577
1571
  Parameters
1578
1572
  ----------
1579
- models: list[NIM]
1580
- List of NIM containers running models in sidecars.
1581
- backend: str
1582
- Compute provider to run the NIM container.
1573
+ packages : Dict[str, str], default {}
1574
+ Packages to use for this flow. The key is the name of the package
1575
+ and the value is the version to use.
1576
+ libraries : Dict[str, str], default {}
1577
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1578
+ python : str, optional, default None
1579
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1580
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1581
+ disabled : bool, default False
1582
+ If set to True, disables Conda.
1583
1583
  """
1584
1584
  ...
1585
1585