ob-metaflow-stubs 6.0.3.106rc1__py2.py3-none-any.whl → 6.0.3.107__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +597 -597
  2. metaflow-stubs/cards.pyi +4 -4
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +1 -1
  7. metaflow-stubs/clone_util.pyi +1 -1
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +1 -1
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +2 -2
  13. metaflow-stubs/info_file.pyi +1 -1
  14. metaflow-stubs/metadata/metadata.pyi +2 -2
  15. metaflow-stubs/metadata/util.pyi +1 -1
  16. metaflow-stubs/metaflow_config.pyi +1 -1
  17. metaflow-stubs/metaflow_current.pyi +64 -64
  18. metaflow-stubs/mflog/mflog.pyi +1 -1
  19. metaflow-stubs/multicore_utils.pyi +1 -1
  20. metaflow-stubs/parameters.pyi +4 -4
  21. metaflow-stubs/plugins/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  24. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  30. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  31. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +5 -5
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +1 -1
  36. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  37. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +1 -1
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +1 -1
  52. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  59. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  60. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  61. metaflow-stubs/plugins/cards/card_client.pyi +1 -1
  62. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  74. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  75. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  76. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  82. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  83. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  84. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  85. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  101. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  102. metaflow-stubs/plugins/package_cli.pyi +1 -1
  103. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/perimeters.pyi +1 -1
  105. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  111. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  112. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  113. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  114. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  118. metaflow-stubs/plugins/tag_cli.pyi +3 -3
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
  121. metaflow-stubs/procpoll.pyi +1 -1
  122. metaflow-stubs/profilers/__init__.pyi +1 -1
  123. metaflow-stubs/pylint_wrapper.pyi +1 -1
  124. metaflow-stubs/runner/__init__.pyi +1 -1
  125. metaflow-stubs/runner/deployer.pyi +3 -3
  126. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  127. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  128. metaflow-stubs/runner/nbrun.pyi +1 -1
  129. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  130. metaflow-stubs/runner/utils.pyi +1 -1
  131. metaflow-stubs/system/__init__.pyi +3 -3
  132. metaflow-stubs/system/system_logger.pyi +2 -2
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +1 -1
  135. metaflow-stubs/tuple_util.pyi +1 -1
  136. {ob_metaflow_stubs-6.0.3.106rc1.dist-info → ob_metaflow_stubs-6.0.3.107.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.107.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.106rc1.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.106rc1.dist-info → ob_metaflow_stubs-6.0.3.107.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.106rc1.dist-info → ob_metaflow_stubs-6.0.3.107.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-10-07T18:32:56.944750 #
4
+ # Generated on 2024-10-07T18:39:02.357763 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.runner.metaflow_runner
12
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
- import metaflow._vendor.click.types
14
11
  import metaflow.metaflow_current
15
- import metaflow.parameters
12
+ import metaflow._vendor.click.types
16
13
  import metaflow.client.core
14
+ import datetime
15
+ import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
17
16
  import typing
18
- import metaflow.datastore.inputs
19
17
  import metaflow.flowspec
18
+ import metaflow.parameters
19
+ import metaflow.datastore.inputs
20
+ import metaflow.runner.metaflow_runner
20
21
  import metaflow.events
21
- import datetime
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -513,21 +513,117 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
513
513
  ...
514
514
 
515
515
  @typing.overload
516
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
516
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
517
517
  """
518
- Decorator prototype for all step decorators. This function gets specialized
519
- and imported for all decorators types by _import_plugin_decorators().
518
+ Specifies that the step will success under all circumstances.
519
+
520
+ The decorator will create an optional artifact, specified by `var`, which
521
+ contains the exception raised. You can use it to detect the presence
522
+ of errors, indicating that all happy-path artifacts produced by the step
523
+ are missing.
524
+
525
+ Parameters
526
+ ----------
527
+ var : str, optional, default None
528
+ Name of the artifact in which to store the caught exception.
529
+ If not specified, the exception is not stored.
530
+ print_exception : bool, default True
531
+ Determines whether or not the exception is printed to
532
+ stdout when caught.
520
533
  """
521
534
  ...
522
535
 
523
536
  @typing.overload
524
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
537
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
525
538
  ...
526
539
 
527
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
540
+ @typing.overload
541
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
542
+ ...
543
+
544
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
528
545
  """
529
- Decorator prototype for all step decorators. This function gets specialized
530
- and imported for all decorators types by _import_plugin_decorators().
546
+ Specifies that the step will success under all circumstances.
547
+
548
+ The decorator will create an optional artifact, specified by `var`, which
549
+ contains the exception raised. You can use it to detect the presence
550
+ of errors, indicating that all happy-path artifacts produced by the step
551
+ are missing.
552
+
553
+ Parameters
554
+ ----------
555
+ var : str, optional, default None
556
+ Name of the artifact in which to store the caught exception.
557
+ If not specified, the exception is not stored.
558
+ print_exception : bool, default True
559
+ Determines whether or not the exception is printed to
560
+ stdout when caught.
561
+ """
562
+ ...
563
+
564
+ @typing.overload
565
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
566
+ """
567
+ Specifies the PyPI packages for the step.
568
+
569
+ Information in this decorator will augment any
570
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
571
+ you can use `@pypi_base` to set packages required by all
572
+ steps and use `@pypi` to specify step-specific overrides.
573
+
574
+ Parameters
575
+ ----------
576
+ packages : Dict[str, str], default: {}
577
+ Packages to use for this step. The key is the name of the package
578
+ and the value is the version to use.
579
+ python : str, optional, default: None
580
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
581
+ that the version used will correspond to the version of the Python interpreter used to start the run.
582
+ """
583
+ ...
584
+
585
+ @typing.overload
586
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
587
+ ...
588
+
589
+ @typing.overload
590
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
591
+ ...
592
+
593
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
594
+ """
595
+ Specifies the PyPI packages for the step.
596
+
597
+ Information in this decorator will augment any
598
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
599
+ you can use `@pypi_base` to set packages required by all
600
+ steps and use `@pypi` to specify step-specific overrides.
601
+
602
+ Parameters
603
+ ----------
604
+ packages : Dict[str, str], default: {}
605
+ Packages to use for this step. The key is the name of the package
606
+ and the value is the version to use.
607
+ python : str, optional, default: None
608
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
609
+ that the version used will correspond to the version of the Python interpreter used to start the run.
610
+ """
611
+ ...
612
+
613
+ @typing.overload
614
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
615
+ """
616
+ Internal decorator to support Fast bakery
617
+ """
618
+ ...
619
+
620
+ @typing.overload
621
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
622
+ ...
623
+
624
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
625
+ """
626
+ Internal decorator to support Fast bakery
531
627
  """
532
628
  ...
533
629
 
@@ -592,290 +688,193 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
592
688
  """
593
689
  ...
594
690
 
595
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
596
- """
597
- Specifies that this step should execute on Kubernetes.
598
-
599
- Parameters
600
- ----------
601
- cpu : int, default 1
602
- Number of CPUs required for this step. If `@resources` is
603
- also present, the maximum value from all decorators is used.
604
- memory : int, default 4096
605
- Memory size (in MB) required for this step. If
606
- `@resources` is also present, the maximum value from all decorators is
607
- used.
608
- disk : int, default 10240
609
- Disk size (in MB) required for this step. If
610
- `@resources` is also present, the maximum value from all decorators is
611
- used.
612
- image : str, optional, default None
613
- Docker image to use when launching on Kubernetes. If not specified, and
614
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
615
- not, a default Docker image mapping to the current version of Python is used.
616
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
617
- If given, the imagePullPolicy to be applied to the Docker image of the step.
618
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
619
- Kubernetes service account to use when launching pod in Kubernetes.
620
- secrets : List[str], optional, default None
621
- Kubernetes secrets to use when launching pod in Kubernetes. These
622
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
623
- in Metaflow configuration.
624
- node_selector: Union[Dict[str,str], str], optional, default None
625
- Kubernetes node selector(s) to apply to the pod running the task.
626
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
627
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
628
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
629
- Kubernetes namespace to use when launching pod in Kubernetes.
630
- gpu : int, optional, default None
631
- Number of GPUs required for this step. A value of zero implies that
632
- the scheduled node should not have GPUs.
633
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
634
- The vendor of the GPUs to be used for this step.
635
- tolerations : List[str], default []
636
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
637
- Kubernetes tolerations to use when launching pod in Kubernetes.
638
- use_tmpfs : bool, default False
639
- This enables an explicit tmpfs mount for this step.
640
- tmpfs_tempdir : bool, default True
641
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
642
- tmpfs_size : int, optional, default: None
643
- The value for the size (in MiB) of the tmpfs mount for this step.
644
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
645
- memory allocated for this step.
646
- tmpfs_path : str, optional, default /metaflow_temp
647
- Path to tmpfs mount for this step.
648
- persistent_volume_claims : Dict[str, str], optional, default None
649
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
650
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
651
- shared_memory: int, optional
652
- Shared memory size (in MiB) required for this step
653
- port: int, optional
654
- Port number to specify in the Kubernetes job object
655
- compute_pool : str, optional, default None
656
- Compute pool to be used for for this step.
657
- If not specified, any accessible compute pool within the perimeter is used.
658
- """
659
- ...
660
-
661
691
  @typing.overload
662
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
692
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
663
693
  """
664
- Specifies a timeout for your step.
665
-
666
- This decorator is useful if this step may hang indefinitely.
694
+ Specifies the resources needed when executing this step.
667
695
 
668
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
669
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
670
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
696
+ Use `@resources` to specify the resource requirements
697
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
671
698
 
672
- Note that all the values specified in parameters are added together so if you specify
673
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
699
+ You can choose the compute layer on the command line by executing e.g.
700
+ ```
701
+ python myflow.py run --with batch
702
+ ```
703
+ or
704
+ ```
705
+ python myflow.py run --with kubernetes
706
+ ```
707
+ which executes the flow on the desired system using the
708
+ requirements specified in `@resources`.
674
709
 
675
710
  Parameters
676
711
  ----------
677
- seconds : int, default 0
678
- Number of seconds to wait prior to timing out.
679
- minutes : int, default 0
680
- Number of minutes to wait prior to timing out.
681
- hours : int, default 0
682
- Number of hours to wait prior to timing out.
712
+ cpu : int, default 1
713
+ Number of CPUs required for this step.
714
+ gpu : int, default 0
715
+ Number of GPUs required for this step.
716
+ disk : int, optional, default None
717
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
718
+ memory : int, default 4096
719
+ Memory size (in MB) required for this step.
720
+ shared_memory : int, optional, default None
721
+ The value for the size (in MiB) of the /dev/shm volume for this step.
722
+ This parameter maps to the `--shm-size` option in Docker.
683
723
  """
684
724
  ...
685
725
 
686
726
  @typing.overload
687
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
727
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
688
728
  ...
689
729
 
690
730
  @typing.overload
691
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
731
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
692
732
  ...
693
733
 
694
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
734
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
695
735
  """
696
- Specifies a timeout for your step.
697
-
698
- This decorator is useful if this step may hang indefinitely.
736
+ Specifies the resources needed when executing this step.
699
737
 
700
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
701
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
702
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
738
+ Use `@resources` to specify the resource requirements
739
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
703
740
 
704
- Note that all the values specified in parameters are added together so if you specify
705
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
741
+ You can choose the compute layer on the command line by executing e.g.
742
+ ```
743
+ python myflow.py run --with batch
744
+ ```
745
+ or
746
+ ```
747
+ python myflow.py run --with kubernetes
748
+ ```
749
+ which executes the flow on the desired system using the
750
+ requirements specified in `@resources`.
706
751
 
707
752
  Parameters
708
753
  ----------
709
- seconds : int, default 0
710
- Number of seconds to wait prior to timing out.
711
- minutes : int, default 0
712
- Number of minutes to wait prior to timing out.
713
- hours : int, default 0
714
- Number of hours to wait prior to timing out.
754
+ cpu : int, default 1
755
+ Number of CPUs required for this step.
756
+ gpu : int, default 0
757
+ Number of GPUs required for this step.
758
+ disk : int, optional, default None
759
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
760
+ memory : int, default 4096
761
+ Memory size (in MB) required for this step.
762
+ shared_memory : int, optional, default None
763
+ The value for the size (in MiB) of the /dev/shm volume for this step.
764
+ This parameter maps to the `--shm-size` option in Docker.
715
765
  """
716
766
  ...
717
767
 
718
768
  @typing.overload
719
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
769
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
720
770
  """
721
- Specifies environment variables to be set prior to the execution of a step.
771
+ Enables loading / saving of models within a step.
722
772
 
723
- Parameters
724
- ----------
725
- vars : Dict[str, str], default {}
726
- Dictionary of environment variables to set.
727
- """
728
- ...
729
-
730
- @typing.overload
731
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
732
- ...
733
-
734
- @typing.overload
735
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
736
- ...
737
-
738
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
739
- """
740
- Specifies environment variables to be set prior to the execution of a step.
741
773
 
742
774
  Parameters
743
775
  ----------
744
- vars : Dict[str, str], default {}
745
- Dictionary of environment variables to set.
746
- """
747
- ...
748
-
749
- @typing.overload
750
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
751
- """
752
- Specifies secrets to be retrieved and injected as environment variables prior to
753
- the execution of a step.
776
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
777
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
778
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
779
+ - `current.checkpoint`
780
+ - `current.model`
781
+ - `current.huggingface_hub`
782
+
783
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
784
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
785
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
786
+
787
+ temp_dir_root : str, default: None
788
+ The root directory under which `current.model.loaded` will store loaded models
789
+
790
+
754
791
 
755
- Parameters
756
- ----------
757
- sources : List[Union[str, Dict[str, Any]]], default: []
758
- List of secret specs, defining how the secrets are to be retrieved
759
792
  """
760
793
  ...
761
794
 
762
795
  @typing.overload
763
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
796
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
764
797
  ...
765
798
 
766
799
  @typing.overload
767
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
800
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
768
801
  ...
769
802
 
770
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
803
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
771
804
  """
772
- Specifies secrets to be retrieved and injected as environment variables prior to
773
- the execution of a step.
805
+ Enables loading / saving of models within a step.
806
+
774
807
 
775
808
  Parameters
776
809
  ----------
777
- sources : List[Union[str, Dict[str, Any]]], default: []
778
- List of secret specs, defining how the secrets are to be retrieved
779
- """
780
- ...
781
-
782
- @typing.overload
783
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
784
- """
785
- Specifies that the step will success under all circumstances.
810
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
811
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
812
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
813
+ - `current.checkpoint`
814
+ - `current.model`
815
+ - `current.huggingface_hub`
816
+
817
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
818
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
819
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
820
+
821
+ temp_dir_root : str, default: None
822
+ The root directory under which `current.model.loaded` will store loaded models
823
+
786
824
 
787
- The decorator will create an optional artifact, specified by `var`, which
788
- contains the exception raised. You can use it to detect the presence
789
- of errors, indicating that all happy-path artifacts produced by the step
790
- are missing.
791
825
 
792
- Parameters
793
- ----------
794
- var : str, optional, default None
795
- Name of the artifact in which to store the caught exception.
796
- If not specified, the exception is not stored.
797
- print_exception : bool, default True
798
- Determines whether or not the exception is printed to
799
- stdout when caught.
800
826
  """
801
827
  ...
802
828
 
803
829
  @typing.overload
804
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
830
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
831
+ """
832
+ Decorator prototype for all step decorators. This function gets specialized
833
+ and imported for all decorators types by _import_plugin_decorators().
834
+ """
805
835
  ...
806
836
 
807
837
  @typing.overload
808
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
838
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
809
839
  ...
810
840
 
811
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
841
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
812
842
  """
813
- Specifies that the step will success under all circumstances.
814
-
815
- The decorator will create an optional artifact, specified by `var`, which
816
- contains the exception raised. You can use it to detect the presence
817
- of errors, indicating that all happy-path artifacts produced by the step
818
- are missing.
819
-
820
- Parameters
821
- ----------
822
- var : str, optional, default None
823
- Name of the artifact in which to store the caught exception.
824
- If not specified, the exception is not stored.
825
- print_exception : bool, default True
826
- Determines whether or not the exception is printed to
827
- stdout when caught.
843
+ Decorator prototype for all step decorators. This function gets specialized
844
+ and imported for all decorators types by _import_plugin_decorators().
828
845
  """
829
846
  ...
830
847
 
831
848
  @typing.overload
832
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
849
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
833
850
  """
834
- Creates a human-readable report, a Metaflow Card, after this step completes.
835
-
836
- Note that you may add multiple `@card` decorators in a step with different parameters.
851
+ Specifies secrets to be retrieved and injected as environment variables prior to
852
+ the execution of a step.
837
853
 
838
854
  Parameters
839
855
  ----------
840
- type : str, default 'default'
841
- Card type.
842
- id : str, optional, default None
843
- If multiple cards are present, use this id to identify this card.
844
- options : Dict[str, Any], default {}
845
- Options passed to the card. The contents depend on the card type.
846
- timeout : int, default 45
847
- Interrupt reporting if it takes more than this many seconds.
848
-
849
-
856
+ sources : List[Union[str, Dict[str, Any]]], default: []
857
+ List of secret specs, defining how the secrets are to be retrieved
850
858
  """
851
859
  ...
852
860
 
853
861
  @typing.overload
854
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
862
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
855
863
  ...
856
864
 
857
865
  @typing.overload
858
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
866
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
859
867
  ...
860
868
 
861
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
869
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
862
870
  """
863
- Creates a human-readable report, a Metaflow Card, after this step completes.
864
-
865
- Note that you may add multiple `@card` decorators in a step with different parameters.
871
+ Specifies secrets to be retrieved and injected as environment variables prior to
872
+ the execution of a step.
866
873
 
867
874
  Parameters
868
875
  ----------
869
- type : str, default 'default'
870
- Card type.
871
- id : str, optional, default None
872
- If multiple cards are present, use this id to identify this card.
873
- options : Dict[str, Any], default {}
874
- Options passed to the card. The contents depend on the card type.
875
- timeout : int, default 45
876
- Interrupt reporting if it takes more than this many seconds.
877
-
878
-
876
+ sources : List[Union[str, Dict[str, Any]]], default: []
877
+ List of secret specs, defining how the secrets are to be retrieved
879
878
  """
880
879
  ...
881
880
 
@@ -937,63 +936,59 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
937
936
  ...
938
937
 
939
938
  @typing.overload
940
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
939
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
941
940
  """
942
- Enables loading / saving of models within a step.
943
-
944
-
945
- Parameters
946
- ----------
947
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
948
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
949
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
950
- - `current.checkpoint`
951
- - `current.model`
952
- - `current.huggingface_hub`
953
-
954
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
955
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
956
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
941
+ Specifies a timeout for your step.
957
942
 
958
- temp_dir_root : str, default: None
959
- The root directory under which `current.model.loaded` will store loaded models
943
+ This decorator is useful if this step may hang indefinitely.
960
944
 
945
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
946
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
947
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
961
948
 
949
+ Note that all the values specified in parameters are added together so if you specify
950
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
962
951
 
952
+ Parameters
953
+ ----------
954
+ seconds : int, default 0
955
+ Number of seconds to wait prior to timing out.
956
+ minutes : int, default 0
957
+ Number of minutes to wait prior to timing out.
958
+ hours : int, default 0
959
+ Number of hours to wait prior to timing out.
963
960
  """
964
961
  ...
965
962
 
966
963
  @typing.overload
967
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
964
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
968
965
  ...
969
966
 
970
967
  @typing.overload
971
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
968
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
972
969
  ...
973
970
 
974
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
971
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
975
972
  """
976
- Enables loading / saving of models within a step.
977
-
978
-
979
- Parameters
980
- ----------
981
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
982
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
983
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
984
- - `current.checkpoint`
985
- - `current.model`
986
- - `current.huggingface_hub`
987
-
988
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
989
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
990
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
973
+ Specifies a timeout for your step.
991
974
 
992
- temp_dir_root : str, default: None
993
- The root directory under which `current.model.loaded` will store loaded models
975
+ This decorator is useful if this step may hang indefinitely.
994
976
 
977
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
978
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
979
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
995
980
 
981
+ Note that all the values specified in parameters are added together so if you specify
982
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
996
983
 
984
+ Parameters
985
+ ----------
986
+ seconds : int, default 0
987
+ Number of seconds to wait prior to timing out.
988
+ minutes : int, default 0
989
+ Number of minutes to wait prior to timing out.
990
+ hours : int, default 0
991
+ Number of hours to wait prior to timing out.
997
992
  """
998
993
  ...
999
994
 
@@ -1051,187 +1046,294 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1051
1046
  ...
1052
1047
 
1053
1048
  @typing.overload
1054
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1049
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1055
1050
  """
1056
- Specifies the PyPI packages for the step.
1051
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1057
1052
 
1058
- Information in this decorator will augment any
1059
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1060
- you can use `@pypi_base` to set packages required by all
1061
- steps and use `@pypi` to specify step-specific overrides.
1053
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1062
1054
 
1063
1055
  Parameters
1064
1056
  ----------
1065
- packages : Dict[str, str], default: {}
1066
- Packages to use for this step. The key is the name of the package
1067
- and the value is the version to use.
1068
- python : str, optional, default: None
1069
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1070
- that the version used will correspond to the version of the Python interpreter used to start the run.
1057
+ type : str, default 'default'
1058
+ Card type.
1059
+ id : str, optional, default None
1060
+ If multiple cards are present, use this id to identify this card.
1061
+ options : Dict[str, Any], default {}
1062
+ Options passed to the card. The contents depend on the card type.
1063
+ timeout : int, default 45
1064
+ Interrupt reporting if it takes more than this many seconds.
1065
+
1066
+
1071
1067
  """
1072
1068
  ...
1073
1069
 
1074
1070
  @typing.overload
1075
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1071
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1076
1072
  ...
1077
1073
 
1078
1074
  @typing.overload
1079
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1075
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1080
1076
  ...
1081
1077
 
1082
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1078
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1083
1079
  """
1084
- Specifies the PyPI packages for the step.
1080
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1085
1081
 
1086
- Information in this decorator will augment any
1087
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1088
- you can use `@pypi_base` to set packages required by all
1089
- steps and use `@pypi` to specify step-specific overrides.
1082
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1090
1083
 
1091
1084
  Parameters
1092
1085
  ----------
1093
- packages : Dict[str, str], default: {}
1094
- Packages to use for this step. The key is the name of the package
1095
- and the value is the version to use.
1096
- python : str, optional, default: None
1097
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1098
- that the version used will correspond to the version of the Python interpreter used to start the run.
1086
+ type : str, default 'default'
1087
+ Card type.
1088
+ id : str, optional, default None
1089
+ If multiple cards are present, use this id to identify this card.
1090
+ options : Dict[str, Any], default {}
1091
+ Options passed to the card. The contents depend on the card type.
1092
+ timeout : int, default 45
1093
+ Interrupt reporting if it takes more than this many seconds.
1094
+
1095
+
1099
1096
  """
1100
1097
  ...
1101
1098
 
1102
1099
  @typing.overload
1103
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1100
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1104
1101
  """
1105
- Internal decorator to support Fast bakery
1102
+ Specifies environment variables to be set prior to the execution of a step.
1103
+
1104
+ Parameters
1105
+ ----------
1106
+ vars : Dict[str, str], default {}
1107
+ Dictionary of environment variables to set.
1106
1108
  """
1107
1109
  ...
1108
1110
 
1109
1111
  @typing.overload
1110
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1112
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1111
1113
  ...
1112
1114
 
1113
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1115
+ @typing.overload
1116
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1117
+ ...
1118
+
1119
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1114
1120
  """
1115
- Internal decorator to support Fast bakery
1121
+ Specifies environment variables to be set prior to the execution of a step.
1122
+
1123
+ Parameters
1124
+ ----------
1125
+ vars : Dict[str, str], default {}
1126
+ Dictionary of environment variables to set.
1116
1127
  """
1117
1128
  ...
1118
1129
 
1119
- @typing.overload
1120
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1130
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1121
1131
  """
1122
- Specifies the resources needed when executing this step.
1123
-
1124
- Use `@resources` to specify the resource requirements
1125
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1126
-
1127
- You can choose the compute layer on the command line by executing e.g.
1128
- ```
1129
- python myflow.py run --with batch
1130
- ```
1131
- or
1132
- ```
1133
- python myflow.py run --with kubernetes
1134
- ```
1135
- which executes the flow on the desired system using the
1136
- requirements specified in `@resources`.
1132
+ Specifies that this step should execute on Kubernetes.
1137
1133
 
1138
1134
  Parameters
1139
1135
  ----------
1140
1136
  cpu : int, default 1
1141
- Number of CPUs required for this step.
1142
- gpu : int, default 0
1143
- Number of GPUs required for this step.
1144
- disk : int, optional, default None
1145
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1137
+ Number of CPUs required for this step. If `@resources` is
1138
+ also present, the maximum value from all decorators is used.
1146
1139
  memory : int, default 4096
1147
- Memory size (in MB) required for this step.
1148
- shared_memory : int, optional, default None
1149
- The value for the size (in MiB) of the /dev/shm volume for this step.
1150
- This parameter maps to the `--shm-size` option in Docker.
1140
+ Memory size (in MB) required for this step. If
1141
+ `@resources` is also present, the maximum value from all decorators is
1142
+ used.
1143
+ disk : int, default 10240
1144
+ Disk size (in MB) required for this step. If
1145
+ `@resources` is also present, the maximum value from all decorators is
1146
+ used.
1147
+ image : str, optional, default None
1148
+ Docker image to use when launching on Kubernetes. If not specified, and
1149
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1150
+ not, a default Docker image mapping to the current version of Python is used.
1151
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1152
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1153
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1154
+ Kubernetes service account to use when launching pod in Kubernetes.
1155
+ secrets : List[str], optional, default None
1156
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1157
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1158
+ in Metaflow configuration.
1159
+ node_selector: Union[Dict[str,str], str], optional, default None
1160
+ Kubernetes node selector(s) to apply to the pod running the task.
1161
+ Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
1162
+ or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
1163
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1164
+ Kubernetes namespace to use when launching pod in Kubernetes.
1165
+ gpu : int, optional, default None
1166
+ Number of GPUs required for this step. A value of zero implies that
1167
+ the scheduled node should not have GPUs.
1168
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1169
+ The vendor of the GPUs to be used for this step.
1170
+ tolerations : List[str], default []
1171
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1172
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1173
+ use_tmpfs : bool, default False
1174
+ This enables an explicit tmpfs mount for this step.
1175
+ tmpfs_tempdir : bool, default True
1176
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1177
+ tmpfs_size : int, optional, default: None
1178
+ The value for the size (in MiB) of the tmpfs mount for this step.
1179
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1180
+ memory allocated for this step.
1181
+ tmpfs_path : str, optional, default /metaflow_temp
1182
+ Path to tmpfs mount for this step.
1183
+ persistent_volume_claims : Dict[str, str], optional, default None
1184
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1185
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1186
+ shared_memory: int, optional
1187
+ Shared memory size (in MiB) required for this step
1188
+ port: int, optional
1189
+ Port number to specify in the Kubernetes job object
1190
+ compute_pool : str, optional, default None
1191
+ Compute pool to be used for for this step.
1192
+ If not specified, any accessible compute pool within the perimeter is used.
1151
1193
  """
1152
1194
  ...
1153
1195
 
1154
1196
  @typing.overload
1155
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1197
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1198
+ """
1199
+ Specifies the times when the flow should be run when running on a
1200
+ production scheduler.
1201
+
1202
+ Parameters
1203
+ ----------
1204
+ hourly : bool, default False
1205
+ Run the workflow hourly.
1206
+ daily : bool, default True
1207
+ Run the workflow daily.
1208
+ weekly : bool, default False
1209
+ Run the workflow weekly.
1210
+ cron : str, optional, default None
1211
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1212
+ specified by this expression.
1213
+ timezone : str, optional, default None
1214
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1215
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1216
+ """
1156
1217
  ...
1157
1218
 
1158
1219
  @typing.overload
1159
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1220
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1160
1221
  ...
1161
1222
 
1162
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1223
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1163
1224
  """
1164
- Specifies the resources needed when executing this step.
1225
+ Specifies the times when the flow should be run when running on a
1226
+ production scheduler.
1165
1227
 
1166
- Use `@resources` to specify the resource requirements
1167
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1228
+ Parameters
1229
+ ----------
1230
+ hourly : bool, default False
1231
+ Run the workflow hourly.
1232
+ daily : bool, default True
1233
+ Run the workflow daily.
1234
+ weekly : bool, default False
1235
+ Run the workflow weekly.
1236
+ cron : str, optional, default None
1237
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1238
+ specified by this expression.
1239
+ timezone : str, optional, default None
1240
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1241
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1242
+ """
1243
+ ...
1244
+
1245
+ @typing.overload
1246
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1247
+ """
1248
+ Specifies the event(s) that this flow depends on.
1168
1249
 
1169
- You can choose the compute layer on the command line by executing e.g.
1170
1250
  ```
1171
- python myflow.py run --with batch
1251
+ @trigger(event='foo')
1172
1252
  ```
1173
1253
  or
1174
1254
  ```
1175
- python myflow.py run --with kubernetes
1255
+ @trigger(events=['foo', 'bar'])
1256
+ ```
1257
+
1258
+ Additionally, you can specify the parameter mappings
1259
+ to map event payload to Metaflow parameters for the flow.
1260
+ ```
1261
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1262
+ ```
1263
+ or
1264
+ ```
1265
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1266
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1267
+ ```
1268
+
1269
+ 'parameters' can also be a list of strings and tuples like so:
1270
+ ```
1271
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1272
+ ```
1273
+ This is equivalent to:
1274
+ ```
1275
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1176
1276
  ```
1177
- which executes the flow on the desired system using the
1178
- requirements specified in `@resources`.
1179
1277
 
1180
1278
  Parameters
1181
1279
  ----------
1182
- cpu : int, default 1
1183
- Number of CPUs required for this step.
1184
- gpu : int, default 0
1185
- Number of GPUs required for this step.
1186
- disk : int, optional, default None
1187
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1188
- memory : int, default 4096
1189
- Memory size (in MB) required for this step.
1190
- shared_memory : int, optional, default None
1191
- The value for the size (in MiB) of the /dev/shm volume for this step.
1192
- This parameter maps to the `--shm-size` option in Docker.
1280
+ event : Union[str, Dict[str, Any]], optional, default None
1281
+ Event dependency for this flow.
1282
+ events : List[Union[str, Dict[str, Any]]], default []
1283
+ Events dependency for this flow.
1284
+ options : Dict[str, Any], default {}
1285
+ Backend-specific configuration for tuning eventing behavior.
1286
+
1287
+
1193
1288
  """
1194
1289
  ...
1195
1290
 
1196
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1291
+ @typing.overload
1292
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1293
+ ...
1294
+
1295
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1197
1296
  """
1198
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1199
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1200
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1201
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1202
- starts only after all sensors finish.
1297
+ Specifies the event(s) that this flow depends on.
1298
+
1299
+ ```
1300
+ @trigger(event='foo')
1301
+ ```
1302
+ or
1303
+ ```
1304
+ @trigger(events=['foo', 'bar'])
1305
+ ```
1306
+
1307
+ Additionally, you can specify the parameter mappings
1308
+ to map event payload to Metaflow parameters for the flow.
1309
+ ```
1310
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1311
+ ```
1312
+ or
1313
+ ```
1314
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1315
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1316
+ ```
1317
+
1318
+ 'parameters' can also be a list of strings and tuples like so:
1319
+ ```
1320
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1321
+ ```
1322
+ This is equivalent to:
1323
+ ```
1324
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1325
+ ```
1203
1326
 
1204
1327
  Parameters
1205
1328
  ----------
1206
- timeout : int
1207
- Time, in seconds before the task times out and fails. (Default: 3600)
1208
- poke_interval : int
1209
- Time in seconds that the job should wait in between each try. (Default: 60)
1210
- mode : str
1211
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1212
- exponential_backoff : bool
1213
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1214
- pool : str
1215
- the slot pool this task should run in,
1216
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1217
- soft_fail : bool
1218
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1219
- name : str
1220
- Name of the sensor on Airflow
1221
- description : str
1222
- Description of sensor in the Airflow UI
1223
- bucket_key : Union[str, List[str]]
1224
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1225
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1226
- bucket_name : str
1227
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1228
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1229
- wildcard_match : bool
1230
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1231
- aws_conn_id : str
1232
- a reference to the s3 connection on Airflow. (Default: None)
1233
- verify : bool
1234
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1329
+ event : Union[str, Dict[str, Any]], optional, default None
1330
+ Event dependency for this flow.
1331
+ events : List[Union[str, Dict[str, Any]]], default []
1332
+ Events dependency for this flow.
1333
+ options : Dict[str, Any], default {}
1334
+ Backend-specific configuration for tuning eventing behavior.
1335
+
1336
+
1235
1337
  """
1236
1338
  ...
1237
1339
 
@@ -1270,98 +1372,17 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
1270
1372
  """
1271
1373
  Specifies what flows belong to the same project.
1272
1374
 
1273
- A project-specific namespace is created for all flows that
1274
- use the same `@project(name)`.
1275
-
1276
- Parameters
1277
- ----------
1278
- name : str
1279
- Project name. Make sure that the name is unique amongst all
1280
- projects that use the same production scheduler. The name may
1281
- contain only lowercase alphanumeric characters and underscores.
1282
-
1283
-
1284
- """
1285
- ...
1286
-
1287
- @typing.overload
1288
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1289
- """
1290
- Specifies the PyPI packages for all steps of the flow.
1291
-
1292
- Use `@pypi_base` to set common packages required by all
1293
- steps and use `@pypi` to specify step-specific overrides.
1294
- Parameters
1295
- ----------
1296
- packages : Dict[str, str], default: {}
1297
- Packages to use for this flow. The key is the name of the package
1298
- and the value is the version to use.
1299
- python : str, optional, default: None
1300
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1301
- that the version used will correspond to the version of the Python interpreter used to start the run.
1302
- """
1303
- ...
1304
-
1305
- @typing.overload
1306
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1307
- ...
1308
-
1309
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1310
- """
1311
- Specifies the PyPI packages for all steps of the flow.
1312
-
1313
- Use `@pypi_base` to set common packages required by all
1314
- steps and use `@pypi` to specify step-specific overrides.
1315
- Parameters
1316
- ----------
1317
- packages : Dict[str, str], default: {}
1318
- Packages to use for this flow. The key is the name of the package
1319
- and the value is the version to use.
1320
- python : str, optional, default: None
1321
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1322
- that the version used will correspond to the version of the Python interpreter used to start the run.
1323
- """
1324
- ...
1325
-
1326
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1327
- """
1328
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1329
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1330
-
1331
- Parameters
1332
- ----------
1333
- timeout : int
1334
- Time, in seconds before the task times out and fails. (Default: 3600)
1335
- poke_interval : int
1336
- Time in seconds that the job should wait in between each try. (Default: 60)
1337
- mode : str
1338
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1339
- exponential_backoff : bool
1340
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1341
- pool : str
1342
- the slot pool this task should run in,
1343
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1344
- soft_fail : bool
1345
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1346
- name : str
1347
- Name of the sensor on Airflow
1348
- description : str
1349
- Description of sensor in the Airflow UI
1350
- external_dag_id : str
1351
- The dag_id that contains the task you want to wait for.
1352
- external_task_ids : List[str]
1353
- The list of task_ids that you want to wait for.
1354
- If None (default value) the sensor waits for the DAG. (Default: None)
1355
- allowed_states : List[str]
1356
- Iterable of allowed states, (Default: ['success'])
1357
- failed_states : List[str]
1358
- Iterable of failed or dis-allowed states. (Default: None)
1359
- execution_delta : datetime.timedelta
1360
- time difference with the previous execution to look at,
1361
- the default is the same logical date as the current task or DAG. (Default: None)
1362
- check_existence: bool
1363
- Set to True to check if the external task exists or check if
1364
- the DAG to wait for exists. (Default: True)
1375
+ A project-specific namespace is created for all flows that
1376
+ use the same `@project(name)`.
1377
+
1378
+ Parameters
1379
+ ----------
1380
+ name : str
1381
+ Project name. Make sure that the name is unique amongst all
1382
+ projects that use the same production scheduler. The name may
1383
+ contain only lowercase alphanumeric characters and underscores.
1384
+
1385
+
1365
1386
  """
1366
1387
  ...
1367
1388
 
@@ -1469,195 +1490,174 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1469
1490
  ...
1470
1491
 
1471
1492
  @typing.overload
1472
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1493
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1473
1494
  """
1474
- Specifies the Conda environment for all steps of the flow.
1475
-
1476
- Use `@conda_base` to set common libraries required by all
1477
- steps and use `@conda` to specify step-specific additions.
1495
+ Specifies the PyPI packages for all steps of the flow.
1478
1496
 
1497
+ Use `@pypi_base` to set common packages required by all
1498
+ steps and use `@pypi` to specify step-specific overrides.
1479
1499
  Parameters
1480
1500
  ----------
1481
- packages : Dict[str, str], default {}
1501
+ packages : Dict[str, str], default: {}
1482
1502
  Packages to use for this flow. The key is the name of the package
1483
1503
  and the value is the version to use.
1484
- libraries : Dict[str, str], default {}
1485
- Supported for backward compatibility. When used with packages, packages will take precedence.
1486
- python : str, optional, default None
1504
+ python : str, optional, default: None
1487
1505
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1488
1506
  that the version used will correspond to the version of the Python interpreter used to start the run.
1489
- disabled : bool, default False
1490
- If set to True, disables Conda.
1491
1507
  """
1492
1508
  ...
1493
1509
 
1494
1510
  @typing.overload
1495
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1511
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1496
1512
  ...
1497
1513
 
1498
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1514
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1499
1515
  """
1500
- Specifies the Conda environment for all steps of the flow.
1501
-
1502
- Use `@conda_base` to set common libraries required by all
1503
- steps and use `@conda` to specify step-specific additions.
1516
+ Specifies the PyPI packages for all steps of the flow.
1504
1517
 
1518
+ Use `@pypi_base` to set common packages required by all
1519
+ steps and use `@pypi` to specify step-specific overrides.
1505
1520
  Parameters
1506
1521
  ----------
1507
- packages : Dict[str, str], default {}
1522
+ packages : Dict[str, str], default: {}
1508
1523
  Packages to use for this flow. The key is the name of the package
1509
1524
  and the value is the version to use.
1510
- libraries : Dict[str, str], default {}
1511
- Supported for backward compatibility. When used with packages, packages will take precedence.
1512
- python : str, optional, default None
1525
+ python : str, optional, default: None
1513
1526
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1514
1527
  that the version used will correspond to the version of the Python interpreter used to start the run.
1515
- disabled : bool, default False
1516
- If set to True, disables Conda.
1517
1528
  """
1518
1529
  ...
1519
1530
 
1520
- @typing.overload
1521
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1531
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1522
1532
  """
1523
- Specifies the times when the flow should be run when running on a
1524
- production scheduler.
1533
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1534
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1535
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1536
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1537
+ starts only after all sensors finish.
1525
1538
 
1526
1539
  Parameters
1527
1540
  ----------
1528
- hourly : bool, default False
1529
- Run the workflow hourly.
1530
- daily : bool, default True
1531
- Run the workflow daily.
1532
- weekly : bool, default False
1533
- Run the workflow weekly.
1534
- cron : str, optional, default None
1535
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1536
- specified by this expression.
1537
- timezone : str, optional, default None
1538
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1539
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1541
+ timeout : int
1542
+ Time, in seconds before the task times out and fails. (Default: 3600)
1543
+ poke_interval : int
1544
+ Time in seconds that the job should wait in between each try. (Default: 60)
1545
+ mode : str
1546
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1547
+ exponential_backoff : bool
1548
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1549
+ pool : str
1550
+ the slot pool this task should run in,
1551
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1552
+ soft_fail : bool
1553
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1554
+ name : str
1555
+ Name of the sensor on Airflow
1556
+ description : str
1557
+ Description of sensor in the Airflow UI
1558
+ bucket_key : Union[str, List[str]]
1559
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1560
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1561
+ bucket_name : str
1562
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1563
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1564
+ wildcard_match : bool
1565
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1566
+ aws_conn_id : str
1567
+ a reference to the s3 connection on Airflow. (Default: None)
1568
+ verify : bool
1569
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1540
1570
  """
1541
1571
  ...
1542
1572
 
1543
- @typing.overload
1544
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1545
- ...
1546
-
1547
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1573
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1548
1574
  """
1549
- Specifies the times when the flow should be run when running on a
1550
- production scheduler.
1575
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1576
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1551
1577
 
1552
1578
  Parameters
1553
1579
  ----------
1554
- hourly : bool, default False
1555
- Run the workflow hourly.
1556
- daily : bool, default True
1557
- Run the workflow daily.
1558
- weekly : bool, default False
1559
- Run the workflow weekly.
1560
- cron : str, optional, default None
1561
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1562
- specified by this expression.
1563
- timezone : str, optional, default None
1564
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1565
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1580
+ timeout : int
1581
+ Time, in seconds before the task times out and fails. (Default: 3600)
1582
+ poke_interval : int
1583
+ Time in seconds that the job should wait in between each try. (Default: 60)
1584
+ mode : str
1585
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1586
+ exponential_backoff : bool
1587
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1588
+ pool : str
1589
+ the slot pool this task should run in,
1590
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1591
+ soft_fail : bool
1592
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1593
+ name : str
1594
+ Name of the sensor on Airflow
1595
+ description : str
1596
+ Description of sensor in the Airflow UI
1597
+ external_dag_id : str
1598
+ The dag_id that contains the task you want to wait for.
1599
+ external_task_ids : List[str]
1600
+ The list of task_ids that you want to wait for.
1601
+ If None (default value) the sensor waits for the DAG. (Default: None)
1602
+ allowed_states : List[str]
1603
+ Iterable of allowed states, (Default: ['success'])
1604
+ failed_states : List[str]
1605
+ Iterable of failed or dis-allowed states. (Default: None)
1606
+ execution_delta : datetime.timedelta
1607
+ time difference with the previous execution to look at,
1608
+ the default is the same logical date as the current task or DAG. (Default: None)
1609
+ check_existence: bool
1610
+ Set to True to check if the external task exists or check if
1611
+ the DAG to wait for exists. (Default: True)
1566
1612
  """
1567
1613
  ...
1568
1614
 
1569
1615
  @typing.overload
1570
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1616
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1571
1617
  """
1572
- Specifies the event(s) that this flow depends on.
1573
-
1574
- ```
1575
- @trigger(event='foo')
1576
- ```
1577
- or
1578
- ```
1579
- @trigger(events=['foo', 'bar'])
1580
- ```
1581
-
1582
- Additionally, you can specify the parameter mappings
1583
- to map event payload to Metaflow parameters for the flow.
1584
- ```
1585
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1586
- ```
1587
- or
1588
- ```
1589
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1590
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1591
- ```
1618
+ Specifies the Conda environment for all steps of the flow.
1592
1619
 
1593
- 'parameters' can also be a list of strings and tuples like so:
1594
- ```
1595
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1596
- ```
1597
- This is equivalent to:
1598
- ```
1599
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1600
- ```
1620
+ Use `@conda_base` to set common libraries required by all
1621
+ steps and use `@conda` to specify step-specific additions.
1601
1622
 
1602
1623
  Parameters
1603
1624
  ----------
1604
- event : Union[str, Dict[str, Any]], optional, default None
1605
- Event dependency for this flow.
1606
- events : List[Union[str, Dict[str, Any]]], default []
1607
- Events dependency for this flow.
1608
- options : Dict[str, Any], default {}
1609
- Backend-specific configuration for tuning eventing behavior.
1610
-
1611
-
1625
+ packages : Dict[str, str], default {}
1626
+ Packages to use for this flow. The key is the name of the package
1627
+ and the value is the version to use.
1628
+ libraries : Dict[str, str], default {}
1629
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1630
+ python : str, optional, default None
1631
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1632
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1633
+ disabled : bool, default False
1634
+ If set to True, disables Conda.
1612
1635
  """
1613
1636
  ...
1614
1637
 
1615
1638
  @typing.overload
1616
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1639
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1617
1640
  ...
1618
1641
 
1619
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1642
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1620
1643
  """
1621
- Specifies the event(s) that this flow depends on.
1622
-
1623
- ```
1624
- @trigger(event='foo')
1625
- ```
1626
- or
1627
- ```
1628
- @trigger(events=['foo', 'bar'])
1629
- ```
1630
-
1631
- Additionally, you can specify the parameter mappings
1632
- to map event payload to Metaflow parameters for the flow.
1633
- ```
1634
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1635
- ```
1636
- or
1637
- ```
1638
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1639
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1640
- ```
1644
+ Specifies the Conda environment for all steps of the flow.
1641
1645
 
1642
- 'parameters' can also be a list of strings and tuples like so:
1643
- ```
1644
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1645
- ```
1646
- This is equivalent to:
1647
- ```
1648
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1649
- ```
1646
+ Use `@conda_base` to set common libraries required by all
1647
+ steps and use `@conda` to specify step-specific additions.
1650
1648
 
1651
1649
  Parameters
1652
1650
  ----------
1653
- event : Union[str, Dict[str, Any]], optional, default None
1654
- Event dependency for this flow.
1655
- events : List[Union[str, Dict[str, Any]]], default []
1656
- Events dependency for this flow.
1657
- options : Dict[str, Any], default {}
1658
- Backend-specific configuration for tuning eventing behavior.
1659
-
1660
-
1651
+ packages : Dict[str, str], default {}
1652
+ Packages to use for this flow. The key is the name of the package
1653
+ and the value is the version to use.
1654
+ libraries : Dict[str, str], default {}
1655
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1656
+ python : str, optional, default None
1657
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1658
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1659
+ disabled : bool, default False
1660
+ If set to True, disables Conda.
1661
1661
  """
1662
1662
  ...
1663
1663