ob-metaflow-stubs 6.0.3.102rc0__py2.py3-none-any.whl → 6.0.3.102rc1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +601 -601
  2. metaflow-stubs/cards.pyi +4 -4
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +1 -1
  7. metaflow-stubs/clone_util.pyi +1 -1
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +1 -1
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/info_file.pyi +1 -1
  14. metaflow-stubs/metadata/metadata.pyi +1 -1
  15. metaflow-stubs/metadata/util.pyi +1 -1
  16. metaflow-stubs/metaflow_config.pyi +1 -1
  17. metaflow-stubs/metaflow_current.pyi +67 -67
  18. metaflow-stubs/mflog/mflog.pyi +1 -1
  19. metaflow-stubs/multicore_utils.pyi +1 -1
  20. metaflow-stubs/parameters.pyi +3 -3
  21. metaflow-stubs/plugins/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  24. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  30. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  31. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +5 -5
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  36. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  37. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +1 -1
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +1 -1
  52. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  59. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  60. metaflow-stubs/plugins/cards/card_cli.pyi +3 -3
  61. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  62. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  74. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  75. metaflow-stubs/plugins/catch_decorator.pyi +1 -1
  76. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  82. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  83. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  84. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  85. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  101. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  102. metaflow-stubs/plugins/package_cli.pyi +1 -1
  103. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  104. metaflow-stubs/plugins/perimeters.pyi +1 -1
  105. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  106. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  111. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  112. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  113. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  114. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  118. metaflow-stubs/plugins/tag_cli.pyi +3 -3
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
  121. metaflow-stubs/procpoll.pyi +1 -1
  122. metaflow-stubs/profilers/__init__.pyi +1 -1
  123. metaflow-stubs/pylint_wrapper.pyi +1 -1
  124. metaflow-stubs/runner/__init__.pyi +1 -1
  125. metaflow-stubs/runner/deployer.pyi +2 -2
  126. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  127. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  128. metaflow-stubs/runner/nbrun.pyi +1 -1
  129. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  130. metaflow-stubs/runner/utils.pyi +1 -1
  131. metaflow-stubs/system/__init__.pyi +3 -3
  132. metaflow-stubs/system/system_logger.pyi +2 -2
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +1 -1
  135. metaflow-stubs/tuple_util.pyi +1 -1
  136. {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.102rc1.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.102rc1.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.102rc0.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.102rc1.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.102rc1.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.12.22.1+obcheckpoint(0.0.10);ob(v1) #
4
- # Generated on 2024-09-20T18:35:21.800373 #
4
+ # Generated on 2024-09-20T18:51:04.453225 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.events
12
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
- import metaflow.datastore.inputs
11
+ import metaflow.client.core
14
12
  import metaflow.metaflow_current
13
+ import metaflow.parameters
15
14
  import metaflow._vendor.click.types
15
+ import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
16
16
  import metaflow.runner.metaflow_runner
17
+ import metaflow.events
17
18
  import typing
18
- import metaflow.flowspec
19
- import metaflow.client.core
20
- import metaflow.parameters
21
19
  import datetime
20
+ import metaflow.datastore.inputs
21
+ import metaflow.flowspec
22
22
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
23
23
  StepFlag = typing.NewType("StepFlag", bool)
24
24
 
@@ -435,194 +435,70 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
435
435
  ...
436
436
 
437
437
  @typing.overload
438
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
439
- """
440
- Specifies that the step will success under all circumstances.
441
-
442
- The decorator will create an optional artifact, specified by `var`, which
443
- contains the exception raised. You can use it to detect the presence
444
- of errors, indicating that all happy-path artifacts produced by the step
445
- are missing.
446
-
447
- Parameters
448
- ----------
449
- var : str, optional, default None
450
- Name of the artifact in which to store the caught exception.
451
- If not specified, the exception is not stored.
452
- print_exception : bool, default True
453
- Determines whether or not the exception is printed to
454
- stdout when caught.
455
- """
456
- ...
457
-
458
- @typing.overload
459
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
460
- ...
461
-
462
- @typing.overload
463
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
464
- ...
465
-
466
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
467
- """
468
- Specifies that the step will success under all circumstances.
469
-
470
- The decorator will create an optional artifact, specified by `var`, which
471
- contains the exception raised. You can use it to detect the presence
472
- of errors, indicating that all happy-path artifacts produced by the step
473
- are missing.
474
-
475
- Parameters
476
- ----------
477
- var : str, optional, default None
478
- Name of the artifact in which to store the caught exception.
479
- If not specified, the exception is not stored.
480
- print_exception : bool, default True
481
- Determines whether or not the exception is printed to
482
- stdout when caught.
483
- """
484
- ...
485
-
486
- @typing.overload
487
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
438
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
488
439
  """
489
- Specifies the number of times the task corresponding
490
- to a step needs to be retried.
491
-
492
- This decorator is useful for handling transient errors, such as networking issues.
493
- If your task contains operations that can't be retried safely, e.g. database updates,
494
- it is advisable to annotate it with `@retry(times=0)`.
495
-
496
- This can be used in conjunction with the `@catch` decorator. The `@catch`
497
- decorator will execute a no-op task after all retries have been exhausted,
498
- ensuring that the flow execution can continue.
499
-
500
- Parameters
501
- ----------
502
- times : int, default 3
503
- Number of times to retry this task.
504
- minutes_between_retries : int, default 2
505
- Number of minutes between retries.
440
+ Decorator prototype for all step decorators. This function gets specialized
441
+ and imported for all decorators types by _import_plugin_decorators().
506
442
  """
507
443
  ...
508
444
 
509
445
  @typing.overload
510
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
511
- ...
512
-
513
- @typing.overload
514
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
446
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
515
447
  ...
516
448
 
517
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
449
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
518
450
  """
519
- Specifies the number of times the task corresponding
520
- to a step needs to be retried.
521
-
522
- This decorator is useful for handling transient errors, such as networking issues.
523
- If your task contains operations that can't be retried safely, e.g. database updates,
524
- it is advisable to annotate it with `@retry(times=0)`.
525
-
526
- This can be used in conjunction with the `@catch` decorator. The `@catch`
527
- decorator will execute a no-op task after all retries have been exhausted,
528
- ensuring that the flow execution can continue.
529
-
530
- Parameters
531
- ----------
532
- times : int, default 3
533
- Number of times to retry this task.
534
- minutes_between_retries : int, default 2
535
- Number of minutes between retries.
451
+ Decorator prototype for all step decorators. This function gets specialized
452
+ and imported for all decorators types by _import_plugin_decorators().
536
453
  """
537
454
  ...
538
455
 
539
456
  @typing.overload
540
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
457
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
541
458
  """
542
- Specifies the Conda environment for the step.
459
+ Specifies the PyPI packages for the step.
543
460
 
544
461
  Information in this decorator will augment any
545
- attributes set in the `@conda_base` flow-level decorator. Hence,
546
- you can use `@conda_base` to set packages required by all
547
- steps and use `@conda` to specify step-specific overrides.
462
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
463
+ you can use `@pypi_base` to set packages required by all
464
+ steps and use `@pypi` to specify step-specific overrides.
548
465
 
549
466
  Parameters
550
467
  ----------
551
- packages : Dict[str, str], default {}
468
+ packages : Dict[str, str], default: {}
552
469
  Packages to use for this step. The key is the name of the package
553
470
  and the value is the version to use.
554
- libraries : Dict[str, str], default {}
555
- Supported for backward compatibility. When used with packages, packages will take precedence.
556
- python : str, optional, default None
471
+ python : str, optional, default: None
557
472
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
558
473
  that the version used will correspond to the version of the Python interpreter used to start the run.
559
- disabled : bool, default False
560
- If set to True, disables @conda.
561
474
  """
562
475
  ...
563
476
 
564
477
  @typing.overload
565
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
478
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
566
479
  ...
567
480
 
568
481
  @typing.overload
569
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
482
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
570
483
  ...
571
484
 
572
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
485
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
573
486
  """
574
- Specifies the Conda environment for the step.
487
+ Specifies the PyPI packages for the step.
575
488
 
576
489
  Information in this decorator will augment any
577
- attributes set in the `@conda_base` flow-level decorator. Hence,
578
- you can use `@conda_base` to set packages required by all
579
- steps and use `@conda` to specify step-specific overrides.
490
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
491
+ you can use `@pypi_base` to set packages required by all
492
+ steps and use `@pypi` to specify step-specific overrides.
580
493
 
581
494
  Parameters
582
495
  ----------
583
- packages : Dict[str, str], default {}
496
+ packages : Dict[str, str], default: {}
584
497
  Packages to use for this step. The key is the name of the package
585
498
  and the value is the version to use.
586
- libraries : Dict[str, str], default {}
587
- Supported for backward compatibility. When used with packages, packages will take precedence.
588
- python : str, optional, default None
499
+ python : str, optional, default: None
589
500
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
590
501
  that the version used will correspond to the version of the Python interpreter used to start the run.
591
- disabled : bool, default False
592
- If set to True, disables @conda.
593
- """
594
- ...
595
-
596
- @typing.overload
597
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
598
- """
599
- Specifies secrets to be retrieved and injected as environment variables prior to
600
- the execution of a step.
601
-
602
- Parameters
603
- ----------
604
- sources : List[Union[str, Dict[str, Any]]], default: []
605
- List of secret specs, defining how the secrets are to be retrieved
606
- """
607
- ...
608
-
609
- @typing.overload
610
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
611
- ...
612
-
613
- @typing.overload
614
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
615
- ...
616
-
617
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
618
- """
619
- Specifies secrets to be retrieved and injected as environment variables prior to
620
- the execution of a step.
621
-
622
- Parameters
623
- ----------
624
- sources : List[Union[str, Dict[str, Any]]], default: []
625
- List of secret specs, defining how the secrets are to be retrieved
626
502
  """
627
503
  ...
628
504
 
@@ -684,129 +560,164 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
684
560
  ...
685
561
 
686
562
  @typing.overload
687
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
563
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
688
564
  """
689
- Specifies environment variables to be set prior to the execution of a step.
565
+ Creates a human-readable report, a Metaflow Card, after this step completes.
566
+
567
+ Note that you may add multiple `@card` decorators in a step with different parameters.
690
568
 
691
569
  Parameters
692
570
  ----------
693
- vars : Dict[str, str], default {}
694
- Dictionary of environment variables to set.
571
+ type : str, default 'default'
572
+ Card type.
573
+ id : str, optional, default None
574
+ If multiple cards are present, use this id to identify this card.
575
+ options : Dict[str, Any], default {}
576
+ Options passed to the card. The contents depend on the card type.
577
+ timeout : int, default 45
578
+ Interrupt reporting if it takes more than this many seconds.
579
+
580
+
695
581
  """
696
582
  ...
697
583
 
698
584
  @typing.overload
699
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
585
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
700
586
  ...
701
587
 
702
588
  @typing.overload
703
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
589
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
704
590
  ...
705
591
 
706
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
592
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
707
593
  """
708
- Specifies environment variables to be set prior to the execution of a step.
594
+ Creates a human-readable report, a Metaflow Card, after this step completes.
595
+
596
+ Note that you may add multiple `@card` decorators in a step with different parameters.
709
597
 
710
598
  Parameters
711
599
  ----------
712
- vars : Dict[str, str], default {}
713
- Dictionary of environment variables to set.
600
+ type : str, default 'default'
601
+ Card type.
602
+ id : str, optional, default None
603
+ If multiple cards are present, use this id to identify this card.
604
+ options : Dict[str, Any], default {}
605
+ Options passed to the card. The contents depend on the card type.
606
+ timeout : int, default 45
607
+ Interrupt reporting if it takes more than this many seconds.
608
+
609
+
714
610
  """
715
611
  ...
716
612
 
717
613
  @typing.overload
718
- def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
614
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
719
615
  """
720
- Specifies the resources needed when executing this step.
721
-
722
- Use `@resources` to specify the resource requirements
723
- independently of the specific compute layer (`@batch`, `@kubernetes`).
616
+ Enables loading / saving of models within a step.
724
617
 
725
- You can choose the compute layer on the command line by executing e.g.
726
- ```
727
- python myflow.py run --with batch
728
- ```
729
- or
730
- ```
731
- python myflow.py run --with kubernetes
732
- ```
733
- which executes the flow on the desired system using the
734
- requirements specified in `@resources`.
735
618
 
736
619
  Parameters
737
620
  ----------
738
- cpu : int, default 1
739
- Number of CPUs required for this step.
740
- gpu : int, default 0
741
- Number of GPUs required for this step.
742
- disk : int, optional, default None
743
- Disk size (in MB) required for this step. Only applies on Kubernetes.
744
- memory : int, default 4096
745
- Memory size (in MB) required for this step.
746
- shared_memory : int, optional, default None
747
- The value for the size (in MiB) of the /dev/shm volume for this step.
748
- This parameter maps to the `--shm-size` option in Docker.
749
- """
750
- ...
751
-
752
- @typing.overload
753
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
754
- ...
621
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
622
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
623
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
624
+ - `current.checkpoint`
625
+ - `current.model`
626
+ - `current.huggingface_hub`
627
+
628
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
629
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
630
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
631
+
632
+ temp_dir_root : str, default: None
633
+ The root directory under which `current.model.loaded` will store loaded models
634
+
635
+
636
+
637
+ """
638
+ ...
755
639
 
756
640
  @typing.overload
757
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
641
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
758
642
  ...
759
643
 
760
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
644
+ @typing.overload
645
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
646
+ ...
647
+
648
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
761
649
  """
762
- Specifies the resources needed when executing this step.
650
+ Enables loading / saving of models within a step.
763
651
 
764
- Use `@resources` to specify the resource requirements
765
- independently of the specific compute layer (`@batch`, `@kubernetes`).
766
652
 
767
- You can choose the compute layer on the command line by executing e.g.
768
- ```
769
- python myflow.py run --with batch
770
- ```
771
- or
772
- ```
773
- python myflow.py run --with kubernetes
774
- ```
775
- which executes the flow on the desired system using the
776
- requirements specified in `@resources`.
653
+ Parameters
654
+ ----------
655
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
656
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
657
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
658
+ - `current.checkpoint`
659
+ - `current.model`
660
+ - `current.huggingface_hub`
661
+
662
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
663
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
664
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
665
+
666
+ temp_dir_root : str, default: None
667
+ The root directory under which `current.model.loaded` will store loaded models
668
+
669
+
670
+
671
+ """
672
+ ...
673
+
674
+ @typing.overload
675
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
676
+ """
677
+ Specifies secrets to be retrieved and injected as environment variables prior to
678
+ the execution of a step.
777
679
 
778
680
  Parameters
779
681
  ----------
780
- cpu : int, default 1
781
- Number of CPUs required for this step.
782
- gpu : int, default 0
783
- Number of GPUs required for this step.
784
- disk : int, optional, default None
785
- Disk size (in MB) required for this step. Only applies on Kubernetes.
786
- memory : int, default 4096
787
- Memory size (in MB) required for this step.
788
- shared_memory : int, optional, default None
789
- The value for the size (in MiB) of the /dev/shm volume for this step.
790
- This parameter maps to the `--shm-size` option in Docker.
682
+ sources : List[Union[str, Dict[str, Any]]], default: []
683
+ List of secret specs, defining how the secrets are to be retrieved
791
684
  """
792
685
  ...
793
686
 
794
687
  @typing.overload
795
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
688
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
689
+ ...
690
+
691
+ @typing.overload
692
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
693
+ ...
694
+
695
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
796
696
  """
797
- Decorator prototype for all step decorators. This function gets specialized
798
- and imported for all decorators types by _import_plugin_decorators().
697
+ Specifies secrets to be retrieved and injected as environment variables prior to
698
+ the execution of a step.
699
+
700
+ Parameters
701
+ ----------
702
+ sources : List[Union[str, Dict[str, Any]]], default: []
703
+ List of secret specs, defining how the secrets are to be retrieved
799
704
  """
800
705
  ...
801
706
 
802
707
  @typing.overload
803
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
708
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
709
+ """
710
+ Internal decorator to support Fast bakery
711
+ """
804
712
  ...
805
713
 
806
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
714
+ @typing.overload
715
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
716
+ ...
717
+
718
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
807
719
  """
808
- Decorator prototype for all step decorators. This function gets specialized
809
- and imported for all decorators types by _import_plugin_decorators().
720
+ Internal decorator to support Fast bakery
810
721
  """
811
722
  ...
812
723
 
@@ -877,395 +788,337 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
877
788
  ...
878
789
 
879
790
  @typing.overload
880
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
791
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
881
792
  """
882
- Specifies the PyPI packages for the step.
793
+ Specifies that the step will success under all circumstances.
794
+
795
+ The decorator will create an optional artifact, specified by `var`, which
796
+ contains the exception raised. You can use it to detect the presence
797
+ of errors, indicating that all happy-path artifacts produced by the step
798
+ are missing.
799
+
800
+ Parameters
801
+ ----------
802
+ var : str, optional, default None
803
+ Name of the artifact in which to store the caught exception.
804
+ If not specified, the exception is not stored.
805
+ print_exception : bool, default True
806
+ Determines whether or not the exception is printed to
807
+ stdout when caught.
808
+ """
809
+ ...
810
+
811
+ @typing.overload
812
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
813
+ ...
814
+
815
+ @typing.overload
816
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
817
+ ...
818
+
819
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
820
+ """
821
+ Specifies that the step will success under all circumstances.
822
+
823
+ The decorator will create an optional artifact, specified by `var`, which
824
+ contains the exception raised. You can use it to detect the presence
825
+ of errors, indicating that all happy-path artifacts produced by the step
826
+ are missing.
827
+
828
+ Parameters
829
+ ----------
830
+ var : str, optional, default None
831
+ Name of the artifact in which to store the caught exception.
832
+ If not specified, the exception is not stored.
833
+ print_exception : bool, default True
834
+ Determines whether or not the exception is printed to
835
+ stdout when caught.
836
+ """
837
+ ...
838
+
839
+ @typing.overload
840
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
841
+ """
842
+ Specifies the Conda environment for the step.
883
843
 
884
844
  Information in this decorator will augment any
885
- attributes set in the `@pyi_base` flow-level decorator. Hence,
886
- you can use `@pypi_base` to set packages required by all
887
- steps and use `@pypi` to specify step-specific overrides.
845
+ attributes set in the `@conda_base` flow-level decorator. Hence,
846
+ you can use `@conda_base` to set packages required by all
847
+ steps and use `@conda` to specify step-specific overrides.
888
848
 
889
849
  Parameters
890
850
  ----------
891
- packages : Dict[str, str], default: {}
851
+ packages : Dict[str, str], default {}
892
852
  Packages to use for this step. The key is the name of the package
893
853
  and the value is the version to use.
894
- python : str, optional, default: None
854
+ libraries : Dict[str, str], default {}
855
+ Supported for backward compatibility. When used with packages, packages will take precedence.
856
+ python : str, optional, default None
895
857
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
896
858
  that the version used will correspond to the version of the Python interpreter used to start the run.
859
+ disabled : bool, default False
860
+ If set to True, disables @conda.
897
861
  """
898
862
  ...
899
863
 
900
864
  @typing.overload
901
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
865
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
902
866
  ...
903
867
 
904
868
  @typing.overload
905
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
869
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
906
870
  ...
907
871
 
908
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
872
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
909
873
  """
910
- Specifies the PyPI packages for the step.
874
+ Specifies the Conda environment for the step.
911
875
 
912
876
  Information in this decorator will augment any
913
- attributes set in the `@pyi_base` flow-level decorator. Hence,
914
- you can use `@pypi_base` to set packages required by all
915
- steps and use `@pypi` to specify step-specific overrides.
877
+ attributes set in the `@conda_base` flow-level decorator. Hence,
878
+ you can use `@conda_base` to set packages required by all
879
+ steps and use `@conda` to specify step-specific overrides.
916
880
 
917
881
  Parameters
918
882
  ----------
919
- packages : Dict[str, str], default: {}
883
+ packages : Dict[str, str], default {}
920
884
  Packages to use for this step. The key is the name of the package
921
885
  and the value is the version to use.
922
- python : str, optional, default: None
886
+ libraries : Dict[str, str], default {}
887
+ Supported for backward compatibility. When used with packages, packages will take precedence.
888
+ python : str, optional, default None
923
889
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
924
890
  that the version used will correspond to the version of the Python interpreter used to start the run.
891
+ disabled : bool, default False
892
+ If set to True, disables @conda.
925
893
  """
926
894
  ...
927
895
 
928
896
  @typing.overload
929
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
897
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
930
898
  """
931
- Creates a human-readable report, a Metaflow Card, after this step completes.
899
+ Specifies the number of times the task corresponding
900
+ to a step needs to be retried.
932
901
 
933
- Note that you may add multiple `@card` decorators in a step with different parameters.
902
+ This decorator is useful for handling transient errors, such as networking issues.
903
+ If your task contains operations that can't be retried safely, e.g. database updates,
904
+ it is advisable to annotate it with `@retry(times=0)`.
905
+
906
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
907
+ decorator will execute a no-op task after all retries have been exhausted,
908
+ ensuring that the flow execution can continue.
934
909
 
935
910
  Parameters
936
911
  ----------
937
- type : str, default 'default'
938
- Card type.
939
- id : str, optional, default None
940
- If multiple cards are present, use this id to identify this card.
941
- options : Dict[str, Any], default {}
942
- Options passed to the card. The contents depend on the card type.
943
- timeout : int, default 45
944
- Interrupt reporting if it takes more than this many seconds.
945
-
946
-
912
+ times : int, default 3
913
+ Number of times to retry this task.
914
+ minutes_between_retries : int, default 2
915
+ Number of minutes between retries.
947
916
  """
948
917
  ...
949
918
 
950
919
  @typing.overload
951
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
920
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
952
921
  ...
953
922
 
954
923
  @typing.overload
955
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
924
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
956
925
  ...
957
926
 
958
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
927
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
959
928
  """
960
- Creates a human-readable report, a Metaflow Card, after this step completes.
929
+ Specifies the number of times the task corresponding
930
+ to a step needs to be retried.
961
931
 
962
- Note that you may add multiple `@card` decorators in a step with different parameters.
932
+ This decorator is useful for handling transient errors, such as networking issues.
933
+ If your task contains operations that can't be retried safely, e.g. database updates,
934
+ it is advisable to annotate it with `@retry(times=0)`.
935
+
936
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
937
+ decorator will execute a no-op task after all retries have been exhausted,
938
+ ensuring that the flow execution can continue.
963
939
 
964
940
  Parameters
965
941
  ----------
966
- type : str, default 'default'
967
- Card type.
968
- id : str, optional, default None
969
- If multiple cards are present, use this id to identify this card.
970
- options : Dict[str, Any], default {}
971
- Options passed to the card. The contents depend on the card type.
972
- timeout : int, default 45
973
- Interrupt reporting if it takes more than this many seconds.
974
-
975
-
976
- """
977
- ...
942
+ times : int, default 3
943
+ Number of times to retry this task.
944
+ minutes_between_retries : int, default 2
945
+ Number of minutes between retries.
946
+ """
947
+ ...
978
948
 
979
949
  @typing.overload
980
- def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
950
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
981
951
  """
982
- Enables checkpointing for a step.
983
-
952
+ Specifies environment variables to be set prior to the execution of a step.
984
953
 
985
954
  Parameters
986
955
  ----------
987
- load_policy : str, default: "fresh"
988
- The policy for loading the checkpoint. The following policies are supported:
989
- - "eager": Loads the the latest available checkpoint within the namespace.
990
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
991
- will be loaded at the start of the task.
992
- - "none": Do not load any checkpoint
993
- - "fresh": Loads the lastest checkpoint created within the running Task.
994
- This mode helps loading checkpoints across various retry attempts of the same task.
995
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
996
- created within the task will be loaded when the task is retries execution on failure.
997
-
998
- temp_dir_root : str, default: None
999
- The root directory under which `current.checkpoint.directory` will be created.
1000
-
1001
-
1002
-
956
+ vars : Dict[str, str], default {}
957
+ Dictionary of environment variables to set.
1003
958
  """
1004
959
  ...
1005
960
 
1006
961
  @typing.overload
1007
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
962
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1008
963
  ...
1009
964
 
1010
965
  @typing.overload
1011
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
966
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1012
967
  ...
1013
968
 
1014
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
969
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1015
970
  """
1016
- Enables checkpointing for a step.
1017
-
971
+ Specifies environment variables to be set prior to the execution of a step.
1018
972
 
1019
973
  Parameters
1020
974
  ----------
1021
- load_policy : str, default: "fresh"
1022
- The policy for loading the checkpoint. The following policies are supported:
1023
- - "eager": Loads the the latest available checkpoint within the namespace.
1024
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1025
- will be loaded at the start of the task.
1026
- - "none": Do not load any checkpoint
1027
- - "fresh": Loads the lastest checkpoint created within the running Task.
1028
- This mode helps loading checkpoints across various retry attempts of the same task.
1029
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1030
- created within the task will be loaded when the task is retries execution on failure.
1031
-
1032
- temp_dir_root : str, default: None
1033
- The root directory under which `current.checkpoint.directory` will be created.
1034
-
1035
-
1036
-
975
+ vars : Dict[str, str], default {}
976
+ Dictionary of environment variables to set.
1037
977
  """
1038
978
  ...
1039
979
 
1040
980
  @typing.overload
1041
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
981
+ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1042
982
  """
1043
- Enables loading / saving of models within a step.
983
+ Specifies the resources needed when executing this step.
1044
984
 
985
+ Use `@resources` to specify the resource requirements
986
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
987
+
988
+ You can choose the compute layer on the command line by executing e.g.
989
+ ```
990
+ python myflow.py run --with batch
991
+ ```
992
+ or
993
+ ```
994
+ python myflow.py run --with kubernetes
995
+ ```
996
+ which executes the flow on the desired system using the
997
+ requirements specified in `@resources`.
1045
998
 
1046
999
  Parameters
1047
1000
  ----------
1048
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1049
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1050
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
1051
- - `current.checkpoint`
1052
- - `current.model`
1053
- - `current.huggingface_hub`
1054
-
1055
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1056
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1057
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1058
-
1059
- temp_dir_root : str, default: None
1060
- The root directory under which `current.model.loaded` will store loaded models
1061
-
1062
-
1063
-
1001
+ cpu : int, default 1
1002
+ Number of CPUs required for this step.
1003
+ gpu : int, default 0
1004
+ Number of GPUs required for this step.
1005
+ disk : int, optional, default None
1006
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1007
+ memory : int, default 4096
1008
+ Memory size (in MB) required for this step.
1009
+ shared_memory : int, optional, default None
1010
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1011
+ This parameter maps to the `--shm-size` option in Docker.
1064
1012
  """
1065
1013
  ...
1066
1014
 
1067
1015
  @typing.overload
1068
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1016
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1069
1017
  ...
1070
1018
 
1071
1019
  @typing.overload
1072
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1020
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1073
1021
  ...
1074
1022
 
1075
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1023
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1076
1024
  """
1077
- Enables loading / saving of models within a step.
1078
-
1079
-
1080
- Parameters
1081
- ----------
1082
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1083
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1084
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
1085
- - `current.checkpoint`
1086
- - `current.model`
1087
- - `current.huggingface_hub`
1088
-
1089
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1090
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1091
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1092
-
1093
- temp_dir_root : str, default: None
1094
- The root directory under which `current.model.loaded` will store loaded models
1095
-
1025
+ Specifies the resources needed when executing this step.
1096
1026
 
1027
+ Use `@resources` to specify the resource requirements
1028
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1097
1029
 
1098
- """
1099
- ...
1100
-
1101
- @typing.overload
1102
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1103
- """
1104
- Internal decorator to support Fast bakery
1105
- """
1106
- ...
1107
-
1108
- @typing.overload
1109
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1110
- ...
1111
-
1112
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1113
- """
1114
- Internal decorator to support Fast bakery
1115
- """
1116
- ...
1117
-
1118
- @typing.overload
1119
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1120
- """
1121
- Specifies the times when the flow should be run when running on a
1122
- production scheduler.
1030
+ You can choose the compute layer on the command line by executing e.g.
1031
+ ```
1032
+ python myflow.py run --with batch
1033
+ ```
1034
+ or
1035
+ ```
1036
+ python myflow.py run --with kubernetes
1037
+ ```
1038
+ which executes the flow on the desired system using the
1039
+ requirements specified in `@resources`.
1123
1040
 
1124
1041
  Parameters
1125
1042
  ----------
1126
- hourly : bool, default False
1127
- Run the workflow hourly.
1128
- daily : bool, default True
1129
- Run the workflow daily.
1130
- weekly : bool, default False
1131
- Run the workflow weekly.
1132
- cron : str, optional, default None
1133
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1134
- specified by this expression.
1135
- timezone : str, optional, default None
1136
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1137
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1043
+ cpu : int, default 1
1044
+ Number of CPUs required for this step.
1045
+ gpu : int, default 0
1046
+ Number of GPUs required for this step.
1047
+ disk : int, optional, default None
1048
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1049
+ memory : int, default 4096
1050
+ Memory size (in MB) required for this step.
1051
+ shared_memory : int, optional, default None
1052
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1053
+ This parameter maps to the `--shm-size` option in Docker.
1138
1054
  """
1139
1055
  ...
1140
1056
 
1141
1057
  @typing.overload
1142
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1143
- ...
1144
-
1145
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1058
+ def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1146
1059
  """
1147
- Specifies the times when the flow should be run when running on a
1148
- production scheduler.
1060
+ Enables checkpointing for a step.
1061
+
1149
1062
 
1150
1063
  Parameters
1151
1064
  ----------
1152
- hourly : bool, default False
1153
- Run the workflow hourly.
1154
- daily : bool, default True
1155
- Run the workflow daily.
1156
- weekly : bool, default False
1157
- Run the workflow weekly.
1158
- cron : str, optional, default None
1159
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1160
- specified by this expression.
1161
- timezone : str, optional, default None
1162
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1163
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1164
- """
1165
- ...
1166
-
1167
- @typing.overload
1168
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1169
- """
1170
- Specifies the event(s) that this flow depends on.
1171
-
1172
- ```
1173
- @trigger(event='foo')
1174
- ```
1175
- or
1176
- ```
1177
- @trigger(events=['foo', 'bar'])
1178
- ```
1179
-
1180
- Additionally, you can specify the parameter mappings
1181
- to map event payload to Metaflow parameters for the flow.
1182
- ```
1183
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1184
- ```
1185
- or
1186
- ```
1187
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1188
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1189
- ```
1065
+ load_policy : str, default: "fresh"
1066
+ The policy for loading the checkpoint. The following policies are supported:
1067
+ - "eager": Loads the the latest available checkpoint within the namespace.
1068
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1069
+ will be loaded at the start of the task.
1070
+ - "none": Do not load any checkpoint
1071
+ - "fresh": Loads the lastest checkpoint created within the running Task.
1072
+ This mode helps loading checkpoints across various retry attempts of the same task.
1073
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1074
+ created within the task will be loaded when the task is retries execution on failure.
1190
1075
 
1191
- 'parameters' can also be a list of strings and tuples like so:
1192
- ```
1193
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1194
- ```
1195
- This is equivalent to:
1196
- ```
1197
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1198
- ```
1076
+ temp_dir_root : str, default: None
1077
+ The root directory under which `current.checkpoint.directory` will be created.
1199
1078
 
1200
- Parameters
1201
- ----------
1202
- event : Union[str, Dict[str, Any]], optional, default None
1203
- Event dependency for this flow.
1204
- events : List[Union[str, Dict[str, Any]]], default []
1205
- Events dependency for this flow.
1206
- options : Dict[str, Any], default {}
1207
- Backend-specific configuration for tuning eventing behavior.
1208
1079
 
1209
1080
 
1210
1081
  """
1211
1082
  ...
1212
1083
 
1213
1084
  @typing.overload
1214
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1085
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1215
1086
  ...
1216
1087
 
1217
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1088
+ @typing.overload
1089
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1090
+ ...
1091
+
1092
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
1218
1093
  """
1219
- Specifies the event(s) that this flow depends on.
1220
-
1221
- ```
1222
- @trigger(event='foo')
1223
- ```
1224
- or
1225
- ```
1226
- @trigger(events=['foo', 'bar'])
1227
- ```
1228
-
1229
- Additionally, you can specify the parameter mappings
1230
- to map event payload to Metaflow parameters for the flow.
1231
- ```
1232
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1233
- ```
1234
- or
1235
- ```
1236
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1237
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1238
- ```
1094
+ Enables checkpointing for a step.
1239
1095
 
1240
- 'parameters' can also be a list of strings and tuples like so:
1241
- ```
1242
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1243
- ```
1244
- This is equivalent to:
1245
- ```
1246
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1247
- ```
1248
1096
 
1249
1097
  Parameters
1250
1098
  ----------
1251
- event : Union[str, Dict[str, Any]], optional, default None
1252
- Event dependency for this flow.
1253
- events : List[Union[str, Dict[str, Any]]], default []
1254
- Events dependency for this flow.
1255
- options : Dict[str, Any], default {}
1256
- Backend-specific configuration for tuning eventing behavior.
1099
+ load_policy : str, default: "fresh"
1100
+ The policy for loading the checkpoint. The following policies are supported:
1101
+ - "eager": Loads the the latest available checkpoint within the namespace.
1102
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1103
+ will be loaded at the start of the task.
1104
+ - "none": Do not load any checkpoint
1105
+ - "fresh": Loads the lastest checkpoint created within the running Task.
1106
+ This mode helps loading checkpoints across various retry attempts of the same task.
1107
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1108
+ created within the task will be loaded when the task is retries execution on failure.
1109
+
1110
+ temp_dir_root : str, default: None
1111
+ The root directory under which `current.checkpoint.directory` will be created.
1112
+
1257
1113
 
1258
1114
 
1259
1115
  """
1260
1116
  ...
1261
1117
 
1262
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1118
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1263
1119
  """
1264
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1265
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1266
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1267
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1268
- starts only after all sensors finish.
1120
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1121
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1269
1122
 
1270
1123
  Parameters
1271
1124
  ----------
@@ -1286,57 +1139,21 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1286
1139
  Name of the sensor on Airflow
1287
1140
  description : str
1288
1141
  Description of sensor in the Airflow UI
1289
- bucket_key : Union[str, List[str]]
1290
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1291
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1292
- bucket_name : str
1293
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1294
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1295
- wildcard_match : bool
1296
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1297
- aws_conn_id : str
1298
- a reference to the s3 connection on Airflow. (Default: None)
1299
- verify : bool
1300
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1301
- """
1302
- ...
1303
-
1304
- @typing.overload
1305
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1306
- """
1307
- Specifies the PyPI packages for all steps of the flow.
1308
-
1309
- Use `@pypi_base` to set common packages required by all
1310
- steps and use `@pypi` to specify step-specific overrides.
1311
- Parameters
1312
- ----------
1313
- packages : Dict[str, str], default: {}
1314
- Packages to use for this flow. The key is the name of the package
1315
- and the value is the version to use.
1316
- python : str, optional, default: None
1317
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1318
- that the version used will correspond to the version of the Python interpreter used to start the run.
1319
- """
1320
- ...
1321
-
1322
- @typing.overload
1323
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1324
- ...
1325
-
1326
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1327
- """
1328
- Specifies the PyPI packages for all steps of the flow.
1329
-
1330
- Use `@pypi_base` to set common packages required by all
1331
- steps and use `@pypi` to specify step-specific overrides.
1332
- Parameters
1333
- ----------
1334
- packages : Dict[str, str], default: {}
1335
- Packages to use for this flow. The key is the name of the package
1336
- and the value is the version to use.
1337
- python : str, optional, default: None
1338
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1339
- that the version used will correspond to the version of the Python interpreter used to start the run.
1142
+ external_dag_id : str
1143
+ The dag_id that contains the task you want to wait for.
1144
+ external_task_ids : List[str]
1145
+ The list of task_ids that you want to wait for.
1146
+ If None (default value) the sensor waits for the DAG. (Default: None)
1147
+ allowed_states : List[str]
1148
+ Iterable of allowed states, (Default: ['success'])
1149
+ failed_states : List[str]
1150
+ Iterable of failed or dis-allowed states. (Default: None)
1151
+ execution_delta : datetime.timedelta
1152
+ time difference with the previous execution to look at,
1153
+ the default is the same logical date as the current task or DAG. (Default: None)
1154
+ check_existence: bool
1155
+ Set to True to check if the external task exists or check if
1156
+ the DAG to wait for exists. (Default: True)
1340
1157
  """
1341
1158
  ...
1342
1159
 
@@ -1389,6 +1206,79 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1389
1206
  """
1390
1207
  ...
1391
1208
 
1209
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1210
+ """
1211
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1212
+
1213
+ User code call
1214
+ -----------
1215
+ @nim(
1216
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1217
+ backend='managed'
1218
+ )
1219
+
1220
+ Valid backend options
1221
+ ---------------------
1222
+ - 'managed': Outerbounds selects a compute provider based on the model.
1223
+ - 🚧 'dataplane': Run in your account.
1224
+
1225
+ Valid model options
1226
+ ----------------
1227
+ - 'meta/llama3-8b-instruct': 8B parameter model
1228
+ - 'meta/llama3-70b-instruct': 70B parameter model
1229
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1230
+
1231
+ Parameters
1232
+ ----------
1233
+ models: list[NIM]
1234
+ List of NIM containers running models in sidecars.
1235
+ backend: str
1236
+ Compute provider to run the NIM container.
1237
+ """
1238
+ ...
1239
+
1240
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1241
+ """
1242
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1243
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1244
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1245
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1246
+ starts only after all sensors finish.
1247
+
1248
+ Parameters
1249
+ ----------
1250
+ timeout : int
1251
+ Time, in seconds before the task times out and fails. (Default: 3600)
1252
+ poke_interval : int
1253
+ Time in seconds that the job should wait in between each try. (Default: 60)
1254
+ mode : str
1255
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1256
+ exponential_backoff : bool
1257
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1258
+ pool : str
1259
+ the slot pool this task should run in,
1260
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1261
+ soft_fail : bool
1262
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1263
+ name : str
1264
+ Name of the sensor on Airflow
1265
+ description : str
1266
+ Description of sensor in the Airflow UI
1267
+ bucket_key : Union[str, List[str]]
1268
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1269
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1270
+ bucket_name : str
1271
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1272
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1273
+ wildcard_match : bool
1274
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1275
+ aws_conn_id : str
1276
+ a reference to the s3 connection on Airflow. (Default: None)
1277
+ verify : bool
1278
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1279
+ """
1280
+ ...
1281
+
1392
1282
  @typing.overload
1393
1283
  def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1394
1284
  """
@@ -1492,34 +1382,91 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1492
1382
  """
1493
1383
  ...
1494
1384
 
1495
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1385
+ @typing.overload
1386
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1496
1387
  """
1497
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1388
+ Specifies the times when the flow should be run when running on a
1389
+ production scheduler.
1498
1390
 
1499
- User code call
1500
- -----------
1501
- @nim(
1502
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1503
- backend='managed'
1504
- )
1391
+ Parameters
1392
+ ----------
1393
+ hourly : bool, default False
1394
+ Run the workflow hourly.
1395
+ daily : bool, default True
1396
+ Run the workflow daily.
1397
+ weekly : bool, default False
1398
+ Run the workflow weekly.
1399
+ cron : str, optional, default None
1400
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1401
+ specified by this expression.
1402
+ timezone : str, optional, default None
1403
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1404
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1405
+ """
1406
+ ...
1407
+
1408
+ @typing.overload
1409
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1410
+ ...
1411
+
1412
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1413
+ """
1414
+ Specifies the times when the flow should be run when running on a
1415
+ production scheduler.
1505
1416
 
1506
- Valid backend options
1507
- ---------------------
1508
- - 'managed': Outerbounds selects a compute provider based on the model.
1509
- - 🚧 'dataplane': Run in your account.
1417
+ Parameters
1418
+ ----------
1419
+ hourly : bool, default False
1420
+ Run the workflow hourly.
1421
+ daily : bool, default True
1422
+ Run the workflow daily.
1423
+ weekly : bool, default False
1424
+ Run the workflow weekly.
1425
+ cron : str, optional, default None
1426
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1427
+ specified by this expression.
1428
+ timezone : str, optional, default None
1429
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1430
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1431
+ """
1432
+ ...
1433
+
1434
+ @typing.overload
1435
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1436
+ """
1437
+ Specifies the PyPI packages for all steps of the flow.
1510
1438
 
1511
- Valid model options
1512
- ----------------
1513
- - 'meta/llama3-8b-instruct': 8B parameter model
1514
- - 'meta/llama3-70b-instruct': 70B parameter model
1515
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1439
+ Use `@pypi_base` to set common packages required by all
1440
+ steps and use `@pypi` to specify step-specific overrides.
1441
+ Parameters
1442
+ ----------
1443
+ packages : Dict[str, str], default: {}
1444
+ Packages to use for this flow. The key is the name of the package
1445
+ and the value is the version to use.
1446
+ python : str, optional, default: None
1447
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1448
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1449
+ """
1450
+ ...
1451
+
1452
+ @typing.overload
1453
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1454
+ ...
1455
+
1456
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1457
+ """
1458
+ Specifies the PyPI packages for all steps of the flow.
1516
1459
 
1460
+ Use `@pypi_base` to set common packages required by all
1461
+ steps and use `@pypi` to specify step-specific overrides.
1517
1462
  Parameters
1518
1463
  ----------
1519
- models: list[NIM]
1520
- List of NIM containers running models in sidecars.
1521
- backend: str
1522
- Compute provider to run the NIM container.
1464
+ packages : Dict[str, str], default: {}
1465
+ Packages to use for this flow. The key is the name of the package
1466
+ and the value is the version to use.
1467
+ python : str, optional, default: None
1468
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1469
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1523
1470
  """
1524
1471
  ...
1525
1472
 
@@ -1541,45 +1488,98 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
1541
1488
  """
1542
1489
  ...
1543
1490
 
1544
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1491
+ @typing.overload
1492
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1545
1493
  """
1546
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1547
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1494
+ Specifies the event(s) that this flow depends on.
1495
+
1496
+ ```
1497
+ @trigger(event='foo')
1498
+ ```
1499
+ or
1500
+ ```
1501
+ @trigger(events=['foo', 'bar'])
1502
+ ```
1503
+
1504
+ Additionally, you can specify the parameter mappings
1505
+ to map event payload to Metaflow parameters for the flow.
1506
+ ```
1507
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1508
+ ```
1509
+ or
1510
+ ```
1511
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1512
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1513
+ ```
1514
+
1515
+ 'parameters' can also be a list of strings and tuples like so:
1516
+ ```
1517
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1518
+ ```
1519
+ This is equivalent to:
1520
+ ```
1521
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1522
+ ```
1548
1523
 
1549
1524
  Parameters
1550
1525
  ----------
1551
- timeout : int
1552
- Time, in seconds before the task times out and fails. (Default: 3600)
1553
- poke_interval : int
1554
- Time in seconds that the job should wait in between each try. (Default: 60)
1555
- mode : str
1556
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1557
- exponential_backoff : bool
1558
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1559
- pool : str
1560
- the slot pool this task should run in,
1561
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1562
- soft_fail : bool
1563
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1564
- name : str
1565
- Name of the sensor on Airflow
1566
- description : str
1567
- Description of sensor in the Airflow UI
1568
- external_dag_id : str
1569
- The dag_id that contains the task you want to wait for.
1570
- external_task_ids : List[str]
1571
- The list of task_ids that you want to wait for.
1572
- If None (default value) the sensor waits for the DAG. (Default: None)
1573
- allowed_states : List[str]
1574
- Iterable of allowed states, (Default: ['success'])
1575
- failed_states : List[str]
1576
- Iterable of failed or dis-allowed states. (Default: None)
1577
- execution_delta : datetime.timedelta
1578
- time difference with the previous execution to look at,
1579
- the default is the same logical date as the current task or DAG. (Default: None)
1580
- check_existence: bool
1581
- Set to True to check if the external task exists or check if
1582
- the DAG to wait for exists. (Default: True)
1526
+ event : Union[str, Dict[str, Any]], optional, default None
1527
+ Event dependency for this flow.
1528
+ events : List[Union[str, Dict[str, Any]]], default []
1529
+ Events dependency for this flow.
1530
+ options : Dict[str, Any], default {}
1531
+ Backend-specific configuration for tuning eventing behavior.
1532
+
1533
+
1534
+ """
1535
+ ...
1536
+
1537
+ @typing.overload
1538
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1539
+ ...
1540
+
1541
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1542
+ """
1543
+ Specifies the event(s) that this flow depends on.
1544
+
1545
+ ```
1546
+ @trigger(event='foo')
1547
+ ```
1548
+ or
1549
+ ```
1550
+ @trigger(events=['foo', 'bar'])
1551
+ ```
1552
+
1553
+ Additionally, you can specify the parameter mappings
1554
+ to map event payload to Metaflow parameters for the flow.
1555
+ ```
1556
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1557
+ ```
1558
+ or
1559
+ ```
1560
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1561
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1562
+ ```
1563
+
1564
+ 'parameters' can also be a list of strings and tuples like so:
1565
+ ```
1566
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1567
+ ```
1568
+ This is equivalent to:
1569
+ ```
1570
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1571
+ ```
1572
+
1573
+ Parameters
1574
+ ----------
1575
+ event : Union[str, Dict[str, Any]], optional, default None
1576
+ Event dependency for this flow.
1577
+ events : List[Union[str, Dict[str, Any]]], default []
1578
+ Events dependency for this flow.
1579
+ options : Dict[str, Any], default {}
1580
+ Backend-specific configuration for tuning eventing behavior.
1581
+
1582
+
1583
1583
  """
1584
1584
  ...
1585
1585