ob-metaflow-stubs 5.7.1__py2.py3-none-any.whl → 5.8.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +382 -382
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +3 -3
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +5 -5
- metaflow-stubs/metadata/metadata.pyi +2 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +17 -17
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +6 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +4 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +5 -5
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +5 -5
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-5.7.1.dist-info → ob_metaflow_stubs-5.8.1.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-5.8.1.dist-info/RECORD +138 -0
- ob_metaflow_stubs-5.7.1.dist-info/RECORD +0 -138
- {ob_metaflow_stubs-5.7.1.dist-info → ob_metaflow_stubs-5.8.1.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-5.7.1.dist-info → ob_metaflow_stubs-5.8.1.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,23 +1,23 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.19.1+ob(v1) #
|
4
|
+
# Generated on 2024-09-06T08:14:24.914849 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow._vendor.click.types
|
12
|
-
import metaflow.datastore.inputs
|
13
|
-
import metaflow.metaflow_current
|
14
|
-
import metaflow.events
|
15
|
-
import datetime
|
16
11
|
import metaflow.parameters
|
17
|
-
import metaflow.flowspec
|
18
|
-
import metaflow.client.core
|
19
12
|
import typing
|
13
|
+
import datetime
|
20
14
|
import metaflow.runner.metaflow_runner
|
15
|
+
import metaflow.datastore.inputs
|
16
|
+
import metaflow.flowspec
|
17
|
+
import metaflow.events
|
18
|
+
import metaflow.client.core
|
19
|
+
import metaflow._vendor.click.types
|
20
|
+
import metaflow.metaflow_current
|
21
21
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
22
22
|
StepFlag = typing.NewType("StepFlag", bool)
|
23
23
|
|
@@ -499,6 +499,55 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
499
499
|
"""
|
500
500
|
...
|
501
501
|
|
502
|
+
@typing.overload
|
503
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
504
|
+
"""
|
505
|
+
Specifies the PyPI packages for the step.
|
506
|
+
|
507
|
+
Information in this decorator will augment any
|
508
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
509
|
+
you can use `@pypi_base` to set packages required by all
|
510
|
+
steps and use `@pypi` to specify step-specific overrides.
|
511
|
+
|
512
|
+
Parameters
|
513
|
+
----------
|
514
|
+
packages : Dict[str, str], default: {}
|
515
|
+
Packages to use for this step. The key is the name of the package
|
516
|
+
and the value is the version to use.
|
517
|
+
python : str, optional, default: None
|
518
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
519
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
520
|
+
"""
|
521
|
+
...
|
522
|
+
|
523
|
+
@typing.overload
|
524
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
525
|
+
...
|
526
|
+
|
527
|
+
@typing.overload
|
528
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
529
|
+
...
|
530
|
+
|
531
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
532
|
+
"""
|
533
|
+
Specifies the PyPI packages for the step.
|
534
|
+
|
535
|
+
Information in this decorator will augment any
|
536
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
537
|
+
you can use `@pypi_base` to set packages required by all
|
538
|
+
steps and use `@pypi` to specify step-specific overrides.
|
539
|
+
|
540
|
+
Parameters
|
541
|
+
----------
|
542
|
+
packages : Dict[str, str], default: {}
|
543
|
+
Packages to use for this step. The key is the name of the package
|
544
|
+
and the value is the version to use.
|
545
|
+
python : str, optional, default: None
|
546
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
547
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
548
|
+
"""
|
549
|
+
...
|
550
|
+
|
502
551
|
@typing.overload
|
503
552
|
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
504
553
|
"""
|
@@ -518,6 +567,174 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
518
567
|
"""
|
519
568
|
...
|
520
569
|
|
570
|
+
@typing.overload
|
571
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
572
|
+
"""
|
573
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
574
|
+
|
575
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
576
|
+
|
577
|
+
Parameters
|
578
|
+
----------
|
579
|
+
type : str, default 'default'
|
580
|
+
Card type.
|
581
|
+
id : str, optional, default None
|
582
|
+
If multiple cards are present, use this id to identify this card.
|
583
|
+
options : Dict[str, Any], default {}
|
584
|
+
Options passed to the card. The contents depend on the card type.
|
585
|
+
timeout : int, default 45
|
586
|
+
Interrupt reporting if it takes more than this many seconds.
|
587
|
+
|
588
|
+
|
589
|
+
"""
|
590
|
+
...
|
591
|
+
|
592
|
+
@typing.overload
|
593
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
594
|
+
...
|
595
|
+
|
596
|
+
@typing.overload
|
597
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
598
|
+
...
|
599
|
+
|
600
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
601
|
+
"""
|
602
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
603
|
+
|
604
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
605
|
+
|
606
|
+
Parameters
|
607
|
+
----------
|
608
|
+
type : str, default 'default'
|
609
|
+
Card type.
|
610
|
+
id : str, optional, default None
|
611
|
+
If multiple cards are present, use this id to identify this card.
|
612
|
+
options : Dict[str, Any], default {}
|
613
|
+
Options passed to the card. The contents depend on the card type.
|
614
|
+
timeout : int, default 45
|
615
|
+
Interrupt reporting if it takes more than this many seconds.
|
616
|
+
|
617
|
+
|
618
|
+
"""
|
619
|
+
...
|
620
|
+
|
621
|
+
@typing.overload
|
622
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
623
|
+
"""
|
624
|
+
Specifies the number of times the task corresponding
|
625
|
+
to a step needs to be retried.
|
626
|
+
|
627
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
628
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
629
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
630
|
+
|
631
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
632
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
633
|
+
ensuring that the flow execution can continue.
|
634
|
+
|
635
|
+
Parameters
|
636
|
+
----------
|
637
|
+
times : int, default 3
|
638
|
+
Number of times to retry this task.
|
639
|
+
minutes_between_retries : int, default 2
|
640
|
+
Number of minutes between retries.
|
641
|
+
"""
|
642
|
+
...
|
643
|
+
|
644
|
+
@typing.overload
|
645
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
646
|
+
...
|
647
|
+
|
648
|
+
@typing.overload
|
649
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
650
|
+
...
|
651
|
+
|
652
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
653
|
+
"""
|
654
|
+
Specifies the number of times the task corresponding
|
655
|
+
to a step needs to be retried.
|
656
|
+
|
657
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
658
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
659
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
660
|
+
|
661
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
662
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
663
|
+
ensuring that the flow execution can continue.
|
664
|
+
|
665
|
+
Parameters
|
666
|
+
----------
|
667
|
+
times : int, default 3
|
668
|
+
Number of times to retry this task.
|
669
|
+
minutes_between_retries : int, default 2
|
670
|
+
Number of minutes between retries.
|
671
|
+
"""
|
672
|
+
...
|
673
|
+
|
674
|
+
@typing.overload
|
675
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
676
|
+
"""
|
677
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
678
|
+
the execution of a step.
|
679
|
+
|
680
|
+
Parameters
|
681
|
+
----------
|
682
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
683
|
+
List of secret specs, defining how the secrets are to be retrieved
|
684
|
+
"""
|
685
|
+
...
|
686
|
+
|
687
|
+
@typing.overload
|
688
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
689
|
+
...
|
690
|
+
|
691
|
+
@typing.overload
|
692
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
693
|
+
...
|
694
|
+
|
695
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
696
|
+
"""
|
697
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
698
|
+
the execution of a step.
|
699
|
+
|
700
|
+
Parameters
|
701
|
+
----------
|
702
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
703
|
+
List of secret specs, defining how the secrets are to be retrieved
|
704
|
+
"""
|
705
|
+
...
|
706
|
+
|
707
|
+
@typing.overload
|
708
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
709
|
+
"""
|
710
|
+
Specifies environment variables to be set prior to the execution of a step.
|
711
|
+
|
712
|
+
Parameters
|
713
|
+
----------
|
714
|
+
vars : Dict[str, str], default {}
|
715
|
+
Dictionary of environment variables to set.
|
716
|
+
"""
|
717
|
+
...
|
718
|
+
|
719
|
+
@typing.overload
|
720
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
721
|
+
...
|
722
|
+
|
723
|
+
@typing.overload
|
724
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
725
|
+
...
|
726
|
+
|
727
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
728
|
+
"""
|
729
|
+
Specifies environment variables to be set prior to the execution of a step.
|
730
|
+
|
731
|
+
Parameters
|
732
|
+
----------
|
733
|
+
vars : Dict[str, str], default {}
|
734
|
+
Dictionary of environment variables to set.
|
735
|
+
"""
|
736
|
+
...
|
737
|
+
|
521
738
|
@typing.overload
|
522
739
|
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
523
740
|
"""
|
@@ -575,23 +792,6 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
575
792
|
"""
|
576
793
|
...
|
577
794
|
|
578
|
-
@typing.overload
|
579
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
580
|
-
"""
|
581
|
-
Internal decorator to support Fast bakery
|
582
|
-
"""
|
583
|
-
...
|
584
|
-
|
585
|
-
@typing.overload
|
586
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
587
|
-
...
|
588
|
-
|
589
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
590
|
-
"""
|
591
|
-
Internal decorator to support Fast bakery
|
592
|
-
"""
|
593
|
-
...
|
594
|
-
|
595
795
|
@typing.overload
|
596
796
|
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
597
797
|
"""
|
@@ -670,119 +870,9 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
670
870
|
...
|
671
871
|
|
672
872
|
@typing.overload
|
673
|
-
def
|
873
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
674
874
|
"""
|
675
|
-
Specifies the
|
676
|
-
|
677
|
-
Information in this decorator will augment any
|
678
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
679
|
-
you can use `@conda_base` to set packages required by all
|
680
|
-
steps and use `@conda` to specify step-specific overrides.
|
681
|
-
|
682
|
-
Parameters
|
683
|
-
----------
|
684
|
-
packages : Dict[str, str], default {}
|
685
|
-
Packages to use for this step. The key is the name of the package
|
686
|
-
and the value is the version to use.
|
687
|
-
libraries : Dict[str, str], default {}
|
688
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
689
|
-
python : str, optional, default None
|
690
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
691
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
692
|
-
disabled : bool, default False
|
693
|
-
If set to True, disables @conda.
|
694
|
-
"""
|
695
|
-
...
|
696
|
-
|
697
|
-
@typing.overload
|
698
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
699
|
-
...
|
700
|
-
|
701
|
-
@typing.overload
|
702
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
703
|
-
...
|
704
|
-
|
705
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
706
|
-
"""
|
707
|
-
Specifies the Conda environment for the step.
|
708
|
-
|
709
|
-
Information in this decorator will augment any
|
710
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
711
|
-
you can use `@conda_base` to set packages required by all
|
712
|
-
steps and use `@conda` to specify step-specific overrides.
|
713
|
-
|
714
|
-
Parameters
|
715
|
-
----------
|
716
|
-
packages : Dict[str, str], default {}
|
717
|
-
Packages to use for this step. The key is the name of the package
|
718
|
-
and the value is the version to use.
|
719
|
-
libraries : Dict[str, str], default {}
|
720
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
721
|
-
python : str, optional, default None
|
722
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
723
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
724
|
-
disabled : bool, default False
|
725
|
-
If set to True, disables @conda.
|
726
|
-
"""
|
727
|
-
...
|
728
|
-
|
729
|
-
@typing.overload
|
730
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
731
|
-
"""
|
732
|
-
Specifies the number of times the task corresponding
|
733
|
-
to a step needs to be retried.
|
734
|
-
|
735
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
736
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
737
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
738
|
-
|
739
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
740
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
741
|
-
ensuring that the flow execution can continue.
|
742
|
-
|
743
|
-
Parameters
|
744
|
-
----------
|
745
|
-
times : int, default 3
|
746
|
-
Number of times to retry this task.
|
747
|
-
minutes_between_retries : int, default 2
|
748
|
-
Number of minutes between retries.
|
749
|
-
"""
|
750
|
-
...
|
751
|
-
|
752
|
-
@typing.overload
|
753
|
-
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
754
|
-
...
|
755
|
-
|
756
|
-
@typing.overload
|
757
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
758
|
-
...
|
759
|
-
|
760
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
761
|
-
"""
|
762
|
-
Specifies the number of times the task corresponding
|
763
|
-
to a step needs to be retried.
|
764
|
-
|
765
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
766
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
767
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
768
|
-
|
769
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
770
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
771
|
-
ensuring that the flow execution can continue.
|
772
|
-
|
773
|
-
Parameters
|
774
|
-
----------
|
775
|
-
times : int, default 3
|
776
|
-
Number of times to retry this task.
|
777
|
-
minutes_between_retries : int, default 2
|
778
|
-
Number of minutes between retries.
|
779
|
-
"""
|
780
|
-
...
|
781
|
-
|
782
|
-
@typing.overload
|
783
|
-
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
784
|
-
"""
|
785
|
-
Specifies that the step will success under all circumstances.
|
875
|
+
Specifies that the step will success under all circumstances.
|
786
876
|
|
787
877
|
The decorator will create an optional artifact, specified by `var`, which
|
788
878
|
contains the exception raised. You can use it to detect the presence
|
@@ -829,166 +919,125 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
829
919
|
...
|
830
920
|
|
831
921
|
@typing.overload
|
832
|
-
def
|
833
|
-
"""
|
834
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
835
|
-
the execution of a step.
|
836
|
-
|
837
|
-
Parameters
|
838
|
-
----------
|
839
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
840
|
-
List of secret specs, defining how the secrets are to be retrieved
|
841
|
-
"""
|
842
|
-
...
|
843
|
-
|
844
|
-
@typing.overload
|
845
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
846
|
-
...
|
847
|
-
|
848
|
-
@typing.overload
|
849
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
850
|
-
...
|
851
|
-
|
852
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
853
|
-
"""
|
854
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
855
|
-
the execution of a step.
|
856
|
-
|
857
|
-
Parameters
|
858
|
-
----------
|
859
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
860
|
-
List of secret specs, defining how the secrets are to be retrieved
|
861
|
-
"""
|
862
|
-
...
|
863
|
-
|
864
|
-
@typing.overload
|
865
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
922
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
866
923
|
"""
|
867
|
-
|
868
|
-
|
869
|
-
Parameters
|
870
|
-
----------
|
871
|
-
vars : Dict[str, str], default {}
|
872
|
-
Dictionary of environment variables to set.
|
924
|
+
Internal decorator to support Fast bakery
|
873
925
|
"""
|
874
926
|
...
|
875
927
|
|
876
928
|
@typing.overload
|
877
|
-
def
|
878
|
-
...
|
879
|
-
|
880
|
-
@typing.overload
|
881
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
929
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
882
930
|
...
|
883
931
|
|
884
|
-
def
|
932
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
885
933
|
"""
|
886
|
-
|
887
|
-
|
888
|
-
Parameters
|
889
|
-
----------
|
890
|
-
vars : Dict[str, str], default {}
|
891
|
-
Dictionary of environment variables to set.
|
934
|
+
Internal decorator to support Fast bakery
|
892
935
|
"""
|
893
936
|
...
|
894
937
|
|
895
938
|
@typing.overload
|
896
|
-
def
|
939
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
897
940
|
"""
|
898
|
-
|
941
|
+
Specifies the Conda environment for the step.
|
899
942
|
|
900
|
-
|
943
|
+
Information in this decorator will augment any
|
944
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
945
|
+
you can use `@conda_base` to set packages required by all
|
946
|
+
steps and use `@conda` to specify step-specific overrides.
|
901
947
|
|
902
948
|
Parameters
|
903
949
|
----------
|
904
|
-
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
|
950
|
+
packages : Dict[str, str], default {}
|
951
|
+
Packages to use for this step. The key is the name of the package
|
952
|
+
and the value is the version to use.
|
953
|
+
libraries : Dict[str, str], default {}
|
954
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
955
|
+
python : str, optional, default None
|
956
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
957
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
958
|
+
disabled : bool, default False
|
959
|
+
If set to True, disables @conda.
|
914
960
|
"""
|
915
961
|
...
|
916
962
|
|
917
963
|
@typing.overload
|
918
|
-
def
|
964
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
919
965
|
...
|
920
966
|
|
921
967
|
@typing.overload
|
922
|
-
def
|
923
|
-
...
|
924
|
-
|
925
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
926
|
-
"""
|
927
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
928
|
-
|
929
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
930
|
-
|
931
|
-
Parameters
|
932
|
-
----------
|
933
|
-
type : str, default 'default'
|
934
|
-
Card type.
|
935
|
-
id : str, optional, default None
|
936
|
-
If multiple cards are present, use this id to identify this card.
|
937
|
-
options : Dict[str, Any], default {}
|
938
|
-
Options passed to the card. The contents depend on the card type.
|
939
|
-
timeout : int, default 45
|
940
|
-
Interrupt reporting if it takes more than this many seconds.
|
941
|
-
|
942
|
-
|
943
|
-
"""
|
968
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
944
969
|
...
|
945
970
|
|
946
|
-
|
947
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
971
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
948
972
|
"""
|
949
|
-
Specifies the
|
973
|
+
Specifies the Conda environment for the step.
|
950
974
|
|
951
975
|
Information in this decorator will augment any
|
952
|
-
attributes set in the `@
|
953
|
-
you can use `@
|
954
|
-
steps and use `@
|
976
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
977
|
+
you can use `@conda_base` to set packages required by all
|
978
|
+
steps and use `@conda` to specify step-specific overrides.
|
955
979
|
|
956
980
|
Parameters
|
957
981
|
----------
|
958
|
-
packages : Dict[str, str], default
|
982
|
+
packages : Dict[str, str], default {}
|
959
983
|
Packages to use for this step. The key is the name of the package
|
960
984
|
and the value is the version to use.
|
961
|
-
|
985
|
+
libraries : Dict[str, str], default {}
|
986
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
987
|
+
python : str, optional, default None
|
962
988
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
963
989
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
990
|
+
disabled : bool, default False
|
991
|
+
If set to True, disables @conda.
|
964
992
|
"""
|
965
993
|
...
|
966
994
|
|
967
995
|
@typing.overload
|
968
|
-
def
|
996
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
997
|
+
"""
|
998
|
+
Specifies the times when the flow should be run when running on a
|
999
|
+
production scheduler.
|
1000
|
+
|
1001
|
+
Parameters
|
1002
|
+
----------
|
1003
|
+
hourly : bool, default False
|
1004
|
+
Run the workflow hourly.
|
1005
|
+
daily : bool, default True
|
1006
|
+
Run the workflow daily.
|
1007
|
+
weekly : bool, default False
|
1008
|
+
Run the workflow weekly.
|
1009
|
+
cron : str, optional, default None
|
1010
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1011
|
+
specified by this expression.
|
1012
|
+
timezone : str, optional, default None
|
1013
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1014
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1015
|
+
"""
|
969
1016
|
...
|
970
1017
|
|
971
1018
|
@typing.overload
|
972
|
-
def
|
1019
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
973
1020
|
...
|
974
1021
|
|
975
|
-
def
|
1022
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
976
1023
|
"""
|
977
|
-
Specifies the
|
978
|
-
|
979
|
-
Information in this decorator will augment any
|
980
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
981
|
-
you can use `@pypi_base` to set packages required by all
|
982
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1024
|
+
Specifies the times when the flow should be run when running on a
|
1025
|
+
production scheduler.
|
983
1026
|
|
984
1027
|
Parameters
|
985
1028
|
----------
|
986
|
-
|
987
|
-
|
988
|
-
|
989
|
-
|
990
|
-
|
991
|
-
|
1029
|
+
hourly : bool, default False
|
1030
|
+
Run the workflow hourly.
|
1031
|
+
daily : bool, default True
|
1032
|
+
Run the workflow daily.
|
1033
|
+
weekly : bool, default False
|
1034
|
+
Run the workflow weekly.
|
1035
|
+
cron : str, optional, default None
|
1036
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1037
|
+
specified by this expression.
|
1038
|
+
timezone : str, optional, default None
|
1039
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1040
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
992
1041
|
"""
|
993
1042
|
...
|
994
1043
|
|
@@ -1041,6 +1090,24 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
1041
1090
|
"""
|
1042
1091
|
...
|
1043
1092
|
|
1093
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1094
|
+
"""
|
1095
|
+
Specifies what flows belong to the same project.
|
1096
|
+
|
1097
|
+
A project-specific namespace is created for all flows that
|
1098
|
+
use the same `@project(name)`.
|
1099
|
+
|
1100
|
+
Parameters
|
1101
|
+
----------
|
1102
|
+
name : str
|
1103
|
+
Project name. Make sure that the name is unique amongst all
|
1104
|
+
projects that use the same production scheduler. The name may
|
1105
|
+
contain only lowercase alphanumeric characters and underscores.
|
1106
|
+
|
1107
|
+
|
1108
|
+
"""
|
1109
|
+
...
|
1110
|
+
|
1044
1111
|
@typing.overload
|
1045
1112
|
def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1046
1113
|
"""
|
@@ -1144,34 +1211,45 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1144
1211
|
"""
|
1145
1212
|
...
|
1146
1213
|
|
1147
|
-
def
|
1214
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1148
1215
|
"""
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1154
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1155
|
-
backend='managed'
|
1156
|
-
)
|
1157
|
-
|
1158
|
-
Valid backend options
|
1159
|
-
---------------------
|
1160
|
-
- 'managed': Outerbounds selects a compute provider based on the model.
|
1161
|
-
- 🚧 'dataplane': Run in your account.
|
1162
|
-
|
1163
|
-
Valid model options
|
1164
|
-
----------------
|
1165
|
-
- 'meta/llama3-8b-instruct': 8B parameter model
|
1166
|
-
- 'meta/llama3-70b-instruct': 70B parameter model
|
1167
|
-
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1216
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1217
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1218
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1219
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1220
|
+
starts only after all sensors finish.
|
1168
1221
|
|
1169
1222
|
Parameters
|
1170
1223
|
----------
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1224
|
+
timeout : int
|
1225
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1226
|
+
poke_interval : int
|
1227
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1228
|
+
mode : str
|
1229
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1230
|
+
exponential_backoff : bool
|
1231
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1232
|
+
pool : str
|
1233
|
+
the slot pool this task should run in,
|
1234
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1235
|
+
soft_fail : bool
|
1236
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1237
|
+
name : str
|
1238
|
+
Name of the sensor on Airflow
|
1239
|
+
description : str
|
1240
|
+
Description of sensor in the Airflow UI
|
1241
|
+
bucket_key : Union[str, List[str]]
|
1242
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1243
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1244
|
+
bucket_name : str
|
1245
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1246
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1247
|
+
wildcard_match : bool
|
1248
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1249
|
+
aws_conn_id : str
|
1250
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1251
|
+
verify : bool
|
1252
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1175
1253
|
"""
|
1176
1254
|
...
|
1177
1255
|
|
@@ -1214,6 +1292,37 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
|
|
1214
1292
|
"""
|
1215
1293
|
...
|
1216
1294
|
|
1295
|
+
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1296
|
+
"""
|
1297
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1298
|
+
|
1299
|
+
User code call
|
1300
|
+
-----------
|
1301
|
+
@nim(
|
1302
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1303
|
+
backend='managed'
|
1304
|
+
)
|
1305
|
+
|
1306
|
+
Valid backend options
|
1307
|
+
---------------------
|
1308
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
1309
|
+
- 🚧 'dataplane': Run in your account.
|
1310
|
+
|
1311
|
+
Valid model options
|
1312
|
+
----------------
|
1313
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
1314
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
1315
|
+
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1316
|
+
|
1317
|
+
Parameters
|
1318
|
+
----------
|
1319
|
+
models: list[NIM]
|
1320
|
+
List of NIM containers running models in sidecars.
|
1321
|
+
backend: str
|
1322
|
+
Compute provider to run the NIM container.
|
1323
|
+
"""
|
1324
|
+
...
|
1325
|
+
|
1217
1326
|
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1218
1327
|
"""
|
1219
1328
|
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
@@ -1256,24 +1365,6 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1256
1365
|
"""
|
1257
1366
|
...
|
1258
1367
|
|
1259
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1260
|
-
"""
|
1261
|
-
Specifies what flows belong to the same project.
|
1262
|
-
|
1263
|
-
A project-specific namespace is created for all flows that
|
1264
|
-
use the same `@project(name)`.
|
1265
|
-
|
1266
|
-
Parameters
|
1267
|
-
----------
|
1268
|
-
name : str
|
1269
|
-
Project name. Make sure that the name is unique amongst all
|
1270
|
-
projects that use the same production scheduler. The name may
|
1271
|
-
contain only lowercase alphanumeric characters and underscores.
|
1272
|
-
|
1273
|
-
|
1274
|
-
"""
|
1275
|
-
...
|
1276
|
-
|
1277
1368
|
@typing.overload
|
1278
1369
|
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1279
1370
|
"""
|
@@ -1369,97 +1460,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1369
1460
|
"""
|
1370
1461
|
...
|
1371
1462
|
|
1372
|
-
@typing.overload
|
1373
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1374
|
-
"""
|
1375
|
-
Specifies the times when the flow should be run when running on a
|
1376
|
-
production scheduler.
|
1377
|
-
|
1378
|
-
Parameters
|
1379
|
-
----------
|
1380
|
-
hourly : bool, default False
|
1381
|
-
Run the workflow hourly.
|
1382
|
-
daily : bool, default True
|
1383
|
-
Run the workflow daily.
|
1384
|
-
weekly : bool, default False
|
1385
|
-
Run the workflow weekly.
|
1386
|
-
cron : str, optional, default None
|
1387
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1388
|
-
specified by this expression.
|
1389
|
-
timezone : str, optional, default None
|
1390
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1391
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1392
|
-
"""
|
1393
|
-
...
|
1394
|
-
|
1395
|
-
@typing.overload
|
1396
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1397
|
-
...
|
1398
|
-
|
1399
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1400
|
-
"""
|
1401
|
-
Specifies the times when the flow should be run when running on a
|
1402
|
-
production scheduler.
|
1403
|
-
|
1404
|
-
Parameters
|
1405
|
-
----------
|
1406
|
-
hourly : bool, default False
|
1407
|
-
Run the workflow hourly.
|
1408
|
-
daily : bool, default True
|
1409
|
-
Run the workflow daily.
|
1410
|
-
weekly : bool, default False
|
1411
|
-
Run the workflow weekly.
|
1412
|
-
cron : str, optional, default None
|
1413
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1414
|
-
specified by this expression.
|
1415
|
-
timezone : str, optional, default None
|
1416
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1417
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1418
|
-
"""
|
1419
|
-
...
|
1420
|
-
|
1421
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1422
|
-
"""
|
1423
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1424
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1425
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1426
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1427
|
-
starts only after all sensors finish.
|
1428
|
-
|
1429
|
-
Parameters
|
1430
|
-
----------
|
1431
|
-
timeout : int
|
1432
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1433
|
-
poke_interval : int
|
1434
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1435
|
-
mode : str
|
1436
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1437
|
-
exponential_backoff : bool
|
1438
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1439
|
-
pool : str
|
1440
|
-
the slot pool this task should run in,
|
1441
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1442
|
-
soft_fail : bool
|
1443
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1444
|
-
name : str
|
1445
|
-
Name of the sensor on Airflow
|
1446
|
-
description : str
|
1447
|
-
Description of sensor in the Airflow UI
|
1448
|
-
bucket_key : Union[str, List[str]]
|
1449
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1450
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1451
|
-
bucket_name : str
|
1452
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1453
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1454
|
-
wildcard_match : bool
|
1455
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1456
|
-
aws_conn_id : str
|
1457
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1458
|
-
verify : bool
|
1459
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1460
|
-
"""
|
1461
|
-
...
|
1462
|
-
|
1463
1463
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
1464
1464
|
"""
|
1465
1465
|
Switch namespace to the one provided.
|