ob-metaflow-stubs 6.0.3.102rc0__py2.py3-none-any.whl → 6.0.3.102rc1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +601 -601
- metaflow-stubs/cards.pyi +4 -4
- metaflow-stubs/cli.pyi +1 -1
- metaflow-stubs/client/__init__.pyi +3 -3
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +1 -1
- metaflow-stubs/clone_util.pyi +1 -1
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +1 -1
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/info_file.pyi +1 -1
- metaflow-stubs/metadata/metadata.pyi +1 -1
- metaflow-stubs/metadata/util.pyi +1 -1
- metaflow-stubs/metaflow_config.pyi +1 -1
- metaflow-stubs/metaflow_current.pyi +67 -67
- metaflow-stubs/mflog/mflog.pyi +1 -1
- metaflow-stubs/multicore_utils.pyi +1 -1
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
- metaflow-stubs/plugins/airflow/exception.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +5 -5
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
- metaflow-stubs/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +1 -1
- metaflow-stubs/plugins/azure/__init__.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
- metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/cards/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_cli.pyi +3 -3
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
- metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +1 -1
- metaflow-stubs/plugins/catch_decorator.pyi +1 -1
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
- metaflow-stubs/plugins/debug_logger.pyi +1 -1
- metaflow-stubs/plugins/debug_monitor.pyi +1 -1
- metaflow-stubs/plugins/environment_decorator.pyi +1 -1
- metaflow-stubs/plugins/events_decorator.pyi +1 -1
- metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
- metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
- metaflow-stubs/plugins/logs_cli.pyi +2 -2
- metaflow-stubs/plugins/package_cli.pyi +1 -1
- metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
- metaflow-stubs/plugins/perimeters.pyi +1 -1
- metaflow-stubs/plugins/project_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +1 -1
- metaflow-stubs/plugins/resources_decorator.pyi +1 -1
- metaflow-stubs/plugins/retry_decorator.pyi +1 -1
- metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
- metaflow-stubs/plugins/storage_executor.pyi +1 -1
- metaflow-stubs/plugins/tag_cli.pyi +3 -3
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
- metaflow-stubs/procpoll.pyi +1 -1
- metaflow-stubs/profilers/__init__.pyi +1 -1
- metaflow-stubs/pylint_wrapper.pyi +1 -1
- metaflow-stubs/runner/__init__.pyi +1 -1
- metaflow-stubs/runner/deployer.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +1 -1
- metaflow-stubs/runner/nbrun.pyi +1 -1
- metaflow-stubs/runner/subprocess_manager.pyi +1 -1
- metaflow-stubs/runner/utils.pyi +1 -1
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +1 -1
- metaflow-stubs/tuple_util.pyi +1 -1
- {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.102rc1.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.102rc1.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.102rc0.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.102rc1.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.102rc0.dist-info → ob_metaflow_stubs-6.0.3.102rc1.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,24 +1,24 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
3
|
# MF version: 2.12.22.1+obcheckpoint(0.0.10);ob(v1) #
|
4
|
-
# Generated on 2024-09-20T18:
|
4
|
+
# Generated on 2024-09-20T18:51:04.453225 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.
|
12
|
-
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
|
13
|
-
import metaflow.datastore.inputs
|
11
|
+
import metaflow.client.core
|
14
12
|
import metaflow.metaflow_current
|
13
|
+
import metaflow.parameters
|
15
14
|
import metaflow._vendor.click.types
|
15
|
+
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
|
16
16
|
import metaflow.runner.metaflow_runner
|
17
|
+
import metaflow.events
|
17
18
|
import typing
|
18
|
-
import metaflow.flowspec
|
19
|
-
import metaflow.client.core
|
20
|
-
import metaflow.parameters
|
21
19
|
import datetime
|
20
|
+
import metaflow.datastore.inputs
|
21
|
+
import metaflow.flowspec
|
22
22
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
23
23
|
StepFlag = typing.NewType("StepFlag", bool)
|
24
24
|
|
@@ -435,194 +435,70 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
435
435
|
...
|
436
436
|
|
437
437
|
@typing.overload
|
438
|
-
def
|
439
|
-
"""
|
440
|
-
Specifies that the step will success under all circumstances.
|
441
|
-
|
442
|
-
The decorator will create an optional artifact, specified by `var`, which
|
443
|
-
contains the exception raised. You can use it to detect the presence
|
444
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
445
|
-
are missing.
|
446
|
-
|
447
|
-
Parameters
|
448
|
-
----------
|
449
|
-
var : str, optional, default None
|
450
|
-
Name of the artifact in which to store the caught exception.
|
451
|
-
If not specified, the exception is not stored.
|
452
|
-
print_exception : bool, default True
|
453
|
-
Determines whether or not the exception is printed to
|
454
|
-
stdout when caught.
|
455
|
-
"""
|
456
|
-
...
|
457
|
-
|
458
|
-
@typing.overload
|
459
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
460
|
-
...
|
461
|
-
|
462
|
-
@typing.overload
|
463
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
464
|
-
...
|
465
|
-
|
466
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
467
|
-
"""
|
468
|
-
Specifies that the step will success under all circumstances.
|
469
|
-
|
470
|
-
The decorator will create an optional artifact, specified by `var`, which
|
471
|
-
contains the exception raised. You can use it to detect the presence
|
472
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
473
|
-
are missing.
|
474
|
-
|
475
|
-
Parameters
|
476
|
-
----------
|
477
|
-
var : str, optional, default None
|
478
|
-
Name of the artifact in which to store the caught exception.
|
479
|
-
If not specified, the exception is not stored.
|
480
|
-
print_exception : bool, default True
|
481
|
-
Determines whether or not the exception is printed to
|
482
|
-
stdout when caught.
|
483
|
-
"""
|
484
|
-
...
|
485
|
-
|
486
|
-
@typing.overload
|
487
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
438
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
488
439
|
"""
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
493
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
494
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
495
|
-
|
496
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
497
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
498
|
-
ensuring that the flow execution can continue.
|
499
|
-
|
500
|
-
Parameters
|
501
|
-
----------
|
502
|
-
times : int, default 3
|
503
|
-
Number of times to retry this task.
|
504
|
-
minutes_between_retries : int, default 2
|
505
|
-
Number of minutes between retries.
|
440
|
+
Decorator prototype for all step decorators. This function gets specialized
|
441
|
+
and imported for all decorators types by _import_plugin_decorators().
|
506
442
|
"""
|
507
443
|
...
|
508
444
|
|
509
445
|
@typing.overload
|
510
|
-
def
|
511
|
-
...
|
512
|
-
|
513
|
-
@typing.overload
|
514
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
446
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
515
447
|
...
|
516
448
|
|
517
|
-
def
|
449
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
518
450
|
"""
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
523
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
524
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
525
|
-
|
526
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
527
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
528
|
-
ensuring that the flow execution can continue.
|
529
|
-
|
530
|
-
Parameters
|
531
|
-
----------
|
532
|
-
times : int, default 3
|
533
|
-
Number of times to retry this task.
|
534
|
-
minutes_between_retries : int, default 2
|
535
|
-
Number of minutes between retries.
|
451
|
+
Decorator prototype for all step decorators. This function gets specialized
|
452
|
+
and imported for all decorators types by _import_plugin_decorators().
|
536
453
|
"""
|
537
454
|
...
|
538
455
|
|
539
456
|
@typing.overload
|
540
|
-
def
|
457
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
541
458
|
"""
|
542
|
-
Specifies the
|
459
|
+
Specifies the PyPI packages for the step.
|
543
460
|
|
544
461
|
Information in this decorator will augment any
|
545
|
-
attributes set in the `@
|
546
|
-
you can use `@
|
547
|
-
steps and use `@
|
462
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
463
|
+
you can use `@pypi_base` to set packages required by all
|
464
|
+
steps and use `@pypi` to specify step-specific overrides.
|
548
465
|
|
549
466
|
Parameters
|
550
467
|
----------
|
551
|
-
packages : Dict[str, str], default {}
|
468
|
+
packages : Dict[str, str], default: {}
|
552
469
|
Packages to use for this step. The key is the name of the package
|
553
470
|
and the value is the version to use.
|
554
|
-
|
555
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
556
|
-
python : str, optional, default None
|
471
|
+
python : str, optional, default: None
|
557
472
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
558
473
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
559
|
-
disabled : bool, default False
|
560
|
-
If set to True, disables @conda.
|
561
474
|
"""
|
562
475
|
...
|
563
476
|
|
564
477
|
@typing.overload
|
565
|
-
def
|
478
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
566
479
|
...
|
567
480
|
|
568
481
|
@typing.overload
|
569
|
-
def
|
482
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
570
483
|
...
|
571
484
|
|
572
|
-
def
|
485
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
573
486
|
"""
|
574
|
-
Specifies the
|
487
|
+
Specifies the PyPI packages for the step.
|
575
488
|
|
576
489
|
Information in this decorator will augment any
|
577
|
-
attributes set in the `@
|
578
|
-
you can use `@
|
579
|
-
steps and use `@
|
490
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
491
|
+
you can use `@pypi_base` to set packages required by all
|
492
|
+
steps and use `@pypi` to specify step-specific overrides.
|
580
493
|
|
581
494
|
Parameters
|
582
495
|
----------
|
583
|
-
packages : Dict[str, str], default {}
|
496
|
+
packages : Dict[str, str], default: {}
|
584
497
|
Packages to use for this step. The key is the name of the package
|
585
498
|
and the value is the version to use.
|
586
|
-
|
587
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
588
|
-
python : str, optional, default None
|
499
|
+
python : str, optional, default: None
|
589
500
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
590
501
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
591
|
-
disabled : bool, default False
|
592
|
-
If set to True, disables @conda.
|
593
|
-
"""
|
594
|
-
...
|
595
|
-
|
596
|
-
@typing.overload
|
597
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
598
|
-
"""
|
599
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
600
|
-
the execution of a step.
|
601
|
-
|
602
|
-
Parameters
|
603
|
-
----------
|
604
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
605
|
-
List of secret specs, defining how the secrets are to be retrieved
|
606
|
-
"""
|
607
|
-
...
|
608
|
-
|
609
|
-
@typing.overload
|
610
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
611
|
-
...
|
612
|
-
|
613
|
-
@typing.overload
|
614
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
615
|
-
...
|
616
|
-
|
617
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
618
|
-
"""
|
619
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
620
|
-
the execution of a step.
|
621
|
-
|
622
|
-
Parameters
|
623
|
-
----------
|
624
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
625
|
-
List of secret specs, defining how the secrets are to be retrieved
|
626
502
|
"""
|
627
503
|
...
|
628
504
|
|
@@ -684,129 +560,164 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
684
560
|
...
|
685
561
|
|
686
562
|
@typing.overload
|
687
|
-
def
|
563
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
688
564
|
"""
|
689
|
-
|
565
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
566
|
+
|
567
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
690
568
|
|
691
569
|
Parameters
|
692
570
|
----------
|
693
|
-
|
694
|
-
|
571
|
+
type : str, default 'default'
|
572
|
+
Card type.
|
573
|
+
id : str, optional, default None
|
574
|
+
If multiple cards are present, use this id to identify this card.
|
575
|
+
options : Dict[str, Any], default {}
|
576
|
+
Options passed to the card. The contents depend on the card type.
|
577
|
+
timeout : int, default 45
|
578
|
+
Interrupt reporting if it takes more than this many seconds.
|
579
|
+
|
580
|
+
|
695
581
|
"""
|
696
582
|
...
|
697
583
|
|
698
584
|
@typing.overload
|
699
|
-
def
|
585
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
700
586
|
...
|
701
587
|
|
702
588
|
@typing.overload
|
703
|
-
def
|
589
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
704
590
|
...
|
705
591
|
|
706
|
-
def
|
592
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
707
593
|
"""
|
708
|
-
|
594
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
595
|
+
|
596
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
709
597
|
|
710
598
|
Parameters
|
711
599
|
----------
|
712
|
-
|
713
|
-
|
600
|
+
type : str, default 'default'
|
601
|
+
Card type.
|
602
|
+
id : str, optional, default None
|
603
|
+
If multiple cards are present, use this id to identify this card.
|
604
|
+
options : Dict[str, Any], default {}
|
605
|
+
Options passed to the card. The contents depend on the card type.
|
606
|
+
timeout : int, default 45
|
607
|
+
Interrupt reporting if it takes more than this many seconds.
|
608
|
+
|
609
|
+
|
714
610
|
"""
|
715
611
|
...
|
716
612
|
|
717
613
|
@typing.overload
|
718
|
-
def
|
614
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
719
615
|
"""
|
720
|
-
|
721
|
-
|
722
|
-
Use `@resources` to specify the resource requirements
|
723
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
616
|
+
Enables loading / saving of models within a step.
|
724
617
|
|
725
|
-
You can choose the compute layer on the command line by executing e.g.
|
726
|
-
```
|
727
|
-
python myflow.py run --with batch
|
728
|
-
```
|
729
|
-
or
|
730
|
-
```
|
731
|
-
python myflow.py run --with kubernetes
|
732
|
-
```
|
733
|
-
which executes the flow on the desired system using the
|
734
|
-
requirements specified in `@resources`.
|
735
618
|
|
736
619
|
Parameters
|
737
620
|
----------
|
738
|
-
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
621
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
622
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
623
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
624
|
+
- `current.checkpoint`
|
625
|
+
- `current.model`
|
626
|
+
- `current.huggingface_hub`
|
627
|
+
|
628
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
629
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
630
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
631
|
+
|
632
|
+
temp_dir_root : str, default: None
|
633
|
+
The root directory under which `current.model.loaded` will store loaded models
|
634
|
+
|
635
|
+
|
636
|
+
|
637
|
+
"""
|
638
|
+
...
|
755
639
|
|
756
640
|
@typing.overload
|
757
|
-
def
|
641
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
758
642
|
...
|
759
643
|
|
760
|
-
|
644
|
+
@typing.overload
|
645
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
646
|
+
...
|
647
|
+
|
648
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
761
649
|
"""
|
762
|
-
|
650
|
+
Enables loading / saving of models within a step.
|
763
651
|
|
764
|
-
Use `@resources` to specify the resource requirements
|
765
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
766
652
|
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
653
|
+
Parameters
|
654
|
+
----------
|
655
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
656
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
657
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
658
|
+
- `current.checkpoint`
|
659
|
+
- `current.model`
|
660
|
+
- `current.huggingface_hub`
|
661
|
+
|
662
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
663
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
664
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
665
|
+
|
666
|
+
temp_dir_root : str, default: None
|
667
|
+
The root directory under which `current.model.loaded` will store loaded models
|
668
|
+
|
669
|
+
|
670
|
+
|
671
|
+
"""
|
672
|
+
...
|
673
|
+
|
674
|
+
@typing.overload
|
675
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
676
|
+
"""
|
677
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
678
|
+
the execution of a step.
|
777
679
|
|
778
680
|
Parameters
|
779
681
|
----------
|
780
|
-
|
781
|
-
|
782
|
-
gpu : int, default 0
|
783
|
-
Number of GPUs required for this step.
|
784
|
-
disk : int, optional, default None
|
785
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
786
|
-
memory : int, default 4096
|
787
|
-
Memory size (in MB) required for this step.
|
788
|
-
shared_memory : int, optional, default None
|
789
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
790
|
-
This parameter maps to the `--shm-size` option in Docker.
|
682
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
683
|
+
List of secret specs, defining how the secrets are to be retrieved
|
791
684
|
"""
|
792
685
|
...
|
793
686
|
|
794
687
|
@typing.overload
|
795
|
-
def
|
688
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
689
|
+
...
|
690
|
+
|
691
|
+
@typing.overload
|
692
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
693
|
+
...
|
694
|
+
|
695
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
796
696
|
"""
|
797
|
-
|
798
|
-
|
697
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
698
|
+
the execution of a step.
|
699
|
+
|
700
|
+
Parameters
|
701
|
+
----------
|
702
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
703
|
+
List of secret specs, defining how the secrets are to be retrieved
|
799
704
|
"""
|
800
705
|
...
|
801
706
|
|
802
707
|
@typing.overload
|
803
|
-
def
|
708
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
709
|
+
"""
|
710
|
+
Internal decorator to support Fast bakery
|
711
|
+
"""
|
804
712
|
...
|
805
713
|
|
806
|
-
|
714
|
+
@typing.overload
|
715
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
716
|
+
...
|
717
|
+
|
718
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
807
719
|
"""
|
808
|
-
|
809
|
-
and imported for all decorators types by _import_plugin_decorators().
|
720
|
+
Internal decorator to support Fast bakery
|
810
721
|
"""
|
811
722
|
...
|
812
723
|
|
@@ -877,395 +788,337 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
877
788
|
...
|
878
789
|
|
879
790
|
@typing.overload
|
880
|
-
def
|
791
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
881
792
|
"""
|
882
|
-
Specifies the
|
793
|
+
Specifies that the step will success under all circumstances.
|
794
|
+
|
795
|
+
The decorator will create an optional artifact, specified by `var`, which
|
796
|
+
contains the exception raised. You can use it to detect the presence
|
797
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
798
|
+
are missing.
|
799
|
+
|
800
|
+
Parameters
|
801
|
+
----------
|
802
|
+
var : str, optional, default None
|
803
|
+
Name of the artifact in which to store the caught exception.
|
804
|
+
If not specified, the exception is not stored.
|
805
|
+
print_exception : bool, default True
|
806
|
+
Determines whether or not the exception is printed to
|
807
|
+
stdout when caught.
|
808
|
+
"""
|
809
|
+
...
|
810
|
+
|
811
|
+
@typing.overload
|
812
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
813
|
+
...
|
814
|
+
|
815
|
+
@typing.overload
|
816
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
817
|
+
...
|
818
|
+
|
819
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
820
|
+
"""
|
821
|
+
Specifies that the step will success under all circumstances.
|
822
|
+
|
823
|
+
The decorator will create an optional artifact, specified by `var`, which
|
824
|
+
contains the exception raised. You can use it to detect the presence
|
825
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
826
|
+
are missing.
|
827
|
+
|
828
|
+
Parameters
|
829
|
+
----------
|
830
|
+
var : str, optional, default None
|
831
|
+
Name of the artifact in which to store the caught exception.
|
832
|
+
If not specified, the exception is not stored.
|
833
|
+
print_exception : bool, default True
|
834
|
+
Determines whether or not the exception is printed to
|
835
|
+
stdout when caught.
|
836
|
+
"""
|
837
|
+
...
|
838
|
+
|
839
|
+
@typing.overload
|
840
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
841
|
+
"""
|
842
|
+
Specifies the Conda environment for the step.
|
883
843
|
|
884
844
|
Information in this decorator will augment any
|
885
|
-
attributes set in the `@
|
886
|
-
you can use `@
|
887
|
-
steps and use `@
|
845
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
846
|
+
you can use `@conda_base` to set packages required by all
|
847
|
+
steps and use `@conda` to specify step-specific overrides.
|
888
848
|
|
889
849
|
Parameters
|
890
850
|
----------
|
891
|
-
packages : Dict[str, str], default
|
851
|
+
packages : Dict[str, str], default {}
|
892
852
|
Packages to use for this step. The key is the name of the package
|
893
853
|
and the value is the version to use.
|
894
|
-
|
854
|
+
libraries : Dict[str, str], default {}
|
855
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
856
|
+
python : str, optional, default None
|
895
857
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
896
858
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
859
|
+
disabled : bool, default False
|
860
|
+
If set to True, disables @conda.
|
897
861
|
"""
|
898
862
|
...
|
899
863
|
|
900
864
|
@typing.overload
|
901
|
-
def
|
865
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
902
866
|
...
|
903
867
|
|
904
868
|
@typing.overload
|
905
|
-
def
|
869
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
906
870
|
...
|
907
871
|
|
908
|
-
def
|
872
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
909
873
|
"""
|
910
|
-
Specifies the
|
874
|
+
Specifies the Conda environment for the step.
|
911
875
|
|
912
876
|
Information in this decorator will augment any
|
913
|
-
attributes set in the `@
|
914
|
-
you can use `@
|
915
|
-
steps and use `@
|
877
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
878
|
+
you can use `@conda_base` to set packages required by all
|
879
|
+
steps and use `@conda` to specify step-specific overrides.
|
916
880
|
|
917
881
|
Parameters
|
918
882
|
----------
|
919
|
-
packages : Dict[str, str], default
|
883
|
+
packages : Dict[str, str], default {}
|
920
884
|
Packages to use for this step. The key is the name of the package
|
921
885
|
and the value is the version to use.
|
922
|
-
|
886
|
+
libraries : Dict[str, str], default {}
|
887
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
888
|
+
python : str, optional, default None
|
923
889
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
924
890
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
891
|
+
disabled : bool, default False
|
892
|
+
If set to True, disables @conda.
|
925
893
|
"""
|
926
894
|
...
|
927
895
|
|
928
896
|
@typing.overload
|
929
|
-
def
|
897
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
930
898
|
"""
|
931
|
-
|
899
|
+
Specifies the number of times the task corresponding
|
900
|
+
to a step needs to be retried.
|
932
901
|
|
933
|
-
|
902
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
903
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
904
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
905
|
+
|
906
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
907
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
908
|
+
ensuring that the flow execution can continue.
|
934
909
|
|
935
910
|
Parameters
|
936
911
|
----------
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
|
941
|
-
options : Dict[str, Any], default {}
|
942
|
-
Options passed to the card. The contents depend on the card type.
|
943
|
-
timeout : int, default 45
|
944
|
-
Interrupt reporting if it takes more than this many seconds.
|
945
|
-
|
946
|
-
|
912
|
+
times : int, default 3
|
913
|
+
Number of times to retry this task.
|
914
|
+
minutes_between_retries : int, default 2
|
915
|
+
Number of minutes between retries.
|
947
916
|
"""
|
948
917
|
...
|
949
918
|
|
950
919
|
@typing.overload
|
951
|
-
def
|
920
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
952
921
|
...
|
953
922
|
|
954
923
|
@typing.overload
|
955
|
-
def
|
924
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
956
925
|
...
|
957
926
|
|
958
|
-
def
|
927
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
959
928
|
"""
|
960
|
-
|
929
|
+
Specifies the number of times the task corresponding
|
930
|
+
to a step needs to be retried.
|
961
931
|
|
962
|
-
|
932
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
933
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
934
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
935
|
+
|
936
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
937
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
938
|
+
ensuring that the flow execution can continue.
|
963
939
|
|
964
940
|
Parameters
|
965
941
|
----------
|
966
|
-
|
967
|
-
|
968
|
-
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
timeout : int, default 45
|
973
|
-
Interrupt reporting if it takes more than this many seconds.
|
974
|
-
|
975
|
-
|
976
|
-
"""
|
977
|
-
...
|
942
|
+
times : int, default 3
|
943
|
+
Number of times to retry this task.
|
944
|
+
minutes_between_retries : int, default 2
|
945
|
+
Number of minutes between retries.
|
946
|
+
"""
|
947
|
+
...
|
978
948
|
|
979
949
|
@typing.overload
|
980
|
-
def
|
950
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
981
951
|
"""
|
982
|
-
|
983
|
-
|
952
|
+
Specifies environment variables to be set prior to the execution of a step.
|
984
953
|
|
985
954
|
Parameters
|
986
955
|
----------
|
987
|
-
|
988
|
-
|
989
|
-
- "eager": Loads the the latest available checkpoint within the namespace.
|
990
|
-
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
991
|
-
will be loaded at the start of the task.
|
992
|
-
- "none": Do not load any checkpoint
|
993
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
994
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
995
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
996
|
-
created within the task will be loaded when the task is retries execution on failure.
|
997
|
-
|
998
|
-
temp_dir_root : str, default: None
|
999
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
1000
|
-
|
1001
|
-
|
1002
|
-
|
956
|
+
vars : Dict[str, str], default {}
|
957
|
+
Dictionary of environment variables to set.
|
1003
958
|
"""
|
1004
959
|
...
|
1005
960
|
|
1006
961
|
@typing.overload
|
1007
|
-
def
|
962
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1008
963
|
...
|
1009
964
|
|
1010
965
|
@typing.overload
|
1011
|
-
def
|
966
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1012
967
|
...
|
1013
968
|
|
1014
|
-
def
|
969
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1015
970
|
"""
|
1016
|
-
|
1017
|
-
|
971
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1018
972
|
|
1019
973
|
Parameters
|
1020
974
|
----------
|
1021
|
-
|
1022
|
-
|
1023
|
-
- "eager": Loads the the latest available checkpoint within the namespace.
|
1024
|
-
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
1025
|
-
will be loaded at the start of the task.
|
1026
|
-
- "none": Do not load any checkpoint
|
1027
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
1028
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
1029
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
1030
|
-
created within the task will be loaded when the task is retries execution on failure.
|
1031
|
-
|
1032
|
-
temp_dir_root : str, default: None
|
1033
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
1034
|
-
|
1035
|
-
|
1036
|
-
|
975
|
+
vars : Dict[str, str], default {}
|
976
|
+
Dictionary of environment variables to set.
|
1037
977
|
"""
|
1038
978
|
...
|
1039
979
|
|
1040
980
|
@typing.overload
|
1041
|
-
def
|
981
|
+
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1042
982
|
"""
|
1043
|
-
|
983
|
+
Specifies the resources needed when executing this step.
|
1044
984
|
|
985
|
+
Use `@resources` to specify the resource requirements
|
986
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
987
|
+
|
988
|
+
You can choose the compute layer on the command line by executing e.g.
|
989
|
+
```
|
990
|
+
python myflow.py run --with batch
|
991
|
+
```
|
992
|
+
or
|
993
|
+
```
|
994
|
+
python myflow.py run --with kubernetes
|
995
|
+
```
|
996
|
+
which executes the flow on the desired system using the
|
997
|
+
requirements specified in `@resources`.
|
1045
998
|
|
1046
999
|
Parameters
|
1047
1000
|
----------
|
1048
|
-
|
1049
|
-
|
1050
|
-
|
1051
|
-
|
1052
|
-
|
1053
|
-
|
1054
|
-
|
1055
|
-
|
1056
|
-
|
1057
|
-
|
1058
|
-
|
1059
|
-
temp_dir_root : str, default: None
|
1060
|
-
The root directory under which `current.model.loaded` will store loaded models
|
1061
|
-
|
1062
|
-
|
1063
|
-
|
1001
|
+
cpu : int, default 1
|
1002
|
+
Number of CPUs required for this step.
|
1003
|
+
gpu : int, default 0
|
1004
|
+
Number of GPUs required for this step.
|
1005
|
+
disk : int, optional, default None
|
1006
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1007
|
+
memory : int, default 4096
|
1008
|
+
Memory size (in MB) required for this step.
|
1009
|
+
shared_memory : int, optional, default None
|
1010
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1011
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1064
1012
|
"""
|
1065
1013
|
...
|
1066
1014
|
|
1067
1015
|
@typing.overload
|
1068
|
-
def
|
1016
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1069
1017
|
...
|
1070
1018
|
|
1071
1019
|
@typing.overload
|
1072
|
-
def
|
1020
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1073
1021
|
...
|
1074
1022
|
|
1075
|
-
def
|
1023
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
1076
1024
|
"""
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1080
|
-
Parameters
|
1081
|
-
----------
|
1082
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
1083
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
1084
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
1085
|
-
- `current.checkpoint`
|
1086
|
-
- `current.model`
|
1087
|
-
- `current.huggingface_hub`
|
1088
|
-
|
1089
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
1090
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
1091
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
1092
|
-
|
1093
|
-
temp_dir_root : str, default: None
|
1094
|
-
The root directory under which `current.model.loaded` will store loaded models
|
1095
|
-
|
1025
|
+
Specifies the resources needed when executing this step.
|
1096
1026
|
|
1027
|
+
Use `@resources` to specify the resource requirements
|
1028
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1097
1029
|
|
1098
|
-
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
@typing.overload
|
1109
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1110
|
-
...
|
1111
|
-
|
1112
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1113
|
-
"""
|
1114
|
-
Internal decorator to support Fast bakery
|
1115
|
-
"""
|
1116
|
-
...
|
1117
|
-
|
1118
|
-
@typing.overload
|
1119
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1120
|
-
"""
|
1121
|
-
Specifies the times when the flow should be run when running on a
|
1122
|
-
production scheduler.
|
1030
|
+
You can choose the compute layer on the command line by executing e.g.
|
1031
|
+
```
|
1032
|
+
python myflow.py run --with batch
|
1033
|
+
```
|
1034
|
+
or
|
1035
|
+
```
|
1036
|
+
python myflow.py run --with kubernetes
|
1037
|
+
```
|
1038
|
+
which executes the flow on the desired system using the
|
1039
|
+
requirements specified in `@resources`.
|
1123
1040
|
|
1124
1041
|
Parameters
|
1125
1042
|
----------
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1043
|
+
cpu : int, default 1
|
1044
|
+
Number of CPUs required for this step.
|
1045
|
+
gpu : int, default 0
|
1046
|
+
Number of GPUs required for this step.
|
1047
|
+
disk : int, optional, default None
|
1048
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1049
|
+
memory : int, default 4096
|
1050
|
+
Memory size (in MB) required for this step.
|
1051
|
+
shared_memory : int, optional, default None
|
1052
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1053
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1138
1054
|
"""
|
1139
1055
|
...
|
1140
1056
|
|
1141
1057
|
@typing.overload
|
1142
|
-
def
|
1143
|
-
...
|
1144
|
-
|
1145
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1058
|
+
def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1146
1059
|
"""
|
1147
|
-
|
1148
|
-
|
1060
|
+
Enables checkpointing for a step.
|
1061
|
+
|
1149
1062
|
|
1150
1063
|
Parameters
|
1151
1064
|
----------
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1163
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1164
|
-
"""
|
1165
|
-
...
|
1166
|
-
|
1167
|
-
@typing.overload
|
1168
|
-
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1169
|
-
"""
|
1170
|
-
Specifies the event(s) that this flow depends on.
|
1171
|
-
|
1172
|
-
```
|
1173
|
-
@trigger(event='foo')
|
1174
|
-
```
|
1175
|
-
or
|
1176
|
-
```
|
1177
|
-
@trigger(events=['foo', 'bar'])
|
1178
|
-
```
|
1179
|
-
|
1180
|
-
Additionally, you can specify the parameter mappings
|
1181
|
-
to map event payload to Metaflow parameters for the flow.
|
1182
|
-
```
|
1183
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1184
|
-
```
|
1185
|
-
or
|
1186
|
-
```
|
1187
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1188
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1189
|
-
```
|
1065
|
+
load_policy : str, default: "fresh"
|
1066
|
+
The policy for loading the checkpoint. The following policies are supported:
|
1067
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
1068
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
1069
|
+
will be loaded at the start of the task.
|
1070
|
+
- "none": Do not load any checkpoint
|
1071
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
1072
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
1073
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
1074
|
+
created within the task will be loaded when the task is retries execution on failure.
|
1190
1075
|
|
1191
|
-
|
1192
|
-
|
1193
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1194
|
-
```
|
1195
|
-
This is equivalent to:
|
1196
|
-
```
|
1197
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1198
|
-
```
|
1076
|
+
temp_dir_root : str, default: None
|
1077
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
1199
1078
|
|
1200
|
-
Parameters
|
1201
|
-
----------
|
1202
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1203
|
-
Event dependency for this flow.
|
1204
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1205
|
-
Events dependency for this flow.
|
1206
|
-
options : Dict[str, Any], default {}
|
1207
|
-
Backend-specific configuration for tuning eventing behavior.
|
1208
1079
|
|
1209
1080
|
|
1210
1081
|
"""
|
1211
1082
|
...
|
1212
1083
|
|
1213
1084
|
@typing.overload
|
1214
|
-
def
|
1085
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1215
1086
|
...
|
1216
1087
|
|
1217
|
-
|
1088
|
+
@typing.overload
|
1089
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1090
|
+
...
|
1091
|
+
|
1092
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = "fresh", temp_dir_root: str = None):
|
1218
1093
|
"""
|
1219
|
-
|
1220
|
-
|
1221
|
-
```
|
1222
|
-
@trigger(event='foo')
|
1223
|
-
```
|
1224
|
-
or
|
1225
|
-
```
|
1226
|
-
@trigger(events=['foo', 'bar'])
|
1227
|
-
```
|
1228
|
-
|
1229
|
-
Additionally, you can specify the parameter mappings
|
1230
|
-
to map event payload to Metaflow parameters for the flow.
|
1231
|
-
```
|
1232
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1233
|
-
```
|
1234
|
-
or
|
1235
|
-
```
|
1236
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1237
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1238
|
-
```
|
1094
|
+
Enables checkpointing for a step.
|
1239
1095
|
|
1240
|
-
'parameters' can also be a list of strings and tuples like so:
|
1241
|
-
```
|
1242
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1243
|
-
```
|
1244
|
-
This is equivalent to:
|
1245
|
-
```
|
1246
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1247
|
-
```
|
1248
1096
|
|
1249
1097
|
Parameters
|
1250
1098
|
----------
|
1251
|
-
|
1252
|
-
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
|
1099
|
+
load_policy : str, default: "fresh"
|
1100
|
+
The policy for loading the checkpoint. The following policies are supported:
|
1101
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
1102
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
1103
|
+
will be loaded at the start of the task.
|
1104
|
+
- "none": Do not load any checkpoint
|
1105
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
1106
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
1107
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
1108
|
+
created within the task will be loaded when the task is retries execution on failure.
|
1109
|
+
|
1110
|
+
temp_dir_root : str, default: None
|
1111
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
1112
|
+
|
1257
1113
|
|
1258
1114
|
|
1259
1115
|
"""
|
1260
1116
|
...
|
1261
1117
|
|
1262
|
-
def
|
1118
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1263
1119
|
"""
|
1264
|
-
The `@
|
1265
|
-
|
1266
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1267
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1268
|
-
starts only after all sensors finish.
|
1120
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1121
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1269
1122
|
|
1270
1123
|
Parameters
|
1271
1124
|
----------
|
@@ -1286,57 +1139,21 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1286
1139
|
Name of the sensor on Airflow
|
1287
1140
|
description : str
|
1288
1141
|
Description of sensor in the Airflow UI
|
1289
|
-
|
1290
|
-
The
|
1291
|
-
|
1292
|
-
|
1293
|
-
|
1294
|
-
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1298
|
-
|
1299
|
-
|
1300
|
-
|
1301
|
-
|
1302
|
-
|
1303
|
-
|
1304
|
-
@typing.overload
|
1305
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1306
|
-
"""
|
1307
|
-
Specifies the PyPI packages for all steps of the flow.
|
1308
|
-
|
1309
|
-
Use `@pypi_base` to set common packages required by all
|
1310
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1311
|
-
Parameters
|
1312
|
-
----------
|
1313
|
-
packages : Dict[str, str], default: {}
|
1314
|
-
Packages to use for this flow. The key is the name of the package
|
1315
|
-
and the value is the version to use.
|
1316
|
-
python : str, optional, default: None
|
1317
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1318
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1319
|
-
"""
|
1320
|
-
...
|
1321
|
-
|
1322
|
-
@typing.overload
|
1323
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1324
|
-
...
|
1325
|
-
|
1326
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1327
|
-
"""
|
1328
|
-
Specifies the PyPI packages for all steps of the flow.
|
1329
|
-
|
1330
|
-
Use `@pypi_base` to set common packages required by all
|
1331
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1332
|
-
Parameters
|
1333
|
-
----------
|
1334
|
-
packages : Dict[str, str], default: {}
|
1335
|
-
Packages to use for this flow. The key is the name of the package
|
1336
|
-
and the value is the version to use.
|
1337
|
-
python : str, optional, default: None
|
1338
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1339
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1142
|
+
external_dag_id : str
|
1143
|
+
The dag_id that contains the task you want to wait for.
|
1144
|
+
external_task_ids : List[str]
|
1145
|
+
The list of task_ids that you want to wait for.
|
1146
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1147
|
+
allowed_states : List[str]
|
1148
|
+
Iterable of allowed states, (Default: ['success'])
|
1149
|
+
failed_states : List[str]
|
1150
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1151
|
+
execution_delta : datetime.timedelta
|
1152
|
+
time difference with the previous execution to look at,
|
1153
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1154
|
+
check_existence: bool
|
1155
|
+
Set to True to check if the external task exists or check if
|
1156
|
+
the DAG to wait for exists. (Default: True)
|
1340
1157
|
"""
|
1341
1158
|
...
|
1342
1159
|
|
@@ -1389,6 +1206,79 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
1389
1206
|
"""
|
1390
1207
|
...
|
1391
1208
|
|
1209
|
+
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1210
|
+
"""
|
1211
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1212
|
+
|
1213
|
+
User code call
|
1214
|
+
-----------
|
1215
|
+
@nim(
|
1216
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1217
|
+
backend='managed'
|
1218
|
+
)
|
1219
|
+
|
1220
|
+
Valid backend options
|
1221
|
+
---------------------
|
1222
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
1223
|
+
- 🚧 'dataplane': Run in your account.
|
1224
|
+
|
1225
|
+
Valid model options
|
1226
|
+
----------------
|
1227
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
1228
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
1229
|
+
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1230
|
+
|
1231
|
+
Parameters
|
1232
|
+
----------
|
1233
|
+
models: list[NIM]
|
1234
|
+
List of NIM containers running models in sidecars.
|
1235
|
+
backend: str
|
1236
|
+
Compute provider to run the NIM container.
|
1237
|
+
"""
|
1238
|
+
...
|
1239
|
+
|
1240
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1241
|
+
"""
|
1242
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1243
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1244
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1245
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1246
|
+
starts only after all sensors finish.
|
1247
|
+
|
1248
|
+
Parameters
|
1249
|
+
----------
|
1250
|
+
timeout : int
|
1251
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1252
|
+
poke_interval : int
|
1253
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1254
|
+
mode : str
|
1255
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1256
|
+
exponential_backoff : bool
|
1257
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1258
|
+
pool : str
|
1259
|
+
the slot pool this task should run in,
|
1260
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1261
|
+
soft_fail : bool
|
1262
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1263
|
+
name : str
|
1264
|
+
Name of the sensor on Airflow
|
1265
|
+
description : str
|
1266
|
+
Description of sensor in the Airflow UI
|
1267
|
+
bucket_key : Union[str, List[str]]
|
1268
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1269
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1270
|
+
bucket_name : str
|
1271
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1272
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1273
|
+
wildcard_match : bool
|
1274
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1275
|
+
aws_conn_id : str
|
1276
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1277
|
+
verify : bool
|
1278
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1279
|
+
"""
|
1280
|
+
...
|
1281
|
+
|
1392
1282
|
@typing.overload
|
1393
1283
|
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1394
1284
|
"""
|
@@ -1492,34 +1382,91 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1492
1382
|
"""
|
1493
1383
|
...
|
1494
1384
|
|
1495
|
-
|
1385
|
+
@typing.overload
|
1386
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1496
1387
|
"""
|
1497
|
-
|
1388
|
+
Specifies the times when the flow should be run when running on a
|
1389
|
+
production scheduler.
|
1498
1390
|
|
1499
|
-
|
1500
|
-
|
1501
|
-
|
1502
|
-
|
1503
|
-
|
1504
|
-
|
1391
|
+
Parameters
|
1392
|
+
----------
|
1393
|
+
hourly : bool, default False
|
1394
|
+
Run the workflow hourly.
|
1395
|
+
daily : bool, default True
|
1396
|
+
Run the workflow daily.
|
1397
|
+
weekly : bool, default False
|
1398
|
+
Run the workflow weekly.
|
1399
|
+
cron : str, optional, default None
|
1400
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1401
|
+
specified by this expression.
|
1402
|
+
timezone : str, optional, default None
|
1403
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1404
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1405
|
+
"""
|
1406
|
+
...
|
1407
|
+
|
1408
|
+
@typing.overload
|
1409
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1410
|
+
...
|
1411
|
+
|
1412
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1413
|
+
"""
|
1414
|
+
Specifies the times when the flow should be run when running on a
|
1415
|
+
production scheduler.
|
1505
1416
|
|
1506
|
-
|
1507
|
-
|
1508
|
-
|
1509
|
-
|
1417
|
+
Parameters
|
1418
|
+
----------
|
1419
|
+
hourly : bool, default False
|
1420
|
+
Run the workflow hourly.
|
1421
|
+
daily : bool, default True
|
1422
|
+
Run the workflow daily.
|
1423
|
+
weekly : bool, default False
|
1424
|
+
Run the workflow weekly.
|
1425
|
+
cron : str, optional, default None
|
1426
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1427
|
+
specified by this expression.
|
1428
|
+
timezone : str, optional, default None
|
1429
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1430
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1431
|
+
"""
|
1432
|
+
...
|
1433
|
+
|
1434
|
+
@typing.overload
|
1435
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1436
|
+
"""
|
1437
|
+
Specifies the PyPI packages for all steps of the flow.
|
1510
1438
|
|
1511
|
-
|
1512
|
-
|
1513
|
-
|
1514
|
-
|
1515
|
-
|
1439
|
+
Use `@pypi_base` to set common packages required by all
|
1440
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1441
|
+
Parameters
|
1442
|
+
----------
|
1443
|
+
packages : Dict[str, str], default: {}
|
1444
|
+
Packages to use for this flow. The key is the name of the package
|
1445
|
+
and the value is the version to use.
|
1446
|
+
python : str, optional, default: None
|
1447
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1448
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1449
|
+
"""
|
1450
|
+
...
|
1451
|
+
|
1452
|
+
@typing.overload
|
1453
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1454
|
+
...
|
1455
|
+
|
1456
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1457
|
+
"""
|
1458
|
+
Specifies the PyPI packages for all steps of the flow.
|
1516
1459
|
|
1460
|
+
Use `@pypi_base` to set common packages required by all
|
1461
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1517
1462
|
Parameters
|
1518
1463
|
----------
|
1519
|
-
|
1520
|
-
|
1521
|
-
|
1522
|
-
|
1464
|
+
packages : Dict[str, str], default: {}
|
1465
|
+
Packages to use for this flow. The key is the name of the package
|
1466
|
+
and the value is the version to use.
|
1467
|
+
python : str, optional, default: None
|
1468
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1469
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1523
1470
|
"""
|
1524
1471
|
...
|
1525
1472
|
|
@@ -1541,45 +1488,98 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
|
|
1541
1488
|
"""
|
1542
1489
|
...
|
1543
1490
|
|
1544
|
-
|
1491
|
+
@typing.overload
|
1492
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1545
1493
|
"""
|
1546
|
-
|
1547
|
-
|
1494
|
+
Specifies the event(s) that this flow depends on.
|
1495
|
+
|
1496
|
+
```
|
1497
|
+
@trigger(event='foo')
|
1498
|
+
```
|
1499
|
+
or
|
1500
|
+
```
|
1501
|
+
@trigger(events=['foo', 'bar'])
|
1502
|
+
```
|
1503
|
+
|
1504
|
+
Additionally, you can specify the parameter mappings
|
1505
|
+
to map event payload to Metaflow parameters for the flow.
|
1506
|
+
```
|
1507
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1508
|
+
```
|
1509
|
+
or
|
1510
|
+
```
|
1511
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1512
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1513
|
+
```
|
1514
|
+
|
1515
|
+
'parameters' can also be a list of strings and tuples like so:
|
1516
|
+
```
|
1517
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1518
|
+
```
|
1519
|
+
This is equivalent to:
|
1520
|
+
```
|
1521
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1522
|
+
```
|
1548
1523
|
|
1549
1524
|
Parameters
|
1550
1525
|
----------
|
1551
|
-
|
1552
|
-
|
1553
|
-
|
1554
|
-
|
1555
|
-
|
1556
|
-
|
1557
|
-
|
1558
|
-
|
1559
|
-
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1563
|
-
|
1564
|
-
|
1565
|
-
|
1566
|
-
|
1567
|
-
|
1568
|
-
|
1569
|
-
|
1570
|
-
|
1571
|
-
|
1572
|
-
|
1573
|
-
|
1574
|
-
|
1575
|
-
|
1576
|
-
|
1577
|
-
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
|
1582
|
-
|
1526
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1527
|
+
Event dependency for this flow.
|
1528
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1529
|
+
Events dependency for this flow.
|
1530
|
+
options : Dict[str, Any], default {}
|
1531
|
+
Backend-specific configuration for tuning eventing behavior.
|
1532
|
+
|
1533
|
+
|
1534
|
+
"""
|
1535
|
+
...
|
1536
|
+
|
1537
|
+
@typing.overload
|
1538
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1539
|
+
...
|
1540
|
+
|
1541
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1542
|
+
"""
|
1543
|
+
Specifies the event(s) that this flow depends on.
|
1544
|
+
|
1545
|
+
```
|
1546
|
+
@trigger(event='foo')
|
1547
|
+
```
|
1548
|
+
or
|
1549
|
+
```
|
1550
|
+
@trigger(events=['foo', 'bar'])
|
1551
|
+
```
|
1552
|
+
|
1553
|
+
Additionally, you can specify the parameter mappings
|
1554
|
+
to map event payload to Metaflow parameters for the flow.
|
1555
|
+
```
|
1556
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1557
|
+
```
|
1558
|
+
or
|
1559
|
+
```
|
1560
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1561
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1562
|
+
```
|
1563
|
+
|
1564
|
+
'parameters' can also be a list of strings and tuples like so:
|
1565
|
+
```
|
1566
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1567
|
+
```
|
1568
|
+
This is equivalent to:
|
1569
|
+
```
|
1570
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1571
|
+
```
|
1572
|
+
|
1573
|
+
Parameters
|
1574
|
+
----------
|
1575
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1576
|
+
Event dependency for this flow.
|
1577
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1578
|
+
Events dependency for this flow.
|
1579
|
+
options : Dict[str, Any], default {}
|
1580
|
+
Backend-specific configuration for tuning eventing behavior.
|
1581
|
+
|
1582
|
+
|
1583
1583
|
"""
|
1584
1584
|
...
|
1585
1585
|
|