ob-metaflow-stubs 6.0.3.103rc3__py2.py3-none-any.whl → 6.0.3.105__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +836 -498
- metaflow-stubs/cards.pyi +211 -5
- metaflow-stubs/cli.pyi +23 -3
- metaflow-stubs/client/__init__.pyi +129 -4
- metaflow-stubs/client/core.pyi +227 -7
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +5 -2
- metaflow-stubs/events.pyi +21 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +71 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +157 -4
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +8 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +39 -36
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +70 -4
- metaflow-stubs/plugins/__init__.pyi +13 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +45 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +45 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +18 -5
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +33 -5
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +103 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +62 -4
- metaflow-stubs/plugins/cards/card_client.pyi +33 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +42 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +36 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
- metaflow-stubs/plugins/cards/exception.pyi +8 -2
- metaflow-stubs/plugins/catch_decorator.pyi +20 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
- metaflow-stubs/plugins/datatools/local.pyi +16 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +82 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +10 -2
- metaflow-stubs/plugins/events_decorator.pyi +106 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +23 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +12 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +100 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
- metaflow-stubs/plugins/logs_cli.pyi +4 -4
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +59 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +33 -2
- metaflow-stubs/plugins/retry_decorator.pyi +21 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
- metaflow-stubs/plugins/storage_executor.pyi +6 -2
- metaflow-stubs/plugins/tag_cli.pyi +36 -5
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +9 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +71 -132
- metaflow-stubs/runner/metaflow_runner.pyi +117 -9
- metaflow-stubs/runner/nbdeploy.pyi +66 -2
- metaflow-stubs/runner/nbrun.pyi +79 -2
- metaflow-stubs/runner/subprocess_manager.pyi +16 -4
- metaflow-stubs/runner/utils.pyi +32 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.105.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.103rc3.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/top_level.txt +0 -0
@@ -1,18 +1,21 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.707737 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.exception
|
12
11
|
import io
|
13
12
|
import metaflow.plugins.datatools.s3.s3
|
13
|
+
import metaflow.exception
|
14
14
|
|
15
15
|
class RangeInfo(tuple, metaclass=type):
|
16
|
+
"""
|
17
|
+
RangeInfo(total_size, request_offset, request_length)
|
18
|
+
"""
|
16
19
|
@staticmethod
|
17
20
|
def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
|
18
21
|
"""
|
@@ -34,6 +37,59 @@ class RangeInfo(tuple, metaclass=type):
|
|
34
37
|
...
|
35
38
|
|
36
39
|
class S3(object, metaclass=type):
|
40
|
+
"""
|
41
|
+
The Metaflow S3 client.
|
42
|
+
|
43
|
+
This object manages the connection to S3 and a temporary diretory that is used
|
44
|
+
to download objects. Note that in most cases when the data fits in memory, no local
|
45
|
+
disk IO is needed as operations are cached by the operating system, which makes
|
46
|
+
operations fast as long as there is enough memory available.
|
47
|
+
|
48
|
+
The easiest way is to use this object as a context manager:
|
49
|
+
```
|
50
|
+
with S3() as s3:
|
51
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
52
|
+
print(data)
|
53
|
+
```
|
54
|
+
The context manager takes care of creating and deleting a temporary directory
|
55
|
+
automatically. Without a context manager, you must call `.close()` to delete
|
56
|
+
the directory explicitly:
|
57
|
+
```
|
58
|
+
s3 = S3()
|
59
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
60
|
+
s3.close()
|
61
|
+
```
|
62
|
+
You can customize the location of the temporary directory with `tmproot`. It
|
63
|
+
defaults to the current working directory.
|
64
|
+
|
65
|
+
To make it easier to deal with object locations, the client can be initialized
|
66
|
+
with an S3 path prefix. There are three ways to handle locations:
|
67
|
+
|
68
|
+
1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
|
69
|
+
initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
|
70
|
+
with the current run ID. This mode makes it easy to version data based
|
71
|
+
on the run ID consistently. You can use the `bucket` and `prefix` to
|
72
|
+
override parts of `DATATOOLS_S3ROOT`.
|
73
|
+
|
74
|
+
2. Specify an S3 prefix explicitly with `s3root`,
|
75
|
+
e.g. `S3(s3root='s3://mybucket/some/path')`.
|
76
|
+
|
77
|
+
3. Specify nothing, i.e. `S3()`, in which case all operations require
|
78
|
+
a full S3 url prefixed with `s3://`.
|
79
|
+
|
80
|
+
Parameters
|
81
|
+
----------
|
82
|
+
tmproot : str, default: '.'
|
83
|
+
Where to store the temporary directory.
|
84
|
+
bucket : str, optional
|
85
|
+
Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
|
86
|
+
prefix : str, optional
|
87
|
+
Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
|
88
|
+
run : FlowSpec or Run, optional
|
89
|
+
Derive path prefix from the current or a past run ID, e.g. S3(run=self).
|
90
|
+
s3root : str, optional
|
91
|
+
If `run` is not specified, use this as the S3 prefix.
|
92
|
+
"""
|
37
93
|
@classmethod
|
38
94
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
39
95
|
...
|
@@ -325,6 +381,9 @@ class S3(object, metaclass=type):
|
|
325
381
|
...
|
326
382
|
|
327
383
|
class S3GetObject(tuple, metaclass=type):
|
384
|
+
"""
|
385
|
+
S3GetObject(key, offset, length)
|
386
|
+
"""
|
328
387
|
@staticmethod
|
329
388
|
def __new__(_cls, key: str, offset: int, length: int):
|
330
389
|
"""
|
@@ -346,6 +405,13 @@ class S3GetObject(tuple, metaclass=type):
|
|
346
405
|
...
|
347
406
|
|
348
407
|
class S3Object(object, metaclass=type):
|
408
|
+
"""
|
409
|
+
This object represents a path or an object in S3,
|
410
|
+
with an optional local copy.
|
411
|
+
|
412
|
+
`S3Object`s are not instantiated directly, but they are returned
|
413
|
+
by many methods of the `S3` client.
|
414
|
+
"""
|
349
415
|
def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[metaflow.plugins.datatools.s3.s3.RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
|
350
416
|
...
|
351
417
|
@property
|
@@ -549,6 +615,9 @@ class S3Object(object, metaclass=type):
|
|
549
615
|
...
|
550
616
|
|
551
617
|
class S3PutObject(tuple, metaclass=type):
|
618
|
+
"""
|
619
|
+
S3PutObject(key, value, path, content_type, encryption, metadata)
|
620
|
+
"""
|
552
621
|
@staticmethod
|
553
622
|
def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
|
554
623
|
"""
|
@@ -1,24 +1,32 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.665728 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
+
import metaflow.metaflow_current
|
11
12
|
import metaflow.flowspec
|
12
13
|
import typing
|
13
|
-
import metaflow.exception
|
14
|
-
import metaflow.datastore.inputs
|
15
|
-
import metaflow.metaflow_current
|
16
14
|
import io
|
15
|
+
import metaflow.datastore.inputs
|
17
16
|
import metaflow.plugins.datatools.s3.s3
|
17
|
+
import metaflow.exception
|
18
18
|
|
19
19
|
TYPE_CHECKING: bool
|
20
20
|
|
21
21
|
class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
|
22
|
+
"""
|
23
|
+
Main class from which all Flows should inherit.
|
24
|
+
|
25
|
+
Attributes
|
26
|
+
----------
|
27
|
+
index
|
28
|
+
input
|
29
|
+
"""
|
22
30
|
def __init__(self, use_cli = True):
|
23
31
|
"""
|
24
32
|
Construct a FlowSpec
|
@@ -290,6 +298,9 @@ def ensure_unicode(x):
|
|
290
298
|
...
|
291
299
|
|
292
300
|
class S3GetObject(tuple, metaclass=type):
|
301
|
+
"""
|
302
|
+
S3GetObject(key, offset, length)
|
303
|
+
"""
|
293
304
|
@staticmethod
|
294
305
|
def __new__(_cls, key: str, offset: int, length: int):
|
295
306
|
"""
|
@@ -311,6 +322,9 @@ class S3GetObject(tuple, metaclass=type):
|
|
311
322
|
...
|
312
323
|
|
313
324
|
class S3PutObject(tuple, metaclass=type):
|
325
|
+
"""
|
326
|
+
S3PutObject(key, value, path, content_type, encryption, metadata)
|
327
|
+
"""
|
314
328
|
@staticmethod
|
315
329
|
def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
|
316
330
|
"""
|
@@ -332,6 +346,9 @@ class S3PutObject(tuple, metaclass=type):
|
|
332
346
|
...
|
333
347
|
|
334
348
|
class RangeInfo(tuple, metaclass=type):
|
349
|
+
"""
|
350
|
+
RangeInfo(total_size, request_offset, request_length)
|
351
|
+
"""
|
335
352
|
@staticmethod
|
336
353
|
def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
|
337
354
|
"""
|
@@ -371,6 +388,13 @@ class MetaflowS3InvalidRange(metaflow.exception.MetaflowException, metaclass=typ
|
|
371
388
|
...
|
372
389
|
|
373
390
|
class S3Object(object, metaclass=type):
|
391
|
+
"""
|
392
|
+
This object represents a path or an object in S3,
|
393
|
+
with an optional local copy.
|
394
|
+
|
395
|
+
`S3Object`s are not instantiated directly, but they are returned
|
396
|
+
by many methods of the `S3` client.
|
397
|
+
"""
|
374
398
|
def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
|
375
399
|
...
|
376
400
|
@property
|
@@ -587,6 +611,59 @@ class S3Client(object, metaclass=type):
|
|
587
611
|
...
|
588
612
|
|
589
613
|
class S3(object, metaclass=type):
|
614
|
+
"""
|
615
|
+
The Metaflow S3 client.
|
616
|
+
|
617
|
+
This object manages the connection to S3 and a temporary diretory that is used
|
618
|
+
to download objects. Note that in most cases when the data fits in memory, no local
|
619
|
+
disk IO is needed as operations are cached by the operating system, which makes
|
620
|
+
operations fast as long as there is enough memory available.
|
621
|
+
|
622
|
+
The easiest way is to use this object as a context manager:
|
623
|
+
```
|
624
|
+
with S3() as s3:
|
625
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
626
|
+
print(data)
|
627
|
+
```
|
628
|
+
The context manager takes care of creating and deleting a temporary directory
|
629
|
+
automatically. Without a context manager, you must call `.close()` to delete
|
630
|
+
the directory explicitly:
|
631
|
+
```
|
632
|
+
s3 = S3()
|
633
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
634
|
+
s3.close()
|
635
|
+
```
|
636
|
+
You can customize the location of the temporary directory with `tmproot`. It
|
637
|
+
defaults to the current working directory.
|
638
|
+
|
639
|
+
To make it easier to deal with object locations, the client can be initialized
|
640
|
+
with an S3 path prefix. There are three ways to handle locations:
|
641
|
+
|
642
|
+
1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
|
643
|
+
initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
|
644
|
+
with the current run ID. This mode makes it easy to version data based
|
645
|
+
on the run ID consistently. You can use the `bucket` and `prefix` to
|
646
|
+
override parts of `DATATOOLS_S3ROOT`.
|
647
|
+
|
648
|
+
2. Specify an S3 prefix explicitly with `s3root`,
|
649
|
+
e.g. `S3(s3root='s3://mybucket/some/path')`.
|
650
|
+
|
651
|
+
3. Specify nothing, i.e. `S3()`, in which case all operations require
|
652
|
+
a full S3 url prefixed with `s3://`.
|
653
|
+
|
654
|
+
Parameters
|
655
|
+
----------
|
656
|
+
tmproot : str, default: '.'
|
657
|
+
Where to store the temporary directory.
|
658
|
+
bucket : str, optional
|
659
|
+
Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
|
660
|
+
prefix : str, optional
|
661
|
+
Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
|
662
|
+
run : FlowSpec or Run, optional
|
663
|
+
Derive path prefix from the current or a past run ID, e.g. S3(run=self).
|
664
|
+
s3root : str, optional
|
665
|
+
If `run` is not specified, use this as the S3 prefix.
|
666
|
+
"""
|
590
667
|
@classmethod
|
591
668
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
592
669
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.722422 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.681694 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.673410 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.673629 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.672000 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -11,6 +11,14 @@ if typing.TYPE_CHECKING:
|
|
11
11
|
import metaflow.decorators
|
12
12
|
|
13
13
|
class EnvironmentDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
14
|
+
"""
|
15
|
+
Specifies environment variables to be set prior to the execution of a step.
|
16
|
+
|
17
|
+
Parameters
|
18
|
+
----------
|
19
|
+
vars : Dict[str, str], default {}
|
20
|
+
Dictionary of environment variables to set.
|
21
|
+
"""
|
14
22
|
def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
|
15
23
|
...
|
16
24
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.672590 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -21,11 +21,115 @@ class MetaflowException(Exception, metaclass=type):
|
|
21
21
|
...
|
22
22
|
|
23
23
|
class TriggerDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
24
|
+
"""
|
25
|
+
Specifies the event(s) that this flow depends on.
|
26
|
+
|
27
|
+
```
|
28
|
+
@trigger(event='foo')
|
29
|
+
```
|
30
|
+
or
|
31
|
+
```
|
32
|
+
@trigger(events=['foo', 'bar'])
|
33
|
+
```
|
34
|
+
|
35
|
+
Additionally, you can specify the parameter mappings
|
36
|
+
to map event payload to Metaflow parameters for the flow.
|
37
|
+
```
|
38
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
39
|
+
```
|
40
|
+
or
|
41
|
+
```
|
42
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
43
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
44
|
+
```
|
45
|
+
|
46
|
+
'parameters' can also be a list of strings and tuples like so:
|
47
|
+
```
|
48
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
49
|
+
```
|
50
|
+
This is equivalent to:
|
51
|
+
```
|
52
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
53
|
+
```
|
54
|
+
|
55
|
+
Parameters
|
56
|
+
----------
|
57
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
58
|
+
Event dependency for this flow.
|
59
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
60
|
+
Events dependency for this flow.
|
61
|
+
options : Dict[str, Any], default {}
|
62
|
+
Backend-specific configuration for tuning eventing behavior.
|
63
|
+
|
64
|
+
MF Add To Current
|
65
|
+
-----------------
|
66
|
+
trigger -> metaflow.events.Trigger
|
67
|
+
Returns `Trigger` if the current run is triggered by an event
|
68
|
+
|
69
|
+
@@ Returns
|
70
|
+
-------
|
71
|
+
Trigger
|
72
|
+
`Trigger` if triggered by an event
|
73
|
+
"""
|
24
74
|
def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
|
25
75
|
...
|
26
76
|
...
|
27
77
|
|
28
78
|
class TriggerOnFinishDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
79
|
+
"""
|
80
|
+
Specifies the flow(s) that this flow depends on.
|
81
|
+
|
82
|
+
```
|
83
|
+
@trigger_on_finish(flow='FooFlow')
|
84
|
+
```
|
85
|
+
or
|
86
|
+
```
|
87
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
88
|
+
```
|
89
|
+
This decorator respects the @project decorator and triggers the flow
|
90
|
+
when upstream runs within the same namespace complete successfully
|
91
|
+
|
92
|
+
Additionally, you can specify project aware upstream flow dependencies
|
93
|
+
by specifying the fully qualified project_flow_name.
|
94
|
+
```
|
95
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
96
|
+
```
|
97
|
+
or
|
98
|
+
```
|
99
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
100
|
+
```
|
101
|
+
|
102
|
+
You can also specify just the project or project branch (other values will be
|
103
|
+
inferred from the current project or project branch):
|
104
|
+
```
|
105
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
106
|
+
```
|
107
|
+
|
108
|
+
Note that `branch` is typically one of:
|
109
|
+
- `prod`
|
110
|
+
- `user.bob`
|
111
|
+
- `test.my_experiment`
|
112
|
+
- `prod.staging`
|
113
|
+
|
114
|
+
Parameters
|
115
|
+
----------
|
116
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
117
|
+
Upstream flow dependency for this flow.
|
118
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
119
|
+
Upstream flow dependencies for this flow.
|
120
|
+
options : Dict[str, Any], default {}
|
121
|
+
Backend-specific configuration for tuning eventing behavior.
|
122
|
+
|
123
|
+
MF Add To Current
|
124
|
+
-----------------
|
125
|
+
trigger -> metaflow.events.Trigger
|
126
|
+
Returns `Trigger` if the current run is triggered by an event
|
127
|
+
|
128
|
+
@@ Returns
|
129
|
+
-------
|
130
|
+
Trigger
|
131
|
+
`Trigger` if triggered by an event
|
132
|
+
"""
|
29
133
|
def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
|
30
134
|
...
|
31
135
|
def get_top_level_options(self):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.671217 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.709378 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -15,6 +15,27 @@ if typing.TYPE_CHECKING:
|
|
15
15
|
current: metaflow.metaflow_current.Current
|
16
16
|
|
17
17
|
class ParallelDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
18
|
+
"""
|
19
|
+
MF Add To Current
|
20
|
+
-----------------
|
21
|
+
parallel -> metaflow.metaflow_current.Parallel
|
22
|
+
|
23
|
+
@@ Returns
|
24
|
+
-------
|
25
|
+
Parallel
|
26
|
+
`namedtuple` with the following fields:
|
27
|
+
- main_ip : str
|
28
|
+
The IP address of the control task.
|
29
|
+
- num_nodes : int
|
30
|
+
The total number of tasks created by @parallel
|
31
|
+
- node_index : int
|
32
|
+
The index of the current task in all the @parallel tasks.
|
33
|
+
- control_task_id : Optional[str]
|
34
|
+
The task ID of the control task. Available to all tasks.
|
35
|
+
|
36
|
+
is_parallel -> bool
|
37
|
+
True if the current step is a @parallel step.
|
38
|
+
"""
|
18
39
|
def __init__(self, attributes = None, statically_defined = False):
|
19
40
|
...
|
20
41
|
def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.670764 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.708947 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,8 +9,8 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.plugins.secrets
|
12
|
-
import abc
|
13
12
|
import metaflow.exception
|
13
|
+
import abc
|
14
14
|
|
15
15
|
class MetaflowException(Exception, metaclass=type):
|
16
16
|
def __init__(self, msg = "", lineno = None):
|
@@ -34,15 +34,27 @@ def get_credentials(scopes, *args, **kwargs):
|
|
34
34
|
GCP_SECRET_MANAGER_PREFIX: None
|
35
35
|
|
36
36
|
class MetaflowGcpSecretsManagerBadResponse(metaflow.exception.MetaflowException, metaclass=type):
|
37
|
+
"""
|
38
|
+
Raised when the response from GCP Secrets Manager is not valid in some way
|
39
|
+
"""
|
37
40
|
...
|
38
41
|
|
39
42
|
class MetaflowGcpSecretsManagerDuplicateKey(metaflow.exception.MetaflowException, metaclass=type):
|
43
|
+
"""
|
44
|
+
Raised when the response from GCP Secrets Manager contains duplicate keys
|
45
|
+
"""
|
40
46
|
...
|
41
47
|
|
42
48
|
class MetaflowGcpSecretsManagerJSONParseError(metaflow.exception.MetaflowException, metaclass=type):
|
49
|
+
"""
|
50
|
+
Raised when the SecretString response from GCP Secrets Manager is not valid JSON
|
51
|
+
"""
|
43
52
|
...
|
44
53
|
|
45
54
|
class MetaflowGcpSecretsManagerNotJSONObject(metaflow.exception.MetaflowException, metaclass=type):
|
55
|
+
"""
|
56
|
+
Raised when the SecretString response from GCP Secrets Manager is not valid JSON dictionary
|
57
|
+
"""
|
46
58
|
...
|
47
59
|
|
48
60
|
class GcpSecretManagerSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.708215 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.708027 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.708498 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.666742 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.668278 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.703507 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|