ob-metaflow-stubs 6.0.3.104__py2.py3-none-any.whl → 6.0.3.106__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +853 -515
- metaflow-stubs/cards.pyi +211 -5
- metaflow-stubs/cli.pyi +22 -2
- metaflow-stubs/client/__init__.pyi +128 -3
- metaflow-stubs/client/core.pyi +226 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +5 -2
- metaflow-stubs/events.pyi +20 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +70 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +156 -3
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +8 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +22 -19
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +68 -2
- metaflow-stubs/plugins/__init__.pyi +14 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +44 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +44 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +32 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +102 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +63 -5
- metaflow-stubs/plugins/cards/card_client.pyi +34 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +41 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
- metaflow-stubs/plugins/cards/exception.pyi +8 -2
- metaflow-stubs/plugins/catch_decorator.pyi +19 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
- metaflow-stubs/plugins/datatools/local.pyi +16 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +83 -6
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +10 -2
- metaflow-stubs/plugins/events_decorator.pyi +106 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +11 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +99 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +59 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +33 -2
- metaflow-stubs/plugins/retry_decorator.pyi +21 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
- metaflow-stubs/plugins/storage_executor.pyi +6 -2
- metaflow-stubs/plugins/tag_cli.pyi +37 -6
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +70 -131
- metaflow-stubs/runner/metaflow_runner.pyi +118 -10
- metaflow-stubs/runner/nbdeploy.pyi +66 -2
- metaflow-stubs/runner/nbrun.pyi +79 -2
- metaflow-stubs/runner/subprocess_manager.pyi +16 -4
- metaflow-stubs/runner/utils.pyi +32 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.106.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.104.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.939404 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.957670 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -83,6 +83,13 @@ class WarningComponent(metaflow.plugins.cards.card_modules.basic.ErrorComponent,
|
|
83
83
|
...
|
84
84
|
|
85
85
|
class ComponentStore(object, metaclass=type):
|
86
|
+
"""
|
87
|
+
The `ComponentStore` object helps store the components for a single card in memory.
|
88
|
+
This class has combination of a array/dictionary like interfaces to access/change the stored components.
|
89
|
+
|
90
|
+
It exposes the `append` /`extend` methods (like an array) to add components.
|
91
|
+
It also exposes the `__getitem__`/`__setitem__` methods (like a dictionary) to access the components by their Ids.
|
92
|
+
"""
|
86
93
|
def __init__(self, logger, card_type = None, components = None, user_set_id = None):
|
87
94
|
...
|
88
95
|
@property
|
@@ -118,6 +125,33 @@ class ComponentStore(object, metaclass=type):
|
|
118
125
|
...
|
119
126
|
|
120
127
|
class CardComponentManager(object, metaclass=type):
|
128
|
+
"""
|
129
|
+
This class manages the card's state for a single card.
|
130
|
+
- It uses the `ComponentStore` to manage the storage of the components
|
131
|
+
- It exposes methods to add, remove and access the components.
|
132
|
+
- It exposes a `refresh` method that will allow refreshing a card with new data
|
133
|
+
for realtime(ish) updates.
|
134
|
+
- The `CardComponentCollector` exposes convenience methods similar to this class for a default
|
135
|
+
editable card. These methods include :
|
136
|
+
- `append`
|
137
|
+
- `extend`
|
138
|
+
- `clear`
|
139
|
+
- `refresh`
|
140
|
+
- `components`
|
141
|
+
- `__iter__`
|
142
|
+
|
143
|
+
## Usage Patterns :
|
144
|
+
|
145
|
+
```python
|
146
|
+
current.card["mycardid"].append(component, id="comp123")
|
147
|
+
current.card["mycardid"].extend([component])
|
148
|
+
current.card["mycardid"].refresh(data) # refreshes the card with new data
|
149
|
+
current.card["mycardid"].components["comp123"] # returns the component with id "comp123"
|
150
|
+
current.card["mycardid"].components["comp123"].update()
|
151
|
+
current.card["mycardid"].components.clear() # Wipe all the components
|
152
|
+
del current.card["mycardid"].components["mycomponentid"] # Delete a component
|
153
|
+
```
|
154
|
+
"""
|
121
155
|
def __init__(self, card_uuid, decorator_attributes, card_creator, components = None, logger = None, no_warnings = False, user_set_card_id = None, runtime_card = False, card_options = None, refresh_interval = 5):
|
122
156
|
...
|
123
157
|
def append(self, component, id = None):
|
@@ -136,6 +170,25 @@ class CardComponentManager(object, metaclass=type):
|
|
136
170
|
...
|
137
171
|
|
138
172
|
class CardComponentCollector(object, metaclass=type):
|
173
|
+
"""
|
174
|
+
This class helps collect `MetaflowCardComponent`s during runtime execution
|
175
|
+
|
176
|
+
### Usage with `current`
|
177
|
+
`current.card` is of type `CardComponentCollector`
|
178
|
+
|
179
|
+
### Main Usage TLDR
|
180
|
+
- [x] `current.card.append` customizes the default editable card.
|
181
|
+
- [x] Only one card can be default editable in a step.
|
182
|
+
- [x] The card class must have `ALLOW_USER_COMPONENTS=True` to be considered default editable.
|
183
|
+
- [x] Classes with `ALLOW_USER_COMPONENTS=False` are never default editable.
|
184
|
+
- [x] The user can specify an `id` argument to a card, in which case the card is editable through `current.card[id].append`.
|
185
|
+
- [x] A card with an id can be also default editable, if there are no other cards that are eligible to be default editable.
|
186
|
+
- [x] If multiple default-editable cards exist but only one card doesn't have an id, the card without an id is considered to be default editable.
|
187
|
+
- [x] If we can't resolve a single default editable card through the above rules, `current.card`.append calls show a warning but the call doesn't fail.
|
188
|
+
- [x] A card that is not default editable can be still edited through:
|
189
|
+
- [x] its `current.card['myid']`
|
190
|
+
- [x] by looking it up by its type, e.g. `current.card.get(type='pytorch')`.
|
191
|
+
"""
|
139
192
|
def __init__(self, logger = None, card_creator = None):
|
140
193
|
...
|
141
194
|
@staticmethod
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.940837 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -20,6 +20,9 @@ class MetaflowException(Exception, metaclass=type):
|
|
20
20
|
TYPE_CHECK_REGEX: str
|
21
21
|
|
22
22
|
class CardClassFoundException(metaflow.exception.MetaflowException, metaclass=type):
|
23
|
+
"""
|
24
|
+
This exception is raised with MetaflowCard class is not present for a particular card type.
|
25
|
+
"""
|
23
26
|
def __init__(self, card_name):
|
24
27
|
...
|
25
28
|
...
|
@@ -30,6 +33,9 @@ class TypeRequiredException(metaflow.exception.MetaflowException, metaclass=type
|
|
30
33
|
...
|
31
34
|
|
32
35
|
class CardNotPresentException(metaflow.exception.MetaflowException, metaclass=type):
|
36
|
+
"""
|
37
|
+
This exception is raised with a card is not present in the datastore.
|
38
|
+
"""
|
33
39
|
def __init__(self, pathspec, card_type = None, card_hash = None, card_id = None):
|
34
40
|
...
|
35
41
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.930084 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -44,6 +44,23 @@ class FailureHandledByCatch(metaflow.exception.MetaflowException, metaclass=type
|
|
44
44
|
...
|
45
45
|
|
46
46
|
class CatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
47
|
+
"""
|
48
|
+
Specifies that the step will success under all circumstances.
|
49
|
+
|
50
|
+
The decorator will create an optional artifact, specified by `var`, which
|
51
|
+
contains the exception raised. You can use it to detect the presence
|
52
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
53
|
+
are missing.
|
54
|
+
|
55
|
+
Parameters
|
56
|
+
----------
|
57
|
+
var : str, optional, default None
|
58
|
+
Name of the artifact in which to store the caught exception.
|
59
|
+
If not specified, the exception is not stored.
|
60
|
+
print_exception : bool, default True
|
61
|
+
Determines whether or not the exception is printed to
|
62
|
+
stdout when caught.
|
63
|
+
"""
|
47
64
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
48
65
|
...
|
49
66
|
def task_exception(self, exception, step, flow, graph, retry_count, max_user_code_retries):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.928799 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,8 +9,8 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.exception
|
12
|
-
import metaflow.plugins.datatools.s3.s3
|
13
12
|
import io
|
13
|
+
import metaflow.plugins.datatools.s3.s3
|
14
14
|
|
15
15
|
def read_in_chunks(dst, src, src_sz, max_chunk_size):
|
16
16
|
...
|
@@ -22,6 +22,13 @@ class MetaflowLocalURLException(metaflow.exception.MetaflowException, metaclass=
|
|
22
22
|
...
|
23
23
|
|
24
24
|
class Local(object, metaclass=type):
|
25
|
+
"""
|
26
|
+
This class allows you to access the local filesystem in a way similar to the S3 datatools
|
27
|
+
client. It is a stripped down version for now and only implements the functionality needed
|
28
|
+
for this use case.
|
29
|
+
|
30
|
+
In the future, we may want to allow it to be used in a way similar to the S3() client.
|
31
|
+
"""
|
25
32
|
@classmethod
|
26
33
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
27
34
|
...
|
@@ -47,6 +54,59 @@ class MetaflowS3Exception(metaflow.exception.MetaflowException, metaclass=type):
|
|
47
54
|
...
|
48
55
|
|
49
56
|
class S3(object, metaclass=type):
|
57
|
+
"""
|
58
|
+
The Metaflow S3 client.
|
59
|
+
|
60
|
+
This object manages the connection to S3 and a temporary diretory that is used
|
61
|
+
to download objects. Note that in most cases when the data fits in memory, no local
|
62
|
+
disk IO is needed as operations are cached by the operating system, which makes
|
63
|
+
operations fast as long as there is enough memory available.
|
64
|
+
|
65
|
+
The easiest way is to use this object as a context manager:
|
66
|
+
```
|
67
|
+
with S3() as s3:
|
68
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
69
|
+
print(data)
|
70
|
+
```
|
71
|
+
The context manager takes care of creating and deleting a temporary directory
|
72
|
+
automatically. Without a context manager, you must call `.close()` to delete
|
73
|
+
the directory explicitly:
|
74
|
+
```
|
75
|
+
s3 = S3()
|
76
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
77
|
+
s3.close()
|
78
|
+
```
|
79
|
+
You can customize the location of the temporary directory with `tmproot`. It
|
80
|
+
defaults to the current working directory.
|
81
|
+
|
82
|
+
To make it easier to deal with object locations, the client can be initialized
|
83
|
+
with an S3 path prefix. There are three ways to handle locations:
|
84
|
+
|
85
|
+
1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
|
86
|
+
initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
|
87
|
+
with the current run ID. This mode makes it easy to version data based
|
88
|
+
on the run ID consistently. You can use the `bucket` and `prefix` to
|
89
|
+
override parts of `DATATOOLS_S3ROOT`.
|
90
|
+
|
91
|
+
2. Specify an S3 prefix explicitly with `s3root`,
|
92
|
+
e.g. `S3(s3root='s3://mybucket/some/path')`.
|
93
|
+
|
94
|
+
3. Specify nothing, i.e. `S3()`, in which case all operations require
|
95
|
+
a full S3 url prefixed with `s3://`.
|
96
|
+
|
97
|
+
Parameters
|
98
|
+
----------
|
99
|
+
tmproot : str, default: '.'
|
100
|
+
Where to store the temporary directory.
|
101
|
+
bucket : str, optional
|
102
|
+
Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
|
103
|
+
prefix : str, optional
|
104
|
+
Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
|
105
|
+
run : FlowSpec or Run, optional
|
106
|
+
Derive path prefix from the current or a past run ID, e.g. S3(run=self).
|
107
|
+
s3root : str, optional
|
108
|
+
If `run` is not specified, use this as the S3 prefix.
|
109
|
+
"""
|
50
110
|
@classmethod
|
51
111
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
52
112
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.922519 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -28,6 +28,13 @@ class MetaflowLocalNotFound(metaflow.exception.MetaflowException, metaclass=type
|
|
28
28
|
...
|
29
29
|
|
30
30
|
class LocalObject(object, metaclass=type):
|
31
|
+
"""
|
32
|
+
This object represents a local object. It is a very thin wrapper
|
33
|
+
to allow it to be used in the same way as the S3Object (only as needed
|
34
|
+
in the IncludeFile use case)
|
35
|
+
|
36
|
+
Get or list calls return one or more of LocalObjects.
|
37
|
+
"""
|
31
38
|
def __init__(self, url, path):
|
32
39
|
...
|
33
40
|
@property
|
@@ -59,6 +66,13 @@ class LocalObject(object, metaclass=type):
|
|
59
66
|
...
|
60
67
|
|
61
68
|
class Local(object, metaclass=type):
|
69
|
+
"""
|
70
|
+
This class allows you to access the local filesystem in a way similar to the S3 datatools
|
71
|
+
client. It is a stripped down version for now and only implements the functionality needed
|
72
|
+
for this use case.
|
73
|
+
|
74
|
+
In the future, we may want to allow it to be used in a way similar to the S3() client.
|
75
|
+
"""
|
62
76
|
@classmethod
|
63
77
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
64
78
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.954629 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,10 +9,13 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.exception
|
12
|
-
import metaflow.plugins.datatools.s3.s3
|
13
12
|
import io
|
13
|
+
import metaflow.plugins.datatools.s3.s3
|
14
14
|
|
15
15
|
class RangeInfo(tuple, metaclass=type):
|
16
|
+
"""
|
17
|
+
RangeInfo(total_size, request_offset, request_length)
|
18
|
+
"""
|
16
19
|
@staticmethod
|
17
20
|
def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
|
18
21
|
"""
|
@@ -34,6 +37,59 @@ class RangeInfo(tuple, metaclass=type):
|
|
34
37
|
...
|
35
38
|
|
36
39
|
class S3(object, metaclass=type):
|
40
|
+
"""
|
41
|
+
The Metaflow S3 client.
|
42
|
+
|
43
|
+
This object manages the connection to S3 and a temporary diretory that is used
|
44
|
+
to download objects. Note that in most cases when the data fits in memory, no local
|
45
|
+
disk IO is needed as operations are cached by the operating system, which makes
|
46
|
+
operations fast as long as there is enough memory available.
|
47
|
+
|
48
|
+
The easiest way is to use this object as a context manager:
|
49
|
+
```
|
50
|
+
with S3() as s3:
|
51
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
52
|
+
print(data)
|
53
|
+
```
|
54
|
+
The context manager takes care of creating and deleting a temporary directory
|
55
|
+
automatically. Without a context manager, you must call `.close()` to delete
|
56
|
+
the directory explicitly:
|
57
|
+
```
|
58
|
+
s3 = S3()
|
59
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
60
|
+
s3.close()
|
61
|
+
```
|
62
|
+
You can customize the location of the temporary directory with `tmproot`. It
|
63
|
+
defaults to the current working directory.
|
64
|
+
|
65
|
+
To make it easier to deal with object locations, the client can be initialized
|
66
|
+
with an S3 path prefix. There are three ways to handle locations:
|
67
|
+
|
68
|
+
1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
|
69
|
+
initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
|
70
|
+
with the current run ID. This mode makes it easy to version data based
|
71
|
+
on the run ID consistently. You can use the `bucket` and `prefix` to
|
72
|
+
override parts of `DATATOOLS_S3ROOT`.
|
73
|
+
|
74
|
+
2. Specify an S3 prefix explicitly with `s3root`,
|
75
|
+
e.g. `S3(s3root='s3://mybucket/some/path')`.
|
76
|
+
|
77
|
+
3. Specify nothing, i.e. `S3()`, in which case all operations require
|
78
|
+
a full S3 url prefixed with `s3://`.
|
79
|
+
|
80
|
+
Parameters
|
81
|
+
----------
|
82
|
+
tmproot : str, default: '.'
|
83
|
+
Where to store the temporary directory.
|
84
|
+
bucket : str, optional
|
85
|
+
Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
|
86
|
+
prefix : str, optional
|
87
|
+
Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
|
88
|
+
run : FlowSpec or Run, optional
|
89
|
+
Derive path prefix from the current or a past run ID, e.g. S3(run=self).
|
90
|
+
s3root : str, optional
|
91
|
+
If `run` is not specified, use this as the S3 prefix.
|
92
|
+
"""
|
37
93
|
@classmethod
|
38
94
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
39
95
|
...
|
@@ -325,6 +381,9 @@ class S3(object, metaclass=type):
|
|
325
381
|
...
|
326
382
|
|
327
383
|
class S3GetObject(tuple, metaclass=type):
|
384
|
+
"""
|
385
|
+
S3GetObject(key, offset, length)
|
386
|
+
"""
|
328
387
|
@staticmethod
|
329
388
|
def __new__(_cls, key: str, offset: int, length: int):
|
330
389
|
"""
|
@@ -346,6 +405,13 @@ class S3GetObject(tuple, metaclass=type):
|
|
346
405
|
...
|
347
406
|
|
348
407
|
class S3Object(object, metaclass=type):
|
408
|
+
"""
|
409
|
+
This object represents a path or an object in S3,
|
410
|
+
with an optional local copy.
|
411
|
+
|
412
|
+
`S3Object`s are not instantiated directly, but they are returned
|
413
|
+
by many methods of the `S3` client.
|
414
|
+
"""
|
349
415
|
def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[metaflow.plugins.datatools.s3.s3.RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
|
350
416
|
...
|
351
417
|
@property
|
@@ -549,6 +615,9 @@ class S3Object(object, metaclass=type):
|
|
549
615
|
...
|
550
616
|
|
551
617
|
class S3PutObject(tuple, metaclass=type):
|
618
|
+
"""
|
619
|
+
S3PutObject(key, value, path, content_type, encryption, metadata)
|
620
|
+
"""
|
552
621
|
@staticmethod
|
553
622
|
def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
|
554
623
|
"""
|
@@ -1,24 +1,32 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.925677 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
+
import metaflow.exception
|
11
12
|
import metaflow.metaflow_current
|
12
|
-
import io
|
13
|
-
import metaflow.datastore.inputs
|
14
|
-
import metaflow.flowspec
|
15
13
|
import typing
|
16
|
-
import metaflow.exception
|
17
14
|
import metaflow.plugins.datatools.s3.s3
|
15
|
+
import metaflow.datastore.inputs
|
16
|
+
import metaflow.flowspec
|
17
|
+
import io
|
18
18
|
|
19
19
|
TYPE_CHECKING: bool
|
20
20
|
|
21
21
|
class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
|
22
|
+
"""
|
23
|
+
Main class from which all Flows should inherit.
|
24
|
+
|
25
|
+
Attributes
|
26
|
+
----------
|
27
|
+
index
|
28
|
+
input
|
29
|
+
"""
|
22
30
|
def __init__(self, use_cli = True):
|
23
31
|
"""
|
24
32
|
Construct a FlowSpec
|
@@ -290,6 +298,9 @@ def ensure_unicode(x):
|
|
290
298
|
...
|
291
299
|
|
292
300
|
class S3GetObject(tuple, metaclass=type):
|
301
|
+
"""
|
302
|
+
S3GetObject(key, offset, length)
|
303
|
+
"""
|
293
304
|
@staticmethod
|
294
305
|
def __new__(_cls, key: str, offset: int, length: int):
|
295
306
|
"""
|
@@ -311,6 +322,9 @@ class S3GetObject(tuple, metaclass=type):
|
|
311
322
|
...
|
312
323
|
|
313
324
|
class S3PutObject(tuple, metaclass=type):
|
325
|
+
"""
|
326
|
+
S3PutObject(key, value, path, content_type, encryption, metadata)
|
327
|
+
"""
|
314
328
|
@staticmethod
|
315
329
|
def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
|
316
330
|
"""
|
@@ -332,6 +346,9 @@ class S3PutObject(tuple, metaclass=type):
|
|
332
346
|
...
|
333
347
|
|
334
348
|
class RangeInfo(tuple, metaclass=type):
|
349
|
+
"""
|
350
|
+
RangeInfo(total_size, request_offset, request_length)
|
351
|
+
"""
|
335
352
|
@staticmethod
|
336
353
|
def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
|
337
354
|
"""
|
@@ -371,6 +388,13 @@ class MetaflowS3InvalidRange(metaflow.exception.MetaflowException, metaclass=typ
|
|
371
388
|
...
|
372
389
|
|
373
390
|
class S3Object(object, metaclass=type):
|
391
|
+
"""
|
392
|
+
This object represents a path or an object in S3,
|
393
|
+
with an optional local copy.
|
394
|
+
|
395
|
+
`S3Object`s are not instantiated directly, but they are returned
|
396
|
+
by many methods of the `S3` client.
|
397
|
+
"""
|
374
398
|
def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
|
375
399
|
...
|
376
400
|
@property
|
@@ -587,6 +611,59 @@ class S3Client(object, metaclass=type):
|
|
587
611
|
...
|
588
612
|
|
589
613
|
class S3(object, metaclass=type):
|
614
|
+
"""
|
615
|
+
The Metaflow S3 client.
|
616
|
+
|
617
|
+
This object manages the connection to S3 and a temporary diretory that is used
|
618
|
+
to download objects. Note that in most cases when the data fits in memory, no local
|
619
|
+
disk IO is needed as operations are cached by the operating system, which makes
|
620
|
+
operations fast as long as there is enough memory available.
|
621
|
+
|
622
|
+
The easiest way is to use this object as a context manager:
|
623
|
+
```
|
624
|
+
with S3() as s3:
|
625
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
626
|
+
print(data)
|
627
|
+
```
|
628
|
+
The context manager takes care of creating and deleting a temporary directory
|
629
|
+
automatically. Without a context manager, you must call `.close()` to delete
|
630
|
+
the directory explicitly:
|
631
|
+
```
|
632
|
+
s3 = S3()
|
633
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
634
|
+
s3.close()
|
635
|
+
```
|
636
|
+
You can customize the location of the temporary directory with `tmproot`. It
|
637
|
+
defaults to the current working directory.
|
638
|
+
|
639
|
+
To make it easier to deal with object locations, the client can be initialized
|
640
|
+
with an S3 path prefix. There are three ways to handle locations:
|
641
|
+
|
642
|
+
1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
|
643
|
+
initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
|
644
|
+
with the current run ID. This mode makes it easy to version data based
|
645
|
+
on the run ID consistently. You can use the `bucket` and `prefix` to
|
646
|
+
override parts of `DATATOOLS_S3ROOT`.
|
647
|
+
|
648
|
+
2. Specify an S3 prefix explicitly with `s3root`,
|
649
|
+
e.g. `S3(s3root='s3://mybucket/some/path')`.
|
650
|
+
|
651
|
+
3. Specify nothing, i.e. `S3()`, in which case all operations require
|
652
|
+
a full S3 url prefixed with `s3://`.
|
653
|
+
|
654
|
+
Parameters
|
655
|
+
----------
|
656
|
+
tmproot : str, default: '.'
|
657
|
+
Where to store the temporary directory.
|
658
|
+
bucket : str, optional
|
659
|
+
Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
|
660
|
+
prefix : str, optional
|
661
|
+
Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
|
662
|
+
run : FlowSpec or Run, optional
|
663
|
+
Derive path prefix from the current or a past run ID, e.g. S3(run=self).
|
664
|
+
s3root : str, optional
|
665
|
+
If `run` is not specified, use this as the S3 prefix.
|
666
|
+
"""
|
590
667
|
@classmethod
|
591
668
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
592
669
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.974266 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.941772 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.933395 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.933634 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.930468 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -11,6 +11,14 @@ if typing.TYPE_CHECKING:
|
|
11
11
|
import metaflow.decorators
|
12
12
|
|
13
13
|
class EnvironmentDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
14
|
+
"""
|
15
|
+
Specifies environment variables to be set prior to the execution of a step.
|
16
|
+
|
17
|
+
Parameters
|
18
|
+
----------
|
19
|
+
vars : Dict[str, str], default {}
|
20
|
+
Dictionary of environment variables to set.
|
21
|
+
"""
|
14
22
|
def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
|
15
23
|
...
|
16
24
|
...
|