nextmv 0.35.0.dev1__py3-none-any.whl → 0.36.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nextmv/__about__.py +1 -1
- nextmv/__init__.py +1 -0
- nextmv/_serialization.py +3 -3
- nextmv/base_model.py +2 -2
- nextmv/cloud/acceptance_test.py +4 -5
- nextmv/cloud/application.py +116 -109
- nextmv/cloud/batch_experiment.py +17 -17
- nextmv/cloud/client.py +13 -13
- nextmv/cloud/ensemble.py +2 -3
- nextmv/cloud/input_set.py +7 -8
- nextmv/cloud/instance.py +3 -4
- nextmv/cloud/package.py +18 -12
- nextmv/cloud/scenario.py +6 -10
- nextmv/default_app/README.md +2 -2
- nextmv/input.py +38 -44
- nextmv/local/application.py +38 -38
- nextmv/local/executor.py +47 -17
- nextmv/local/runner.py +10 -10
- nextmv/manifest.py +76 -94
- nextmv/model.py +6 -6
- nextmv/options.py +12 -12
- nextmv/output.py +46 -61
- nextmv/polling.py +2 -2
- nextmv/run.py +58 -58
- {nextmv-0.35.0.dev1.dist-info → nextmv-0.36.0.dist-info}/METADATA +3 -4
- nextmv-0.36.0.dist-info/RECORD +49 -0
- nextmv/default_app/src/main.py +0 -37
- nextmv-0.35.0.dev1.dist-info/RECORD +0 -50
- {nextmv-0.35.0.dev1.dist-info → nextmv-0.36.0.dist-info}/WHEEL +0 -0
- {nextmv-0.35.0.dev1.dist-info → nextmv-0.36.0.dist-info}/licenses/LICENSE +0 -0
nextmv/cloud/client.py
CHANGED
|
@@ -16,7 +16,7 @@ get_size(obj)
|
|
|
16
16
|
|
|
17
17
|
import os
|
|
18
18
|
from dataclasses import dataclass, field
|
|
19
|
-
from typing import IO, Any
|
|
19
|
+
from typing import IO, Any
|
|
20
20
|
from urllib.parse import urljoin
|
|
21
21
|
|
|
22
22
|
import requests
|
|
@@ -98,7 +98,7 @@ class Client:
|
|
|
98
98
|
>>> print(response.json())
|
|
99
99
|
"""
|
|
100
100
|
|
|
101
|
-
api_key:
|
|
101
|
+
api_key: str | None = None
|
|
102
102
|
"""API key to use for authenticating with the Nextmv Cloud API. If not
|
|
103
103
|
provided, the client will look for the NEXTMV_API_KEY environment
|
|
104
104
|
variable."""
|
|
@@ -117,7 +117,7 @@ class Client:
|
|
|
117
117
|
seconds."""
|
|
118
118
|
configuration_file: str = "~/.nextmv/config.yaml"
|
|
119
119
|
"""Path to the configuration file used by the Nextmv CLI."""
|
|
120
|
-
headers:
|
|
120
|
+
headers: dict[str, str] | None = None
|
|
121
121
|
"""Headers to use for requests to the Nextmv Cloud API."""
|
|
122
122
|
max_retries: int = 10
|
|
123
123
|
"""Maximum number of retries to use for requests to the Nextmv Cloud
|
|
@@ -196,11 +196,11 @@ class Client:
|
|
|
196
196
|
self,
|
|
197
197
|
method: str,
|
|
198
198
|
endpoint: str,
|
|
199
|
-
data:
|
|
200
|
-
headers:
|
|
201
|
-
payload:
|
|
202
|
-
query_params:
|
|
203
|
-
json_configurations:
|
|
199
|
+
data: Any | None = None,
|
|
200
|
+
headers: dict[str, str] | None = None,
|
|
201
|
+
payload: dict[str, Any] | None = None,
|
|
202
|
+
query_params: dict[str, Any] | None = None,
|
|
203
|
+
json_configurations: dict[str, Any] | None = None,
|
|
204
204
|
) -> requests.Response:
|
|
205
205
|
"""
|
|
206
206
|
Makes a request to the Nextmv Cloud API.
|
|
@@ -324,10 +324,10 @@ class Client:
|
|
|
324
324
|
|
|
325
325
|
def upload_to_presigned_url(
|
|
326
326
|
self,
|
|
327
|
-
data:
|
|
327
|
+
data: dict[str, Any] | str | None,
|
|
328
328
|
url: str,
|
|
329
|
-
json_configurations:
|
|
330
|
-
tar_file:
|
|
329
|
+
json_configurations: dict[str, Any] | None = None,
|
|
330
|
+
tar_file: str | None = None,
|
|
331
331
|
) -> None:
|
|
332
332
|
"""
|
|
333
333
|
Uploads data to a presigned URL.
|
|
@@ -369,7 +369,7 @@ class Client:
|
|
|
369
369
|
>>> client.upload_to_presigned_url(data=input_data, url="PRE_SIGNED_URL") # doctest: +SKIP
|
|
370
370
|
"""
|
|
371
371
|
|
|
372
|
-
upload_data:
|
|
372
|
+
upload_data: str | None = None
|
|
373
373
|
if data is not None:
|
|
374
374
|
if isinstance(data, dict):
|
|
375
375
|
upload_data = deflated_serialize_json(data, json_configurations=json_configurations)
|
|
@@ -433,7 +433,7 @@ class Client:
|
|
|
433
433
|
}
|
|
434
434
|
|
|
435
435
|
|
|
436
|
-
def get_size(obj:
|
|
436
|
+
def get_size(obj: dict[str, Any] | IO[bytes] | str, json_configurations: dict[str, Any] | None = None) -> int:
|
|
437
437
|
"""
|
|
438
438
|
Finds the size of an object in bytes.
|
|
439
439
|
|
nextmv/cloud/ensemble.py
CHANGED
|
@@ -22,7 +22,6 @@ EnsembleDefinition
|
|
|
22
22
|
|
|
23
23
|
from datetime import datetime
|
|
24
24
|
from enum import Enum
|
|
25
|
-
from typing import Optional
|
|
26
25
|
|
|
27
26
|
from nextmv.base_model import BaseModel
|
|
28
27
|
|
|
@@ -56,9 +55,9 @@ class RunGroup(BaseModel):
|
|
|
56
55
|
"""The unique identifier of the run group."""
|
|
57
56
|
instance_id: str
|
|
58
57
|
"""ID of the app instance that this run group executes on."""
|
|
59
|
-
options:
|
|
58
|
+
options: dict | None = None
|
|
60
59
|
"""Runtime options/parameters for the application."""
|
|
61
|
-
repetitions:
|
|
60
|
+
repetitions: int | None = None
|
|
62
61
|
"""The number of times the run is to be repeated on the instance and with
|
|
63
62
|
the options defined in the run group"""
|
|
64
63
|
|
nextmv/cloud/input_set.py
CHANGED
|
@@ -11,7 +11,6 @@ InputSet
|
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
13
|
from datetime import datetime
|
|
14
|
-
from typing import Optional
|
|
15
14
|
|
|
16
15
|
from nextmv.base_model import BaseModel
|
|
17
16
|
from nextmv.run import Format
|
|
@@ -59,19 +58,19 @@ class ManagedInput(BaseModel):
|
|
|
59
58
|
id: str
|
|
60
59
|
"""ID of the input."""
|
|
61
60
|
|
|
62
|
-
name:
|
|
61
|
+
name: str | None = None
|
|
63
62
|
"""Name of the input."""
|
|
64
|
-
description:
|
|
63
|
+
description: str | None = None
|
|
65
64
|
"""Description of the input."""
|
|
66
|
-
run_id:
|
|
65
|
+
run_id: str | None = None
|
|
67
66
|
"""ID of the run that created the input."""
|
|
68
|
-
upload_id:
|
|
67
|
+
upload_id: str | None = None
|
|
69
68
|
"""ID of the upload that created the input."""
|
|
70
|
-
format:
|
|
69
|
+
format: Format | None = None
|
|
71
70
|
"""Format of the input."""
|
|
72
|
-
created_at:
|
|
71
|
+
created_at: datetime | None = None
|
|
73
72
|
"""Creation time of the input."""
|
|
74
|
-
updated_at:
|
|
73
|
+
updated_at: datetime | None = None
|
|
75
74
|
"""Last update time of the input."""
|
|
76
75
|
|
|
77
76
|
|
nextmv/cloud/instance.py
CHANGED
|
@@ -13,7 +13,6 @@ Instance
|
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
from datetime import datetime
|
|
16
|
-
from typing import Optional
|
|
17
16
|
|
|
18
17
|
from nextmv.base_model import BaseModel
|
|
19
18
|
|
|
@@ -48,11 +47,11 @@ class InstanceConfiguration(BaseModel):
|
|
|
48
47
|
... )
|
|
49
48
|
"""
|
|
50
49
|
|
|
51
|
-
execution_class:
|
|
50
|
+
execution_class: str | None = None
|
|
52
51
|
"""Execution class for the instance."""
|
|
53
|
-
options:
|
|
52
|
+
options: dict | None = None
|
|
54
53
|
"""Options of the app that the instance uses."""
|
|
55
|
-
secrets_collection_id:
|
|
54
|
+
secrets_collection_id: str | None = None
|
|
56
55
|
"""ID of the secrets collection that the instance uses."""
|
|
57
56
|
|
|
58
57
|
|
nextmv/cloud/package.py
CHANGED
|
@@ -8,7 +8,6 @@ import shutil
|
|
|
8
8
|
import subprocess
|
|
9
9
|
import tarfile
|
|
10
10
|
import tempfile
|
|
11
|
-
from typing import Optional
|
|
12
11
|
|
|
13
12
|
from nextmv.logger import log
|
|
14
13
|
from nextmv.manifest import MANIFEST_FILE_NAME, Manifest, ManifestBuild, ManifestType
|
|
@@ -24,8 +23,8 @@ _MANDATORY_FILES_PER_TYPE = {
|
|
|
24
23
|
def _package(
|
|
25
24
|
app_dir: str,
|
|
26
25
|
manifest: Manifest,
|
|
27
|
-
model:
|
|
28
|
-
model_configuration:
|
|
26
|
+
model: Model | None = None,
|
|
27
|
+
model_configuration: ModelConfiguration | None = None,
|
|
29
28
|
verbose: bool = False,
|
|
30
29
|
) -> tuple[str, str]:
|
|
31
30
|
"""Package the app into a tarball."""
|
|
@@ -75,7 +74,7 @@ def _package(
|
|
|
75
74
|
|
|
76
75
|
def _run_build_command(
|
|
77
76
|
app_dir: str,
|
|
78
|
-
manifest_build:
|
|
77
|
+
manifest_build: ManifestBuild | None = None,
|
|
79
78
|
verbose: bool = False,
|
|
80
79
|
) -> None:
|
|
81
80
|
"""Run the build command specified in the manifest."""
|
|
@@ -118,7 +117,7 @@ def _get_shell_command_elements(pre_push_command):
|
|
|
118
117
|
|
|
119
118
|
def _run_pre_push_command(
|
|
120
119
|
app_dir: str,
|
|
121
|
-
pre_push_command:
|
|
120
|
+
pre_push_command: str | None = None,
|
|
122
121
|
verbose: bool = False,
|
|
123
122
|
) -> None:
|
|
124
123
|
"""Run the pre-push command specified in the manifest."""
|
|
@@ -205,8 +204,15 @@ def __find_files(
|
|
|
205
204
|
def __confirm_mandatory_files(manifest: Manifest, present_files: list[str]) -> None:
|
|
206
205
|
"""Confirm that all mandatory files are present in the given list of files."""
|
|
207
206
|
|
|
208
|
-
mandatory_files = _MANDATORY_FILES_PER_TYPE[manifest.type]
|
|
209
207
|
found_files = {os.path.normpath(file): True for file in present_files}
|
|
208
|
+
|
|
209
|
+
# Check for mandatory files (if a custom execution config is provided we check the
|
|
210
|
+
# custom entrypoint instead)
|
|
211
|
+
mandatory_files = []
|
|
212
|
+
if manifest.execution is None or manifest.execution.entrypoint is None:
|
|
213
|
+
mandatory_files = _MANDATORY_FILES_PER_TYPE[manifest.type]
|
|
214
|
+
else:
|
|
215
|
+
mandatory_files.append(os.path.normpath(manifest.execution.entrypoint))
|
|
210
216
|
missing_files = [file for file in mandatory_files if file not in found_files]
|
|
211
217
|
|
|
212
218
|
if missing_files:
|
|
@@ -217,8 +223,8 @@ def __handle_python(
|
|
|
217
223
|
app_dir: str,
|
|
218
224
|
temp_dir: str,
|
|
219
225
|
manifest: Manifest,
|
|
220
|
-
model:
|
|
221
|
-
model_configuration:
|
|
226
|
+
model: Model | None = None,
|
|
227
|
+
model_configuration: ModelConfiguration | None = None,
|
|
222
228
|
verbose: bool = False,
|
|
223
229
|
) -> None:
|
|
224
230
|
"""Handles the Python-specific packaging logic."""
|
|
@@ -413,10 +419,10 @@ def __confirm_python_version(output: str) -> None:
|
|
|
413
419
|
except ValueError:
|
|
414
420
|
major, minor = map(int, version.split("."))
|
|
415
421
|
|
|
416
|
-
if major == 3 and minor >=
|
|
422
|
+
if major == 3 and minor >= 10:
|
|
417
423
|
return
|
|
418
424
|
|
|
419
|
-
raise Exception("python version 3.
|
|
425
|
+
raise Exception("python version 3.10 or higher is required")
|
|
420
426
|
|
|
421
427
|
|
|
422
428
|
def __confirm_python_bundling_version(version: str) -> None:
|
|
@@ -425,9 +431,9 @@ def __confirm_python_bundling_version(version: str) -> None:
|
|
|
425
431
|
match = re_version.fullmatch(version)
|
|
426
432
|
if match:
|
|
427
433
|
major, minor = int(match.group(1)), int(match.group(2))
|
|
428
|
-
if major == 3 and minor >=
|
|
434
|
+
if major == 3 and minor >= 10:
|
|
429
435
|
return
|
|
430
|
-
raise Exception(f"python version 3.
|
|
436
|
+
raise Exception(f"python version 3.10 or higher is required for bundling, got {version}")
|
|
431
437
|
|
|
432
438
|
|
|
433
439
|
def __compress_tar(source: str, target: str) -> tuple[str, int]:
|
nextmv/cloud/scenario.py
CHANGED
|
@@ -15,7 +15,7 @@ Scenario
|
|
|
15
15
|
import itertools
|
|
16
16
|
from dataclasses import dataclass
|
|
17
17
|
from enum import Enum
|
|
18
|
-
from typing import Any
|
|
18
|
+
from typing import Any
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
@dataclass
|
|
@@ -158,11 +158,7 @@ class ScenarioInput:
|
|
|
158
158
|
Type of input for the scenario. This is used to determine how the input
|
|
159
159
|
should be processed.
|
|
160
160
|
"""
|
|
161
|
-
scenario_input_data:
|
|
162
|
-
str, # Input set ID
|
|
163
|
-
list[str], # List of Input IDs
|
|
164
|
-
list[dict[str, Any]], # Raw data
|
|
165
|
-
]
|
|
161
|
+
scenario_input_data: str | list[str] | list[dict[str, Any]]
|
|
166
162
|
"""
|
|
167
163
|
Input data for the scenario. This can be a single input set ID (`str`), a
|
|
168
164
|
list of input IDs (`list[str]`), or raw data (`list[dict[str, Any]]`).
|
|
@@ -252,12 +248,12 @@ class Scenario:
|
|
|
252
248
|
instance_id: str
|
|
253
249
|
"""ID of the instance to be used for the scenario."""
|
|
254
250
|
|
|
255
|
-
scenario_id:
|
|
251
|
+
scenario_id: str | None = None
|
|
256
252
|
"""
|
|
257
253
|
Optional ID of the scenario. The default value will be set as
|
|
258
254
|
`scenario-<index>` if not set.
|
|
259
255
|
"""
|
|
260
|
-
configuration:
|
|
256
|
+
configuration: list[ScenarioConfiguration] | None = None
|
|
261
257
|
"""Optional configuration for the scenario. Use this attribute to configure
|
|
262
258
|
variation of options for the scenario.
|
|
263
259
|
"""
|
|
@@ -300,8 +296,8 @@ class Scenario:
|
|
|
300
296
|
if self.configuration is None or len(self.configuration) == 0:
|
|
301
297
|
return [{}]
|
|
302
298
|
|
|
303
|
-
keys, value_lists = zip(*((config.name, config.values) for config in self.configuration))
|
|
304
|
-
combinations = [dict(zip(keys, values)) for values in itertools.product(*value_lists)]
|
|
299
|
+
keys, value_lists = zip(*((config.name, config.values) for config in self.configuration), strict=False)
|
|
300
|
+
combinations = [dict(zip(keys, values, strict=False)) for values in itertools.product(*value_lists)]
|
|
305
301
|
|
|
306
302
|
return combinations
|
|
307
303
|
|
nextmv/default_app/README.md
CHANGED
|
@@ -22,11 +22,11 @@ A sample input file is also provided as `input.json`.
|
|
|
22
22
|
1. Install packages.
|
|
23
23
|
|
|
24
24
|
```bash
|
|
25
|
-
|
|
25
|
+
pip install -r requirements.txt
|
|
26
26
|
```
|
|
27
27
|
|
|
28
28
|
2. Run the app.
|
|
29
29
|
|
|
30
30
|
```bash
|
|
31
|
-
cat input.json |
|
|
31
|
+
cat input.json | python main.py
|
|
32
32
|
```
|
nextmv/input.py
CHANGED
|
@@ -35,7 +35,7 @@ import sys
|
|
|
35
35
|
from collections.abc import Callable
|
|
36
36
|
from dataclasses import dataclass
|
|
37
37
|
from enum import Enum
|
|
38
|
-
from typing import Any
|
|
38
|
+
from typing import Any
|
|
39
39
|
|
|
40
40
|
from nextmv._serialization import serialize_json
|
|
41
41
|
from nextmv.deprecated import deprecated
|
|
@@ -155,17 +155,17 @@ class DataFile:
|
|
|
155
155
|
|
|
156
156
|
The `loader` function should return the data that will be used in the model.
|
|
157
157
|
"""
|
|
158
|
-
loader_kwargs:
|
|
158
|
+
loader_kwargs: dict[str, Any] | None = None
|
|
159
159
|
"""
|
|
160
160
|
Optional keyword arguments to pass to the loader function. This can be used
|
|
161
161
|
to customize the behavior of the loader.
|
|
162
162
|
"""
|
|
163
|
-
loader_args:
|
|
163
|
+
loader_args: list[Any] | None = None
|
|
164
164
|
"""
|
|
165
165
|
Optional positional arguments to pass to the loader function. This can be
|
|
166
166
|
used to customize the behavior of the loader.
|
|
167
167
|
"""
|
|
168
|
-
input_data_key:
|
|
168
|
+
input_data_key: str | None = None
|
|
169
169
|
"""
|
|
170
170
|
Use this parameter to set a custom key to represent your file.
|
|
171
171
|
|
|
@@ -180,8 +180,8 @@ class DataFile:
|
|
|
180
180
|
|
|
181
181
|
def json_data_file(
|
|
182
182
|
name: str,
|
|
183
|
-
json_configurations:
|
|
184
|
-
input_data_key:
|
|
183
|
+
json_configurations: dict[str, Any] | None = None,
|
|
184
|
+
input_data_key: str | None = None,
|
|
185
185
|
) -> DataFile:
|
|
186
186
|
"""
|
|
187
187
|
This is a convenience function to create a `DataFile` that reads JSON data.
|
|
@@ -231,7 +231,7 @@ def json_data_file(
|
|
|
231
231
|
|
|
232
232
|
json_configurations = json_configurations or {}
|
|
233
233
|
|
|
234
|
-
def loader(file_path: str) ->
|
|
234
|
+
def loader(file_path: str) -> dict[str, Any] | Any:
|
|
235
235
|
with open(file_path, encoding="utf-8") as f:
|
|
236
236
|
return json.load(f, **json_configurations)
|
|
237
237
|
|
|
@@ -244,8 +244,8 @@ def json_data_file(
|
|
|
244
244
|
|
|
245
245
|
def csv_data_file(
|
|
246
246
|
name: str,
|
|
247
|
-
csv_configurations:
|
|
248
|
-
input_data_key:
|
|
247
|
+
csv_configurations: dict[str, Any] | None = None,
|
|
248
|
+
input_data_key: str | None = None,
|
|
249
249
|
) -> DataFile:
|
|
250
250
|
"""
|
|
251
251
|
This is a convenience function to create a `DataFile` that reads CSV data.
|
|
@@ -306,7 +306,7 @@ def csv_data_file(
|
|
|
306
306
|
)
|
|
307
307
|
|
|
308
308
|
|
|
309
|
-
def text_data_file(name: str, input_data_key:
|
|
309
|
+
def text_data_file(name: str, input_data_key: str | None = None) -> DataFile:
|
|
310
310
|
"""
|
|
311
311
|
This is a convenience function to create a `DataFile` that reads utf-8
|
|
312
312
|
encoded text data.
|
|
@@ -408,13 +408,7 @@ class Input:
|
|
|
408
408
|
If the `input_format` is not one of the supported formats.
|
|
409
409
|
"""
|
|
410
410
|
|
|
411
|
-
data:
|
|
412
|
-
Union[dict[str, Any], Any], # JSON
|
|
413
|
-
str, # TEXT
|
|
414
|
-
list[dict[str, Any]], # CSV
|
|
415
|
-
dict[str, list[dict[str, Any]]], # CSV_ARCHIVE
|
|
416
|
-
dict[str, Any], # MULTI_FILE
|
|
417
|
-
]
|
|
411
|
+
data: dict[str, Any] | Any | str | list[dict[str, Any]] | dict[str, list[dict[str, Any]]] | dict[str, Any]
|
|
418
412
|
"""
|
|
419
413
|
The actual data.
|
|
420
414
|
|
|
@@ -427,14 +421,14 @@ class Input:
|
|
|
427
421
|
- For `MULTI_FILE`: `dict[str, Any]`
|
|
428
422
|
"""
|
|
429
423
|
|
|
430
|
-
input_format:
|
|
424
|
+
input_format: InputFormat | None = InputFormat.JSON
|
|
431
425
|
"""
|
|
432
426
|
Format of the input data.
|
|
433
427
|
|
|
434
428
|
Default is `InputFormat.JSON`.
|
|
435
429
|
"""
|
|
436
430
|
|
|
437
|
-
options:
|
|
431
|
+
options: Options | None = None
|
|
438
432
|
"""
|
|
439
433
|
Options that the `Input` was created with.
|
|
440
434
|
|
|
@@ -558,7 +552,7 @@ class InputLoader:
|
|
|
558
552
|
def load(
|
|
559
553
|
self,
|
|
560
554
|
input_format: InputFormat = InputFormat.JSON,
|
|
561
|
-
options:
|
|
555
|
+
options: Options | None = None,
|
|
562
556
|
*args,
|
|
563
557
|
**kwargs,
|
|
564
558
|
) -> Input:
|
|
@@ -636,7 +630,7 @@ class LocalInputLoader(InputLoader):
|
|
|
636
630
|
with open(path, encoding="utf-8") as f:
|
|
637
631
|
return f.read().rstrip("\n")
|
|
638
632
|
|
|
639
|
-
def _read_csv(path: str, csv_configurations:
|
|
633
|
+
def _read_csv(path: str, csv_configurations: dict[str, Any] | None) -> list[dict[str, Any]]:
|
|
640
634
|
"""
|
|
641
635
|
Read a CSV file and return its contents as a list of dictionaries.
|
|
642
636
|
|
|
@@ -655,7 +649,7 @@ class LocalInputLoader(InputLoader):
|
|
|
655
649
|
with open(path, encoding="utf-8") as f:
|
|
656
650
|
return list(csv.DictReader(f, **csv_configurations))
|
|
657
651
|
|
|
658
|
-
def _read_json(path: str, _) ->
|
|
652
|
+
def _read_json(path: str, _) -> dict[str, Any] | Any:
|
|
659
653
|
"""
|
|
660
654
|
Read a JSON file and return its parsed contents.
|
|
661
655
|
|
|
@@ -704,11 +698,11 @@ class LocalInputLoader(InputLoader):
|
|
|
704
698
|
|
|
705
699
|
def load(
|
|
706
700
|
self,
|
|
707
|
-
input_format:
|
|
708
|
-
options:
|
|
709
|
-
path:
|
|
710
|
-
csv_configurations:
|
|
711
|
-
data_files:
|
|
701
|
+
input_format: InputFormat | None = InputFormat.JSON,
|
|
702
|
+
options: Options | None = None,
|
|
703
|
+
path: str | None = None,
|
|
704
|
+
csv_configurations: dict[str, Any] | None = None,
|
|
705
|
+
data_files: list[DataFile] | None = None,
|
|
712
706
|
) -> Input:
|
|
713
707
|
"""
|
|
714
708
|
Load the input data. The input data can be in various formats. For
|
|
@@ -789,11 +783,11 @@ class LocalInputLoader(InputLoader):
|
|
|
789
783
|
|
|
790
784
|
def _load_utf8_encoded(
|
|
791
785
|
self,
|
|
792
|
-
csv_configurations:
|
|
793
|
-
path:
|
|
794
|
-
input_format:
|
|
786
|
+
csv_configurations: dict[str, Any] | None,
|
|
787
|
+
path: str | None = None,
|
|
788
|
+
input_format: InputFormat | None = InputFormat.JSON,
|
|
795
789
|
use_file_reader: bool = False,
|
|
796
|
-
) ->
|
|
790
|
+
) -> dict[str, Any] | str | list[dict[str, Any]]:
|
|
797
791
|
"""
|
|
798
792
|
Load a utf-8 encoded file from stdin or filesystem.
|
|
799
793
|
|
|
@@ -831,8 +825,8 @@ class LocalInputLoader(InputLoader):
|
|
|
831
825
|
|
|
832
826
|
def _load_archive(
|
|
833
827
|
self,
|
|
834
|
-
csv_configurations:
|
|
835
|
-
path:
|
|
828
|
+
csv_configurations: dict[str, Any] | None,
|
|
829
|
+
path: str | None = None,
|
|
836
830
|
) -> dict[str, list[dict[str, Any]]]:
|
|
837
831
|
"""
|
|
838
832
|
Load CSV files from a directory.
|
|
@@ -887,7 +881,7 @@ class LocalInputLoader(InputLoader):
|
|
|
887
881
|
def _load_multi_file(
|
|
888
882
|
self,
|
|
889
883
|
data_files: list[DataFile],
|
|
890
|
-
path:
|
|
884
|
+
path: str | None = None,
|
|
891
885
|
) -> dict[str, Any]:
|
|
892
886
|
"""
|
|
893
887
|
Load multiple files from a directory.
|
|
@@ -961,10 +955,10 @@ class LocalInputLoader(InputLoader):
|
|
|
961
955
|
|
|
962
956
|
|
|
963
957
|
def load_local(
|
|
964
|
-
input_format:
|
|
965
|
-
options:
|
|
966
|
-
path:
|
|
967
|
-
csv_configurations:
|
|
958
|
+
input_format: InputFormat | None = InputFormat.JSON,
|
|
959
|
+
options: Options | None = None,
|
|
960
|
+
path: str | None = None,
|
|
961
|
+
csv_configurations: dict[str, Any] | None = None,
|
|
968
962
|
) -> Input:
|
|
969
963
|
"""
|
|
970
964
|
!!! warning
|
|
@@ -1016,12 +1010,12 @@ _LOCAL_INPUT_LOADER = LocalInputLoader()
|
|
|
1016
1010
|
|
|
1017
1011
|
|
|
1018
1012
|
def load(
|
|
1019
|
-
input_format:
|
|
1020
|
-
options:
|
|
1021
|
-
path:
|
|
1022
|
-
csv_configurations:
|
|
1023
|
-
loader:
|
|
1024
|
-
data_files:
|
|
1013
|
+
input_format: InputFormat | None = InputFormat.JSON,
|
|
1014
|
+
options: Options | None = None,
|
|
1015
|
+
path: str | None = None,
|
|
1016
|
+
csv_configurations: dict[str, Any] | None = None,
|
|
1017
|
+
loader: InputLoader | None = _LOCAL_INPUT_LOADER,
|
|
1018
|
+
data_files: list[DataFile] | None = None,
|
|
1025
1019
|
) -> Input:
|
|
1026
1020
|
"""
|
|
1027
1021
|
Load input data using the specified loader.
|