nextmv 0.35.0__py3-none-any.whl → 0.35.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nextmv/__about__.py +1 -1
- nextmv/__init__.py +0 -1
- nextmv/_serialization.py +3 -3
- nextmv/base_model.py +2 -2
- nextmv/cloud/acceptance_test.py +5 -4
- nextmv/cloud/application.py +109 -116
- nextmv/cloud/batch_experiment.py +17 -17
- nextmv/cloud/client.py +13 -13
- nextmv/cloud/ensemble.py +3 -2
- nextmv/cloud/input_set.py +8 -7
- nextmv/cloud/instance.py +4 -3
- nextmv/cloud/package.py +12 -18
- nextmv/cloud/scenario.py +10 -6
- nextmv/input.py +44 -38
- nextmv/local/application.py +39 -39
- nextmv/local/executor.py +17 -47
- nextmv/local/runner.py +10 -10
- nextmv/manifest.py +79 -69
- nextmv/model.py +6 -6
- nextmv/options.py +12 -12
- nextmv/output.py +61 -46
- nextmv/polling.py +2 -2
- nextmv/run.py +58 -58
- {nextmv-0.35.0.dist-info → nextmv-0.35.0.dev0.dist-info}/METADATA +4 -3
- nextmv-0.35.0.dev0.dist-info/RECORD +50 -0
- nextmv-0.35.0.dist-info/RECORD +0 -50
- {nextmv-0.35.0.dist-info → nextmv-0.35.0.dev0.dist-info}/WHEEL +0 -0
- {nextmv-0.35.0.dist-info → nextmv-0.35.0.dev0.dist-info}/licenses/LICENSE +0 -0
nextmv/cloud/client.py
CHANGED
|
@@ -16,7 +16,7 @@ get_size(obj)
|
|
|
16
16
|
|
|
17
17
|
import os
|
|
18
18
|
from dataclasses import dataclass, field
|
|
19
|
-
from typing import IO, Any
|
|
19
|
+
from typing import IO, Any, Optional, Union
|
|
20
20
|
from urllib.parse import urljoin
|
|
21
21
|
|
|
22
22
|
import requests
|
|
@@ -98,7 +98,7 @@ class Client:
|
|
|
98
98
|
>>> print(response.json())
|
|
99
99
|
"""
|
|
100
100
|
|
|
101
|
-
api_key: str
|
|
101
|
+
api_key: Optional[str] = None
|
|
102
102
|
"""API key to use for authenticating with the Nextmv Cloud API. If not
|
|
103
103
|
provided, the client will look for the NEXTMV_API_KEY environment
|
|
104
104
|
variable."""
|
|
@@ -117,7 +117,7 @@ class Client:
|
|
|
117
117
|
seconds."""
|
|
118
118
|
configuration_file: str = "~/.nextmv/config.yaml"
|
|
119
119
|
"""Path to the configuration file used by the Nextmv CLI."""
|
|
120
|
-
headers: dict[str, str]
|
|
120
|
+
headers: Optional[dict[str, str]] = None
|
|
121
121
|
"""Headers to use for requests to the Nextmv Cloud API."""
|
|
122
122
|
max_retries: int = 10
|
|
123
123
|
"""Maximum number of retries to use for requests to the Nextmv Cloud
|
|
@@ -196,11 +196,11 @@ class Client:
|
|
|
196
196
|
self,
|
|
197
197
|
method: str,
|
|
198
198
|
endpoint: str,
|
|
199
|
-
data: Any
|
|
200
|
-
headers: dict[str, str]
|
|
201
|
-
payload: dict[str, Any]
|
|
202
|
-
query_params: dict[str, Any]
|
|
203
|
-
json_configurations: dict[str, Any]
|
|
199
|
+
data: Optional[Any] = None,
|
|
200
|
+
headers: Optional[dict[str, str]] = None,
|
|
201
|
+
payload: Optional[dict[str, Any]] = None,
|
|
202
|
+
query_params: Optional[dict[str, Any]] = None,
|
|
203
|
+
json_configurations: Optional[dict[str, Any]] = None,
|
|
204
204
|
) -> requests.Response:
|
|
205
205
|
"""
|
|
206
206
|
Makes a request to the Nextmv Cloud API.
|
|
@@ -324,10 +324,10 @@ class Client:
|
|
|
324
324
|
|
|
325
325
|
def upload_to_presigned_url(
|
|
326
326
|
self,
|
|
327
|
-
data: dict[str, Any]
|
|
327
|
+
data: Optional[Union[dict[str, Any], str]],
|
|
328
328
|
url: str,
|
|
329
|
-
json_configurations: dict[str, Any]
|
|
330
|
-
tar_file: str
|
|
329
|
+
json_configurations: Optional[dict[str, Any]] = None,
|
|
330
|
+
tar_file: Optional[str] = None,
|
|
331
331
|
) -> None:
|
|
332
332
|
"""
|
|
333
333
|
Uploads data to a presigned URL.
|
|
@@ -369,7 +369,7 @@ class Client:
|
|
|
369
369
|
>>> client.upload_to_presigned_url(data=input_data, url="PRE_SIGNED_URL") # doctest: +SKIP
|
|
370
370
|
"""
|
|
371
371
|
|
|
372
|
-
upload_data: str
|
|
372
|
+
upload_data: Optional[str] = None
|
|
373
373
|
if data is not None:
|
|
374
374
|
if isinstance(data, dict):
|
|
375
375
|
upload_data = deflated_serialize_json(data, json_configurations=json_configurations)
|
|
@@ -433,7 +433,7 @@ class Client:
|
|
|
433
433
|
}
|
|
434
434
|
|
|
435
435
|
|
|
436
|
-
def get_size(obj: dict[str, Any]
|
|
436
|
+
def get_size(obj: Union[dict[str, Any], IO[bytes], str], json_configurations: Optional[dict[str, Any]] = None) -> int:
|
|
437
437
|
"""
|
|
438
438
|
Finds the size of an object in bytes.
|
|
439
439
|
|
nextmv/cloud/ensemble.py
CHANGED
|
@@ -22,6 +22,7 @@ EnsembleDefinition
|
|
|
22
22
|
|
|
23
23
|
from datetime import datetime
|
|
24
24
|
from enum import Enum
|
|
25
|
+
from typing import Optional
|
|
25
26
|
|
|
26
27
|
from nextmv.base_model import BaseModel
|
|
27
28
|
|
|
@@ -55,9 +56,9 @@ class RunGroup(BaseModel):
|
|
|
55
56
|
"""The unique identifier of the run group."""
|
|
56
57
|
instance_id: str
|
|
57
58
|
"""ID of the app instance that this run group executes on."""
|
|
58
|
-
options: dict
|
|
59
|
+
options: Optional[dict] = None
|
|
59
60
|
"""Runtime options/parameters for the application."""
|
|
60
|
-
repetitions: int
|
|
61
|
+
repetitions: Optional[int] = None
|
|
61
62
|
"""The number of times the run is to be repeated on the instance and with
|
|
62
63
|
the options defined in the run group"""
|
|
63
64
|
|
nextmv/cloud/input_set.py
CHANGED
|
@@ -11,6 +11,7 @@ InputSet
|
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
13
|
from datetime import datetime
|
|
14
|
+
from typing import Optional
|
|
14
15
|
|
|
15
16
|
from nextmv.base_model import BaseModel
|
|
16
17
|
from nextmv.run import Format
|
|
@@ -58,19 +59,19 @@ class ManagedInput(BaseModel):
|
|
|
58
59
|
id: str
|
|
59
60
|
"""ID of the input."""
|
|
60
61
|
|
|
61
|
-
name: str
|
|
62
|
+
name: Optional[str] = None
|
|
62
63
|
"""Name of the input."""
|
|
63
|
-
description: str
|
|
64
|
+
description: Optional[str] = None
|
|
64
65
|
"""Description of the input."""
|
|
65
|
-
run_id: str
|
|
66
|
+
run_id: Optional[str] = None
|
|
66
67
|
"""ID of the run that created the input."""
|
|
67
|
-
upload_id: str
|
|
68
|
+
upload_id: Optional[str] = None
|
|
68
69
|
"""ID of the upload that created the input."""
|
|
69
|
-
format: Format
|
|
70
|
+
format: Optional[Format] = None
|
|
70
71
|
"""Format of the input."""
|
|
71
|
-
created_at: datetime
|
|
72
|
+
created_at: Optional[datetime] = None
|
|
72
73
|
"""Creation time of the input."""
|
|
73
|
-
updated_at: datetime
|
|
74
|
+
updated_at: Optional[datetime] = None
|
|
74
75
|
"""Last update time of the input."""
|
|
75
76
|
|
|
76
77
|
|
nextmv/cloud/instance.py
CHANGED
|
@@ -13,6 +13,7 @@ Instance
|
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
from datetime import datetime
|
|
16
|
+
from typing import Optional
|
|
16
17
|
|
|
17
18
|
from nextmv.base_model import BaseModel
|
|
18
19
|
|
|
@@ -47,11 +48,11 @@ class InstanceConfiguration(BaseModel):
|
|
|
47
48
|
... )
|
|
48
49
|
"""
|
|
49
50
|
|
|
50
|
-
execution_class: str
|
|
51
|
+
execution_class: Optional[str] = None
|
|
51
52
|
"""Execution class for the instance."""
|
|
52
|
-
options: dict
|
|
53
|
+
options: Optional[dict] = None
|
|
53
54
|
"""Options of the app that the instance uses."""
|
|
54
|
-
secrets_collection_id: str
|
|
55
|
+
secrets_collection_id: Optional[str] = None
|
|
55
56
|
"""ID of the secrets collection that the instance uses."""
|
|
56
57
|
|
|
57
58
|
|
nextmv/cloud/package.py
CHANGED
|
@@ -8,6 +8,7 @@ import shutil
|
|
|
8
8
|
import subprocess
|
|
9
9
|
import tarfile
|
|
10
10
|
import tempfile
|
|
11
|
+
from typing import Optional
|
|
11
12
|
|
|
12
13
|
from nextmv.logger import log
|
|
13
14
|
from nextmv.manifest import MANIFEST_FILE_NAME, Manifest, ManifestBuild, ManifestType
|
|
@@ -23,8 +24,8 @@ _MANDATORY_FILES_PER_TYPE = {
|
|
|
23
24
|
def _package(
|
|
24
25
|
app_dir: str,
|
|
25
26
|
manifest: Manifest,
|
|
26
|
-
model: Model
|
|
27
|
-
model_configuration: ModelConfiguration
|
|
27
|
+
model: Optional[Model] = None,
|
|
28
|
+
model_configuration: Optional[ModelConfiguration] = None,
|
|
28
29
|
verbose: bool = False,
|
|
29
30
|
) -> tuple[str, str]:
|
|
30
31
|
"""Package the app into a tarball."""
|
|
@@ -74,7 +75,7 @@ def _package(
|
|
|
74
75
|
|
|
75
76
|
def _run_build_command(
|
|
76
77
|
app_dir: str,
|
|
77
|
-
manifest_build: ManifestBuild
|
|
78
|
+
manifest_build: Optional[ManifestBuild] = None,
|
|
78
79
|
verbose: bool = False,
|
|
79
80
|
) -> None:
|
|
80
81
|
"""Run the build command specified in the manifest."""
|
|
@@ -117,7 +118,7 @@ def _get_shell_command_elements(pre_push_command):
|
|
|
117
118
|
|
|
118
119
|
def _run_pre_push_command(
|
|
119
120
|
app_dir: str,
|
|
120
|
-
pre_push_command: str
|
|
121
|
+
pre_push_command: Optional[str] = None,
|
|
121
122
|
verbose: bool = False,
|
|
122
123
|
) -> None:
|
|
123
124
|
"""Run the pre-push command specified in the manifest."""
|
|
@@ -204,15 +205,8 @@ def __find_files(
|
|
|
204
205
|
def __confirm_mandatory_files(manifest: Manifest, present_files: list[str]) -> None:
|
|
205
206
|
"""Confirm that all mandatory files are present in the given list of files."""
|
|
206
207
|
|
|
208
|
+
mandatory_files = _MANDATORY_FILES_PER_TYPE[manifest.type]
|
|
207
209
|
found_files = {os.path.normpath(file): True for file in present_files}
|
|
208
|
-
|
|
209
|
-
# Check for mandatory files (if a custom execution config is provided we check the
|
|
210
|
-
# custom entrypoint instead)
|
|
211
|
-
mandatory_files = []
|
|
212
|
-
if manifest.execution is None or manifest.execution.entrypoint is None:
|
|
213
|
-
mandatory_files = _MANDATORY_FILES_PER_TYPE[manifest.type]
|
|
214
|
-
else:
|
|
215
|
-
mandatory_files.append(os.path.normpath(manifest.execution.entrypoint))
|
|
216
210
|
missing_files = [file for file in mandatory_files if file not in found_files]
|
|
217
211
|
|
|
218
212
|
if missing_files:
|
|
@@ -223,8 +217,8 @@ def __handle_python(
|
|
|
223
217
|
app_dir: str,
|
|
224
218
|
temp_dir: str,
|
|
225
219
|
manifest: Manifest,
|
|
226
|
-
model: Model
|
|
227
|
-
model_configuration: ModelConfiguration
|
|
220
|
+
model: Optional[Model] = None,
|
|
221
|
+
model_configuration: Optional[ModelConfiguration] = None,
|
|
228
222
|
verbose: bool = False,
|
|
229
223
|
) -> None:
|
|
230
224
|
"""Handles the Python-specific packaging logic."""
|
|
@@ -419,10 +413,10 @@ def __confirm_python_version(output: str) -> None:
|
|
|
419
413
|
except ValueError:
|
|
420
414
|
major, minor = map(int, version.split("."))
|
|
421
415
|
|
|
422
|
-
if major == 3 and minor >=
|
|
416
|
+
if major == 3 and minor >= 9:
|
|
423
417
|
return
|
|
424
418
|
|
|
425
|
-
raise Exception("python version 3.
|
|
419
|
+
raise Exception("python version 3.9 or higher is required")
|
|
426
420
|
|
|
427
421
|
|
|
428
422
|
def __confirm_python_bundling_version(version: str) -> None:
|
|
@@ -431,9 +425,9 @@ def __confirm_python_bundling_version(version: str) -> None:
|
|
|
431
425
|
match = re_version.fullmatch(version)
|
|
432
426
|
if match:
|
|
433
427
|
major, minor = int(match.group(1)), int(match.group(2))
|
|
434
|
-
if major == 3 and minor >=
|
|
428
|
+
if major == 3 and minor >= 9:
|
|
435
429
|
return
|
|
436
|
-
raise Exception(f"python version 3.
|
|
430
|
+
raise Exception(f"python version 3.9 or higher is required for bundling, got {version}")
|
|
437
431
|
|
|
438
432
|
|
|
439
433
|
def __compress_tar(source: str, target: str) -> tuple[str, int]:
|
nextmv/cloud/scenario.py
CHANGED
|
@@ -15,7 +15,7 @@ Scenario
|
|
|
15
15
|
import itertools
|
|
16
16
|
from dataclasses import dataclass
|
|
17
17
|
from enum import Enum
|
|
18
|
-
from typing import Any
|
|
18
|
+
from typing import Any, Optional, Union
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
@dataclass
|
|
@@ -158,7 +158,11 @@ class ScenarioInput:
|
|
|
158
158
|
Type of input for the scenario. This is used to determine how the input
|
|
159
159
|
should be processed.
|
|
160
160
|
"""
|
|
161
|
-
scenario_input_data:
|
|
161
|
+
scenario_input_data: Union[
|
|
162
|
+
str, # Input set ID
|
|
163
|
+
list[str], # List of Input IDs
|
|
164
|
+
list[dict[str, Any]], # Raw data
|
|
165
|
+
]
|
|
162
166
|
"""
|
|
163
167
|
Input data for the scenario. This can be a single input set ID (`str`), a
|
|
164
168
|
list of input IDs (`list[str]`), or raw data (`list[dict[str, Any]]`).
|
|
@@ -248,12 +252,12 @@ class Scenario:
|
|
|
248
252
|
instance_id: str
|
|
249
253
|
"""ID of the instance to be used for the scenario."""
|
|
250
254
|
|
|
251
|
-
scenario_id: str
|
|
255
|
+
scenario_id: Optional[str] = None
|
|
252
256
|
"""
|
|
253
257
|
Optional ID of the scenario. The default value will be set as
|
|
254
258
|
`scenario-<index>` if not set.
|
|
255
259
|
"""
|
|
256
|
-
configuration: list[ScenarioConfiguration]
|
|
260
|
+
configuration: Optional[list[ScenarioConfiguration]] = None
|
|
257
261
|
"""Optional configuration for the scenario. Use this attribute to configure
|
|
258
262
|
variation of options for the scenario.
|
|
259
263
|
"""
|
|
@@ -296,8 +300,8 @@ class Scenario:
|
|
|
296
300
|
if self.configuration is None or len(self.configuration) == 0:
|
|
297
301
|
return [{}]
|
|
298
302
|
|
|
299
|
-
keys, value_lists = zip(*((config.name, config.values) for config in self.configuration)
|
|
300
|
-
combinations = [dict(zip(keys, values
|
|
303
|
+
keys, value_lists = zip(*((config.name, config.values) for config in self.configuration))
|
|
304
|
+
combinations = [dict(zip(keys, values)) for values in itertools.product(*value_lists)]
|
|
301
305
|
|
|
302
306
|
return combinations
|
|
303
307
|
|
nextmv/input.py
CHANGED
|
@@ -35,7 +35,7 @@ import sys
|
|
|
35
35
|
from collections.abc import Callable
|
|
36
36
|
from dataclasses import dataclass
|
|
37
37
|
from enum import Enum
|
|
38
|
-
from typing import Any
|
|
38
|
+
from typing import Any, Optional, Union
|
|
39
39
|
|
|
40
40
|
from nextmv._serialization import serialize_json
|
|
41
41
|
from nextmv.deprecated import deprecated
|
|
@@ -155,17 +155,17 @@ class DataFile:
|
|
|
155
155
|
|
|
156
156
|
The `loader` function should return the data that will be used in the model.
|
|
157
157
|
"""
|
|
158
|
-
loader_kwargs: dict[str, Any]
|
|
158
|
+
loader_kwargs: Optional[dict[str, Any]] = None
|
|
159
159
|
"""
|
|
160
160
|
Optional keyword arguments to pass to the loader function. This can be used
|
|
161
161
|
to customize the behavior of the loader.
|
|
162
162
|
"""
|
|
163
|
-
loader_args: list[Any]
|
|
163
|
+
loader_args: Optional[list[Any]] = None
|
|
164
164
|
"""
|
|
165
165
|
Optional positional arguments to pass to the loader function. This can be
|
|
166
166
|
used to customize the behavior of the loader.
|
|
167
167
|
"""
|
|
168
|
-
input_data_key: str
|
|
168
|
+
input_data_key: Optional[str] = None
|
|
169
169
|
"""
|
|
170
170
|
Use this parameter to set a custom key to represent your file.
|
|
171
171
|
|
|
@@ -180,8 +180,8 @@ class DataFile:
|
|
|
180
180
|
|
|
181
181
|
def json_data_file(
|
|
182
182
|
name: str,
|
|
183
|
-
json_configurations: dict[str, Any]
|
|
184
|
-
input_data_key: str
|
|
183
|
+
json_configurations: Optional[dict[str, Any]] = None,
|
|
184
|
+
input_data_key: Optional[str] = None,
|
|
185
185
|
) -> DataFile:
|
|
186
186
|
"""
|
|
187
187
|
This is a convenience function to create a `DataFile` that reads JSON data.
|
|
@@ -231,7 +231,7 @@ def json_data_file(
|
|
|
231
231
|
|
|
232
232
|
json_configurations = json_configurations or {}
|
|
233
233
|
|
|
234
|
-
def loader(file_path: str) -> dict[str, Any]
|
|
234
|
+
def loader(file_path: str) -> Union[dict[str, Any], Any]:
|
|
235
235
|
with open(file_path, encoding="utf-8") as f:
|
|
236
236
|
return json.load(f, **json_configurations)
|
|
237
237
|
|
|
@@ -244,8 +244,8 @@ def json_data_file(
|
|
|
244
244
|
|
|
245
245
|
def csv_data_file(
|
|
246
246
|
name: str,
|
|
247
|
-
csv_configurations: dict[str, Any]
|
|
248
|
-
input_data_key: str
|
|
247
|
+
csv_configurations: Optional[dict[str, Any]] = None,
|
|
248
|
+
input_data_key: Optional[str] = None,
|
|
249
249
|
) -> DataFile:
|
|
250
250
|
"""
|
|
251
251
|
This is a convenience function to create a `DataFile` that reads CSV data.
|
|
@@ -306,7 +306,7 @@ def csv_data_file(
|
|
|
306
306
|
)
|
|
307
307
|
|
|
308
308
|
|
|
309
|
-
def text_data_file(name: str, input_data_key: str
|
|
309
|
+
def text_data_file(name: str, input_data_key: Optional[str] = None) -> DataFile:
|
|
310
310
|
"""
|
|
311
311
|
This is a convenience function to create a `DataFile` that reads utf-8
|
|
312
312
|
encoded text data.
|
|
@@ -408,7 +408,13 @@ class Input:
|
|
|
408
408
|
If the `input_format` is not one of the supported formats.
|
|
409
409
|
"""
|
|
410
410
|
|
|
411
|
-
data:
|
|
411
|
+
data: Union[
|
|
412
|
+
Union[dict[str, Any], Any], # JSON
|
|
413
|
+
str, # TEXT
|
|
414
|
+
list[dict[str, Any]], # CSV
|
|
415
|
+
dict[str, list[dict[str, Any]]], # CSV_ARCHIVE
|
|
416
|
+
dict[str, Any], # MULTI_FILE
|
|
417
|
+
]
|
|
412
418
|
"""
|
|
413
419
|
The actual data.
|
|
414
420
|
|
|
@@ -421,14 +427,14 @@ class Input:
|
|
|
421
427
|
- For `MULTI_FILE`: `dict[str, Any]`
|
|
422
428
|
"""
|
|
423
429
|
|
|
424
|
-
input_format: InputFormat
|
|
430
|
+
input_format: Optional[InputFormat] = InputFormat.JSON
|
|
425
431
|
"""
|
|
426
432
|
Format of the input data.
|
|
427
433
|
|
|
428
434
|
Default is `InputFormat.JSON`.
|
|
429
435
|
"""
|
|
430
436
|
|
|
431
|
-
options: Options
|
|
437
|
+
options: Optional[Options] = None
|
|
432
438
|
"""
|
|
433
439
|
Options that the `Input` was created with.
|
|
434
440
|
|
|
@@ -552,7 +558,7 @@ class InputLoader:
|
|
|
552
558
|
def load(
|
|
553
559
|
self,
|
|
554
560
|
input_format: InputFormat = InputFormat.JSON,
|
|
555
|
-
options: Options
|
|
561
|
+
options: Optional[Options] = None,
|
|
556
562
|
*args,
|
|
557
563
|
**kwargs,
|
|
558
564
|
) -> Input:
|
|
@@ -630,7 +636,7 @@ class LocalInputLoader(InputLoader):
|
|
|
630
636
|
with open(path, encoding="utf-8") as f:
|
|
631
637
|
return f.read().rstrip("\n")
|
|
632
638
|
|
|
633
|
-
def _read_csv(path: str, csv_configurations: dict[str, Any]
|
|
639
|
+
def _read_csv(path: str, csv_configurations: Optional[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
634
640
|
"""
|
|
635
641
|
Read a CSV file and return its contents as a list of dictionaries.
|
|
636
642
|
|
|
@@ -649,7 +655,7 @@ class LocalInputLoader(InputLoader):
|
|
|
649
655
|
with open(path, encoding="utf-8") as f:
|
|
650
656
|
return list(csv.DictReader(f, **csv_configurations))
|
|
651
657
|
|
|
652
|
-
def _read_json(path: str, _) -> dict[str, Any]
|
|
658
|
+
def _read_json(path: str, _) -> Union[dict[str, Any], Any]:
|
|
653
659
|
"""
|
|
654
660
|
Read a JSON file and return its parsed contents.
|
|
655
661
|
|
|
@@ -698,11 +704,11 @@ class LocalInputLoader(InputLoader):
|
|
|
698
704
|
|
|
699
705
|
def load(
|
|
700
706
|
self,
|
|
701
|
-
input_format: InputFormat
|
|
702
|
-
options: Options
|
|
703
|
-
path: str
|
|
704
|
-
csv_configurations: dict[str, Any]
|
|
705
|
-
data_files: list[DataFile]
|
|
707
|
+
input_format: Optional[InputFormat] = InputFormat.JSON,
|
|
708
|
+
options: Optional[Options] = None,
|
|
709
|
+
path: Optional[str] = None,
|
|
710
|
+
csv_configurations: Optional[dict[str, Any]] = None,
|
|
711
|
+
data_files: Optional[list[DataFile]] = None,
|
|
706
712
|
) -> Input:
|
|
707
713
|
"""
|
|
708
714
|
Load the input data. The input data can be in various formats. For
|
|
@@ -783,11 +789,11 @@ class LocalInputLoader(InputLoader):
|
|
|
783
789
|
|
|
784
790
|
def _load_utf8_encoded(
|
|
785
791
|
self,
|
|
786
|
-
csv_configurations: dict[str, Any]
|
|
787
|
-
path: str
|
|
788
|
-
input_format: InputFormat
|
|
792
|
+
csv_configurations: Optional[dict[str, Any]],
|
|
793
|
+
path: Optional[str] = None,
|
|
794
|
+
input_format: Optional[InputFormat] = InputFormat.JSON,
|
|
789
795
|
use_file_reader: bool = False,
|
|
790
|
-
) -> dict[str, Any]
|
|
796
|
+
) -> Union[dict[str, Any], str, list[dict[str, Any]]]:
|
|
791
797
|
"""
|
|
792
798
|
Load a utf-8 encoded file from stdin or filesystem.
|
|
793
799
|
|
|
@@ -825,8 +831,8 @@ class LocalInputLoader(InputLoader):
|
|
|
825
831
|
|
|
826
832
|
def _load_archive(
|
|
827
833
|
self,
|
|
828
|
-
csv_configurations: dict[str, Any]
|
|
829
|
-
path: str
|
|
834
|
+
csv_configurations: Optional[dict[str, Any]],
|
|
835
|
+
path: Optional[str] = None,
|
|
830
836
|
) -> dict[str, list[dict[str, Any]]]:
|
|
831
837
|
"""
|
|
832
838
|
Load CSV files from a directory.
|
|
@@ -881,7 +887,7 @@ class LocalInputLoader(InputLoader):
|
|
|
881
887
|
def _load_multi_file(
|
|
882
888
|
self,
|
|
883
889
|
data_files: list[DataFile],
|
|
884
|
-
path: str
|
|
890
|
+
path: Optional[str] = None,
|
|
885
891
|
) -> dict[str, Any]:
|
|
886
892
|
"""
|
|
887
893
|
Load multiple files from a directory.
|
|
@@ -955,10 +961,10 @@ class LocalInputLoader(InputLoader):
|
|
|
955
961
|
|
|
956
962
|
|
|
957
963
|
def load_local(
|
|
958
|
-
input_format: InputFormat
|
|
959
|
-
options: Options
|
|
960
|
-
path: str
|
|
961
|
-
csv_configurations: dict[str, Any]
|
|
964
|
+
input_format: Optional[InputFormat] = InputFormat.JSON,
|
|
965
|
+
options: Optional[Options] = None,
|
|
966
|
+
path: Optional[str] = None,
|
|
967
|
+
csv_configurations: Optional[dict[str, Any]] = None,
|
|
962
968
|
) -> Input:
|
|
963
969
|
"""
|
|
964
970
|
!!! warning
|
|
@@ -1010,12 +1016,12 @@ _LOCAL_INPUT_LOADER = LocalInputLoader()
|
|
|
1010
1016
|
|
|
1011
1017
|
|
|
1012
1018
|
def load(
|
|
1013
|
-
input_format: InputFormat
|
|
1014
|
-
options: Options
|
|
1015
|
-
path: str
|
|
1016
|
-
csv_configurations: dict[str, Any]
|
|
1017
|
-
loader: InputLoader
|
|
1018
|
-
data_files: list[DataFile]
|
|
1019
|
+
input_format: Optional[InputFormat] = InputFormat.JSON,
|
|
1020
|
+
options: Optional[Options] = None,
|
|
1021
|
+
path: Optional[str] = None,
|
|
1022
|
+
csv_configurations: Optional[dict[str, Any]] = None,
|
|
1023
|
+
loader: Optional[InputLoader] = _LOCAL_INPUT_LOADER,
|
|
1024
|
+
data_files: Optional[list[DataFile]] = None,
|
|
1019
1025
|
) -> Input:
|
|
1020
1026
|
"""
|
|
1021
1027
|
Load input data using the specified loader.
|