nextmv 0.40.0__py3-none-any.whl → 1.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nextmv/__about__.py +1 -1
- nextmv/__init__.py +2 -0
- nextmv/cli/CONTRIBUTING.md +511 -0
- nextmv/cli/cloud/__init__.py +45 -0
- nextmv/cli/cloud/acceptance/__init__.py +27 -0
- nextmv/cli/cloud/acceptance/create.py +393 -0
- nextmv/cli/cloud/acceptance/delete.py +68 -0
- nextmv/cli/cloud/acceptance/get.py +104 -0
- nextmv/cli/cloud/acceptance/list.py +62 -0
- nextmv/cli/cloud/acceptance/update.py +95 -0
- nextmv/cli/cloud/account/__init__.py +28 -0
- nextmv/cli/cloud/account/create.py +83 -0
- nextmv/cli/cloud/account/delete.py +60 -0
- nextmv/cli/cloud/account/get.py +66 -0
- nextmv/cli/cloud/account/update.py +70 -0
- nextmv/cli/cloud/app/__init__.py +35 -0
- nextmv/cli/cloud/app/create.py +141 -0
- nextmv/cli/cloud/app/delete.py +58 -0
- nextmv/cli/cloud/app/exists.py +44 -0
- nextmv/cli/cloud/app/get.py +66 -0
- nextmv/cli/cloud/app/list.py +61 -0
- nextmv/cli/cloud/app/push.py +137 -0
- nextmv/cli/cloud/app/update.py +124 -0
- nextmv/cli/cloud/batch/__init__.py +29 -0
- nextmv/cli/cloud/batch/create.py +454 -0
- nextmv/cli/cloud/batch/delete.py +68 -0
- nextmv/cli/cloud/batch/get.py +104 -0
- nextmv/cli/cloud/batch/list.py +63 -0
- nextmv/cli/cloud/batch/metadata.py +66 -0
- nextmv/cli/cloud/batch/update.py +95 -0
- nextmv/cli/cloud/data/__init__.py +26 -0
- nextmv/cli/cloud/data/upload.py +162 -0
- nextmv/cli/cloud/ensemble/__init__.py +31 -0
- nextmv/cli/cloud/ensemble/create.py +414 -0
- nextmv/cli/cloud/ensemble/delete.py +67 -0
- nextmv/cli/cloud/ensemble/get.py +65 -0
- nextmv/cli/cloud/ensemble/update.py +103 -0
- nextmv/cli/cloud/input_set/__init__.py +30 -0
- nextmv/cli/cloud/input_set/create.py +168 -0
- nextmv/cli/cloud/input_set/get.py +63 -0
- nextmv/cli/cloud/input_set/list.py +63 -0
- nextmv/cli/cloud/input_set/update.py +123 -0
- nextmv/cli/cloud/instance/__init__.py +35 -0
- nextmv/cli/cloud/instance/create.py +290 -0
- nextmv/cli/cloud/instance/delete.py +62 -0
- nextmv/cli/cloud/instance/exists.py +39 -0
- nextmv/cli/cloud/instance/get.py +62 -0
- nextmv/cli/cloud/instance/list.py +60 -0
- nextmv/cli/cloud/instance/update.py +216 -0
- nextmv/cli/cloud/managed_input/__init__.py +31 -0
- nextmv/cli/cloud/managed_input/create.py +146 -0
- nextmv/cli/cloud/managed_input/delete.py +65 -0
- nextmv/cli/cloud/managed_input/get.py +63 -0
- nextmv/cli/cloud/managed_input/list.py +60 -0
- nextmv/cli/cloud/managed_input/update.py +97 -0
- nextmv/cli/cloud/run/__init__.py +37 -0
- nextmv/cli/cloud/run/cancel.py +37 -0
- nextmv/cli/cloud/run/create.py +530 -0
- nextmv/cli/cloud/run/get.py +199 -0
- nextmv/cli/cloud/run/input.py +86 -0
- nextmv/cli/cloud/run/list.py +80 -0
- nextmv/cli/cloud/run/logs.py +167 -0
- nextmv/cli/cloud/run/metadata.py +67 -0
- nextmv/cli/cloud/run/track.py +501 -0
- nextmv/cli/cloud/scenario/__init__.py +29 -0
- nextmv/cli/cloud/scenario/create.py +451 -0
- nextmv/cli/cloud/scenario/delete.py +65 -0
- nextmv/cli/cloud/scenario/get.py +102 -0
- nextmv/cli/cloud/scenario/list.py +63 -0
- nextmv/cli/cloud/scenario/metadata.py +67 -0
- nextmv/cli/cloud/scenario/update.py +93 -0
- nextmv/cli/cloud/secrets/__init__.py +33 -0
- nextmv/cli/cloud/secrets/create.py +206 -0
- nextmv/cli/cloud/secrets/delete.py +67 -0
- nextmv/cli/cloud/secrets/get.py +66 -0
- nextmv/cli/cloud/secrets/list.py +60 -0
- nextmv/cli/cloud/secrets/update.py +147 -0
- nextmv/cli/cloud/upload/__init__.py +22 -0
- nextmv/cli/cloud/upload/create.py +39 -0
- nextmv/cli/cloud/version/__init__.py +33 -0
- nextmv/cli/cloud/version/create.py +97 -0
- nextmv/cli/cloud/version/delete.py +62 -0
- nextmv/cli/cloud/version/exists.py +39 -0
- nextmv/cli/cloud/version/get.py +62 -0
- nextmv/cli/cloud/version/list.py +60 -0
- nextmv/cli/cloud/version/update.py +92 -0
- nextmv/cli/community/__init__.py +24 -0
- nextmv/cli/community/clone.py +3 -3
- nextmv/cli/community/list.py +1 -1
- nextmv/cli/configuration/__init__.py +23 -0
- nextmv/cli/configuration/config.py +68 -4
- nextmv/cli/configuration/create.py +14 -15
- nextmv/cli/configuration/delete.py +24 -12
- nextmv/cli/configuration/list.py +1 -1
- nextmv/cli/main.py +58 -16
- nextmv/cli/message.py +153 -0
- nextmv/cli/options.py +168 -0
- nextmv/cli/version.py +20 -1
- nextmv/cloud/__init__.py +4 -1
- nextmv/cloud/acceptance_test.py +19 -18
- nextmv/cloud/account.py +268 -24
- nextmv/cloud/application/__init__.py +955 -0
- nextmv/cloud/application/_acceptance.py +419 -0
- nextmv/cloud/application/_batch_scenario.py +860 -0
- nextmv/cloud/application/_ensemble.py +251 -0
- nextmv/cloud/application/_input_set.py +227 -0
- nextmv/cloud/application/_instance.py +289 -0
- nextmv/cloud/application/_managed_input.py +227 -0
- nextmv/cloud/application/_run.py +1393 -0
- nextmv/cloud/application/_secrets.py +294 -0
- nextmv/cloud/application/_utils.py +54 -0
- nextmv/cloud/application/_version.py +303 -0
- nextmv/cloud/batch_experiment.py +3 -1
- nextmv/cloud/instance.py +11 -1
- nextmv/cloud/integration.py +1 -1
- nextmv/cloud/package.py +50 -9
- nextmv/input.py +20 -36
- nextmv/local/application.py +3 -15
- nextmv/polling.py +54 -16
- nextmv/run.py +83 -27
- {nextmv-0.40.0.dist-info → nextmv-1.0.0.dev0.dist-info}/METADATA +33 -8
- nextmv-1.0.0.dev0.dist-info/RECORD +158 -0
- nextmv/cli/community/community.py +0 -24
- nextmv/cli/configuration/configuration.py +0 -23
- nextmv/cli/error.py +0 -22
- nextmv/cloud/application.py +0 -4204
- nextmv-0.40.0.dist-info/RECORD +0 -66
- {nextmv-0.40.0.dist-info → nextmv-1.0.0.dev0.dist-info}/WHEEL +0 -0
- {nextmv-0.40.0.dist-info → nextmv-1.0.0.dev0.dist-info}/licenses/LICENSE +0 -0
nextmv/cloud/package.py
CHANGED
|
@@ -6,9 +6,12 @@ import platform
|
|
|
6
6
|
import re
|
|
7
7
|
import shutil
|
|
8
8
|
import subprocess
|
|
9
|
+
import sys
|
|
9
10
|
import tarfile
|
|
10
11
|
import tempfile
|
|
11
12
|
|
|
13
|
+
import rich
|
|
14
|
+
|
|
12
15
|
from nextmv.logger import log
|
|
13
16
|
from nextmv.manifest import MANIFEST_FILE_NAME, Manifest, ManifestBuild, ManifestType
|
|
14
17
|
from nextmv.model import Model, ModelConfiguration, _cleanup_python_model
|
|
@@ -21,18 +24,19 @@ _MANDATORY_FILES_PER_TYPE = {
|
|
|
21
24
|
}
|
|
22
25
|
|
|
23
26
|
|
|
24
|
-
def _package(
|
|
27
|
+
def _package( # noqa: C901 # complexity attributed to printing.
|
|
25
28
|
app_dir: str,
|
|
26
29
|
manifest: Manifest,
|
|
27
30
|
model: Model | None = None,
|
|
28
31
|
model_configuration: ModelConfiguration | None = None,
|
|
29
32
|
verbose: bool = False,
|
|
33
|
+
rich_print: bool = False,
|
|
30
34
|
) -> tuple[str, str]:
|
|
31
35
|
"""Package the app into a tarball."""
|
|
32
36
|
|
|
33
37
|
with tempfile.TemporaryDirectory(prefix="nextmv-temp-") as temp_dir:
|
|
34
38
|
if manifest.type == ManifestType.PYTHON:
|
|
35
|
-
__handle_python(app_dir, temp_dir, manifest, model, model_configuration, verbose)
|
|
39
|
+
__handle_python(app_dir, temp_dir, manifest, model, model_configuration, verbose, rich_print)
|
|
36
40
|
|
|
37
41
|
found, missing, files = __find_files(app_dir, manifest.files)
|
|
38
42
|
__confirm_mandatory_files(manifest, found)
|
|
@@ -55,7 +59,13 @@ def _package(
|
|
|
55
59
|
raise Exception(f"error copying asset files {file['absolute_path']}: {e}") from e
|
|
56
60
|
|
|
57
61
|
if verbose:
|
|
58
|
-
|
|
62
|
+
if rich_print:
|
|
63
|
+
rich.print(
|
|
64
|
+
f":clipboard: Copied files listed in [magenta]{MANIFEST_FILE_NAME}[/magenta] manifest.",
|
|
65
|
+
file=sys.stderr,
|
|
66
|
+
)
|
|
67
|
+
else:
|
|
68
|
+
log(f'📋 Copied files listed in "{MANIFEST_FILE_NAME}" manifest.')
|
|
59
69
|
|
|
60
70
|
if manifest.type == ManifestType.PYTHON:
|
|
61
71
|
_cleanup_python_model(app_dir, model_configuration, verbose)
|
|
@@ -66,9 +76,22 @@ def _package(
|
|
|
66
76
|
if verbose:
|
|
67
77
|
try:
|
|
68
78
|
size = __human_friendly_file_size(tar_file)
|
|
69
|
-
|
|
79
|
+
if rich_print:
|
|
80
|
+
rich.print(
|
|
81
|
+
":package: Packaged application "
|
|
82
|
+
f"([magenta]{file_count_msg}[/magenta], [magenta]{size}[/magenta]).",
|
|
83
|
+
file=sys.stderr,
|
|
84
|
+
)
|
|
85
|
+
else:
|
|
86
|
+
log(f"📦 Packaged application ({file_count_msg}, {size}).")
|
|
70
87
|
except Exception:
|
|
71
|
-
|
|
88
|
+
if rich_print:
|
|
89
|
+
rich.print(
|
|
90
|
+
f":package: Packaged application ([magenta]{file_count_msg}[/magenta]).",
|
|
91
|
+
file=sys.stderr,
|
|
92
|
+
)
|
|
93
|
+
else:
|
|
94
|
+
log(f"📦 Packaged application ({file_count_msg}).")
|
|
72
95
|
|
|
73
96
|
return tar_file, output_dir
|
|
74
97
|
|
|
@@ -77,6 +100,7 @@ def _run_build_command(
|
|
|
77
100
|
app_dir: str,
|
|
78
101
|
manifest_build: ManifestBuild | None = None,
|
|
79
102
|
verbose: bool = False,
|
|
103
|
+
rich_print: bool = False,
|
|
80
104
|
) -> None:
|
|
81
105
|
"""Run the build command specified in the manifest."""
|
|
82
106
|
|
|
@@ -85,7 +109,12 @@ def _run_build_command(
|
|
|
85
109
|
|
|
86
110
|
elements = manifest_build.command.split(" ")
|
|
87
111
|
command_str = " ".join(elements)
|
|
88
|
-
|
|
112
|
+
|
|
113
|
+
if verbose:
|
|
114
|
+
if rich_print:
|
|
115
|
+
rich.print(f":construction: Running build command: [magenta]{command_str}[/magenta]", file=sys.stderr)
|
|
116
|
+
else:
|
|
117
|
+
log(f'🚧 Running build command: "{command_str}"')
|
|
89
118
|
try:
|
|
90
119
|
result = subprocess.run(
|
|
91
120
|
elements,
|
|
@@ -120,6 +149,7 @@ def _run_pre_push_command(
|
|
|
120
149
|
app_dir: str,
|
|
121
150
|
pre_push_command: str | None = None,
|
|
122
151
|
verbose: bool = False,
|
|
152
|
+
rich_print: bool = False,
|
|
123
153
|
) -> None:
|
|
124
154
|
"""Run the pre-push command specified in the manifest."""
|
|
125
155
|
|
|
@@ -129,7 +159,11 @@ def _run_pre_push_command(
|
|
|
129
159
|
elements = _get_shell_command_elements(pre_push_command)
|
|
130
160
|
|
|
131
161
|
command_str = " ".join(elements)
|
|
132
|
-
|
|
162
|
+
if verbose:
|
|
163
|
+
if rich_print:
|
|
164
|
+
rich.print(f":hammer: Running pre-push command: [magenta]{command_str}[/magenta]", file=sys.stderr)
|
|
165
|
+
else:
|
|
166
|
+
log(f'🔨 Running pre-push command: "{command_str}"')
|
|
133
167
|
try:
|
|
134
168
|
result = subprocess.run(
|
|
135
169
|
elements,
|
|
@@ -227,16 +261,23 @@ def __handle_python(
|
|
|
227
261
|
model: Model | None = None,
|
|
228
262
|
model_configuration: ModelConfiguration | None = None,
|
|
229
263
|
verbose: bool = False,
|
|
264
|
+
rich_print: bool = False,
|
|
230
265
|
) -> None:
|
|
231
266
|
"""Handles the Python-specific packaging logic."""
|
|
232
267
|
|
|
233
268
|
if model is not None and model_configuration is not None:
|
|
234
269
|
if verbose:
|
|
235
|
-
|
|
270
|
+
if rich_print:
|
|
271
|
+
rich.print(":crystal_ball: Encoding Python model.", file=sys.stderr)
|
|
272
|
+
else:
|
|
273
|
+
log("🔮 Encoding Python model.")
|
|
236
274
|
model.save(app_dir, model_configuration)
|
|
237
275
|
|
|
238
276
|
if verbose:
|
|
239
|
-
|
|
277
|
+
if rich_print:
|
|
278
|
+
rich.print(":snake: Bundling Python dependencies.", file=sys.stderr)
|
|
279
|
+
else:
|
|
280
|
+
log("🐍 Bundling Python dependencies.")
|
|
240
281
|
__install_dependencies(manifest, app_dir, temp_dir)
|
|
241
282
|
|
|
242
283
|
|
nextmv/input.py
CHANGED
|
@@ -77,8 +77,6 @@ class InputFormat(str, Enum):
|
|
|
77
77
|
"""JSON format, utf-8 encoded."""
|
|
78
78
|
TEXT = "text"
|
|
79
79
|
"""Text format, utf-8 encoded."""
|
|
80
|
-
CSV = "csv"
|
|
81
|
-
"""CSV format, utf-8 encoded."""
|
|
82
80
|
CSV_ARCHIVE = "csv-archive"
|
|
83
81
|
"""CSV archive format: multiple CSV files."""
|
|
84
82
|
MULTI_FILE = "multi-file"
|
|
@@ -376,8 +374,6 @@ class Input:
|
|
|
376
374
|
means that the data must be JSON-deserializable, which includes dicts and
|
|
377
375
|
lists.
|
|
378
376
|
- `InputFormat.TEXT`: the data is `str`, and it must be utf-8 encoded.
|
|
379
|
-
- `InputFormat.CSV`: the data is `list[dict[str, Any]]`, where each dict
|
|
380
|
-
represents a row in the CSV.
|
|
381
377
|
- `InputFormat.CSV_ARCHIVE`: the data is `dict[str, list[dict[str, Any]]]`,
|
|
382
378
|
where each key is the name of a CSV file and the value is a list of dicts
|
|
383
379
|
representing the rows in that CSV file.
|
|
@@ -466,12 +462,6 @@ class Input:
|
|
|
466
462
|
"input_format InputFormat.TEXT, supported type is `str`"
|
|
467
463
|
)
|
|
468
464
|
|
|
469
|
-
elif self.input_format == InputFormat.CSV and not isinstance(self.data, list):
|
|
470
|
-
raise ValueError(
|
|
471
|
-
f"unsupported Input.data type: {type(self.data)} with "
|
|
472
|
-
"input_format InputFormat.CSV, supported type is `list`"
|
|
473
|
-
)
|
|
474
|
-
|
|
475
465
|
elif self.input_format == InputFormat.CSV_ARCHIVE and not isinstance(self.data, dict):
|
|
476
466
|
raise ValueError(
|
|
477
467
|
f"unsupported Input.data type: {type(self.data)} with "
|
|
@@ -607,8 +597,6 @@ class LocalInputLoader(InputLoader):
|
|
|
607
597
|
>>> loader = LocalInputLoader()
|
|
608
598
|
>>> # Load JSON from stdin or file
|
|
609
599
|
>>> input_obj = loader.load(input_format=InputFormat.JSON, path="data.json")
|
|
610
|
-
>>> # Load CSV from a file
|
|
611
|
-
>>> input_obj = loader.load(input_format=InputFormat.CSV, path="data.csv")
|
|
612
600
|
"""
|
|
613
601
|
|
|
614
602
|
def _read_text(path: str, _) -> str:
|
|
@@ -672,7 +660,6 @@ class LocalInputLoader(InputLoader):
|
|
|
672
660
|
STDIN_READERS = {
|
|
673
661
|
InputFormat.JSON: lambda _: json.load(sys.stdin),
|
|
674
662
|
InputFormat.TEXT: lambda _: sys.stdin.read().rstrip("\n"),
|
|
675
|
-
InputFormat.CSV: lambda csv_configurations: list(csv.DictReader(sys.stdin, **csv_configurations)),
|
|
676
663
|
}
|
|
677
664
|
"""
|
|
678
665
|
Dictionary of functions to read from standard input.
|
|
@@ -687,7 +674,7 @@ class LocalInputLoader(InputLoader):
|
|
|
687
674
|
FILE_READERS = {
|
|
688
675
|
InputFormat.JSON: _read_json,
|
|
689
676
|
InputFormat.TEXT: _read_text,
|
|
690
|
-
|
|
677
|
+
"CSV": _read_csv,
|
|
691
678
|
}
|
|
692
679
|
"""
|
|
693
680
|
Dictionary of functions to read from files.
|
|
@@ -706,23 +693,23 @@ class LocalInputLoader(InputLoader):
|
|
|
706
693
|
) -> Input:
|
|
707
694
|
"""
|
|
708
695
|
Load the input data. The input data can be in various formats. For
|
|
709
|
-
`InputFormat.JSON
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
696
|
+
`InputFormat.JSON` and `InputFormat.TEXT`, the data can be streamed
|
|
697
|
+
from stdin or read from a file. When the `path` argument is provided
|
|
698
|
+
(and valid), the input data is read from the file specified by `path`,
|
|
699
|
+
otherwise, it is streamed from stdin. For `InputFormat.CSV_ARCHIVE`,
|
|
700
|
+
the input data is read from the directory specified by `path`. If the
|
|
701
|
+
`path` is not provided, the default location `input` is used. The
|
|
702
|
+
directory should contain one or more files, where each file in the
|
|
703
|
+
directory is a CSV file.
|
|
717
704
|
|
|
718
705
|
The `Input` that is returned contains the `data` attribute. This data
|
|
719
706
|
can be of different types, depending on the provided `input_format`:
|
|
720
707
|
|
|
721
708
|
- `InputFormat.JSON`: the data is a `dict[str, Any]`.
|
|
722
709
|
- `InputFormat.TEXT`: the data is a `str`.
|
|
723
|
-
- `InputFormat.
|
|
724
|
-
|
|
725
|
-
|
|
710
|
+
- `InputFormat.CSV_ARCHIVE`: the data is a `dict[str, list[dict[str,
|
|
711
|
+
Any]]]`. Each key is the name of the CSV file, minus the `.csv`
|
|
712
|
+
extension.
|
|
726
713
|
- `InputFormat.MULTI_FILE`: the data is a `dict[str, Any]`, where each
|
|
727
714
|
key is the file name (with extension) and the value is the data read
|
|
728
715
|
from the file. The data can be of any type, depending on the file
|
|
@@ -745,11 +732,11 @@ class LocalInputLoader(InputLoader):
|
|
|
745
732
|
`input_format` is set to `InputFormat.MULTI_FILE`. Each `DataFile`
|
|
746
733
|
instance should have a `name` (the file name with extension) and a
|
|
747
734
|
`loader` function that reads the data from the file. The `loader`
|
|
748
|
-
function should accept the file path as its first argument and
|
|
749
|
-
the data read from the file. The `loader` can also accept
|
|
750
|
-
positional and keyword arguments, which can be provided
|
|
751
|
-
`loader_args` and `loader_kwargs` attributes of the
|
|
752
|
-
instance.
|
|
735
|
+
function should accept the file path as its first argument and
|
|
736
|
+
return the data read from the file. The `loader` can also accept
|
|
737
|
+
additional positional and keyword arguments, which can be provided
|
|
738
|
+
through the `loader_args` and `loader_kwargs` attributes of the
|
|
739
|
+
`DataFile` instance.
|
|
753
740
|
|
|
754
741
|
Returns
|
|
755
742
|
-------
|
|
@@ -766,7 +753,7 @@ class LocalInputLoader(InputLoader):
|
|
|
766
753
|
if csv_configurations is None:
|
|
767
754
|
csv_configurations = {}
|
|
768
755
|
|
|
769
|
-
if input_format in [InputFormat.JSON, InputFormat.TEXT
|
|
756
|
+
if input_format in [InputFormat.JSON, InputFormat.TEXT]:
|
|
770
757
|
data = self._load_utf8_encoded(path=path, input_format=input_format, csv_configurations=csv_configurations)
|
|
771
758
|
elif input_format == InputFormat.CSV_ARCHIVE:
|
|
772
759
|
data = self._load_archive(path=path, csv_configurations=csv_configurations)
|
|
@@ -785,7 +772,7 @@ class LocalInputLoader(InputLoader):
|
|
|
785
772
|
self,
|
|
786
773
|
csv_configurations: dict[str, Any] | None,
|
|
787
774
|
path: str | None = None,
|
|
788
|
-
input_format: InputFormat | None = InputFormat.JSON,
|
|
775
|
+
input_format: InputFormat | str | None = InputFormat.JSON,
|
|
789
776
|
use_file_reader: bool = False,
|
|
790
777
|
) -> dict[str, Any] | str | list[dict[str, Any]]:
|
|
791
778
|
"""
|
|
@@ -871,7 +858,7 @@ class LocalInputLoader(InputLoader):
|
|
|
871
858
|
stripped = file.removesuffix(csv_ext)
|
|
872
859
|
data[stripped] = self._load_utf8_encoded(
|
|
873
860
|
path=os.path.join(dir_path, file),
|
|
874
|
-
input_format=
|
|
861
|
+
input_format="CSV",
|
|
875
862
|
use_file_reader=True,
|
|
876
863
|
csv_configurations=csv_configurations,
|
|
877
864
|
)
|
|
@@ -1034,7 +1021,6 @@ def load(
|
|
|
1034
1021
|
|
|
1035
1022
|
- `InputFormat.JSON`: the data is a `dict[str, Any]`
|
|
1036
1023
|
- `InputFormat.TEXT`: the data is a `str`
|
|
1037
|
-
- `InputFormat.CSV`: the data is a `list[dict[str, Any]]`
|
|
1038
1024
|
- `InputFormat.CSV_ARCHIVE`: the data is a `dict[str, list[dict[str, Any]]]`
|
|
1039
1025
|
Each key is the name of the CSV file, minus the `.csv` extension.
|
|
1040
1026
|
- `InputFormat.MULTI_FILE`: the data is a `dict[str, Any]`
|
|
@@ -1119,8 +1105,6 @@ def load(
|
|
|
1119
1105
|
>>> from nextmv.input import load, InputFormat
|
|
1120
1106
|
>>> # Load JSON from stdin
|
|
1121
1107
|
>>> input_obj = load(input_format=InputFormat.JSON)
|
|
1122
|
-
>>> # Load CSV from a file
|
|
1123
|
-
>>> input_obj = load(input_format=InputFormat.CSV, path="data.csv")
|
|
1124
1108
|
>>> # Load CSV archive from a directory
|
|
1125
1109
|
>>> input_obj = load(input_format=InputFormat.CSV_ARCHIVE, path="input_dir")
|
|
1126
1110
|
"""
|
nextmv/local/application.py
CHANGED
|
@@ -256,8 +256,7 @@ class Application:
|
|
|
256
256
|
`nextmv.InputFormat.MULTI_FILE`, you should use the
|
|
257
257
|
`input_dir_path` argument instead. This argument takes precedence
|
|
258
258
|
over the `input`. If `input_dir_path` is specified, this function
|
|
259
|
-
looks for files in that directory and tars them
|
|
260
|
-
uploaded using the `upload_large_input` method. If both the
|
|
259
|
+
looks for files in that directory and tars them. If both the
|
|
261
260
|
`input_dir_path` and `input` arguments are provided, the `input`
|
|
262
261
|
is ignored.
|
|
263
262
|
|
|
@@ -273,9 +272,6 @@ class Application:
|
|
|
273
272
|
|
|
274
273
|
When working with JSON or text data, use the `input` argument
|
|
275
274
|
directly.
|
|
276
|
-
|
|
277
|
-
In general, if an input is too large, it will be uploaded with the
|
|
278
|
-
`upload_large_input` method.
|
|
279
275
|
name: Optional[str]
|
|
280
276
|
Name of the local run.
|
|
281
277
|
description: Optional[str]
|
|
@@ -399,8 +395,7 @@ class Application:
|
|
|
399
395
|
`nextmv.InputFormat.MULTI_FILE`, you should use the
|
|
400
396
|
`input_dir_path` argument instead. This argument takes precedence
|
|
401
397
|
over the `input`. If `input_dir_path` is specified, this function
|
|
402
|
-
looks for files in that directory and tars them
|
|
403
|
-
uploaded using the `upload_large_input` method. If both the
|
|
398
|
+
looks for files in that directory and tars them. If both the
|
|
404
399
|
`input_dir_path` and `input` arguments are provided, the `input` is
|
|
405
400
|
ignored.
|
|
406
401
|
|
|
@@ -416,9 +411,6 @@ class Application:
|
|
|
416
411
|
|
|
417
412
|
When working with JSON or text data, use the `input` argument
|
|
418
413
|
directly.
|
|
419
|
-
|
|
420
|
-
In general, if an input is too large, it will be uploaded with the
|
|
421
|
-
`upload_large_input` method.
|
|
422
414
|
name: Optional[str]
|
|
423
415
|
Name of the local run.
|
|
424
416
|
description: Optional[str]
|
|
@@ -699,11 +691,7 @@ class Application:
|
|
|
699
691
|
|
|
700
692
|
def polling_func() -> tuple[Any, bool]:
|
|
701
693
|
run_information = self.run_metadata(run_id=run_id)
|
|
702
|
-
if run_information.metadata.
|
|
703
|
-
StatusV2.succeeded,
|
|
704
|
-
StatusV2.failed,
|
|
705
|
-
StatusV2.canceled,
|
|
706
|
-
}:
|
|
694
|
+
if run_information.metadata.run_is_finalized():
|
|
707
695
|
return run_information, True
|
|
708
696
|
|
|
709
697
|
return None, False
|
nextmv/polling.py
CHANGED
|
@@ -128,17 +128,49 @@ class PollingOptions:
|
|
|
128
128
|
return True to stop the polling and False to continue. The function does
|
|
129
129
|
not receive any arguments. The function is called before each poll.
|
|
130
130
|
"""
|
|
131
|
+
sleep_duration_func: Callable[[], float] | None = None
|
|
132
|
+
"""
|
|
133
|
+
Optional function to calculate the sleep duration between polls. If provided,
|
|
134
|
+
this function will be called to determine how long to sleep instead of using
|
|
135
|
+
the default exponential backoff calculation. The function should return a
|
|
136
|
+
float representing the sleep duration in seconds. The function does not
|
|
137
|
+
receive any arguments and is called before each sleep.
|
|
138
|
+
"""
|
|
131
139
|
|
|
132
140
|
|
|
133
141
|
DEFAULT_POLLING_OPTIONS: PollingOptions = PollingOptions()
|
|
134
142
|
"""
|
|
143
|
+
!!! warning
|
|
144
|
+
`DEFAULT_POLLING_OPTIONS` is a mutable global variable. Use the `default_polling_options`
|
|
145
|
+
function to obtain a fresh instance of `PollingOptions` with default settings.
|
|
146
|
+
|
|
135
147
|
Default polling options to use when polling for a run result. This constant
|
|
136
148
|
provides the default values for `PollingOptions` used across the module.
|
|
137
|
-
Using these defaults is recommended for most use cases unless specific timing
|
|
138
|
-
needs are required.
|
|
139
149
|
"""
|
|
140
150
|
|
|
141
151
|
|
|
152
|
+
def default_polling_options() -> PollingOptions:
|
|
153
|
+
"""
|
|
154
|
+
Returns a new instance of PollingOptions with default settings.
|
|
155
|
+
|
|
156
|
+
This function can be used to obtain a fresh set of default polling options
|
|
157
|
+
that can be modified as needed without affecting the global defaults.
|
|
158
|
+
|
|
159
|
+
You can import the `default_polling_options` function directly from `nextmv`:
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from nextmv import default_polling_options
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Returns
|
|
166
|
+
-------
|
|
167
|
+
PollingOptions
|
|
168
|
+
A new instance of PollingOptions with default values.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
return PollingOptions()
|
|
172
|
+
|
|
173
|
+
|
|
142
174
|
def poll( # noqa: C901
|
|
143
175
|
polling_options: PollingOptions,
|
|
144
176
|
polling_func: Callable[[], tuple[Any, bool]],
|
|
@@ -255,23 +287,29 @@ def poll( # noqa: C901
|
|
|
255
287
|
)
|
|
256
288
|
|
|
257
289
|
# Calculate the delay.
|
|
258
|
-
if
|
|
259
|
-
#
|
|
260
|
-
|
|
261
|
-
delay = polling_options.max_delay
|
|
262
|
-
delay += random.uniform(0, polling_options.jitter) # Add jitter.
|
|
290
|
+
if polling_options.sleep_duration_func is not None:
|
|
291
|
+
# Use the custom sleep duration function if provided.
|
|
292
|
+
sleep_duration = polling_options.sleep_duration_func()
|
|
263
293
|
else:
|
|
264
|
-
delay
|
|
265
|
-
|
|
266
|
-
|
|
294
|
+
# Calculate delay using exponential backoff with jitter.
|
|
295
|
+
if max_reached:
|
|
296
|
+
# If we already reached the maximum, we don't want to further calculate the
|
|
297
|
+
# delay to avoid overflows.
|
|
298
|
+
delay = polling_options.max_delay
|
|
299
|
+
else:
|
|
300
|
+
delay = polling_options.delay # Base
|
|
301
|
+
delay += polling_options.backoff * (2**ix) # Add exponential backoff.
|
|
302
|
+
|
|
303
|
+
# We cannot exceed the max delay.
|
|
304
|
+
if delay >= polling_options.max_delay:
|
|
305
|
+
max_reached = True
|
|
306
|
+
delay = polling_options.max_delay
|
|
307
|
+
|
|
308
|
+
# Add jitter.
|
|
309
|
+
delay += random.uniform(0, polling_options.jitter)
|
|
267
310
|
|
|
268
|
-
|
|
269
|
-
if delay >= polling_options.max_delay:
|
|
270
|
-
max_reached = True
|
|
271
|
-
delay = polling_options.max_delay
|
|
311
|
+
sleep_duration = delay
|
|
272
312
|
|
|
273
|
-
# Sleep for the calculated delay.
|
|
274
|
-
sleep_duration = delay
|
|
275
313
|
if polling_options.verbose:
|
|
276
314
|
log(f"polling | sleeping for duration: {sleep_duration}")
|
|
277
315
|
|
nextmv/run.py
CHANGED
|
@@ -687,6 +687,23 @@ class Metadata(BaseModel):
|
|
|
687
687
|
statistics: dict[str, Any] | None = None
|
|
688
688
|
"""User defined statistics of the run."""
|
|
689
689
|
|
|
690
|
+
def run_is_finalized(self) -> bool:
|
|
691
|
+
"""
|
|
692
|
+
Checks if the run has reached a finalized state.
|
|
693
|
+
|
|
694
|
+
Returns
|
|
695
|
+
-------
|
|
696
|
+
bool
|
|
697
|
+
True if the run status is one of `succeeded`, `failed`, or
|
|
698
|
+
`canceled`. False otherwise.
|
|
699
|
+
"""
|
|
700
|
+
|
|
701
|
+
return self.status_v2 in {
|
|
702
|
+
StatusV2.succeeded,
|
|
703
|
+
StatusV2.failed,
|
|
704
|
+
StatusV2.canceled,
|
|
705
|
+
}
|
|
706
|
+
|
|
690
707
|
|
|
691
708
|
class SyncedRun(BaseModel):
|
|
692
709
|
"""
|
|
@@ -1031,6 +1048,43 @@ class RunLog(BaseModel):
|
|
|
1031
1048
|
"""Log of the run."""
|
|
1032
1049
|
|
|
1033
1050
|
|
|
1051
|
+
class TimestampedRunLog(BaseModel):
|
|
1052
|
+
"""
|
|
1053
|
+
Timestamped log entry of a run.
|
|
1054
|
+
|
|
1055
|
+
You can import the `TimestampedRunLog` class directly from `nextmv`:
|
|
1056
|
+
|
|
1057
|
+
```python
|
|
1058
|
+
from nextmv import TimestampedRunLog
|
|
1059
|
+
```
|
|
1060
|
+
|
|
1061
|
+
Parameters
|
|
1062
|
+
----------
|
|
1063
|
+
timestamp : datetime
|
|
1064
|
+
Timestamp of the log entry.
|
|
1065
|
+
log : str
|
|
1066
|
+
Log message.
|
|
1067
|
+
|
|
1068
|
+
Examples
|
|
1069
|
+
--------
|
|
1070
|
+
>>> from nextmv import TimestampedRunLog
|
|
1071
|
+
>>> from datetime import datetime
|
|
1072
|
+
>>> log_entry = TimestampedRunLog(
|
|
1073
|
+
... timestamp=datetime(2023, 1, 1, 12, 0, 0),
|
|
1074
|
+
... log="Optimization started"
|
|
1075
|
+
... )
|
|
1076
|
+
>>> log_entry.timestamp
|
|
1077
|
+
datetime.datetime(2023, 1, 1, 12, 0)
|
|
1078
|
+
>>> log_entry.log
|
|
1079
|
+
'Optimization started'
|
|
1080
|
+
"""
|
|
1081
|
+
|
|
1082
|
+
timestamp: datetime
|
|
1083
|
+
"""Timestamp of the log entry."""
|
|
1084
|
+
log: str
|
|
1085
|
+
"""Log message."""
|
|
1086
|
+
|
|
1087
|
+
|
|
1034
1088
|
class RunQueuing(BaseModel):
|
|
1035
1089
|
"""
|
|
1036
1090
|
RunQueuing configuration for a run.
|
|
@@ -1395,8 +1449,8 @@ class TrackedRun:
|
|
|
1395
1449
|
output : Output or dict[str, Any] or str, optional
|
|
1396
1450
|
The output of the run being tracked. Please note that if the output
|
|
1397
1451
|
format is JSON, then the output data must be JSON serializable. If both
|
|
1398
|
-
`output` and `output_dir_path` are specified, the `output` is ignored,
|
|
1399
|
-
the files in the directory are used instead. Defaults to None.
|
|
1452
|
+
`output` and `output_dir_path` are specified, the `output` is ignored,
|
|
1453
|
+
and the files in the directory are used instead. Defaults to None.
|
|
1400
1454
|
duration : int, optional
|
|
1401
1455
|
The duration of the run being tracked, in milliseconds. This field is
|
|
1402
1456
|
optional. Defaults to None.
|
|
@@ -1404,39 +1458,41 @@ class TrackedRun:
|
|
|
1404
1458
|
An error message if the run failed. You should only specify this if the
|
|
1405
1459
|
run failed (the `status` is `TrackedRunStatus.FAILED`), otherwise an
|
|
1406
1460
|
exception will be raised. This field is optional. Defaults to None.
|
|
1407
|
-
logs : list[str], optional
|
|
1408
|
-
The logs of the run being tracked.
|
|
1409
|
-
the log. This field is optional.
|
|
1461
|
+
logs : str or list[str], optional
|
|
1462
|
+
The logs of the run being tracked. If the logs are provided as a list,
|
|
1463
|
+
each element of the list is a line in the log. This field is optional.
|
|
1464
|
+
Defaults to None.
|
|
1410
1465
|
name : str, optional
|
|
1411
1466
|
Optional name for the run being tracked. Defaults to None.
|
|
1412
1467
|
description : str, optional
|
|
1413
1468
|
Optional description for the run being tracked. Defaults to None.
|
|
1414
1469
|
input_dir_path : str, optional
|
|
1415
1470
|
Path to a directory containing input files. If specified, the calling
|
|
1416
|
-
function will package the files in the directory into a tar file and
|
|
1417
|
-
it as a large input. This is useful for non-JSON input formats,
|
|
1418
|
-
when working with `CSV_ARCHIVE` or `MULTI_FILE`. If both
|
|
1419
|
-
`input_dir_path` are specified, the `input` is ignored, and
|
|
1420
|
-
the directory are used instead. Defaults to None.
|
|
1471
|
+
function will package the files in the directory into a tar file and
|
|
1472
|
+
upload it as a large input. This is useful for non-JSON input formats,
|
|
1473
|
+
such as when working with `CSV_ARCHIVE` or `MULTI_FILE`. If both
|
|
1474
|
+
`input` and `input_dir_path` are specified, the `input` is ignored, and
|
|
1475
|
+
the files in the directory are used instead. Defaults to None.
|
|
1421
1476
|
output_dir_path : str, optional
|
|
1422
1477
|
Path to a directory containing output files. If specified, the calling
|
|
1423
|
-
function will package the files in the directory into a tar file and
|
|
1424
|
-
it as a large output. This is useful for non-JSON output
|
|
1425
|
-
when working with `CSV_ARCHIVE` or `MULTI_FILE`. If
|
|
1426
|
-
`output_dir_path` are specified, the `output` is
|
|
1427
|
-
are saved in the directory instead. Defaults to
|
|
1478
|
+
function will package the files in the directory into a tar file and
|
|
1479
|
+
upload it as a large output. This is useful for non-JSON output
|
|
1480
|
+
formats, such as when working with `CSV_ARCHIVE` or `MULTI_FILE`. If
|
|
1481
|
+
both `output` and `output_dir_path` are specified, the `output` is
|
|
1482
|
+
ignored, and the files are saved in the directory instead. Defaults to
|
|
1483
|
+
None.
|
|
1428
1484
|
statistics : Statistics or dict[str, Any], optional
|
|
1429
1485
|
Statistics of the run being tracked. Only use this field if you want to
|
|
1430
|
-
track statistics for `CSV_ARCHIVE` or `MULTI_FILE` output formats. If
|
|
1431
|
-
are working with `JSON` or `TEXT` output formats, this field will
|
|
1432
|
-
ignored, as the statistics are extracted directly from the `output`.
|
|
1486
|
+
track statistics for `CSV_ARCHIVE` or `MULTI_FILE` output formats. If
|
|
1487
|
+
you are working with `JSON` or `TEXT` output formats, this field will
|
|
1488
|
+
be ignored, as the statistics are extracted directly from the `output`.
|
|
1433
1489
|
This field is optional. Defaults to None.
|
|
1434
1490
|
assets : list[Asset or dict[str, Any]], optional
|
|
1435
|
-
Assets associated with the run being tracked. Only use this field if
|
|
1436
|
-
want to track assets for `CSV_ARCHIVE` or `MULTI_FILE` output
|
|
1437
|
-
If you are working with `JSON` or `TEXT` output formats, this
|
|
1438
|
-
be ignored, as the assets are extracted directly from the
|
|
1439
|
-
This field is optional. Defaults to None.
|
|
1491
|
+
Assets associated with the run being tracked. Only use this field if
|
|
1492
|
+
you want to track assets for `CSV_ARCHIVE` or `MULTI_FILE` output
|
|
1493
|
+
formats. If you are working with `JSON` or `TEXT` output formats, this
|
|
1494
|
+
field will be ignored, as the assets are extracted directly from the
|
|
1495
|
+
`output`. This field is optional. Defaults to None.
|
|
1440
1496
|
|
|
1441
1497
|
Examples
|
|
1442
1498
|
--------
|
|
@@ -1482,8 +1538,8 @@ class TrackedRun:
|
|
|
1482
1538
|
------
|
|
1483
1539
|
ValueError
|
|
1484
1540
|
If the status value is invalid, if an error message is provided for a
|
|
1485
|
-
successful run, or if input/output formats are not JSON or
|
|
1486
|
-
|
|
1541
|
+
successful run, or if input/output formats are not JSON or input/output
|
|
1542
|
+
dicts are not JSON serializable.
|
|
1487
1543
|
"""
|
|
1488
1544
|
|
|
1489
1545
|
status: TrackedRunStatus
|
|
@@ -1508,8 +1564,8 @@ class TrackedRun:
|
|
|
1508
1564
|
error: str | None = None
|
|
1509
1565
|
"""An error message if the run failed. You should only specify this if the
|
|
1510
1566
|
run failed, otherwise an exception will be raised."""
|
|
1511
|
-
logs: list[str] | None = None
|
|
1512
|
-
"""The logs of the run being tracked.
|
|
1567
|
+
logs: str | list[str] | None = None
|
|
1568
|
+
"""The logs of the run being tracked. If a list, each element is a line in
|
|
1513
1569
|
the log."""
|
|
1514
1570
|
name: str | None = None
|
|
1515
1571
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nextmv
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.0.dev0
|
|
4
4
|
Summary: The all-purpose Python SDK for Nextmv
|
|
5
5
|
Project-URL: Homepage, https://www.nextmv.io
|
|
6
6
|
Project-URL: Documentation, https://nextmv-py.docs.nextmv.io/en/latest/nextmv/
|
|
@@ -267,20 +267,45 @@ Description-Content-Type: text/markdown
|
|
|
267
267
|
|
|
268
268
|
Welcome to `nextmv`, the general Python SDK for the Nextmv Platform.
|
|
269
269
|
|
|
270
|
-
📖 To learn more about
|
|
270
|
+
📖 To learn more about `nextmv`, visit the [docs][docs].
|
|
271
271
|
|
|
272
272
|
## Installation
|
|
273
273
|
|
|
274
|
-
Requires Python `>=3.10`. Install using
|
|
274
|
+
Requires Python `>=3.10`. Install using the Python package manager of your
|
|
275
|
+
choice:
|
|
275
276
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
```
|
|
277
|
+
- `pip`
|
|
278
|
+
|
|
279
|
+
```bash
|
|
280
|
+
pip install nextmv
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
- `pipx`
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
pipx install nextmv
|
|
287
|
+
```
|
|
279
288
|
|
|
280
|
-
|
|
289
|
+
- `uv`
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
uv tool install nextmv
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
Install all optional dependencies (recommended) by specifying `"nextmv[all]"`
|
|
296
|
+
instead of just `"nextmv"`.
|
|
297
|
+
|
|
298
|
+
## CLI
|
|
299
|
+
|
|
300
|
+
The Nextmv CLI is installed automatically with the SDK. To verify installation,
|
|
301
|
+
run:
|
|
281
302
|
|
|
282
303
|
```bash
|
|
283
|
-
|
|
304
|
+
nextmv --help
|
|
284
305
|
```
|
|
285
306
|
|
|
307
|
+
If you are contributing to the CLI, please make sure you read the [CLI
|
|
308
|
+
Contributing Guide][cli-contributing].
|
|
309
|
+
|
|
286
310
|
[docs]: https://nextmv-py.docs.nextmv.io/en/latest/nextmv/
|
|
311
|
+
[cli-contributing]: nextmv/cli/CONTRIBUTING.md
|