nextmv 0.31.0__py3-none-any.whl → 0.32.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nextmv/__about__.py +1 -1
- nextmv/__init__.py +4 -9
- nextmv/cloud/application.py +37 -3
- nextmv/cloud/batch_experiment.py +7 -1
- nextmv/default_app/README.md +17 -2
- nextmv/default_app/app.yaml +1 -2
- nextmv/default_app/input.json +5 -0
- nextmv/default_app/main.py +37 -0
- nextmv/input.py +2 -8
- nextmv/local/application.py +250 -224
- nextmv/local/executor.py +7 -12
- nextmv/local/local.py +97 -0
- nextmv/local/runner.py +3 -41
- nextmv/output.py +6 -26
- nextmv/run.py +476 -108
- {nextmv-0.31.0.dist-info → nextmv-0.32.0.dist-info}/METADATA +1 -1
- {nextmv-0.31.0.dist-info → nextmv-0.32.0.dist-info}/RECORD +19 -16
- {nextmv-0.31.0.dist-info → nextmv-0.32.0.dist-info}/WHEEL +0 -0
- {nextmv-0.31.0.dist-info → nextmv-0.32.0.dist-info}/licenses/LICENSE +0 -0
nextmv/local/runner.py
CHANGED
|
@@ -11,8 +11,6 @@ new_run
|
|
|
11
11
|
Function to initialize a new run.
|
|
12
12
|
record_input
|
|
13
13
|
Function to write the input to the appropriate location.
|
|
14
|
-
calculate_files_size
|
|
15
|
-
Function to calculate the total size of files in a directory.
|
|
16
14
|
"""
|
|
17
15
|
|
|
18
16
|
import importlib.util
|
|
@@ -24,7 +22,8 @@ import sys
|
|
|
24
22
|
from datetime import datetime, timezone
|
|
25
23
|
from typing import Any, Optional, Union
|
|
26
24
|
|
|
27
|
-
from nextmv.input import
|
|
25
|
+
from nextmv.input import INPUTS_KEY
|
|
26
|
+
from nextmv.local.local import DEFAULT_INPUT_JSON_FILE, NEXTMV_DIR, RUNS_KEY, calculate_files_size
|
|
28
27
|
from nextmv.manifest import Manifest
|
|
29
28
|
from nextmv.run import Format, FormatInput, Metadata, RunInformation, StatusV2
|
|
30
29
|
from nextmv.safe import safe_id
|
|
@@ -176,7 +175,7 @@ def new_run(
|
|
|
176
175
|
"""
|
|
177
176
|
|
|
178
177
|
# First, ensure the runs directory exists.
|
|
179
|
-
runs_dir = os.path.join(src,
|
|
178
|
+
runs_dir = os.path.join(src, NEXTMV_DIR, RUNS_KEY)
|
|
180
179
|
os.makedirs(runs_dir, exist_ok=True)
|
|
181
180
|
|
|
182
181
|
# Create a new run directory.
|
|
@@ -273,40 +272,3 @@ def record_input(
|
|
|
273
272
|
|
|
274
273
|
# Update the input size in the run information file.
|
|
275
274
|
calculate_files_size(run_dir, run_id, run_inputs_dir, metadata_key="input_size")
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
def calculate_files_size(run_dir: str, run_id: str, dir_path: str, metadata_key: str) -> None:
|
|
279
|
-
"""
|
|
280
|
-
Calculates the total size of the files in a directory, in bytes.
|
|
281
|
-
|
|
282
|
-
The calculated size is stored in the run information metadata under the
|
|
283
|
-
specified key.
|
|
284
|
-
|
|
285
|
-
Parameters
|
|
286
|
-
----------
|
|
287
|
-
run_dir : str
|
|
288
|
-
The path to the run directory.
|
|
289
|
-
run_id : str
|
|
290
|
-
The ID of the run.
|
|
291
|
-
dir_path : str
|
|
292
|
-
The path to the directory whose size is to be calculated.
|
|
293
|
-
metadata_key : str
|
|
294
|
-
The key under which to store the calculated size in the run information
|
|
295
|
-
metadata.
|
|
296
|
-
"""
|
|
297
|
-
|
|
298
|
-
total_size = 0
|
|
299
|
-
for dirpath, _, filenames in os.walk(dir_path):
|
|
300
|
-
for f in filenames:
|
|
301
|
-
fp = os.path.join(dirpath, f)
|
|
302
|
-
# Skip if it is a symbolic link
|
|
303
|
-
if not os.path.islink(fp):
|
|
304
|
-
total_size += os.path.getsize(fp)
|
|
305
|
-
|
|
306
|
-
info_file = os.path.join(run_dir, f"{run_id}.json")
|
|
307
|
-
with open(info_file, "r+") as f:
|
|
308
|
-
info = json.load(f)
|
|
309
|
-
info["metadata"][metadata_key] = total_size
|
|
310
|
-
f.seek(0)
|
|
311
|
-
json.dump(info, f, indent=2)
|
|
312
|
-
f.truncate()
|
nextmv/output.py
CHANGED
|
@@ -41,20 +41,16 @@ Functions
|
|
|
41
41
|
write
|
|
42
42
|
Write the output to the specified destination.
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
ASSETS_KEY
|
|
44
|
+
Attributes
|
|
45
|
+
----------
|
|
46
|
+
ASSETS_KEY : str
|
|
47
47
|
Assets key constant used for identifying assets in the run output.
|
|
48
|
-
STATISTICS_KEY
|
|
48
|
+
STATISTICS_KEY : str
|
|
49
49
|
Statistics key constant used for identifying statistics in the run output.
|
|
50
|
-
SOLUTIONS_KEY
|
|
50
|
+
SOLUTIONS_KEY : str
|
|
51
51
|
Solutions key constant used for identifying solutions in the run output.
|
|
52
|
-
OUTPUTS_KEY
|
|
52
|
+
OUTPUTS_KEY : str
|
|
53
53
|
Outputs key constant used for identifying outputs in the run output.
|
|
54
|
-
LOGS_FILE
|
|
55
|
-
Constant used for identifying the file used for logging.
|
|
56
|
-
DEFAULT_OUTPUT_JSON_FILE
|
|
57
|
-
Constant for the default output JSON file name.
|
|
58
54
|
"""
|
|
59
55
|
|
|
60
56
|
import copy
|
|
@@ -90,22 +86,6 @@ OUTPUTS_KEY = "outputs"
|
|
|
90
86
|
"""
|
|
91
87
|
Outputs key constant used for identifying outputs in the run output.
|
|
92
88
|
"""
|
|
93
|
-
OUTPUT_KEY = "output"
|
|
94
|
-
"""
|
|
95
|
-
Output key constant used for identifying output in the run output.
|
|
96
|
-
"""
|
|
97
|
-
LOGS_KEY = "logs"
|
|
98
|
-
"""
|
|
99
|
-
Logs key constant used for identifying logs in the run output.
|
|
100
|
-
"""
|
|
101
|
-
LOGS_FILE = "stderr.log"
|
|
102
|
-
"""
|
|
103
|
-
Constant used for identifying the file used for logging.
|
|
104
|
-
"""
|
|
105
|
-
DEFAULT_OUTPUT_JSON_FILE = "solution.json"
|
|
106
|
-
"""
|
|
107
|
-
Constant for the default output JSON file name.
|
|
108
|
-
"""
|
|
109
89
|
|
|
110
90
|
|
|
111
91
|
class RunStatistics(BaseModel):
|