nextmv 0.18.0__py3-none-any.whl → 1.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nextmv/__about__.py +1 -1
- nextmv/__entrypoint__.py +8 -13
- nextmv/__init__.py +53 -0
- nextmv/_serialization.py +96 -0
- nextmv/base_model.py +54 -9
- nextmv/cli/CONTRIBUTING.md +511 -0
- nextmv/cli/__init__.py +0 -0
- nextmv/cli/cloud/__init__.py +47 -0
- nextmv/cli/cloud/acceptance/__init__.py +27 -0
- nextmv/cli/cloud/acceptance/create.py +393 -0
- nextmv/cli/cloud/acceptance/delete.py +68 -0
- nextmv/cli/cloud/acceptance/get.py +104 -0
- nextmv/cli/cloud/acceptance/list.py +62 -0
- nextmv/cli/cloud/acceptance/update.py +95 -0
- nextmv/cli/cloud/account/__init__.py +28 -0
- nextmv/cli/cloud/account/create.py +83 -0
- nextmv/cli/cloud/account/delete.py +60 -0
- nextmv/cli/cloud/account/get.py +66 -0
- nextmv/cli/cloud/account/update.py +70 -0
- nextmv/cli/cloud/app/__init__.py +35 -0
- nextmv/cli/cloud/app/create.py +141 -0
- nextmv/cli/cloud/app/delete.py +58 -0
- nextmv/cli/cloud/app/exists.py +44 -0
- nextmv/cli/cloud/app/get.py +66 -0
- nextmv/cli/cloud/app/list.py +61 -0
- nextmv/cli/cloud/app/push.py +137 -0
- nextmv/cli/cloud/app/update.py +124 -0
- nextmv/cli/cloud/batch/__init__.py +29 -0
- nextmv/cli/cloud/batch/create.py +454 -0
- nextmv/cli/cloud/batch/delete.py +68 -0
- nextmv/cli/cloud/batch/get.py +104 -0
- nextmv/cli/cloud/batch/list.py +63 -0
- nextmv/cli/cloud/batch/metadata.py +66 -0
- nextmv/cli/cloud/batch/update.py +95 -0
- nextmv/cli/cloud/data/__init__.py +26 -0
- nextmv/cli/cloud/data/upload.py +162 -0
- nextmv/cli/cloud/ensemble/__init__.py +31 -0
- nextmv/cli/cloud/ensemble/create.py +414 -0
- nextmv/cli/cloud/ensemble/delete.py +67 -0
- nextmv/cli/cloud/ensemble/get.py +65 -0
- nextmv/cli/cloud/ensemble/update.py +103 -0
- nextmv/cli/cloud/input_set/__init__.py +30 -0
- nextmv/cli/cloud/input_set/create.py +170 -0
- nextmv/cli/cloud/input_set/get.py +63 -0
- nextmv/cli/cloud/input_set/list.py +63 -0
- nextmv/cli/cloud/input_set/update.py +123 -0
- nextmv/cli/cloud/instance/__init__.py +35 -0
- nextmv/cli/cloud/instance/create.py +290 -0
- nextmv/cli/cloud/instance/delete.py +62 -0
- nextmv/cli/cloud/instance/exists.py +39 -0
- nextmv/cli/cloud/instance/get.py +62 -0
- nextmv/cli/cloud/instance/list.py +60 -0
- nextmv/cli/cloud/instance/update.py +216 -0
- nextmv/cli/cloud/managed_input/__init__.py +31 -0
- nextmv/cli/cloud/managed_input/create.py +146 -0
- nextmv/cli/cloud/managed_input/delete.py +65 -0
- nextmv/cli/cloud/managed_input/get.py +63 -0
- nextmv/cli/cloud/managed_input/list.py +60 -0
- nextmv/cli/cloud/managed_input/update.py +97 -0
- nextmv/cli/cloud/run/__init__.py +37 -0
- nextmv/cli/cloud/run/cancel.py +37 -0
- nextmv/cli/cloud/run/create.py +530 -0
- nextmv/cli/cloud/run/get.py +199 -0
- nextmv/cli/cloud/run/input.py +86 -0
- nextmv/cli/cloud/run/list.py +80 -0
- nextmv/cli/cloud/run/logs.py +167 -0
- nextmv/cli/cloud/run/metadata.py +67 -0
- nextmv/cli/cloud/run/track.py +501 -0
- nextmv/cli/cloud/scenario/__init__.py +29 -0
- nextmv/cli/cloud/scenario/create.py +451 -0
- nextmv/cli/cloud/scenario/delete.py +65 -0
- nextmv/cli/cloud/scenario/get.py +102 -0
- nextmv/cli/cloud/scenario/list.py +63 -0
- nextmv/cli/cloud/scenario/metadata.py +67 -0
- nextmv/cli/cloud/scenario/update.py +93 -0
- nextmv/cli/cloud/secrets/__init__.py +33 -0
- nextmv/cli/cloud/secrets/create.py +206 -0
- nextmv/cli/cloud/secrets/delete.py +67 -0
- nextmv/cli/cloud/secrets/get.py +66 -0
- nextmv/cli/cloud/secrets/list.py +60 -0
- nextmv/cli/cloud/secrets/update.py +147 -0
- nextmv/cli/cloud/shadow/__init__.py +33 -0
- nextmv/cli/cloud/shadow/create.py +184 -0
- nextmv/cli/cloud/shadow/delete.py +68 -0
- nextmv/cli/cloud/shadow/get.py +61 -0
- nextmv/cli/cloud/shadow/list.py +63 -0
- nextmv/cli/cloud/shadow/metadata.py +66 -0
- nextmv/cli/cloud/shadow/start.py +43 -0
- nextmv/cli/cloud/shadow/stop.py +43 -0
- nextmv/cli/cloud/shadow/update.py +95 -0
- nextmv/cli/cloud/upload/__init__.py +22 -0
- nextmv/cli/cloud/upload/create.py +39 -0
- nextmv/cli/cloud/version/__init__.py +33 -0
- nextmv/cli/cloud/version/create.py +97 -0
- nextmv/cli/cloud/version/delete.py +62 -0
- nextmv/cli/cloud/version/exists.py +39 -0
- nextmv/cli/cloud/version/get.py +62 -0
- nextmv/cli/cloud/version/list.py +60 -0
- nextmv/cli/cloud/version/update.py +92 -0
- nextmv/cli/community/__init__.py +24 -0
- nextmv/cli/community/clone.py +270 -0
- nextmv/cli/community/list.py +265 -0
- nextmv/cli/configuration/__init__.py +23 -0
- nextmv/cli/configuration/config.py +195 -0
- nextmv/cli/configuration/create.py +94 -0
- nextmv/cli/configuration/delete.py +67 -0
- nextmv/cli/configuration/list.py +77 -0
- nextmv/cli/main.py +188 -0
- nextmv/cli/message.py +153 -0
- nextmv/cli/options.py +206 -0
- nextmv/cli/version.py +38 -0
- nextmv/cloud/__init__.py +71 -17
- nextmv/cloud/acceptance_test.py +757 -51
- nextmv/cloud/account.py +406 -17
- nextmv/cloud/application/__init__.py +957 -0
- nextmv/cloud/application/_acceptance.py +419 -0
- nextmv/cloud/application/_batch_scenario.py +860 -0
- nextmv/cloud/application/_ensemble.py +251 -0
- nextmv/cloud/application/_input_set.py +227 -0
- nextmv/cloud/application/_instance.py +289 -0
- nextmv/cloud/application/_managed_input.py +227 -0
- nextmv/cloud/application/_run.py +1393 -0
- nextmv/cloud/application/_secrets.py +294 -0
- nextmv/cloud/application/_shadow.py +314 -0
- nextmv/cloud/application/_utils.py +54 -0
- nextmv/cloud/application/_version.py +303 -0
- nextmv/cloud/assets.py +48 -0
- nextmv/cloud/batch_experiment.py +294 -33
- nextmv/cloud/client.py +307 -66
- nextmv/cloud/ensemble.py +247 -0
- nextmv/cloud/input_set.py +120 -2
- nextmv/cloud/instance.py +133 -8
- nextmv/cloud/integration.py +533 -0
- nextmv/cloud/package.py +168 -53
- nextmv/cloud/scenario.py +410 -0
- nextmv/cloud/secrets.py +234 -0
- nextmv/cloud/shadow.py +190 -0
- nextmv/cloud/url.py +73 -0
- nextmv/cloud/version.py +132 -4
- nextmv/default_app/.gitignore +1 -0
- nextmv/default_app/README.md +32 -0
- nextmv/default_app/app.yaml +12 -0
- nextmv/default_app/input.json +5 -0
- nextmv/default_app/main.py +37 -0
- nextmv/default_app/requirements.txt +2 -0
- nextmv/default_app/src/__init__.py +0 -0
- nextmv/default_app/src/visuals.py +36 -0
- nextmv/deprecated.py +47 -0
- nextmv/input.py +861 -90
- nextmv/local/__init__.py +5 -0
- nextmv/local/application.py +1251 -0
- nextmv/local/executor.py +1042 -0
- nextmv/local/geojson_handler.py +323 -0
- nextmv/local/local.py +97 -0
- nextmv/local/plotly_handler.py +61 -0
- nextmv/local/runner.py +274 -0
- nextmv/logger.py +80 -9
- nextmv/manifest.py +1466 -0
- nextmv/model.py +241 -66
- nextmv/options.py +708 -115
- nextmv/output.py +1301 -274
- nextmv/polling.py +325 -0
- nextmv/run.py +1702 -0
- nextmv/safe.py +145 -0
- nextmv/status.py +122 -0
- nextmv-1.0.0.dev2.dist-info/METADATA +311 -0
- nextmv-1.0.0.dev2.dist-info/RECORD +170 -0
- {nextmv-0.18.0.dist-info → nextmv-1.0.0.dev2.dist-info}/WHEEL +1 -1
- nextmv-1.0.0.dev2.dist-info/entry_points.txt +2 -0
- nextmv/cloud/application.py +0 -1405
- nextmv/cloud/manifest.py +0 -234
- nextmv/cloud/status.py +0 -29
- nextmv-0.18.0.dist-info/METADATA +0 -770
- nextmv-0.18.0.dist-info/RECORD +0 -25
- {nextmv-0.18.0.dist-info → nextmv-1.0.0.dev2.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,1251 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Application module for interacting with local Nextmv applications.
|
|
3
|
+
|
|
4
|
+
This module provides functionality to interact with applications in Nextmv,
|
|
5
|
+
including application management, running applications, and managing inputs.
|
|
6
|
+
|
|
7
|
+
Classes
|
|
8
|
+
-------
|
|
9
|
+
Application
|
|
10
|
+
Class for interacting with local Nextmv applications.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import shutil
|
|
16
|
+
import tempfile
|
|
17
|
+
import webbrowser
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from nextmv import cloud
|
|
23
|
+
from nextmv._serialization import deflated_serialize_json
|
|
24
|
+
from nextmv.base_model import BaseModel
|
|
25
|
+
from nextmv.input import INPUTS_KEY, Input, InputFormat
|
|
26
|
+
from nextmv.local.local import (
|
|
27
|
+
DEFAULT_INPUT_JSON_FILE,
|
|
28
|
+
DEFAULT_OUTPUT_JSON_FILE,
|
|
29
|
+
LOGS_FILE,
|
|
30
|
+
LOGS_KEY,
|
|
31
|
+
NEXTMV_DIR,
|
|
32
|
+
RUNS_KEY,
|
|
33
|
+
)
|
|
34
|
+
from nextmv.local.runner import run
|
|
35
|
+
from nextmv.logger import log
|
|
36
|
+
from nextmv.manifest import Manifest, default_python_manifest
|
|
37
|
+
from nextmv.options import Options
|
|
38
|
+
from nextmv.output import ASSETS_KEY, OUTPUTS_KEY, SOLUTIONS_KEY, STATISTICS_KEY, OutputFormat
|
|
39
|
+
from nextmv.polling import DEFAULT_POLLING_OPTIONS, PollingOptions, poll
|
|
40
|
+
from nextmv.run import (
|
|
41
|
+
ErrorLog,
|
|
42
|
+
Format,
|
|
43
|
+
FormatInput,
|
|
44
|
+
Run,
|
|
45
|
+
RunConfiguration,
|
|
46
|
+
RunInformation,
|
|
47
|
+
RunResult,
|
|
48
|
+
SyncedRun,
|
|
49
|
+
TrackedRun,
|
|
50
|
+
TrackedRunStatus,
|
|
51
|
+
)
|
|
52
|
+
from nextmv.safe import safe_id
|
|
53
|
+
from nextmv.status import StatusV2
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class Application:
|
|
58
|
+
"""
|
|
59
|
+
A decision model that can be executed.
|
|
60
|
+
|
|
61
|
+
You can import the `Application` class directly from `local`:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from nextmv.local import Application
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
This class represents an application in Nextmv, providing methods to
|
|
68
|
+
interact with the application, run it with different inputs, manage
|
|
69
|
+
versions, instances, experiments, and more.
|
|
70
|
+
|
|
71
|
+
Parameters
|
|
72
|
+
----------
|
|
73
|
+
src : str
|
|
74
|
+
Source of the application, when initialized locally. An application's
|
|
75
|
+
source typically refers to the directory containing the `app.yaml`
|
|
76
|
+
manifest.
|
|
77
|
+
description : Optional[str], default=None
|
|
78
|
+
Description of the application.
|
|
79
|
+
|
|
80
|
+
Examples
|
|
81
|
+
--------
|
|
82
|
+
>>> from nextmv.local import Application
|
|
83
|
+
>>> app = Application(src="path/to/app")
|
|
84
|
+
>>> # Retrieve an app's run result
|
|
85
|
+
>>> result = app.run_result("run-id")
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
src: str
|
|
89
|
+
"""
|
|
90
|
+
Source of the application, when initialized locally. An application's
|
|
91
|
+
source typically refers to the directory containing the `app.yaml`
|
|
92
|
+
manifest.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
description: str | None = None
|
|
96
|
+
"""Description of the application."""
|
|
97
|
+
manifest: Manifest | None = None
|
|
98
|
+
"""
|
|
99
|
+
Manifest of the application. A manifest is a file named `app.yaml` that
|
|
100
|
+
must be present at the root of the application's `src` directory. If the
|
|
101
|
+
app is initialized, and a manifest is not present, a default Python
|
|
102
|
+
manifest will be created, using the `nextmv.default_python_manifest`
|
|
103
|
+
function. If you specify this argument, and a manifest file is already
|
|
104
|
+
present in the `src` directory, the provided manifest will override the
|
|
105
|
+
existing one.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
def __post_init__(self):
|
|
109
|
+
"""
|
|
110
|
+
Validate the presence of the manifest in the application.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
if self.manifest is not None:
|
|
114
|
+
self.manifest.to_yaml(self.src)
|
|
115
|
+
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
try:
|
|
119
|
+
manifest = Manifest.from_yaml(self.src)
|
|
120
|
+
self.manifest = manifest
|
|
121
|
+
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
except Exception:
|
|
125
|
+
manifest = default_python_manifest()
|
|
126
|
+
self.manifest = manifest
|
|
127
|
+
manifest.to_yaml(self.src)
|
|
128
|
+
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
@classmethod
|
|
132
|
+
def initialize(
|
|
133
|
+
cls,
|
|
134
|
+
src: str | None = None,
|
|
135
|
+
description: str | None = None,
|
|
136
|
+
destination: str | None = None,
|
|
137
|
+
) -> "Application":
|
|
138
|
+
"""
|
|
139
|
+
Initialize a sample Nextmv application, locally.
|
|
140
|
+
|
|
141
|
+
This method will create a new application in the local file system. The
|
|
142
|
+
application is a dir with the name given by `src` (it becomes the
|
|
143
|
+
_source_ of the app), under the location given by `destination`. If the
|
|
144
|
+
`destination` parameter is not specified, the current working directory
|
|
145
|
+
is used as default. This method will scaffold the application with the
|
|
146
|
+
necessary files and directories to have an opinionated structure for
|
|
147
|
+
your decision model. Once the application is initialized, you are
|
|
148
|
+
encouraged to complete it with the decision model itself, so that the
|
|
149
|
+
application can be run locally.
|
|
150
|
+
|
|
151
|
+
If the `src` parameter is not provided, a random name will be generated
|
|
152
|
+
for the application.
|
|
153
|
+
|
|
154
|
+
Parameters
|
|
155
|
+
----------
|
|
156
|
+
src : str, optional
|
|
157
|
+
Source (ID, name) of the application. Will be generated if not
|
|
158
|
+
provided.
|
|
159
|
+
description : str, optional
|
|
160
|
+
Description of the application.
|
|
161
|
+
destination : str, optional
|
|
162
|
+
Destination directory where the application will be initialized. If
|
|
163
|
+
not provided, the current working directory will be used.
|
|
164
|
+
|
|
165
|
+
Returns
|
|
166
|
+
-------
|
|
167
|
+
Application
|
|
168
|
+
The initialized application instance.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
destination_dir = os.getcwd() if destination is None else destination
|
|
172
|
+
app_id = src if src is not None else safe_id("app")
|
|
173
|
+
|
|
174
|
+
# Create the new directory with the given name.
|
|
175
|
+
app_src = os.path.join(destination_dir, app_id)
|
|
176
|
+
if os.path.exists(app_src):
|
|
177
|
+
raise FileExistsError(f"destination dir for src already exists: {app_src}")
|
|
178
|
+
|
|
179
|
+
os.makedirs(app_src, exist_ok=False)
|
|
180
|
+
|
|
181
|
+
# Get the path to the initial app structure template.
|
|
182
|
+
current_file_dir = os.path.dirname(os.path.abspath(__file__))
|
|
183
|
+
initial_app_structure_path = os.path.join(current_file_dir, "..", "default_app")
|
|
184
|
+
initial_app_structure_path = os.path.normpath(initial_app_structure_path)
|
|
185
|
+
|
|
186
|
+
# Copy everything from initial_app_structure to the new directory.
|
|
187
|
+
if os.path.exists(initial_app_structure_path):
|
|
188
|
+
shutil.copytree(initial_app_structure_path, app_src, dirs_exist_ok=True)
|
|
189
|
+
|
|
190
|
+
return cls(
|
|
191
|
+
src=app_src,
|
|
192
|
+
description=description,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
def list_runs(self) -> list[Run]:
|
|
196
|
+
"""
|
|
197
|
+
List all runs for the application.
|
|
198
|
+
|
|
199
|
+
Returns
|
|
200
|
+
-------
|
|
201
|
+
list[Run]
|
|
202
|
+
A list of all runs associated with the application.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
runs_dir = os.path.join(self.src, NEXTMV_DIR, RUNS_KEY)
|
|
206
|
+
if not os.path.exists(runs_dir):
|
|
207
|
+
raise ValueError(f"`.nextmv/runs` dir does not exist at app source: {self.src}")
|
|
208
|
+
|
|
209
|
+
dirs = os.listdir(runs_dir)
|
|
210
|
+
if not dirs:
|
|
211
|
+
return []
|
|
212
|
+
|
|
213
|
+
run_ids = [d for d in dirs if os.path.isdir(os.path.join(runs_dir, d))]
|
|
214
|
+
if not run_ids:
|
|
215
|
+
return []
|
|
216
|
+
|
|
217
|
+
runs = []
|
|
218
|
+
for run_id in run_ids:
|
|
219
|
+
info = self.run_metadata(run_id=run_id)
|
|
220
|
+
run = info.to_run()
|
|
221
|
+
runs.append(run)
|
|
222
|
+
|
|
223
|
+
return runs
|
|
224
|
+
|
|
225
|
+
def new_run(
|
|
226
|
+
self,
|
|
227
|
+
input: Input | dict[str, Any] | BaseModel | str = None,
|
|
228
|
+
name: str | None = None,
|
|
229
|
+
description: str | None = None,
|
|
230
|
+
options: Options | dict[str, str] | None = None,
|
|
231
|
+
configuration: RunConfiguration | dict[str, Any] | None = None,
|
|
232
|
+
json_configurations: dict[str, Any] | None = None,
|
|
233
|
+
input_dir_path: str | None = None,
|
|
234
|
+
) -> str:
|
|
235
|
+
"""
|
|
236
|
+
Run the application locally with the provided input.
|
|
237
|
+
|
|
238
|
+
This method is the local equivalent to `cloud.Application.new_run`,
|
|
239
|
+
which submits the input to Nextmv Cloud. This method runs the
|
|
240
|
+
application locally using the `src` of the app.
|
|
241
|
+
|
|
242
|
+
Make sure that the `src` attribute is set on the `Application` class
|
|
243
|
+
before running locally, as it is required by the method.
|
|
244
|
+
|
|
245
|
+
Parameters
|
|
246
|
+
----------
|
|
247
|
+
input: Union[Input, dict[str, Any], BaseModel, str]
|
|
248
|
+
Input to use for the run. This can be a `nextmv.Input` object,
|
|
249
|
+
`dict`, `BaseModel` or `str`.
|
|
250
|
+
|
|
251
|
+
If `nextmv.Input` is used, and the `input_format` is either
|
|
252
|
+
`nextmv.InputFormat.JSON` or `nextmv.InputFormat.TEXT`, then the
|
|
253
|
+
input data is extracted from the `.data` property.
|
|
254
|
+
|
|
255
|
+
If you want to work with `nextmv.InputFormat.CSV_ARCHIVE` or
|
|
256
|
+
`nextmv.InputFormat.MULTI_FILE`, you should use the
|
|
257
|
+
`input_dir_path` argument instead. This argument takes precedence
|
|
258
|
+
over the `input`. If `input_dir_path` is specified, this function
|
|
259
|
+
looks for files in that directory and tars them. If both the
|
|
260
|
+
`input_dir_path` and `input` arguments are provided, the `input`
|
|
261
|
+
is ignored.
|
|
262
|
+
|
|
263
|
+
When `input_dir_path` is specified, the `configuration` argument
|
|
264
|
+
must also be provided. More specifically, the
|
|
265
|
+
`RunConfiguration.format.format_input.input_type` parameter
|
|
266
|
+
dictates what kind of input is being submitted to the Nextmv Cloud.
|
|
267
|
+
Make sure that this parameter is specified when working with the
|
|
268
|
+
following input formats:
|
|
269
|
+
|
|
270
|
+
- `nextmv.InputFormat.CSV_ARCHIVE`
|
|
271
|
+
- `nextmv.InputFormat.MULTI_FILE`
|
|
272
|
+
|
|
273
|
+
When working with JSON or text data, use the `input` argument
|
|
274
|
+
directly.
|
|
275
|
+
name: Optional[str]
|
|
276
|
+
Name of the local run.
|
|
277
|
+
description: Optional[str]
|
|
278
|
+
Description of the local run.
|
|
279
|
+
options: Optional[Union[Options, dict[str, str]]]
|
|
280
|
+
Options to use for the run. This can be a `nextmv.Options` object
|
|
281
|
+
or a dict. If a dict is used, the keys must be strings and the
|
|
282
|
+
values must be strings as well. If a `nextmv.Options` object is
|
|
283
|
+
used, the options are extracted from the `.to_cloud_dict()` method.
|
|
284
|
+
Note that specifying `options` overrides the `input.options` (if
|
|
285
|
+
the `input` is of type `nextmv.Input`).
|
|
286
|
+
configuration: Optional[Union[RunConfiguration, dict[str, Any]]]
|
|
287
|
+
Configuration to use for the run. This can be a
|
|
288
|
+
`cloud.RunConfiguration` object or a dict. If the object is used,
|
|
289
|
+
then the `.to_dict()` method is applied to extract the
|
|
290
|
+
configuration.
|
|
291
|
+
json_configurations: Optional[dict[str, Any]]
|
|
292
|
+
Optional configurations for JSON serialization. This is used to
|
|
293
|
+
customize the serialization before data is sent.
|
|
294
|
+
input_dir_path: Optional[str]
|
|
295
|
+
Path to a directory containing input files. This is useful for
|
|
296
|
+
input formats like `nextmv.InputFormat.CSV_ARCHIVE` or
|
|
297
|
+
`nextmv.InputFormat.MULTI_FILE`. If both `input` and
|
|
298
|
+
`input_dir_path` are specified, the `input` is ignored, and the
|
|
299
|
+
files in the directory are used instead.
|
|
300
|
+
|
|
301
|
+
Returns
|
|
302
|
+
-------
|
|
303
|
+
str
|
|
304
|
+
ID (`run_id`) of the local run that was executed.
|
|
305
|
+
|
|
306
|
+
Raises
|
|
307
|
+
------
|
|
308
|
+
ValueError
|
|
309
|
+
If the `src` property for the `Application` is not specified.
|
|
310
|
+
If neither `input` nor `input_dir_path` is specified.
|
|
311
|
+
If `input_dir_path` is specified but `configuration` is not provided.
|
|
312
|
+
FileNotFoundError
|
|
313
|
+
If the manifest.yaml file cannot be found in the specified `src` directory.
|
|
314
|
+
|
|
315
|
+
Examples
|
|
316
|
+
--------
|
|
317
|
+
>>> from nextmv.local import Application
|
|
318
|
+
>>> app = Application(id="my-app", src="/path/to/app")
|
|
319
|
+
>>> run_id = app.new_run(
|
|
320
|
+
... input={"vehicles": [{"id": "v1"}]},
|
|
321
|
+
... options={"duration": "10s"}
|
|
322
|
+
... )
|
|
323
|
+
>>> print(f"Local run completed with ID: {run_id}")
|
|
324
|
+
"""
|
|
325
|
+
|
|
326
|
+
configuration = self.__validate_input_dir_path_and_configuration(input_dir_path, configuration)
|
|
327
|
+
|
|
328
|
+
if self.src is None:
|
|
329
|
+
raise ValueError("`src` property for the `Application` must be specified to run the application locally")
|
|
330
|
+
|
|
331
|
+
if input is None and input_dir_path is None:
|
|
332
|
+
raise ValueError("Either `input` or `input_directory` must be specified")
|
|
333
|
+
|
|
334
|
+
try:
|
|
335
|
+
manifest = Manifest.from_yaml(self.src)
|
|
336
|
+
except FileNotFoundError as e:
|
|
337
|
+
raise FileNotFoundError(
|
|
338
|
+
f"Could not find manifest.yaml in {self.src}. Maybe specify a different `src` dir?"
|
|
339
|
+
) from e
|
|
340
|
+
|
|
341
|
+
input_data = None if input_dir_path else self.__extract_input_data(input)
|
|
342
|
+
options_dict = self.__extract_options_dict(options, json_configurations)
|
|
343
|
+
run_config_dict = self.__extract_run_config(input, configuration, input_dir_path)
|
|
344
|
+
run_id = run(
|
|
345
|
+
app_id=self.src,
|
|
346
|
+
src=self.src,
|
|
347
|
+
manifest=manifest,
|
|
348
|
+
run_config=run_config_dict,
|
|
349
|
+
name=name,
|
|
350
|
+
description=description,
|
|
351
|
+
input_data=input_data,
|
|
352
|
+
inputs_dir_path=input_dir_path,
|
|
353
|
+
options=options_dict,
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
return run_id
|
|
357
|
+
|
|
358
|
+
def new_run_with_result(
|
|
359
|
+
self,
|
|
360
|
+
input: Input | dict[str, Any] | BaseModel | str = None,
|
|
361
|
+
name: str | None = None,
|
|
362
|
+
description: str | None = None,
|
|
363
|
+
run_options: Options | dict[str, str] | None = None,
|
|
364
|
+
polling_options: PollingOptions = DEFAULT_POLLING_OPTIONS,
|
|
365
|
+
configuration: RunConfiguration | dict[str, Any] | None = None,
|
|
366
|
+
json_configurations: dict[str, Any] | None = None,
|
|
367
|
+
input_dir_path: str | None = None,
|
|
368
|
+
output_dir_path: str | None = ".",
|
|
369
|
+
) -> RunResult:
|
|
370
|
+
"""
|
|
371
|
+
Submit an input to start a new local run of the application and poll
|
|
372
|
+
for the result. This is a convenience method that combines the
|
|
373
|
+
`new_run` and `run_result_with_polling` methods, applying
|
|
374
|
+
polling logic to check when the local run succeeded.
|
|
375
|
+
|
|
376
|
+
This method is the local equivalent to
|
|
377
|
+
`cloud.Application.new_run_with_result`, which submits the input to
|
|
378
|
+
Nextmv Cloud. This method runs the application locally using the `src`
|
|
379
|
+
of the app.
|
|
380
|
+
|
|
381
|
+
Make sure that the `src` attribute is set on the `Application` class
|
|
382
|
+
before running locally, as it is required by the method.
|
|
383
|
+
|
|
384
|
+
Parameters
|
|
385
|
+
----------
|
|
386
|
+
input: Union[Input, dict[str, Any], BaseModel, str]
|
|
387
|
+
Input to use for the run. This can be a `nextmv.Input` object,
|
|
388
|
+
`dict`, `BaseModel` or `str`.
|
|
389
|
+
|
|
390
|
+
If `nextmv.Input` is used, and the `input_format` is either
|
|
391
|
+
`nextmv.InputFormat.JSON` or `nextmv.InputFormat.TEXT`, then the
|
|
392
|
+
input data is extracted from the `.data` property.
|
|
393
|
+
|
|
394
|
+
If you want to work with `nextmv.InputFormat.CSV_ARCHIVE` or
|
|
395
|
+
`nextmv.InputFormat.MULTI_FILE`, you should use the
|
|
396
|
+
`input_dir_path` argument instead. This argument takes precedence
|
|
397
|
+
over the `input`. If `input_dir_path` is specified, this function
|
|
398
|
+
looks for files in that directory and tars them. If both the
|
|
399
|
+
`input_dir_path` and `input` arguments are provided, the `input` is
|
|
400
|
+
ignored.
|
|
401
|
+
|
|
402
|
+
When `input_dir_path` is specified, the `configuration` argument
|
|
403
|
+
must also be provided. More specifically, the
|
|
404
|
+
`RunConfiguration.format.format_input.input_type` parameter
|
|
405
|
+
dictates what kind of input is being submitted to the Nextmv Cloud.
|
|
406
|
+
Make sure that this parameter is specified when working with the
|
|
407
|
+
following input formats:
|
|
408
|
+
|
|
409
|
+
- `nextmv.InputFormat.CSV_ARCHIVE`
|
|
410
|
+
- `nextmv.InputFormat.MULTI_FILE`
|
|
411
|
+
|
|
412
|
+
When working with JSON or text data, use the `input` argument
|
|
413
|
+
directly.
|
|
414
|
+
name: Optional[str]
|
|
415
|
+
Name of the local run.
|
|
416
|
+
description: Optional[str]
|
|
417
|
+
Description of the local run.
|
|
418
|
+
run_options: Optional[Union[Options, dict[str, str]]]
|
|
419
|
+
Options to use for the run. This can be a `nextmv.Options` object
|
|
420
|
+
or a dict. If a dict is used, the keys must be strings and the
|
|
421
|
+
values must be strings as well. If a `nextmv.Options` object is
|
|
422
|
+
used, the options are extracted from the `.to_cloud_dict()` method.
|
|
423
|
+
Note that specifying `options` overrides the `input.options` (if
|
|
424
|
+
the `input` is of type `nextmv.Input`).
|
|
425
|
+
polling_options: PollingOptions, default=_DEFAULT_POLLING_OPTIONS
|
|
426
|
+
Options to use when polling for the run result.
|
|
427
|
+
configuration: Optional[Union[RunConfiguration, dict[str, Any]]]
|
|
428
|
+
Configuration to use for the run. This can be a
|
|
429
|
+
`cloud.RunConfiguration` object or a dict. If the object is used,
|
|
430
|
+
then the `.to_dict()` method is applied to extract the
|
|
431
|
+
configuration.
|
|
432
|
+
json_configurations: Optional[dict[str, Any]]
|
|
433
|
+
Optional configurations for JSON serialization. This is used to
|
|
434
|
+
customize the serialization before data is sent.
|
|
435
|
+
input_dir_path: Optional[str]
|
|
436
|
+
Path to a directory containing input files. This is useful for
|
|
437
|
+
input formats like `nextmv.InputFormat.CSV_ARCHIVE` or
|
|
438
|
+
`nextmv.InputFormat.MULTI_FILE`. If both `input` and
|
|
439
|
+
`input_dir_path` are specified, the `input` is ignored, and the
|
|
440
|
+
files in the directory are used instead.
|
|
441
|
+
output_dir_path : Optional[str], default="."
|
|
442
|
+
Path to a directory where non-JSON output files will be saved. This
|
|
443
|
+
is required if the output is non-JSON. If the directory does not
|
|
444
|
+
exist, it will be created. Uses the current directory by default.
|
|
445
|
+
|
|
446
|
+
Returns
|
|
447
|
+
-------
|
|
448
|
+
RunResult
|
|
449
|
+
Result of the run, including output.
|
|
450
|
+
|
|
451
|
+
Raises
|
|
452
|
+
------
|
|
453
|
+
ValueError
|
|
454
|
+
If the `src` property for the `Application` is not specified. If
|
|
455
|
+
neither `input` nor `inputs_dir_path` is specified. If
|
|
456
|
+
`inputs_dir_path` is specified but `configuration` is not provided.
|
|
457
|
+
FileNotFoundError
|
|
458
|
+
If the manifest.yaml file cannot be found in the specified `src`
|
|
459
|
+
directory.
|
|
460
|
+
|
|
461
|
+
Examples
|
|
462
|
+
--------
|
|
463
|
+
>>> from nextmv.local import Application
|
|
464
|
+
>>> app = Application(id="my-app", src="/path/to/app")
|
|
465
|
+
>>> run_result = app.new_run_with_result(
|
|
466
|
+
... input={"vehicles": [{"id": "v1"}]},
|
|
467
|
+
... options={"duration": "10s"}
|
|
468
|
+
... )
|
|
469
|
+
>>> print(f"Local run completed with ID: {run_result.id}")
|
|
470
|
+
"""
|
|
471
|
+
|
|
472
|
+
run_id = self.new_run(
|
|
473
|
+
input=input,
|
|
474
|
+
name=name,
|
|
475
|
+
description=description,
|
|
476
|
+
options=run_options,
|
|
477
|
+
configuration=configuration,
|
|
478
|
+
json_configurations=json_configurations,
|
|
479
|
+
input_dir_path=input_dir_path,
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
return self.run_result_with_polling(
|
|
483
|
+
run_id=run_id,
|
|
484
|
+
polling_options=polling_options,
|
|
485
|
+
output_dir_path=output_dir_path,
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
def run_logs(self, run_id: str) -> str:
|
|
489
|
+
"""
|
|
490
|
+
Get the logs of a local run.
|
|
491
|
+
|
|
492
|
+
If the run does not have any logs, or they are empty, then this method
|
|
493
|
+
simply returns a blank string. This method is equivalent to fetching
|
|
494
|
+
the content of the `.nextmv/runs/{run_id}/logs/logs.log` file.
|
|
495
|
+
|
|
496
|
+
Parameters
|
|
497
|
+
----------
|
|
498
|
+
run_id : str
|
|
499
|
+
ID of the run to retrieve logs for.
|
|
500
|
+
|
|
501
|
+
Returns
|
|
502
|
+
-------
|
|
503
|
+
str
|
|
504
|
+
The contents of the logs file for the run.
|
|
505
|
+
|
|
506
|
+
Raises
|
|
507
|
+
------
|
|
508
|
+
ValueError
|
|
509
|
+
If the `.nextmv/runs` directory does not exist at the application
|
|
510
|
+
source, or if the specified run ID does not exist.
|
|
511
|
+
"""
|
|
512
|
+
|
|
513
|
+
runs_dir = os.path.join(self.src, NEXTMV_DIR, RUNS_KEY)
|
|
514
|
+
if not os.path.exists(runs_dir):
|
|
515
|
+
raise ValueError(f"`.nextmv/runs` dir does not exist at app source: {self.src}")
|
|
516
|
+
|
|
517
|
+
run_dir = os.path.join(runs_dir, run_id)
|
|
518
|
+
if not os.path.exists(run_dir):
|
|
519
|
+
raise ValueError(f"`{run_id}` run dir does not exist at: {runs_dir}")
|
|
520
|
+
|
|
521
|
+
logs_dir = os.path.join(run_dir, LOGS_KEY)
|
|
522
|
+
if not os.path.exists(logs_dir):
|
|
523
|
+
return ""
|
|
524
|
+
|
|
525
|
+
logs_file = os.path.join(logs_dir, LOGS_FILE)
|
|
526
|
+
if not os.path.exists(logs_file):
|
|
527
|
+
return ""
|
|
528
|
+
|
|
529
|
+
with open(logs_file) as f:
|
|
530
|
+
logs = f.read()
|
|
531
|
+
|
|
532
|
+
return logs
|
|
533
|
+
|
|
534
|
+
def run_metadata(self, run_id: str) -> RunInformation:
|
|
535
|
+
"""
|
|
536
|
+
Get the metadata of a local run.
|
|
537
|
+
|
|
538
|
+
This method is the local equivalent to
|
|
539
|
+
`cloud.Application.run_metadata`, which retrieves the metadata of a
|
|
540
|
+
remote run in Nextmv Cloud. This method is used to get the metadata of
|
|
541
|
+
a run that was executed locally using the `new_run` or
|
|
542
|
+
`new_run_with_result` method.
|
|
543
|
+
|
|
544
|
+
Retrieves information about a run without including the run output.
|
|
545
|
+
This is useful when you only need the run's status and metadata.
|
|
546
|
+
|
|
547
|
+
Parameters
|
|
548
|
+
----------
|
|
549
|
+
run_id : str
|
|
550
|
+
ID of the run to retrieve metadata for.
|
|
551
|
+
|
|
552
|
+
Returns
|
|
553
|
+
-------
|
|
554
|
+
RunInformation
|
|
555
|
+
Metadata of the run (run information without output).
|
|
556
|
+
|
|
557
|
+
Raises
|
|
558
|
+
------
|
|
559
|
+
ValueError
|
|
560
|
+
If the `.nextmv/runs` directory does not exist at the application
|
|
561
|
+
source, or if the specified run ID does not exist.
|
|
562
|
+
|
|
563
|
+
Examples
|
|
564
|
+
--------
|
|
565
|
+
>>> metadata = app.run_metadata("run-789")
|
|
566
|
+
>>> print(metadata.metadata.status_v2)
|
|
567
|
+
StatusV2.succeeded
|
|
568
|
+
"""
|
|
569
|
+
|
|
570
|
+
runs_dir = os.path.join(self.src, NEXTMV_DIR, RUNS_KEY)
|
|
571
|
+
if not os.path.exists(runs_dir):
|
|
572
|
+
raise ValueError(f"`.nextmv/runs` dir does not exist at app source: {self.src}")
|
|
573
|
+
|
|
574
|
+
run_dir = os.path.join(runs_dir, run_id)
|
|
575
|
+
if not os.path.exists(run_dir):
|
|
576
|
+
raise ValueError(f"`{run_id}` run dir does not exist at: {runs_dir}")
|
|
577
|
+
|
|
578
|
+
info_file = os.path.join(run_dir, f"{run_id}.json")
|
|
579
|
+
if not os.path.exists(info_file):
|
|
580
|
+
raise ValueError(f"`{info_file}` file does not exist at: {run_dir}")
|
|
581
|
+
|
|
582
|
+
with open(info_file) as f:
|
|
583
|
+
info_dict = json.load(f)
|
|
584
|
+
|
|
585
|
+
info = RunInformation.from_dict(info_dict)
|
|
586
|
+
|
|
587
|
+
return info
|
|
588
|
+
|
|
589
|
+
def run_result(self, run_id: str, output_dir_path: str | None = ".") -> RunResult:
|
|
590
|
+
"""
|
|
591
|
+
Get the local result of a run.
|
|
592
|
+
|
|
593
|
+
This method is the local equivalent to `cloud.Application.run_result`,
|
|
594
|
+
which retrieves the result of a remote run in Nextmv Cloud. This method
|
|
595
|
+
is used to get the result of a run that was executed locally using the
|
|
596
|
+
`new_run` or `new_run_with_result` method.
|
|
597
|
+
|
|
598
|
+
Retrieves the complete result of a run, including the run output.
|
|
599
|
+
|
|
600
|
+
Parameters
|
|
601
|
+
----------
|
|
602
|
+
run_id : str
|
|
603
|
+
ID of the run to get results for.
|
|
604
|
+
output_dir_path : Optional[str], default="."
|
|
605
|
+
Path to a directory where non-JSON output files will be saved. This
|
|
606
|
+
is required if the output is non-JSON. If the directory does not
|
|
607
|
+
exist, it will be created. Uses the current directory by default.
|
|
608
|
+
|
|
609
|
+
Returns
|
|
610
|
+
-------
|
|
611
|
+
RunResult
|
|
612
|
+
Result of the run, including output.
|
|
613
|
+
|
|
614
|
+
Raises
|
|
615
|
+
------
|
|
616
|
+
ValueError
|
|
617
|
+
If the `.nextmv/runs` directory does not exist at the application
|
|
618
|
+
source, or if the specified run ID does not exist.
|
|
619
|
+
|
|
620
|
+
Examples
|
|
621
|
+
--------
|
|
622
|
+
>>> result = app.run_result("run-123")
|
|
623
|
+
>>> print(result.metadata.status_v2)
|
|
624
|
+
'succeeded'
|
|
625
|
+
"""
|
|
626
|
+
|
|
627
|
+
run_information = self.run_metadata(run_id=run_id)
|
|
628
|
+
|
|
629
|
+
return self.__run_result(
|
|
630
|
+
run_id=run_id,
|
|
631
|
+
run_information=run_information,
|
|
632
|
+
output_dir_path=output_dir_path,
|
|
633
|
+
)
|
|
634
|
+
|
|
635
|
+
def run_result_with_polling(
|
|
636
|
+
self,
|
|
637
|
+
run_id: str,
|
|
638
|
+
polling_options: PollingOptions = DEFAULT_POLLING_OPTIONS,
|
|
639
|
+
output_dir_path: str | None = ".",
|
|
640
|
+
) -> RunResult:
|
|
641
|
+
"""
|
|
642
|
+
Get the result of a local run with polling.
|
|
643
|
+
|
|
644
|
+
This method is the local equivalent to
|
|
645
|
+
`cloud.Application.run_result_with_polling`, which retrieves the result
|
|
646
|
+
of a remote run in Nextmv Cloud. This method is used to get the result
|
|
647
|
+
of a run that was executed locally using the `new_run` or
|
|
648
|
+
`new_run_with_result` method.
|
|
649
|
+
|
|
650
|
+
Retrieves the result of a run including the run output. This method
|
|
651
|
+
polls for the result until the run finishes executing or the polling
|
|
652
|
+
strategy is exhausted.
|
|
653
|
+
|
|
654
|
+
Parameters
|
|
655
|
+
----------
|
|
656
|
+
run_id : str
|
|
657
|
+
ID of the run to retrieve the result for.
|
|
658
|
+
polling_options : PollingOptions, default=_DEFAULT_POLLING_OPTIONS
|
|
659
|
+
Options to use when polling for the run result.
|
|
660
|
+
output_dir_path : Optional[str], default="."
|
|
661
|
+
Path to a directory where non-JSON output files will be saved. This
|
|
662
|
+
is required if the output is non-JSON. If the directory does not
|
|
663
|
+
exist, it will be created. Uses the current directory by default.
|
|
664
|
+
|
|
665
|
+
Returns
|
|
666
|
+
-------
|
|
667
|
+
RunResult
|
|
668
|
+
Complete result of the run including output data.
|
|
669
|
+
|
|
670
|
+
Raises
|
|
671
|
+
------
|
|
672
|
+
requests.HTTPError
|
|
673
|
+
If the response status code is not 2xx.
|
|
674
|
+
TimeoutError
|
|
675
|
+
If the run does not complete after the polling strategy is
|
|
676
|
+
exhausted based on time duration.
|
|
677
|
+
RuntimeError
|
|
678
|
+
If the run does not complete after the polling strategy is
|
|
679
|
+
exhausted based on number of tries.
|
|
680
|
+
|
|
681
|
+
Examples
|
|
682
|
+
--------
|
|
683
|
+
>>> from nextmv.cloud import PollingOptions
|
|
684
|
+
>>> # Create custom polling options
|
|
685
|
+
>>> polling_opts = PollingOptions(max_tries=50, max_duration=600)
|
|
686
|
+
>>> # Get run result with polling
|
|
687
|
+
>>> result = app.run_result_with_polling("run-123", polling_opts)
|
|
688
|
+
>>> print(result.output)
|
|
689
|
+
{'solution': {...}}
|
|
690
|
+
"""
|
|
691
|
+
|
|
692
|
+
def polling_func() -> tuple[Any, bool]:
|
|
693
|
+
run_information = self.run_metadata(run_id=run_id)
|
|
694
|
+
if run_information.metadata.run_is_finalized():
|
|
695
|
+
return run_information, True
|
|
696
|
+
|
|
697
|
+
return None, False
|
|
698
|
+
|
|
699
|
+
run_information = poll(polling_options=polling_options, polling_func=polling_func)
|
|
700
|
+
|
|
701
|
+
return self.__run_result(
|
|
702
|
+
run_id=run_id,
|
|
703
|
+
run_information=run_information,
|
|
704
|
+
output_dir_path=output_dir_path,
|
|
705
|
+
)
|
|
706
|
+
|
|
707
|
+
def run_visuals(self, run_id: str) -> None:
|
|
708
|
+
"""
|
|
709
|
+
Open the local run visuals in a web browser.
|
|
710
|
+
|
|
711
|
+
This method opens the visual representation of a locally executed run
|
|
712
|
+
in the default web browser. It assumes that the run was executed locally
|
|
713
|
+
using the `new_run` or `new_run_with_result` method and that
|
|
714
|
+
the necessary visualization files are present.
|
|
715
|
+
|
|
716
|
+
If the run was correctly configured to produce visual assets, then the
|
|
717
|
+
run will contain a `visuals` directory with one or more HTML files.
|
|
718
|
+
Each file is opened in a new tab in the default web browser.
|
|
719
|
+
|
|
720
|
+
Parameters
|
|
721
|
+
----------
|
|
722
|
+
run_id : str
|
|
723
|
+
ID of the local run to visualize.
|
|
724
|
+
|
|
725
|
+
Raises
|
|
726
|
+
------
|
|
727
|
+
ValueError
|
|
728
|
+
If the `.nextmv/runs` directory does not exist at the application
|
|
729
|
+
source, or if the specified run ID does not exist.
|
|
730
|
+
"""
|
|
731
|
+
|
|
732
|
+
runs_dir = os.path.join(self.src, NEXTMV_DIR, RUNS_KEY)
|
|
733
|
+
if not os.path.exists(runs_dir):
|
|
734
|
+
raise ValueError(f"`.nextmv/runs` dir does not exist at app source: {self.src}")
|
|
735
|
+
|
|
736
|
+
run_dir = os.path.join(runs_dir, run_id)
|
|
737
|
+
if not os.path.exists(run_dir):
|
|
738
|
+
raise ValueError(f"`{run_id}` run dir does not exist at: {runs_dir}")
|
|
739
|
+
|
|
740
|
+
visuals_dir = os.path.join(run_dir, "visuals")
|
|
741
|
+
if not os.path.exists(visuals_dir):
|
|
742
|
+
raise ValueError(f"`visuals` dir does not exist at: {run_dir}")
|
|
743
|
+
|
|
744
|
+
for file in os.listdir(visuals_dir):
|
|
745
|
+
if file.endswith(".html"):
|
|
746
|
+
file_path = os.path.join(visuals_dir, file)
|
|
747
|
+
webbrowser.open_new_tab(f"file://{os.path.realpath(file_path)}")
|
|
748
|
+
|
|
749
|
+
def sync( # noqa: C901
|
|
750
|
+
self,
|
|
751
|
+
target: cloud.Application,
|
|
752
|
+
run_ids: list[str] | None = None,
|
|
753
|
+
instance_id: str | None = None,
|
|
754
|
+
verbose: bool | None = False,
|
|
755
|
+
) -> None:
|
|
756
|
+
"""
|
|
757
|
+
Sync the local application to a Nextmv Cloud application target.
|
|
758
|
+
|
|
759
|
+
The `Application` class allows you to perform and handle local
|
|
760
|
+
application runs with methods such as:
|
|
761
|
+
|
|
762
|
+
- `new_run`
|
|
763
|
+
- `new_run_with_result`
|
|
764
|
+
- `run_metadata`
|
|
765
|
+
- `run_result`
|
|
766
|
+
- `run_result_with_polling`
|
|
767
|
+
|
|
768
|
+
The runs produced locally live under `self.src/.nextmv/runs`. This
|
|
769
|
+
method syncs those runs to a Nextmv Cloud application target, making
|
|
770
|
+
them available for remote execution and management.
|
|
771
|
+
|
|
772
|
+
Parameters
|
|
773
|
+
----------
|
|
774
|
+
target : cloud.Application
|
|
775
|
+
Target Nextmv Cloud application where the local application runs
|
|
776
|
+
will be synced to.
|
|
777
|
+
run_ids : Optional[list[str]], default=None
|
|
778
|
+
List of run IDs to sync. If None, all local runs found under
|
|
779
|
+
`self.src/.nextmv/runs` will be synced.
|
|
780
|
+
instance_id : Optional[str], default=None
|
|
781
|
+
Optional instance ID if you want to associate your runs with an
|
|
782
|
+
instance.
|
|
783
|
+
verbose : Optional[bool], default=False
|
|
784
|
+
Whether to print verbose output during the sync process. Useful for
|
|
785
|
+
debugging a large number of runs being synced.
|
|
786
|
+
|
|
787
|
+
Raises
|
|
788
|
+
------
|
|
789
|
+
ValueError
|
|
790
|
+
If the `src` property is not specified.
|
|
791
|
+
ValueError
|
|
792
|
+
If the `client` property is not specified.
|
|
793
|
+
ValueError
|
|
794
|
+
If the application does not exist in Nextmv Cloud.
|
|
795
|
+
ValueError
|
|
796
|
+
If a run does not exist locally.
|
|
797
|
+
requests.HTTPError
|
|
798
|
+
If the response status code is not 2xx.
|
|
799
|
+
"""
|
|
800
|
+
if self.src is None:
|
|
801
|
+
raise ValueError(
|
|
802
|
+
"`src` property for the `Application` must be specified to sync the application to Nextmv Cloud"
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
if target.client is None:
|
|
806
|
+
raise ValueError(
|
|
807
|
+
"`client` property for the target `Application` must be specified to sync the application to Cloud"
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
if not target.exists(target.client, target.id):
|
|
811
|
+
raise ValueError(
|
|
812
|
+
"target Application does not exist in Nextmv Cloud, create it with `cloud.Application.new`"
|
|
813
|
+
)
|
|
814
|
+
if verbose:
|
|
815
|
+
log(f"☁️ Starting sync of local application `{self.src}` to Nextmv Cloud application `{target.id}`.")
|
|
816
|
+
|
|
817
|
+
# Create a temp dir to store the outputs that are written by default to
|
|
818
|
+
# ".". During the sync process, we don't need to keep these outputs, so
|
|
819
|
+
# we can use a temp dir that will be deleted after the sync is done.
|
|
820
|
+
with tempfile.TemporaryDirectory(prefix="nextmv-sync-run-") as temp_results_dir:
|
|
821
|
+
runs_dir = os.path.join(self.src, NEXTMV_DIR, RUNS_KEY)
|
|
822
|
+
if run_ids is None:
|
|
823
|
+
# If runs are not specified, by default we sync all local runs that
|
|
824
|
+
# can be found.
|
|
825
|
+
dirs = os.listdir(runs_dir)
|
|
826
|
+
run_ids = [d for d in dirs if os.path.isdir(os.path.join(runs_dir, d))]
|
|
827
|
+
|
|
828
|
+
if verbose:
|
|
829
|
+
log(f"ℹ️ Found {len(run_ids)} local runs to sync from {runs_dir}.")
|
|
830
|
+
else:
|
|
831
|
+
if verbose:
|
|
832
|
+
log(f"ℹ️ Syncing {len(run_ids)} specified local runs from {runs_dir}.")
|
|
833
|
+
|
|
834
|
+
total = 0
|
|
835
|
+
for run_id in run_ids:
|
|
836
|
+
synced = self.__sync_run(
|
|
837
|
+
target=target,
|
|
838
|
+
run_id=run_id,
|
|
839
|
+
runs_dir=runs_dir,
|
|
840
|
+
temp_dir=temp_results_dir,
|
|
841
|
+
instance_id=instance_id,
|
|
842
|
+
verbose=verbose,
|
|
843
|
+
)
|
|
844
|
+
if synced:
|
|
845
|
+
total += 1
|
|
846
|
+
|
|
847
|
+
if verbose:
|
|
848
|
+
log(
|
|
849
|
+
f"🚀 Process completed, synced local application `{self.src}` to "
|
|
850
|
+
f"Nextmv Cloud application `{target.id}`: "
|
|
851
|
+
f"{total}/{len(run_ids)} runs."
|
|
852
|
+
)
|
|
853
|
+
|
|
854
|
+
def __run_result(
|
|
855
|
+
self,
|
|
856
|
+
run_id: str,
|
|
857
|
+
run_information: RunInformation,
|
|
858
|
+
output_dir_path: str | None = ".",
|
|
859
|
+
) -> RunResult:
|
|
860
|
+
"""
|
|
861
|
+
Get the result of a local run.
|
|
862
|
+
|
|
863
|
+
This is a private method that retrieves the complete result of a run,
|
|
864
|
+
including the output data, from a local source. This method serves as
|
|
865
|
+
the base implementation for retrieving run results, regardless of
|
|
866
|
+
polling strategy.
|
|
867
|
+
|
|
868
|
+
Parameters
|
|
869
|
+
----------
|
|
870
|
+
run_id : str
|
|
871
|
+
ID of the run to retrieve the result for.
|
|
872
|
+
run_information : RunInformation
|
|
873
|
+
Information about the run, including metadata such as output size.
|
|
874
|
+
output_dir_path : Optional[str], default="."
|
|
875
|
+
Path to a directory where non-JSON output files will be saved. This
|
|
876
|
+
is required if the output is non-JSON. If the directory does not
|
|
877
|
+
exist, it will be created. Uses the current directory by default.
|
|
878
|
+
|
|
879
|
+
Returns
|
|
880
|
+
-------
|
|
881
|
+
RunResult
|
|
882
|
+
Result of the run, including all metadata and output data.
|
|
883
|
+
|
|
884
|
+
Raises
|
|
885
|
+
------
|
|
886
|
+
ValueError
|
|
887
|
+
If the output format is not JSON and no output_dir_path is
|
|
888
|
+
provided.
|
|
889
|
+
If the output format is unknown.
|
|
890
|
+
"""
|
|
891
|
+
|
|
892
|
+
result = RunResult.from_dict(run_information.to_dict())
|
|
893
|
+
if result.metadata.error:
|
|
894
|
+
result.error_log = ErrorLog(error=result.metadata.error)
|
|
895
|
+
|
|
896
|
+
if result.metadata.status_v2 != StatusV2.succeeded:
|
|
897
|
+
return result
|
|
898
|
+
|
|
899
|
+
# See whether we can attach the output directly or need to save to the given
|
|
900
|
+
# directory
|
|
901
|
+
output_type = run_information.metadata.format.format_output.output_type
|
|
902
|
+
if output_type != OutputFormat.JSON and (not output_dir_path or output_dir_path == ""):
|
|
903
|
+
raise ValueError(
|
|
904
|
+
"The output format is not JSON: an `output_dir_path` must be provided.",
|
|
905
|
+
)
|
|
906
|
+
|
|
907
|
+
runs_dir = os.path.join(self.src, NEXTMV_DIR, RUNS_KEY)
|
|
908
|
+
solutions_dir = os.path.join(runs_dir, run_id, OUTPUTS_KEY, SOLUTIONS_KEY)
|
|
909
|
+
|
|
910
|
+
if output_type == OutputFormat.JSON:
|
|
911
|
+
with open(os.path.join(solutions_dir, DEFAULT_OUTPUT_JSON_FILE)) as f:
|
|
912
|
+
result.output = json.load(f)
|
|
913
|
+
elif output_type in {OutputFormat.CSV_ARCHIVE, OutputFormat.MULTI_FILE}:
|
|
914
|
+
shutil.copytree(solutions_dir, output_dir_path, dirs_exist_ok=True)
|
|
915
|
+
else:
|
|
916
|
+
raise ValueError(f"Unknown output type: {output_type}")
|
|
917
|
+
|
|
918
|
+
return result
|
|
919
|
+
|
|
920
|
+
def __validate_input_dir_path_and_configuration(
|
|
921
|
+
self,
|
|
922
|
+
input_dir_path: str | None,
|
|
923
|
+
configuration: RunConfiguration | dict[str, Any] | None,
|
|
924
|
+
) -> RunConfiguration:
|
|
925
|
+
"""
|
|
926
|
+
Auxiliary function to validate the directory path and configuration.
|
|
927
|
+
"""
|
|
928
|
+
|
|
929
|
+
if configuration is None:
|
|
930
|
+
if self.manifest.configuration is not None and self.manifest.configuration.content is not None:
|
|
931
|
+
configuration = RunConfiguration(
|
|
932
|
+
format=Format(
|
|
933
|
+
format_input=FormatInput(
|
|
934
|
+
input_type=self.manifest.configuration.content.format,
|
|
935
|
+
),
|
|
936
|
+
),
|
|
937
|
+
)
|
|
938
|
+
elif isinstance(configuration, dict):
|
|
939
|
+
# Forcefully turn the configuration into a RunConfiguration object to
|
|
940
|
+
# make it easier to deal with in the other functions.
|
|
941
|
+
configuration = RunConfiguration.from_dict(configuration)
|
|
942
|
+
|
|
943
|
+
if input_dir_path is None or input_dir_path == "":
|
|
944
|
+
return configuration
|
|
945
|
+
|
|
946
|
+
if configuration is None:
|
|
947
|
+
raise ValueError(
|
|
948
|
+
"If `dir_path` is provided, either a `RunConfiguration` must also be provided or "
|
|
949
|
+
"the application's manifest (app.yaml) must include the format under "
|
|
950
|
+
"`configuration.content.format`.",
|
|
951
|
+
)
|
|
952
|
+
|
|
953
|
+
config_format = configuration.format
|
|
954
|
+
if config_format is None:
|
|
955
|
+
raise ValueError(
|
|
956
|
+
"If `dir_path` is provided, `RunConfiguration.format` must also be provided.",
|
|
957
|
+
)
|
|
958
|
+
|
|
959
|
+
input_type = config_format.format_input
|
|
960
|
+
if input_type is None:
|
|
961
|
+
raise ValueError(
|
|
962
|
+
"If `dir_path` is provided, `RunConfiguration.format.format_input` must also be provided.",
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
if input_type is None or input_type in (InputFormat.JSON, InputFormat.TEXT):
|
|
966
|
+
raise ValueError(
|
|
967
|
+
"If `dir_path` is provided, `RunConfiguration.format.format_input.input_type` must be set to "
|
|
968
|
+
f"a valid type. Valid types are: {[InputFormat.CSV_ARCHIVE, InputFormat.MULTI_FILE]}",
|
|
969
|
+
)
|
|
970
|
+
|
|
971
|
+
return configuration
|
|
972
|
+
|
|
973
|
+
def __extract_input_data(
|
|
974
|
+
self,
|
|
975
|
+
input: Input | dict[str, Any] | BaseModel | str = None,
|
|
976
|
+
) -> dict[str, Any] | str | None:
|
|
977
|
+
"""
|
|
978
|
+
Auxiliary function to extract the input data from the input, based on
|
|
979
|
+
its type.
|
|
980
|
+
"""
|
|
981
|
+
|
|
982
|
+
input_data = None
|
|
983
|
+
if isinstance(input, BaseModel):
|
|
984
|
+
input_data = input.to_dict()
|
|
985
|
+
elif isinstance(input, dict) or isinstance(input, str):
|
|
986
|
+
input_data = input
|
|
987
|
+
elif isinstance(input, Input):
|
|
988
|
+
input_data = input.data
|
|
989
|
+
|
|
990
|
+
return input_data
|
|
991
|
+
|
|
992
|
+
def __extract_options_dict(
|
|
993
|
+
self,
|
|
994
|
+
options: Options | dict[str, str] | None = None,
|
|
995
|
+
json_configurations: dict[str, Any] | None = None,
|
|
996
|
+
) -> dict[str, str]:
|
|
997
|
+
"""
|
|
998
|
+
Auxiliary function to extract the options that will be sent to the
|
|
999
|
+
application for execution.
|
|
1000
|
+
"""
|
|
1001
|
+
|
|
1002
|
+
options_dict = {}
|
|
1003
|
+
if options is not None:
|
|
1004
|
+
if isinstance(options, Options):
|
|
1005
|
+
options_dict = options.to_dict_cloud()
|
|
1006
|
+
elif isinstance(options, dict):
|
|
1007
|
+
for k, v in options.items():
|
|
1008
|
+
if isinstance(v, str):
|
|
1009
|
+
options_dict[k] = v
|
|
1010
|
+
else:
|
|
1011
|
+
options_dict[k] = deflated_serialize_json(v, json_configurations=json_configurations)
|
|
1012
|
+
|
|
1013
|
+
return options_dict
|
|
1014
|
+
|
|
1015
|
+
def __extract_run_config(
|
|
1016
|
+
self,
|
|
1017
|
+
input: Input | dict[str, Any] | BaseModel | str = None,
|
|
1018
|
+
configuration: RunConfiguration | dict[str, Any] | None = None,
|
|
1019
|
+
dir_path: str | None = None,
|
|
1020
|
+
) -> dict[str, Any]:
|
|
1021
|
+
"""
|
|
1022
|
+
Auxiliary function to extract the run configuration that will be sent
|
|
1023
|
+
to the application for execution.
|
|
1024
|
+
"""
|
|
1025
|
+
|
|
1026
|
+
if configuration is not None:
|
|
1027
|
+
configuration_dict = (
|
|
1028
|
+
configuration.to_dict() if isinstance(configuration, RunConfiguration) else configuration
|
|
1029
|
+
)
|
|
1030
|
+
return configuration_dict
|
|
1031
|
+
|
|
1032
|
+
configuration = RunConfiguration()
|
|
1033
|
+
configuration.resolve(input=input, dir_path=dir_path)
|
|
1034
|
+
configuration_dict = configuration.to_dict()
|
|
1035
|
+
|
|
1036
|
+
return configuration_dict
|
|
1037
|
+
|
|
1038
|
+
def __sync_run( # noqa: C901
|
|
1039
|
+
self,
|
|
1040
|
+
target: cloud.Application,
|
|
1041
|
+
run_id: str,
|
|
1042
|
+
runs_dir: str,
|
|
1043
|
+
temp_dir: str,
|
|
1044
|
+
instance_id: str | None = None,
|
|
1045
|
+
verbose: bool | None = False,
|
|
1046
|
+
) -> bool:
|
|
1047
|
+
"""
|
|
1048
|
+
Syncs a local run to a Nextmv Cloud target application. Returns True if
|
|
1049
|
+
the run was synced, False if it was skipped (already synced).
|
|
1050
|
+
"""
|
|
1051
|
+
|
|
1052
|
+
if verbose:
|
|
1053
|
+
log(f"🔄 Syncing local run `{run_id}`... ")
|
|
1054
|
+
|
|
1055
|
+
# For files-based runs, the result files are written by default to ".".
|
|
1056
|
+
# Avoid this using a dedicated temp dir.
|
|
1057
|
+
run_result = self.run_result(run_id, output_dir_path=temp_dir)
|
|
1058
|
+
input_type = run_result.metadata.format.format_input.input_type
|
|
1059
|
+
|
|
1060
|
+
# Skip runs that have already been synced.
|
|
1061
|
+
synced_run, already_synced = run_result.is_synced(app_id=target.id, instance_id=instance_id)
|
|
1062
|
+
if already_synced:
|
|
1063
|
+
if verbose:
|
|
1064
|
+
log(f" ⏭️ Skipping local run `{run_id}`, already synced with {synced_run.to_dict()}.")
|
|
1065
|
+
|
|
1066
|
+
return False
|
|
1067
|
+
|
|
1068
|
+
# Check that it is a valid run with inputs, outputs, logs, etc.
|
|
1069
|
+
if not self.__valid_run_result(run_result, runs_dir, run_id):
|
|
1070
|
+
if verbose:
|
|
1071
|
+
log(f" ❌ Skipping local run `{run_id}`, invalid run (missing inputs, outputs or logs).")
|
|
1072
|
+
|
|
1073
|
+
return False
|
|
1074
|
+
|
|
1075
|
+
status = TrackedRunStatus.SUCCEEDED
|
|
1076
|
+
if run_result.metadata.status_v2 != StatusV2.succeeded:
|
|
1077
|
+
status = TrackedRunStatus.FAILED
|
|
1078
|
+
|
|
1079
|
+
# Read the logs of the run and place each line as an element in a list
|
|
1080
|
+
run_dir = os.path.join(runs_dir, run_id)
|
|
1081
|
+
with open(os.path.join(run_dir, LOGS_KEY, LOGS_FILE)) as f:
|
|
1082
|
+
stderr_logs = f.read()
|
|
1083
|
+
|
|
1084
|
+
# Create the tracked run object and start configuring it.
|
|
1085
|
+
tracked_run = TrackedRun(
|
|
1086
|
+
status=status,
|
|
1087
|
+
duration=int(run_result.metadata.duration),
|
|
1088
|
+
error=run_result.metadata.error,
|
|
1089
|
+
logs=stderr_logs,
|
|
1090
|
+
name=run_result.name,
|
|
1091
|
+
description=run_result.description,
|
|
1092
|
+
)
|
|
1093
|
+
|
|
1094
|
+
# Resolve the input according to its type.
|
|
1095
|
+
inputs_path = os.path.join(run_dir, INPUTS_KEY)
|
|
1096
|
+
if input_type == InputFormat.JSON:
|
|
1097
|
+
with open(os.path.join(inputs_path, DEFAULT_INPUT_JSON_FILE)) as f:
|
|
1098
|
+
tracked_run.input = json.load(f)
|
|
1099
|
+
elif input_type == InputFormat.TEXT:
|
|
1100
|
+
with open(os.path.join(inputs_path, "input")) as f:
|
|
1101
|
+
tracked_run.input = f.read()
|
|
1102
|
+
else:
|
|
1103
|
+
tracked_run.input_dir_path = inputs_path
|
|
1104
|
+
|
|
1105
|
+
# Resolve the output according to its type.
|
|
1106
|
+
output_type = run_result.metadata.format.format_output.output_type
|
|
1107
|
+
if output_type == OutputFormat.JSON:
|
|
1108
|
+
tracked_run.output = run_result.output
|
|
1109
|
+
else:
|
|
1110
|
+
tracked_run.output_dir_path = os.path.join(run_dir, OUTPUTS_KEY, SOLUTIONS_KEY)
|
|
1111
|
+
|
|
1112
|
+
# Resolve the statistics according to their type and presence. If
|
|
1113
|
+
# working with JSON, the statistics should be resolved from the output.
|
|
1114
|
+
if output_type in {OutputFormat.CSV_ARCHIVE, OutputFormat.MULTI_FILE}:
|
|
1115
|
+
stats_file_path = os.path.join(run_dir, OUTPUTS_KEY, STATISTICS_KEY, f"{STATISTICS_KEY}.json")
|
|
1116
|
+
if os.path.exists(stats_file_path):
|
|
1117
|
+
with open(stats_file_path) as f:
|
|
1118
|
+
tracked_run.statistics = json.load(f)
|
|
1119
|
+
|
|
1120
|
+
# Resolve the assets according to their type and presence. If working
|
|
1121
|
+
# with JSON, the assets should be resolved from the output.
|
|
1122
|
+
if output_type in {OutputFormat.CSV_ARCHIVE, OutputFormat.MULTI_FILE}:
|
|
1123
|
+
assets_file_path = os.path.join(run_dir, OUTPUTS_KEY, ASSETS_KEY, f"{ASSETS_KEY}.json")
|
|
1124
|
+
if os.path.exists(assets_file_path):
|
|
1125
|
+
with open(assets_file_path) as f:
|
|
1126
|
+
tracked_run.assets = json.load(f)
|
|
1127
|
+
|
|
1128
|
+
# Actually sync the run by tracking it remotely on Nextmv Cloud.
|
|
1129
|
+
configuration = RunConfiguration(
|
|
1130
|
+
format=Format(
|
|
1131
|
+
format_input=run_result.metadata.format.format_input,
|
|
1132
|
+
format_output=run_result.metadata.format.format_output,
|
|
1133
|
+
),
|
|
1134
|
+
)
|
|
1135
|
+
tracked_id = target.track_run(
|
|
1136
|
+
tracked_run=tracked_run,
|
|
1137
|
+
instance_id=instance_id,
|
|
1138
|
+
configuration=configuration,
|
|
1139
|
+
)
|
|
1140
|
+
|
|
1141
|
+
# Mark the local run as synced by updating the local run info.
|
|
1142
|
+
synced_run = SyncedRun(
|
|
1143
|
+
run_id=tracked_id,
|
|
1144
|
+
synced_at=datetime.now(timezone.utc),
|
|
1145
|
+
app_id=target.id,
|
|
1146
|
+
instance_id=instance_id,
|
|
1147
|
+
)
|
|
1148
|
+
run_result.add_synced_run(synced_run)
|
|
1149
|
+
with open(os.path.join(run_dir, f"{run_id}.json"), "w") as f:
|
|
1150
|
+
json.dump(run_result.to_dict(), f, indent=2)
|
|
1151
|
+
|
|
1152
|
+
if verbose:
|
|
1153
|
+
log(f"✅ Synced local run `{run_id}` as remote run `{synced_run.to_dict()}`.")
|
|
1154
|
+
|
|
1155
|
+
return True
|
|
1156
|
+
|
|
1157
|
+
def __valid_run_result(self, run_result: RunResult, runs_dir: str, run_id: str) -> bool:
|
|
1158
|
+
"""
|
|
1159
|
+
Validate that a run result has all required files and directories.
|
|
1160
|
+
|
|
1161
|
+
This method checks that a local run has the expected directory structure
|
|
1162
|
+
and files, including inputs, outputs, and logs.
|
|
1163
|
+
|
|
1164
|
+
Parameters
|
|
1165
|
+
----------
|
|
1166
|
+
run_result : RunResult
|
|
1167
|
+
The run result to validate.
|
|
1168
|
+
runs_dir : str
|
|
1169
|
+
Path to the runs directory.
|
|
1170
|
+
run_id : str
|
|
1171
|
+
ID of the run to validate.
|
|
1172
|
+
|
|
1173
|
+
Returns
|
|
1174
|
+
-------
|
|
1175
|
+
bool
|
|
1176
|
+
True if the run is valid, False otherwise.
|
|
1177
|
+
"""
|
|
1178
|
+
run_dir = os.path.join(runs_dir, run_id)
|
|
1179
|
+
|
|
1180
|
+
# Check that the run directory exists
|
|
1181
|
+
if not os.path.exists(run_dir):
|
|
1182
|
+
return False
|
|
1183
|
+
|
|
1184
|
+
# Validate inputs
|
|
1185
|
+
if not self.__validate_inputs(run_dir, run_result.metadata.format.format_input.input_type):
|
|
1186
|
+
return False
|
|
1187
|
+
|
|
1188
|
+
# Validate outputs
|
|
1189
|
+
format_output = run_result.metadata.format.format_output
|
|
1190
|
+
if format_output is None or not format_output:
|
|
1191
|
+
return False
|
|
1192
|
+
|
|
1193
|
+
output_type = format_output.output_type
|
|
1194
|
+
if output_type is None or output_type == "":
|
|
1195
|
+
return False
|
|
1196
|
+
|
|
1197
|
+
if not self.__validate_outputs(run_dir, output_type):
|
|
1198
|
+
return False
|
|
1199
|
+
|
|
1200
|
+
# Validate logs
|
|
1201
|
+
if not self.__validate_logs(run_dir):
|
|
1202
|
+
return False
|
|
1203
|
+
|
|
1204
|
+
return True
|
|
1205
|
+
|
|
1206
|
+
def __validate_inputs(self, run_dir: str, input_type: InputFormat) -> bool:
|
|
1207
|
+
"""Validate that the inputs directory and files exist for the given input type."""
|
|
1208
|
+
inputs_path = os.path.join(run_dir, INPUTS_KEY)
|
|
1209
|
+
if not os.path.exists(inputs_path):
|
|
1210
|
+
return False
|
|
1211
|
+
|
|
1212
|
+
if input_type == InputFormat.JSON:
|
|
1213
|
+
input_file = os.path.join(inputs_path, DEFAULT_INPUT_JSON_FILE)
|
|
1214
|
+
|
|
1215
|
+
return os.path.isfile(input_file)
|
|
1216
|
+
|
|
1217
|
+
if input_type == InputFormat.TEXT:
|
|
1218
|
+
input_file = os.path.join(inputs_path, "input")
|
|
1219
|
+
|
|
1220
|
+
return os.path.isfile(input_file)
|
|
1221
|
+
|
|
1222
|
+
# For CSV_ARCHIVE and MULTI_FILE, inputs_path should be a directory
|
|
1223
|
+
return os.path.isdir(inputs_path)
|
|
1224
|
+
|
|
1225
|
+
def __validate_outputs(self, run_dir: str, output_type: OutputFormat) -> bool:
|
|
1226
|
+
"""Validate that the outputs directory and files exist for the given output type."""
|
|
1227
|
+
outputs_dir = os.path.join(run_dir, OUTPUTS_KEY)
|
|
1228
|
+
if not os.path.exists(outputs_dir):
|
|
1229
|
+
return False
|
|
1230
|
+
|
|
1231
|
+
solutions_dir = os.path.join(outputs_dir, SOLUTIONS_KEY)
|
|
1232
|
+
if not os.path.exists(solutions_dir):
|
|
1233
|
+
return False
|
|
1234
|
+
|
|
1235
|
+
if output_type == OutputFormat.JSON:
|
|
1236
|
+
solution_file = os.path.join(solutions_dir, DEFAULT_OUTPUT_JSON_FILE)
|
|
1237
|
+
|
|
1238
|
+
return os.path.isfile(solution_file)
|
|
1239
|
+
|
|
1240
|
+
# For CSV_ARCHIVE and MULTI_FILE, solutions_dir should be a directory
|
|
1241
|
+
return os.path.isdir(solutions_dir)
|
|
1242
|
+
|
|
1243
|
+
def __validate_logs(self, run_dir: str) -> bool:
|
|
1244
|
+
"""Validate that the logs directory and file exist."""
|
|
1245
|
+
logs_dir = os.path.join(run_dir, LOGS_KEY)
|
|
1246
|
+
if not os.path.exists(logs_dir):
|
|
1247
|
+
return False
|
|
1248
|
+
|
|
1249
|
+
logs_file = os.path.join(logs_dir, LOGS_FILE)
|
|
1250
|
+
|
|
1251
|
+
return os.path.isfile(logs_file)
|