jcmsuite 7.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of jcmsuite might be problematic. Click here for more details.
- jcmsuite/__init__.py +22 -0
- jcmsuite/_build_info.py +4 -0
- jcmsuite/context/__init__.py +44 -0
- jcmsuite/context/interface_metadata.py +158 -0
- jcmsuite/context/models.py +213 -0
- jcmsuite/context/scope.py +210 -0
- jcmsuite/context/solver_context.py +569 -0
- jcmsuite/context/subprocess_tools.py +172 -0
- jcmsuite/contract.py +47 -0
- jcmsuite/daemon/__init__.py +19 -0
- jcmsuite/daemon/handler.py +925 -0
- jcmsuite/daemon/internal/__init__.py +4 -0
- jcmsuite/daemon/internal/bundle_protocol.py +64 -0
- jcmsuite/daemon/internal/handler_helpers.py +368 -0
- jcmsuite/daemon/internal/handler_resource_jobs.py +227 -0
- jcmsuite/daemon/internal/handler_wait.py +564 -0
- jcmsuite/daemon/models.py +349 -0
- jcmsuite/daemon/namespaces.py +250 -0
- jcmsuite/daemon/protocol.py +801 -0
- jcmsuite/daemon/resources.py +407 -0
- jcmsuite/daemon/scope.py +53 -0
- jcmsuite/daemon/transport.py +223 -0
- jcmsuite/editing/__init__.py +10 -0
- jcmsuite/editing/editor.py +274 -0
- jcmsuite/exceptions/__init__.py +32 -0
- jcmsuite/exceptions/errors.py +173 -0
- jcmsuite/geo/__init__.py +10 -0
- jcmsuite/geo/mesh.py +302 -0
- jcmsuite/info/__init__.py +10 -0
- jcmsuite/info/runtime_info.py +45 -0
- jcmsuite/internal/__init__.py +2 -0
- jcmsuite/internal/diagnostics.py +74 -0
- jcmsuite/internal/progress.py +80 -0
- jcmsuite/loaders/__init__.py +18 -0
- jcmsuite/loaders/blob_header.py +315 -0
- jcmsuite/loaders/cartesian_field_reader.py +249 -0
- jcmsuite/loaders/cartesian_fields.py +324 -0
- jcmsuite/loaders/nested_mapping.py +69 -0
- jcmsuite/loaders/powerflux.py +136 -0
- jcmsuite/loaders/table.py +637 -0
- jcmsuite/logging/__init__.py +22 -0
- jcmsuite/logging/runtime.py +210 -0
- jcmsuite/results/__init__.py +126 -0
- jcmsuite/results/artifacts.py +806 -0
- jcmsuite/results/discovery.py +82 -0
- jcmsuite/results/field_exports.py +1062 -0
- jcmsuite/results/field_payloads.py +453 -0
- jcmsuite/results/internal/__init__.py +4 -0
- jcmsuite/results/internal/export/__init__.py +4 -0
- jcmsuite/results/internal/export/common.py +56 -0
- jcmsuite/results/internal/export/pandas/__init__.py +4 -0
- jcmsuite/results/internal/export/pandas/common.py +165 -0
- jcmsuite/results/internal/export/pandas/payload.py +480 -0
- jcmsuite/results/internal/export/pandas/rows.py +484 -0
- jcmsuite/results/internal/export/preview.py +178 -0
- jcmsuite/results/internal/export/xarray/__init__.py +4 -0
- jcmsuite/results/internal/export/xarray/export.py +688 -0
- jcmsuite/results/internal/export/xarray/grouped.py +608 -0
- jcmsuite/results/internal/export/xarray/headers.py +290 -0
- jcmsuite/results/internal/export/xarray/runtime.py +99 -0
- jcmsuite/results/internal/export/xarray/shared.py +246 -0
- jcmsuite/results/internal/export/xarray/tabular.py +304 -0
- jcmsuite/results/internal/export/xarray/types.py +28 -0
- jcmsuite/results/internal/models/__init__.py +4 -0
- jcmsuite/results/internal/models/batch.py +1081 -0
- jcmsuite/results/internal/models/batch_postprocess.py +1728 -0
- jcmsuite/results/internal/models/bundle.py +343 -0
- jcmsuite/results/internal/models/core.py +556 -0
- jcmsuite/results/internal/models/submission.py +1157 -0
- jcmsuite/results/internal/resultbag/__init__.py +4 -0
- jcmsuite/results/internal/resultbag/access.py +489 -0
- jcmsuite/results/internal/resultbag/iteration.py +199 -0
- jcmsuite/results/internal/resultbag/query.py +226 -0
- jcmsuite/results/internal/resultbag/raw.py +407 -0
- jcmsuite/results/internal/resultbag/source_files.py +142 -0
- jcmsuite/results/internal/resultbag/store.py +881 -0
- jcmsuite/results/internal/resultbag/utils.py +258 -0
- jcmsuite/results/models.py +76 -0
- jcmsuite/results/opening.py +55 -0
- jcmsuite/results/pandas_export.py +325 -0
- jcmsuite/results/payloads.py +478 -0
- jcmsuite/results/postprocess.py +97 -0
- jcmsuite/results/postprocess_payloads.py +757 -0
- jcmsuite/results/postprocess_record.py +418 -0
- jcmsuite/results/resultbag.py +279 -0
- jcmsuite/results/resultbag_management.py +433 -0
- jcmsuite/results/structured_derivatives.py +387 -0
- jcmsuite/solve/__init__.py +10 -0
- jcmsuite/solve/daemon_bundle_staging.py +100 -0
- jcmsuite/solve/daemon_bundle_wait.py +89 -0
- jcmsuite/solve/orchestration.py +1576 -0
- jcmsuite/solve/orchestration_execution.py +180 -0
- jcmsuite/solve/orchestration_routing.py +128 -0
- jcmsuite/solve/orchestration_support.py +323 -0
- jcmsuite/solve/path_resolution.py +98 -0
- jcmsuite/solve/progress.py +20 -0
- jcmsuite/solve/solver.py +709 -0
- jcmsuite/solve/staging.py +264 -0
- jcmsuite/solve/types.py +25 -0
- jcmsuite/solve/working_dirs.py +227 -0
- jcmsuite/templates/__init__.py +11 -0
- jcmsuite/templates/converter.py +71 -0
- jcmsuite/templates/formatting.py +335 -0
- jcmsuite/templates/formatting_cli.py +47 -0
- jcmsuite/templates/rendering.py +666 -0
- jcmsuite/torch/__init__.py +25 -0
- jcmsuite/torch/autograd.py +488 -0
- jcmsuite/torch/coordinate_policy.py +82 -0
- jcmsuite/torch/core.py +312 -0
- jcmsuite/torch/results/__init__.py +51 -0
- jcmsuite/torch/types.py +84 -0
- jcmsuite/torch/validation.py +208 -0
- jcmsuite/torch/wrappers.py +4332 -0
- jcmsuite/view/__init__.py +10 -0
- jcmsuite/view/viewer.py +122 -0
- jcmsuite-7.4.2.dist-info/METADATA +180 -0
- jcmsuite-7.4.2.dist-info/RECORD +119 -0
- jcmsuite-7.4.2.dist-info/WHEEL +4 -0
- jcmsuite-7.4.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,1576 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# Copyright (c) 2001-2026 JCMwave GmbH
|
|
3
|
+
|
|
4
|
+
"""Internal solve orchestration and execution helpers."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
from collections.abc import Mapping, Sequence
|
|
10
|
+
from logging import Logger
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import TYPE_CHECKING, cast
|
|
13
|
+
|
|
14
|
+
from ..context import BackendMode, SolverContext, resolve_context
|
|
15
|
+
from ..context.subprocess_tools import _call_binary
|
|
16
|
+
from ..daemon.handler import resolve_scoped_handler
|
|
17
|
+
from ..daemon.models import DaemonResource
|
|
18
|
+
from ..exceptions import (
|
|
19
|
+
BackendUnavailableError,
|
|
20
|
+
InvalidConfigurationError,
|
|
21
|
+
SolveExecutionError,
|
|
22
|
+
)
|
|
23
|
+
from ..logging import get_logger
|
|
24
|
+
from ..results import (
|
|
25
|
+
BatchSolveEntry,
|
|
26
|
+
BatchSolveFailure,
|
|
27
|
+
BatchSolveResult,
|
|
28
|
+
BatchSolveSubmission,
|
|
29
|
+
BundledBatchSolveEntry,
|
|
30
|
+
BundledBatchSolveResult,
|
|
31
|
+
BundledBatchSolveSubmission,
|
|
32
|
+
BundledBatchSolveSubmissionEntry,
|
|
33
|
+
BundledSolveEntry,
|
|
34
|
+
BundledSolveResult,
|
|
35
|
+
BundledSolveSubmission,
|
|
36
|
+
Resultbag,
|
|
37
|
+
SolveResult,
|
|
38
|
+
SolveSubmission,
|
|
39
|
+
TemplateRenderInfo,
|
|
40
|
+
)
|
|
41
|
+
from ..results.internal.resultbag.raw import raw_result_from_solve_result
|
|
42
|
+
from ..results.models import SolveKeys
|
|
43
|
+
from .daemon_bundle_staging import (
|
|
44
|
+
resolve_daemon_bundle_batch_dirs,
|
|
45
|
+
stage_daemon_bundles,
|
|
46
|
+
)
|
|
47
|
+
from .daemon_bundle_wait import wait_bundle_batch_entry
|
|
48
|
+
from .orchestration_execution import (
|
|
49
|
+
execute_local_solve_runtime,
|
|
50
|
+
submit_or_wait_daemon_runtime,
|
|
51
|
+
)
|
|
52
|
+
from .orchestration_routing import (
|
|
53
|
+
_stage_bundle_common_runtime_inputs,
|
|
54
|
+
_validate_dispatch_options,
|
|
55
|
+
is_batch_keys,
|
|
56
|
+
is_project_bundle,
|
|
57
|
+
normalize_project_bundle,
|
|
58
|
+
)
|
|
59
|
+
from .orchestration_support import (
|
|
60
|
+
_build_attempt_log_payload,
|
|
61
|
+
_is_pending_daemon_submission,
|
|
62
|
+
_is_retryable_solve_error,
|
|
63
|
+
load_cached_bundle_result,
|
|
64
|
+
load_cached_result,
|
|
65
|
+
prepare_project_inputs,
|
|
66
|
+
resolve_backend_mode,
|
|
67
|
+
resolve_context_backend,
|
|
68
|
+
)
|
|
69
|
+
from .path_resolution import resolve_project_input_path
|
|
70
|
+
from .progress import create_progress_bar
|
|
71
|
+
from .types import (
|
|
72
|
+
BatchOnError,
|
|
73
|
+
BatchSolveKeysInput,
|
|
74
|
+
DispatchMode,
|
|
75
|
+
ProjectBundleInput,
|
|
76
|
+
ProjectPathInput,
|
|
77
|
+
SolveKeysInput,
|
|
78
|
+
SolveMode,
|
|
79
|
+
WaitBreakWhen,
|
|
80
|
+
WaitOnTimeout,
|
|
81
|
+
WorkingDirInput,
|
|
82
|
+
)
|
|
83
|
+
from .working_dirs import (
|
|
84
|
+
resolve_batch_working_dirs,
|
|
85
|
+
resolve_bundle_working_dirs,
|
|
86
|
+
resolve_single_working_dir,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
if TYPE_CHECKING:
|
|
90
|
+
from ..daemon.handler import DaemonHandler
|
|
91
|
+
|
|
92
|
+
_logger: Logger = get_logger(__name__)
|
|
93
|
+
_PARTIAL_RESULT_ATTRIBUTE = "_jcmsuite_partial_result"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _attach_partial_result(exception: Exception, result: SolveResult) -> None:
|
|
97
|
+
"""Attach a partial solve result to an exception for collect-mode recovery."""
|
|
98
|
+
|
|
99
|
+
setattr(exception, _PARTIAL_RESULT_ATTRIBUTE, result)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _extract_partial_result(exception: Exception) -> SolveResult | None:
|
|
103
|
+
"""Return a partial solve result attached to an exception when present."""
|
|
104
|
+
|
|
105
|
+
partial_result: object = getattr(exception, _PARTIAL_RESULT_ATTRIBUTE, None)
|
|
106
|
+
if isinstance(partial_result, SolveResult):
|
|
107
|
+
return partial_result
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
async def solve_impl(
|
|
112
|
+
*,
|
|
113
|
+
project_file: ProjectPathInput | ProjectBundleInput,
|
|
114
|
+
caller_file: Path | None,
|
|
115
|
+
keys: SolveKeysInput | BatchSolveKeysInput,
|
|
116
|
+
resultbag: Resultbag | None,
|
|
117
|
+
recompute: bool,
|
|
118
|
+
max_retries: int,
|
|
119
|
+
on_error: BatchOnError,
|
|
120
|
+
context: SolverContext | None,
|
|
121
|
+
backend: BackendMode | None,
|
|
122
|
+
working_dir: WorkingDirInput,
|
|
123
|
+
suffix_pattern: str | None,
|
|
124
|
+
temporary: bool,
|
|
125
|
+
dispatch: DispatchMode,
|
|
126
|
+
wait_break_when: WaitBreakWhen,
|
|
127
|
+
wait_timeout: float | None,
|
|
128
|
+
wait_on_timeout: WaitOnTimeout,
|
|
129
|
+
wait_progress: bool,
|
|
130
|
+
wait_progress_label: str | None,
|
|
131
|
+
quiet: bool,
|
|
132
|
+
mode: SolveMode,
|
|
133
|
+
) -> (
|
|
134
|
+
SolveResult
|
|
135
|
+
| BatchSolveResult
|
|
136
|
+
| BundledSolveResult
|
|
137
|
+
| BundledBatchSolveResult
|
|
138
|
+
| SolveSubmission
|
|
139
|
+
| BundledSolveSubmission
|
|
140
|
+
| BundledBatchSolveSubmission
|
|
141
|
+
| BatchSolveSubmission
|
|
142
|
+
):
|
|
143
|
+
"""Internal solve implementation shared by sync and async wrappers."""
|
|
144
|
+
|
|
145
|
+
if mode not in ("solve", "postprocess"):
|
|
146
|
+
raise InvalidConfigurationError(
|
|
147
|
+
f"Unsupported solve mode {mode!r}; expected 'solve' or 'postprocess'."
|
|
148
|
+
)
|
|
149
|
+
if mode == "postprocess":
|
|
150
|
+
if resultbag is not None:
|
|
151
|
+
raise InvalidConfigurationError(
|
|
152
|
+
"Resultbag is not supported with mode='postprocess'."
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
_validate_dispatch_options(
|
|
156
|
+
dispatch=dispatch,
|
|
157
|
+
on_error=on_error,
|
|
158
|
+
max_retries=max_retries,
|
|
159
|
+
wait_break_when=wait_break_when,
|
|
160
|
+
wait_timeout=wait_timeout,
|
|
161
|
+
wait_on_timeout=wait_on_timeout,
|
|
162
|
+
wait_progress=wait_progress,
|
|
163
|
+
wait_progress_label=wait_progress_label,
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
scoped_daemon_handler: DaemonHandler | None = resolve_scoped_handler(
|
|
167
|
+
context=context
|
|
168
|
+
)
|
|
169
|
+
effective_context: SolverContext
|
|
170
|
+
if context is not None:
|
|
171
|
+
effective_context = resolve_context(context)
|
|
172
|
+
elif scoped_daemon_handler is not None:
|
|
173
|
+
effective_context = scoped_daemon_handler.context
|
|
174
|
+
else:
|
|
175
|
+
effective_context = resolve_context(None)
|
|
176
|
+
temp_dir_root: Path | None = effective_context.system.temp_dir_root
|
|
177
|
+
|
|
178
|
+
if is_project_bundle(project_file):
|
|
179
|
+
bundle_projects: tuple[Path, ...] = normalize_project_bundle(
|
|
180
|
+
cast(ProjectBundleInput, project_file),
|
|
181
|
+
caller_file=caller_file,
|
|
182
|
+
)
|
|
183
|
+
if dispatch == "submit" and is_batch_keys(keys):
|
|
184
|
+
bundle_batch_keys_for_submit: Sequence[Mapping[str, object]] = cast(
|
|
185
|
+
Sequence[Mapping[str, object]], keys
|
|
186
|
+
)
|
|
187
|
+
return await solve_bundle_batch_submit_impl(
|
|
188
|
+
project_files=bundle_projects,
|
|
189
|
+
caller_file=caller_file,
|
|
190
|
+
keys=bundle_batch_keys_for_submit,
|
|
191
|
+
resultbag=resultbag,
|
|
192
|
+
recompute=recompute,
|
|
193
|
+
max_retries=max_retries,
|
|
194
|
+
on_error=on_error,
|
|
195
|
+
context=context,
|
|
196
|
+
backend=backend,
|
|
197
|
+
working_dir=working_dir,
|
|
198
|
+
suffix_pattern=suffix_pattern,
|
|
199
|
+
temporary=temporary,
|
|
200
|
+
quiet=quiet,
|
|
201
|
+
mode=mode,
|
|
202
|
+
)
|
|
203
|
+
if dispatch == "submit":
|
|
204
|
+
bundle_single_keys_for_submit: Mapping[str, object] | None = cast(
|
|
205
|
+
Mapping[str, object] | None, keys
|
|
206
|
+
)
|
|
207
|
+
return await solve_bundle_submit_impl(
|
|
208
|
+
project_files=bundle_projects,
|
|
209
|
+
caller_file=caller_file,
|
|
210
|
+
keys=bundle_single_keys_for_submit,
|
|
211
|
+
resultbag=resultbag,
|
|
212
|
+
recompute=recompute,
|
|
213
|
+
max_retries=max_retries,
|
|
214
|
+
context=context,
|
|
215
|
+
backend=backend,
|
|
216
|
+
bundle_root=resolve_single_working_dir(
|
|
217
|
+
working_dir=working_dir,
|
|
218
|
+
temporary=temporary,
|
|
219
|
+
temp_dir_root=temp_dir_root,
|
|
220
|
+
),
|
|
221
|
+
suffix_pattern=suffix_pattern,
|
|
222
|
+
quiet=quiet,
|
|
223
|
+
mode=mode,
|
|
224
|
+
)
|
|
225
|
+
if is_batch_keys(keys):
|
|
226
|
+
bundle_batch_keys: Sequence[Mapping[str, object]] = cast(
|
|
227
|
+
Sequence[Mapping[str, object]], keys
|
|
228
|
+
)
|
|
229
|
+
return await solve_bundle_batch_impl(
|
|
230
|
+
project_files=bundle_projects,
|
|
231
|
+
caller_file=caller_file,
|
|
232
|
+
keys=bundle_batch_keys,
|
|
233
|
+
recompute=recompute,
|
|
234
|
+
max_retries=max_retries,
|
|
235
|
+
on_error=on_error,
|
|
236
|
+
context=context,
|
|
237
|
+
backend=backend,
|
|
238
|
+
resultbag=resultbag,
|
|
239
|
+
working_dir=working_dir,
|
|
240
|
+
suffix_pattern=suffix_pattern,
|
|
241
|
+
temporary=temporary,
|
|
242
|
+
wait_break_when=wait_break_when,
|
|
243
|
+
wait_timeout=wait_timeout,
|
|
244
|
+
wait_on_timeout=wait_on_timeout,
|
|
245
|
+
wait_progress=wait_progress,
|
|
246
|
+
wait_progress_label=wait_progress_label,
|
|
247
|
+
quiet=quiet,
|
|
248
|
+
mode=mode,
|
|
249
|
+
)
|
|
250
|
+
bundle_single_keys: Mapping[str, object] | None = cast(
|
|
251
|
+
Mapping[str, object] | None, keys
|
|
252
|
+
)
|
|
253
|
+
return await solve_bundle_impl(
|
|
254
|
+
project_files=bundle_projects,
|
|
255
|
+
caller_file=caller_file,
|
|
256
|
+
keys=bundle_single_keys,
|
|
257
|
+
recompute=recompute,
|
|
258
|
+
max_retries=max_retries,
|
|
259
|
+
context=context,
|
|
260
|
+
backend=backend,
|
|
261
|
+
resultbag=resultbag,
|
|
262
|
+
bundle_root=resolve_single_working_dir(
|
|
263
|
+
working_dir=working_dir,
|
|
264
|
+
temporary=temporary,
|
|
265
|
+
temp_dir_root=temp_dir_root,
|
|
266
|
+
),
|
|
267
|
+
suffix_pattern=suffix_pattern,
|
|
268
|
+
wait_break_when=wait_break_when,
|
|
269
|
+
wait_timeout=wait_timeout,
|
|
270
|
+
wait_on_timeout=wait_on_timeout,
|
|
271
|
+
wait_progress=wait_progress,
|
|
272
|
+
wait_progress_label=wait_progress_label,
|
|
273
|
+
quiet=quiet,
|
|
274
|
+
mode=mode,
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
if is_batch_keys(keys):
|
|
278
|
+
batch_keys: Sequence[Mapping[str, object]] = cast(
|
|
279
|
+
Sequence[Mapping[str, object]], keys
|
|
280
|
+
)
|
|
281
|
+
return await solve_batch_impl(
|
|
282
|
+
project_file=cast(ProjectPathInput, project_file),
|
|
283
|
+
caller_file=caller_file,
|
|
284
|
+
keys=batch_keys,
|
|
285
|
+
resultbag=resultbag,
|
|
286
|
+
recompute=recompute,
|
|
287
|
+
max_retries=max_retries,
|
|
288
|
+
on_error=on_error,
|
|
289
|
+
context=context,
|
|
290
|
+
backend=backend,
|
|
291
|
+
working_dir=working_dir,
|
|
292
|
+
suffix_pattern=suffix_pattern,
|
|
293
|
+
temporary=temporary,
|
|
294
|
+
dispatch=dispatch,
|
|
295
|
+
wait_break_when=wait_break_when,
|
|
296
|
+
wait_timeout=wait_timeout,
|
|
297
|
+
wait_on_timeout=wait_on_timeout,
|
|
298
|
+
wait_progress=wait_progress,
|
|
299
|
+
wait_progress_label=wait_progress_label,
|
|
300
|
+
quiet=quiet,
|
|
301
|
+
mode=mode,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
single_keys: Mapping[str, object] | None = cast(Mapping[str, object] | None, keys)
|
|
305
|
+
single_result: SolveResult = await solve_single_impl(
|
|
306
|
+
project_file=cast(ProjectPathInput, project_file),
|
|
307
|
+
caller_file=caller_file,
|
|
308
|
+
keys=single_keys,
|
|
309
|
+
resultbag=resultbag,
|
|
310
|
+
recompute=recompute,
|
|
311
|
+
max_retries=max_retries,
|
|
312
|
+
context=context,
|
|
313
|
+
backend=backend,
|
|
314
|
+
working_dir=resolve_single_working_dir(
|
|
315
|
+
working_dir=working_dir,
|
|
316
|
+
temporary=temporary,
|
|
317
|
+
temp_dir_root=temp_dir_root,
|
|
318
|
+
),
|
|
319
|
+
suffix_pattern=suffix_pattern,
|
|
320
|
+
dispatch=dispatch,
|
|
321
|
+
wait_break_when=wait_break_when,
|
|
322
|
+
wait_timeout=wait_timeout,
|
|
323
|
+
wait_on_timeout=wait_on_timeout,
|
|
324
|
+
wait_progress=wait_progress,
|
|
325
|
+
wait_progress_label=wait_progress_label,
|
|
326
|
+
quiet=quiet,
|
|
327
|
+
mode=mode,
|
|
328
|
+
)
|
|
329
|
+
if dispatch == "submit":
|
|
330
|
+
if (
|
|
331
|
+
single_result.backend_used != "daemon"
|
|
332
|
+
or single_result.daemon_job_id is None
|
|
333
|
+
):
|
|
334
|
+
raise InvalidConfigurationError(
|
|
335
|
+
"`dispatch='submit'` is only supported for daemon backend solves."
|
|
336
|
+
)
|
|
337
|
+
submission_waiter: DaemonHandler | None = resolve_scoped_handler(
|
|
338
|
+
context=context
|
|
339
|
+
)
|
|
340
|
+
if submission_waiter is None:
|
|
341
|
+
resolved_context_for_submission: SolverContext = resolve_context(context)
|
|
342
|
+
submission_waiter = resolved_context_for_submission._get_daemon_handler()
|
|
343
|
+
if not _is_pending_daemon_submission(
|
|
344
|
+
daemon_handler=submission_waiter,
|
|
345
|
+
daemon_job_id=single_result.daemon_job_id,
|
|
346
|
+
):
|
|
347
|
+
raise InvalidConfigurationError(
|
|
348
|
+
"No daemon submission was created for `dispatch='submit'`; "
|
|
349
|
+
"result was resolved from cache."
|
|
350
|
+
)
|
|
351
|
+
source_project_mappings: tuple[tuple[Path, Path], ...] = ()
|
|
352
|
+
if single_result.project_file is not None:
|
|
353
|
+
source_project_path: Path = resolve_project_input_path(
|
|
354
|
+
cast(ProjectPathInput, project_file),
|
|
355
|
+
caller_file=caller_file,
|
|
356
|
+
)
|
|
357
|
+
source_project_mappings = (
|
|
358
|
+
(single_result.project_file, source_project_path),
|
|
359
|
+
)
|
|
360
|
+
return SolveSubmission(
|
|
361
|
+
job_id=single_result.daemon_job_id,
|
|
362
|
+
result=single_result,
|
|
363
|
+
keys=None if single_keys is None else dict(single_keys),
|
|
364
|
+
index=None,
|
|
365
|
+
)._bind_runtime(
|
|
366
|
+
waiter=submission_waiter,
|
|
367
|
+
resultbag=resultbag,
|
|
368
|
+
suffix_pattern=suffix_pattern,
|
|
369
|
+
source_project_mappings=source_project_mappings,
|
|
370
|
+
)
|
|
371
|
+
return single_result
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
async def solve_bundle_impl(
|
|
375
|
+
*,
|
|
376
|
+
project_files: Sequence[Path],
|
|
377
|
+
caller_file: Path | None,
|
|
378
|
+
keys: Mapping[str, object] | None,
|
|
379
|
+
recompute: bool,
|
|
380
|
+
max_retries: int,
|
|
381
|
+
context: SolverContext | None,
|
|
382
|
+
backend: BackendMode | None,
|
|
383
|
+
resultbag: Resultbag | None,
|
|
384
|
+
bundle_root: Path | None,
|
|
385
|
+
suffix_pattern: str | None,
|
|
386
|
+
wait_break_when: WaitBreakWhen,
|
|
387
|
+
wait_timeout: float | None,
|
|
388
|
+
wait_on_timeout: WaitOnTimeout,
|
|
389
|
+
wait_progress: bool,
|
|
390
|
+
wait_progress_label: str | None,
|
|
391
|
+
quiet: bool,
|
|
392
|
+
mode: SolveMode = "solve",
|
|
393
|
+
) -> BundledSolveResult:
|
|
394
|
+
"""Run one ordered project bundle sequentially and return bundle results."""
|
|
395
|
+
|
|
396
|
+
if max_retries < 0:
|
|
397
|
+
raise InvalidConfigurationError("max_retries must be >= 0.")
|
|
398
|
+
resolved_context: SolverContext
|
|
399
|
+
resolved_backend: BackendMode
|
|
400
|
+
resolved_handler: DaemonHandler | None
|
|
401
|
+
resolved_context, resolved_backend, resolved_handler = resolve_context_backend(
|
|
402
|
+
context=context,
|
|
403
|
+
backend=backend,
|
|
404
|
+
)
|
|
405
|
+
if resolved_backend == "daemon":
|
|
406
|
+
for attempt_index in range(max_retries + 1):
|
|
407
|
+
submission: BundledSolveSubmission = await solve_bundle_submit_impl(
|
|
408
|
+
project_files=project_files,
|
|
409
|
+
caller_file=caller_file,
|
|
410
|
+
keys=keys,
|
|
411
|
+
resultbag=resultbag,
|
|
412
|
+
recompute=recompute,
|
|
413
|
+
max_retries=0,
|
|
414
|
+
context=resolved_context,
|
|
415
|
+
backend=resolved_backend,
|
|
416
|
+
bundle_root=bundle_root,
|
|
417
|
+
suffix_pattern=suffix_pattern,
|
|
418
|
+
quiet=quiet,
|
|
419
|
+
mode=mode,
|
|
420
|
+
)
|
|
421
|
+
try:
|
|
422
|
+
return await submission.wait(
|
|
423
|
+
asynchronous=True,
|
|
424
|
+
break_when=wait_break_when,
|
|
425
|
+
timeout=wait_timeout,
|
|
426
|
+
on_timeout=wait_on_timeout,
|
|
427
|
+
progress=wait_progress,
|
|
428
|
+
progress_label=wait_progress_label,
|
|
429
|
+
)
|
|
430
|
+
except SolveExecutionError:
|
|
431
|
+
# An explicit retry starts a new complete bundle only after the
|
|
432
|
+
# failed job was closed. A timeout leaves its original job live
|
|
433
|
+
# and must never create a second allocation.
|
|
434
|
+
if attempt_index >= max_retries:
|
|
435
|
+
raise
|
|
436
|
+
raise RuntimeError("Bundle retry loop terminated unexpectedly.")
|
|
437
|
+
|
|
438
|
+
resolved_keys: Mapping[str, object] = {} if keys is None else keys
|
|
439
|
+
cached_result: BundledSolveResult | None = load_cached_bundle_result(
|
|
440
|
+
resultbag=resultbag,
|
|
441
|
+
project_paths=project_files,
|
|
442
|
+
keys=resolved_keys,
|
|
443
|
+
recompute=recompute,
|
|
444
|
+
suffix_pattern=suffix_pattern,
|
|
445
|
+
)
|
|
446
|
+
if cached_result is not None:
|
|
447
|
+
entry: BundledSolveEntry
|
|
448
|
+
if keys is None:
|
|
449
|
+
for entry in cached_result.entries:
|
|
450
|
+
entry.result.keys = None
|
|
451
|
+
else:
|
|
452
|
+
for entry in cached_result.entries:
|
|
453
|
+
entry.result.keys = SolveKeys(
|
|
454
|
+
dict(resolved_keys), source=entry.result.project_file
|
|
455
|
+
)
|
|
456
|
+
return cached_result
|
|
457
|
+
if resultbag is not None:
|
|
458
|
+
resultbag.management.clear_attempt_logs(
|
|
459
|
+
resolved_keys,
|
|
460
|
+
suffix_pattern=suffix_pattern,
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
_stage_bundle_common_runtime_inputs(
|
|
464
|
+
project_files=project_files,
|
|
465
|
+
bundle_root=bundle_root,
|
|
466
|
+
)
|
|
467
|
+
project_working_dirs: list[Path | None] = resolve_bundle_working_dirs(
|
|
468
|
+
project_files=project_files,
|
|
469
|
+
bundle_root=bundle_root,
|
|
470
|
+
)
|
|
471
|
+
bundle_entries: list[BundledSolveEntry] = []
|
|
472
|
+
project_index: int
|
|
473
|
+
project_file: Path
|
|
474
|
+
for project_index, project_file in enumerate(project_files):
|
|
475
|
+
single_result: SolveResult = await solve_single_impl(
|
|
476
|
+
project_file=project_file,
|
|
477
|
+
caller_file=caller_file,
|
|
478
|
+
keys=keys,
|
|
479
|
+
resultbag=None,
|
|
480
|
+
recompute=recompute,
|
|
481
|
+
max_retries=max_retries,
|
|
482
|
+
context=context,
|
|
483
|
+
backend=backend,
|
|
484
|
+
working_dir=project_working_dirs[project_index],
|
|
485
|
+
suffix_pattern=suffix_pattern,
|
|
486
|
+
dispatch="wait",
|
|
487
|
+
wait_break_when=wait_break_when,
|
|
488
|
+
wait_timeout=wait_timeout,
|
|
489
|
+
wait_on_timeout=wait_on_timeout,
|
|
490
|
+
wait_progress=wait_progress,
|
|
491
|
+
wait_progress_label=wait_progress_label,
|
|
492
|
+
quiet=quiet,
|
|
493
|
+
mode=mode,
|
|
494
|
+
)
|
|
495
|
+
bundle_entries.append(
|
|
496
|
+
BundledSolveEntry(
|
|
497
|
+
project_file=project_file,
|
|
498
|
+
result=single_result,
|
|
499
|
+
)
|
|
500
|
+
)
|
|
501
|
+
bundle_result: BundledSolveResult = BundledSolveResult(
|
|
502
|
+
entries=tuple(bundle_entries)
|
|
503
|
+
)
|
|
504
|
+
if resultbag is not None:
|
|
505
|
+
resultbag.management.add(
|
|
506
|
+
keys=resolved_keys,
|
|
507
|
+
result=raw_result_from_solve_result(bundle_result),
|
|
508
|
+
log=None,
|
|
509
|
+
suffix_pattern=suffix_pattern,
|
|
510
|
+
)
|
|
511
|
+
return bundle_result
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
async def solve_bundle_batch_impl(
|
|
515
|
+
*,
|
|
516
|
+
project_files: Sequence[Path],
|
|
517
|
+
caller_file: Path | None,
|
|
518
|
+
keys: Sequence[Mapping[str, object]],
|
|
519
|
+
recompute: bool,
|
|
520
|
+
max_retries: int,
|
|
521
|
+
on_error: BatchOnError,
|
|
522
|
+
context: SolverContext | None,
|
|
523
|
+
backend: BackendMode | None,
|
|
524
|
+
resultbag: Resultbag | None,
|
|
525
|
+
working_dir: WorkingDirInput,
|
|
526
|
+
suffix_pattern: str | None,
|
|
527
|
+
temporary: bool,
|
|
528
|
+
wait_break_when: WaitBreakWhen,
|
|
529
|
+
wait_timeout: float | None,
|
|
530
|
+
wait_on_timeout: WaitOnTimeout,
|
|
531
|
+
wait_progress: bool,
|
|
532
|
+
wait_progress_label: str | None,
|
|
533
|
+
quiet: bool,
|
|
534
|
+
mode: SolveMode = "solve",
|
|
535
|
+
) -> BundledBatchSolveResult:
|
|
536
|
+
"""Run batched project keys whose execution unit is one project bundle."""
|
|
537
|
+
|
|
538
|
+
if max_retries < 0:
|
|
539
|
+
raise InvalidConfigurationError("max_retries must be >= 0.")
|
|
540
|
+
resolved_context: SolverContext
|
|
541
|
+
resolved_backend: BackendMode
|
|
542
|
+
resolved_handler: DaemonHandler | None
|
|
543
|
+
resolved_context, resolved_backend, resolved_handler = resolve_context_backend(
|
|
544
|
+
context=context,
|
|
545
|
+
backend=backend,
|
|
546
|
+
)
|
|
547
|
+
if resolved_backend == "daemon":
|
|
548
|
+
bundle_submission: BundledBatchSolveSubmission = (
|
|
549
|
+
await solve_bundle_batch_submit_impl(
|
|
550
|
+
project_files=project_files,
|
|
551
|
+
caller_file=caller_file,
|
|
552
|
+
keys=keys,
|
|
553
|
+
resultbag=resultbag,
|
|
554
|
+
recompute=recompute,
|
|
555
|
+
max_retries=max_retries,
|
|
556
|
+
on_error=on_error,
|
|
557
|
+
context=resolved_context,
|
|
558
|
+
backend=resolved_backend,
|
|
559
|
+
working_dir=working_dir,
|
|
560
|
+
suffix_pattern=suffix_pattern,
|
|
561
|
+
temporary=temporary,
|
|
562
|
+
quiet=quiet,
|
|
563
|
+
mode=mode,
|
|
564
|
+
)
|
|
565
|
+
)
|
|
566
|
+
return await bundle_submission.wait(
|
|
567
|
+
asynchronous=True,
|
|
568
|
+
break_when=wait_break_when,
|
|
569
|
+
timeout=wait_timeout,
|
|
570
|
+
on_timeout=wait_on_timeout,
|
|
571
|
+
on_error=on_error,
|
|
572
|
+
progress=wait_progress,
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
cached_entry_indices: set[int] = set()
|
|
576
|
+
if resultbag is not None and not recompute:
|
|
577
|
+
key_index_for_cache: int
|
|
578
|
+
key_payload_for_cache: Mapping[str, object]
|
|
579
|
+
for key_index_for_cache, key_payload_for_cache in enumerate(keys):
|
|
580
|
+
cached_bundle_result: BundledSolveResult | None = load_cached_bundle_result(
|
|
581
|
+
resultbag=resultbag,
|
|
582
|
+
project_paths=project_files,
|
|
583
|
+
keys=key_payload_for_cache,
|
|
584
|
+
recompute=False,
|
|
585
|
+
suffix_pattern=suffix_pattern,
|
|
586
|
+
)
|
|
587
|
+
if cached_bundle_result is not None:
|
|
588
|
+
cached_entry_indices.add(key_index_for_cache)
|
|
589
|
+
|
|
590
|
+
batch_working_dirs: list[Path | None] = resolve_batch_working_dirs(
|
|
591
|
+
working_dir=working_dir,
|
|
592
|
+
batch_size=len(keys),
|
|
593
|
+
temporary=temporary,
|
|
594
|
+
temp_dir_root=resolved_context.system.temp_dir_root,
|
|
595
|
+
)
|
|
596
|
+
entries: list[BundledBatchSolveEntry] = []
|
|
597
|
+
live_entry_count: int = len(keys) - len(cached_entry_indices)
|
|
598
|
+
progress_bar = create_progress_bar(
|
|
599
|
+
total=live_entry_count,
|
|
600
|
+
enabled=wait_progress and live_entry_count > 0,
|
|
601
|
+
description=wait_progress_label or "solve(bundle batch)",
|
|
602
|
+
)
|
|
603
|
+
try:
|
|
604
|
+
submission_limit: int = _resolve_bundle_batch_submission_limit(
|
|
605
|
+
daemon_handler=resolved_handler,
|
|
606
|
+
batch_size=len(keys),
|
|
607
|
+
)
|
|
608
|
+
submission_semaphore: asyncio.Semaphore = asyncio.Semaphore(submission_limit)
|
|
609
|
+
|
|
610
|
+
async def _solve_one_bundle_entry(
|
|
611
|
+
*,
|
|
612
|
+
key_index: int,
|
|
613
|
+
key_payload: Mapping[str, object],
|
|
614
|
+
) -> BundledBatchSolveEntry:
|
|
615
|
+
key_dict: dict[str, object] = dict(key_payload)
|
|
616
|
+
async with submission_semaphore:
|
|
617
|
+
try:
|
|
618
|
+
bundle_result: BundledSolveResult = await solve_bundle_impl(
|
|
619
|
+
project_files=project_files,
|
|
620
|
+
caller_file=caller_file,
|
|
621
|
+
keys=key_dict,
|
|
622
|
+
recompute=recompute,
|
|
623
|
+
max_retries=max_retries,
|
|
624
|
+
context=resolved_context,
|
|
625
|
+
backend=resolved_backend,
|
|
626
|
+
resultbag=resultbag,
|
|
627
|
+
bundle_root=batch_working_dirs[key_index],
|
|
628
|
+
suffix_pattern=suffix_pattern,
|
|
629
|
+
wait_break_when=wait_break_when,
|
|
630
|
+
wait_timeout=wait_timeout,
|
|
631
|
+
wait_on_timeout=wait_on_timeout,
|
|
632
|
+
wait_progress=wait_progress,
|
|
633
|
+
wait_progress_label=wait_progress_label,
|
|
634
|
+
quiet=quiet,
|
|
635
|
+
mode=mode,
|
|
636
|
+
)
|
|
637
|
+
except Exception as exception:
|
|
638
|
+
if on_error == "raise":
|
|
639
|
+
raise
|
|
640
|
+
return BundledBatchSolveEntry(
|
|
641
|
+
index=key_index,
|
|
642
|
+
keys=key_dict,
|
|
643
|
+
result=None,
|
|
644
|
+
failure=BatchSolveFailure(
|
|
645
|
+
index=key_index,
|
|
646
|
+
keys=key_dict,
|
|
647
|
+
exception_type=type(exception),
|
|
648
|
+
message=str(exception),
|
|
649
|
+
),
|
|
650
|
+
project_files=tuple(project_files),
|
|
651
|
+
)
|
|
652
|
+
return BundledBatchSolveEntry(
|
|
653
|
+
index=key_index,
|
|
654
|
+
keys=key_dict,
|
|
655
|
+
result=bundle_result,
|
|
656
|
+
failure=None,
|
|
657
|
+
project_files=tuple(project_files),
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
key_entries: list[tuple[int, Mapping[str, object]]] = list(enumerate(keys))
|
|
661
|
+
pending_entries: list[BundledBatchSolveEntry] = list(
|
|
662
|
+
await asyncio.gather(
|
|
663
|
+
*(
|
|
664
|
+
_solve_one_bundle_entry(
|
|
665
|
+
key_index=key_index,
|
|
666
|
+
key_payload=key_payload,
|
|
667
|
+
)
|
|
668
|
+
for key_index, key_payload in key_entries
|
|
669
|
+
)
|
|
670
|
+
)
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
for batch_entry in pending_entries:
|
|
674
|
+
entries.append(batch_entry)
|
|
675
|
+
if batch_entry.index not in cached_entry_indices:
|
|
676
|
+
if batch_entry.failure is None:
|
|
677
|
+
progress_bar.update(1)
|
|
678
|
+
else:
|
|
679
|
+
progress_bar.mark_failure(1)
|
|
680
|
+
progress_bar.update(1)
|
|
681
|
+
return BundledBatchSolveResult(entries=tuple(entries))
|
|
682
|
+
finally:
|
|
683
|
+
progress_bar.close()
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
async def solve_bundle_submit_impl(
|
|
687
|
+
*,
|
|
688
|
+
project_files: Sequence[Path],
|
|
689
|
+
caller_file: Path | None,
|
|
690
|
+
keys: Mapping[str, object] | None,
|
|
691
|
+
resultbag: Resultbag | None,
|
|
692
|
+
recompute: bool,
|
|
693
|
+
max_retries: int,
|
|
694
|
+
context: SolverContext | None,
|
|
695
|
+
backend: BackendMode | None,
|
|
696
|
+
bundle_root: Path | None,
|
|
697
|
+
suffix_pattern: str | None,
|
|
698
|
+
quiet: bool,
|
|
699
|
+
mode: SolveMode = "solve",
|
|
700
|
+
_staged: bool = False,
|
|
701
|
+
) -> BundledSolveSubmission:
|
|
702
|
+
"""Prepare every project and submit the ordered bundle as one daemon job."""
|
|
703
|
+
|
|
704
|
+
resolved_context: SolverContext
|
|
705
|
+
resolved_backend: BackendMode
|
|
706
|
+
backend_daemon_handler: DaemonHandler | None
|
|
707
|
+
resolved_context, resolved_backend, backend_daemon_handler = (
|
|
708
|
+
resolve_context_backend(
|
|
709
|
+
context=context,
|
|
710
|
+
backend=backend,
|
|
711
|
+
)
|
|
712
|
+
)
|
|
713
|
+
if resolved_backend != "daemon":
|
|
714
|
+
raise InvalidConfigurationError(
|
|
715
|
+
"`dispatch='submit'` is only supported for daemon backend solves."
|
|
716
|
+
)
|
|
717
|
+
if backend_daemon_handler is None:
|
|
718
|
+
raise BackendUnavailableError(
|
|
719
|
+
"Daemon handler is not available for bundle submission."
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
resolved_keys: Mapping[str, object] = {} if keys is None else keys
|
|
723
|
+
cached_result: BundledSolveResult | None = load_cached_bundle_result(
|
|
724
|
+
resultbag=resultbag,
|
|
725
|
+
project_paths=project_files,
|
|
726
|
+
keys=resolved_keys,
|
|
727
|
+
recompute=recompute,
|
|
728
|
+
suffix_pattern=suffix_pattern,
|
|
729
|
+
)
|
|
730
|
+
if cached_result is not None:
|
|
731
|
+
for entry in cached_result.entries:
|
|
732
|
+
entry.result.keys = (
|
|
733
|
+
None
|
|
734
|
+
if keys is None
|
|
735
|
+
else SolveKeys(dict(keys), source=entry.result.project_file)
|
|
736
|
+
)
|
|
737
|
+
return BundledSolveSubmission(
|
|
738
|
+
job_id=None,
|
|
739
|
+
project_files=tuple(project_files),
|
|
740
|
+
keys=None if keys is None else dict(keys),
|
|
741
|
+
)._bind_runtime(
|
|
742
|
+
waiter=backend_daemon_handler,
|
|
743
|
+
resolved_result=cached_result,
|
|
744
|
+
)
|
|
745
|
+
if resultbag is not None:
|
|
746
|
+
resultbag.management.clear_attempt_logs(
|
|
747
|
+
resolved_keys,
|
|
748
|
+
suffix_pattern=suffix_pattern,
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
if not _staged:
|
|
752
|
+
stage_daemon_bundles(project_files=project_files, bundle_roots=(bundle_root,))
|
|
753
|
+
project_working_dirs: list[Path | None] = resolve_bundle_working_dirs(
|
|
754
|
+
project_files=project_files,
|
|
755
|
+
bundle_root=bundle_root,
|
|
756
|
+
)
|
|
757
|
+
entries: list[BundledSolveEntry] = []
|
|
758
|
+
prepared_files: list[Path] = []
|
|
759
|
+
project_index: int
|
|
760
|
+
project_file: Path
|
|
761
|
+
for project_index, project_file in enumerate(project_files):
|
|
762
|
+
prepared_project: Path
|
|
763
|
+
template_renders: tuple[TemplateRenderInfo, ...]
|
|
764
|
+
prepared_project, template_renders = prepare_project_inputs(
|
|
765
|
+
project_path=project_file,
|
|
766
|
+
working_dir=project_working_dirs[project_index],
|
|
767
|
+
resolved_keys=resolved_keys,
|
|
768
|
+
suffix_pattern=suffix_pattern,
|
|
769
|
+
)
|
|
770
|
+
prepared_files.append(prepared_project)
|
|
771
|
+
entries.append(
|
|
772
|
+
BundledSolveEntry(
|
|
773
|
+
project_file=project_file,
|
|
774
|
+
result=SolveResult(
|
|
775
|
+
project_file=prepared_project,
|
|
776
|
+
backend_used="daemon",
|
|
777
|
+
template_renders=template_renders,
|
|
778
|
+
keys=None if keys is None else resolved_keys,
|
|
779
|
+
_post_process_only=mode == "postprocess",
|
|
780
|
+
),
|
|
781
|
+
)
|
|
782
|
+
)
|
|
783
|
+
bundle_result: BundledSolveResult = BundledSolveResult(entries=tuple(entries))
|
|
784
|
+
job_id: int = backend_daemon_handler._submit_solve_job(
|
|
785
|
+
result=bundle_result,
|
|
786
|
+
project_file=prepared_files,
|
|
787
|
+
source_project_mappings=tuple(zip(prepared_files, project_files, strict=True)),
|
|
788
|
+
keys=dict(resolved_keys),
|
|
789
|
+
resultbag=resultbag,
|
|
790
|
+
suffix_pattern=suffix_pattern,
|
|
791
|
+
mode=mode,
|
|
792
|
+
)
|
|
793
|
+
for entry in entries:
|
|
794
|
+
entry.result.daemon_job_id = job_id
|
|
795
|
+
|
|
796
|
+
async def _wait_bundle_submission(
|
|
797
|
+
*,
|
|
798
|
+
bundle_submission: BundledSolveSubmission,
|
|
799
|
+
daemon: DaemonHandler,
|
|
800
|
+
break_when: WaitBreakWhen,
|
|
801
|
+
timeout: float | None,
|
|
802
|
+
on_timeout: WaitOnTimeout,
|
|
803
|
+
on_error: BatchOnError,
|
|
804
|
+
progress: bool,
|
|
805
|
+
progress_label: str | None,
|
|
806
|
+
) -> BundledSolveResult:
|
|
807
|
+
waited_results: list[object]
|
|
808
|
+
waited_logs: list[str | None]
|
|
809
|
+
waited_results, waited_logs = await daemon._wait_impl(
|
|
810
|
+
job_ids=[job_id],
|
|
811
|
+
break_when="all",
|
|
812
|
+
timeout=timeout,
|
|
813
|
+
on_timeout="raise",
|
|
814
|
+
on_error="raise",
|
|
815
|
+
progress=progress,
|
|
816
|
+
progress_label=progress_label,
|
|
817
|
+
)
|
|
818
|
+
if len(waited_results) != 1 or waited_results[0] is not bundle_result:
|
|
819
|
+
raise InvalidConfigurationError(
|
|
820
|
+
"Daemon wait returned an invalid bundle result."
|
|
821
|
+
)
|
|
822
|
+
return bundle_result
|
|
823
|
+
|
|
824
|
+
return BundledSolveSubmission(
|
|
825
|
+
job_id=job_id,
|
|
826
|
+
project_files=tuple(prepared_files),
|
|
827
|
+
keys=None if keys is None else dict(keys),
|
|
828
|
+
)._bind_runtime(
|
|
829
|
+
waiter=backend_daemon_handler,
|
|
830
|
+
wait_callback=_wait_bundle_submission,
|
|
831
|
+
)
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
async def solve_bundle_batch_submit_impl(
|
|
835
|
+
*,
|
|
836
|
+
project_files: Sequence[Path],
|
|
837
|
+
caller_file: Path | None,
|
|
838
|
+
keys: Sequence[Mapping[str, object]],
|
|
839
|
+
resultbag: Resultbag | None,
|
|
840
|
+
recompute: bool,
|
|
841
|
+
max_retries: int,
|
|
842
|
+
on_error: BatchOnError,
|
|
843
|
+
context: SolverContext | None,
|
|
844
|
+
backend: BackendMode | None,
|
|
845
|
+
working_dir: WorkingDirInput,
|
|
846
|
+
suffix_pattern: str | None,
|
|
847
|
+
temporary: bool,
|
|
848
|
+
quiet: bool,
|
|
849
|
+
mode: SolveMode = "solve",
|
|
850
|
+
) -> BundledBatchSolveSubmission:
|
|
851
|
+
"""Submit one complete ordered bundle for each Project Keys entry."""
|
|
852
|
+
|
|
853
|
+
if on_error not in ("raise", "collect"):
|
|
854
|
+
raise InvalidConfigurationError(
|
|
855
|
+
f"Unsupported batch on_error policy: {on_error!r}"
|
|
856
|
+
)
|
|
857
|
+
|
|
858
|
+
resolved_context: SolverContext
|
|
859
|
+
resolved_backend: BackendMode
|
|
860
|
+
backend_daemon_handler: DaemonHandler | None
|
|
861
|
+
resolved_context, resolved_backend, backend_daemon_handler = (
|
|
862
|
+
resolve_context_backend(
|
|
863
|
+
context=context,
|
|
864
|
+
backend=backend,
|
|
865
|
+
)
|
|
866
|
+
)
|
|
867
|
+
if resolved_backend != "daemon":
|
|
868
|
+
raise InvalidConfigurationError(
|
|
869
|
+
"`dispatch='submit'` is only supported for daemon backend solves."
|
|
870
|
+
)
|
|
871
|
+
if backend_daemon_handler is None:
|
|
872
|
+
raise BackendUnavailableError(
|
|
873
|
+
"Daemon handler is not available for bundle batch submission."
|
|
874
|
+
)
|
|
875
|
+
|
|
876
|
+
cached_indices: set[int] = set()
|
|
877
|
+
if resultbag is not None and not recompute:
|
|
878
|
+
for index, key_payload in enumerate(keys):
|
|
879
|
+
try:
|
|
880
|
+
cached: BundledSolveResult | None = load_cached_bundle_result(
|
|
881
|
+
resultbag=resultbag,
|
|
882
|
+
project_paths=project_files,
|
|
883
|
+
keys=key_payload,
|
|
884
|
+
recompute=False,
|
|
885
|
+
suffix_pattern=suffix_pattern,
|
|
886
|
+
)
|
|
887
|
+
except Exception:
|
|
888
|
+
if on_error == "raise":
|
|
889
|
+
raise
|
|
890
|
+
# Per-entry submission still collects this failure. A cache
|
|
891
|
+
# probe must not turn collect mode into a global exception.
|
|
892
|
+
continue
|
|
893
|
+
if cached is not None:
|
|
894
|
+
cached_indices.add(index)
|
|
895
|
+
batch_working_dirs: list[Path | None] = resolve_daemon_bundle_batch_dirs(
|
|
896
|
+
working_dir=working_dir,
|
|
897
|
+
batch_size=len(keys),
|
|
898
|
+
temp_dir_root=resolved_context.system.temp_dir_root,
|
|
899
|
+
cached_indices=cached_indices,
|
|
900
|
+
project_files=project_files,
|
|
901
|
+
)
|
|
902
|
+
submission_limit: int = _resolve_bundle_batch_submission_limit(
|
|
903
|
+
daemon_handler=backend_daemon_handler,
|
|
904
|
+
batch_size=len(keys),
|
|
905
|
+
)
|
|
906
|
+
submission_semaphore: asyncio.Semaphore = asyncio.Semaphore(submission_limit)
|
|
907
|
+
|
|
908
|
+
async def _submit_one_bundle_entry(
|
|
909
|
+
*,
|
|
910
|
+
key_index: int,
|
|
911
|
+
key_payload: Mapping[str, object],
|
|
912
|
+
) -> BundledBatchSolveSubmissionEntry:
|
|
913
|
+
key_dict: dict[str, object] = dict(key_payload)
|
|
914
|
+
async with submission_semaphore:
|
|
915
|
+
try:
|
|
916
|
+
if key_index not in cached_indices:
|
|
917
|
+
stage_daemon_bundles(
|
|
918
|
+
project_files=project_files,
|
|
919
|
+
bundle_roots=(batch_working_dirs[key_index],),
|
|
920
|
+
excluded_roots=batch_working_dirs,
|
|
921
|
+
)
|
|
922
|
+
bundle_submission: BundledSolveSubmission = (
|
|
923
|
+
await solve_bundle_submit_impl(
|
|
924
|
+
project_files=project_files,
|
|
925
|
+
caller_file=caller_file,
|
|
926
|
+
keys=key_dict,
|
|
927
|
+
resultbag=resultbag,
|
|
928
|
+
recompute=recompute,
|
|
929
|
+
max_retries=max_retries,
|
|
930
|
+
context=resolved_context,
|
|
931
|
+
backend=resolved_backend,
|
|
932
|
+
bundle_root=batch_working_dirs[key_index],
|
|
933
|
+
suffix_pattern=suffix_pattern,
|
|
934
|
+
quiet=quiet,
|
|
935
|
+
mode=mode,
|
|
936
|
+
_staged=True,
|
|
937
|
+
)
|
|
938
|
+
)
|
|
939
|
+
except Exception as exception:
|
|
940
|
+
if on_error == "raise":
|
|
941
|
+
raise
|
|
942
|
+
return BundledBatchSolveSubmissionEntry(
|
|
943
|
+
index=key_index,
|
|
944
|
+
keys=key_dict,
|
|
945
|
+
result=None,
|
|
946
|
+
failure=BatchSolveFailure(
|
|
947
|
+
index=key_index,
|
|
948
|
+
keys=key_dict,
|
|
949
|
+
exception_type=type(exception),
|
|
950
|
+
message=str(exception),
|
|
951
|
+
),
|
|
952
|
+
project_files=tuple(project_files),
|
|
953
|
+
)
|
|
954
|
+
return BundledBatchSolveSubmissionEntry(
|
|
955
|
+
index=key_index,
|
|
956
|
+
keys=key_dict,
|
|
957
|
+
result=bundle_submission,
|
|
958
|
+
failure=None,
|
|
959
|
+
project_files=tuple(project_files),
|
|
960
|
+
)
|
|
961
|
+
|
|
962
|
+
key_entries: list[tuple[int, Mapping[str, object]]] = list(enumerate(keys))
|
|
963
|
+
entries: list[BundledBatchSolveSubmissionEntry] = list(
|
|
964
|
+
await asyncio.gather(
|
|
965
|
+
*(
|
|
966
|
+
_submit_one_bundle_entry(
|
|
967
|
+
key_index=key_index,
|
|
968
|
+
key_payload=key_payload,
|
|
969
|
+
)
|
|
970
|
+
for key_index, key_payload in key_entries
|
|
971
|
+
)
|
|
972
|
+
)
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
async def _wait_batch_bundle_submission(
|
|
976
|
+
*,
|
|
977
|
+
batch_submission: BundledBatchSolveSubmission,
|
|
978
|
+
daemon: DaemonHandler,
|
|
979
|
+
break_when: WaitBreakWhen,
|
|
980
|
+
timeout: float | None,
|
|
981
|
+
on_timeout: WaitOnTimeout,
|
|
982
|
+
on_error: BatchOnError,
|
|
983
|
+
progress: bool,
|
|
984
|
+
progress_label: str | None,
|
|
985
|
+
) -> BundledBatchSolveResult:
|
|
986
|
+
if break_when != "all":
|
|
987
|
+
raise InvalidConfigurationError(
|
|
988
|
+
"BundledBatchSolveSubmission.wait() only supports `break_when='all'`."
|
|
989
|
+
)
|
|
990
|
+
if on_timeout != "raise":
|
|
991
|
+
raise InvalidConfigurationError(
|
|
992
|
+
"BundledBatchSolveSubmission.wait() only supports `on_timeout='raise'`."
|
|
993
|
+
)
|
|
994
|
+
|
|
995
|
+
result_entries: list[BundledBatchSolveEntry] = []
|
|
996
|
+
live_entry_count: int = sum(
|
|
997
|
+
1
|
|
998
|
+
for batch_entry in batch_submission.entries
|
|
999
|
+
if batch_entry.result is not None
|
|
1000
|
+
and batch_entry.result._resolved_result is None
|
|
1001
|
+
)
|
|
1002
|
+
progress_bar = create_progress_bar(
|
|
1003
|
+
total=live_entry_count,
|
|
1004
|
+
enabled=progress and live_entry_count > 0,
|
|
1005
|
+
description=progress_label or "solve(bundle batch)",
|
|
1006
|
+
)
|
|
1007
|
+
try:
|
|
1008
|
+
|
|
1009
|
+
async def _wait_one_bundle_entry(
|
|
1010
|
+
*,
|
|
1011
|
+
batch_entry: BundledBatchSolveSubmissionEntry,
|
|
1012
|
+
) -> BundledBatchSolveEntry:
|
|
1013
|
+
"""Wait for one bundle submission and convert failures when allowed."""
|
|
1014
|
+
|
|
1015
|
+
async def resubmit(
|
|
1016
|
+
entry: BundledBatchSolveSubmissionEntry,
|
|
1017
|
+
) -> BundledBatchSolveSubmissionEntry:
|
|
1018
|
+
return await _submit_one_bundle_entry(
|
|
1019
|
+
key_index=entry.index,
|
|
1020
|
+
key_payload=entry.keys,
|
|
1021
|
+
)
|
|
1022
|
+
|
|
1023
|
+
return await wait_bundle_batch_entry(
|
|
1024
|
+
entry=batch_entry,
|
|
1025
|
+
daemon=daemon,
|
|
1026
|
+
timeout=timeout,
|
|
1027
|
+
on_error=on_error,
|
|
1028
|
+
progress=progress,
|
|
1029
|
+
progress_label=progress_label,
|
|
1030
|
+
max_retries=max_retries,
|
|
1031
|
+
resubmit=resubmit,
|
|
1032
|
+
)
|
|
1033
|
+
|
|
1034
|
+
live_tasks: set[asyncio.Task[BundledBatchSolveEntry]] = set()
|
|
1035
|
+
batch_entry: BundledBatchSolveSubmissionEntry
|
|
1036
|
+
for batch_entry in batch_submission.entries:
|
|
1037
|
+
is_live_entry: bool = (
|
|
1038
|
+
batch_entry.result is not None
|
|
1039
|
+
and batch_entry.result._resolved_result is None
|
|
1040
|
+
)
|
|
1041
|
+
if not is_live_entry:
|
|
1042
|
+
if batch_entry.result is None:
|
|
1043
|
+
if batch_entry.failure is None:
|
|
1044
|
+
raise InvalidConfigurationError(
|
|
1045
|
+
"Batch bundle submission entry has neither result nor failure."
|
|
1046
|
+
)
|
|
1047
|
+
if on_error == "raise":
|
|
1048
|
+
raise InvalidConfigurationError(batch_entry.failure.message)
|
|
1049
|
+
result_entries.append(
|
|
1050
|
+
BundledBatchSolveEntry(
|
|
1051
|
+
index=batch_entry.index,
|
|
1052
|
+
keys=batch_entry.keys,
|
|
1053
|
+
result=None,
|
|
1054
|
+
failure=batch_entry.failure,
|
|
1055
|
+
project_files=batch_entry.project_files,
|
|
1056
|
+
)
|
|
1057
|
+
)
|
|
1058
|
+
else:
|
|
1059
|
+
resolved_bundle_result: BundledSolveResult = (
|
|
1060
|
+
await batch_entry.result.wait(
|
|
1061
|
+
daemon=daemon,
|
|
1062
|
+
asynchronous=True,
|
|
1063
|
+
break_when="all",
|
|
1064
|
+
timeout=timeout,
|
|
1065
|
+
on_timeout="raise",
|
|
1066
|
+
on_error="raise",
|
|
1067
|
+
progress=progress,
|
|
1068
|
+
progress_label=progress_label,
|
|
1069
|
+
)
|
|
1070
|
+
)
|
|
1071
|
+
result_entries.append(
|
|
1072
|
+
BundledBatchSolveEntry(
|
|
1073
|
+
index=batch_entry.index,
|
|
1074
|
+
keys=batch_entry.keys,
|
|
1075
|
+
result=resolved_bundle_result,
|
|
1076
|
+
failure=None,
|
|
1077
|
+
project_files=batch_entry.project_files,
|
|
1078
|
+
)
|
|
1079
|
+
)
|
|
1080
|
+
continue
|
|
1081
|
+
live_tasks.add(
|
|
1082
|
+
asyncio.create_task(_wait_one_bundle_entry(batch_entry=batch_entry))
|
|
1083
|
+
)
|
|
1084
|
+
|
|
1085
|
+
while live_tasks:
|
|
1086
|
+
done_tasks: set[asyncio.Task[BundledBatchSolveEntry]]
|
|
1087
|
+
pending_tasks: set[asyncio.Task[BundledBatchSolveEntry]]
|
|
1088
|
+
done_tasks, pending_tasks = await asyncio.wait(
|
|
1089
|
+
live_tasks,
|
|
1090
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
1091
|
+
)
|
|
1092
|
+
live_tasks = pending_tasks
|
|
1093
|
+
completed_task: asyncio.Task[BundledBatchSolveEntry]
|
|
1094
|
+
for completed_task in done_tasks:
|
|
1095
|
+
try:
|
|
1096
|
+
completed_entry: BundledBatchSolveEntry = (
|
|
1097
|
+
completed_task.result()
|
|
1098
|
+
)
|
|
1099
|
+
except Exception:
|
|
1100
|
+
for pending_task in live_tasks:
|
|
1101
|
+
pending_task.cancel()
|
|
1102
|
+
await asyncio.gather(*live_tasks, return_exceptions=True)
|
|
1103
|
+
raise
|
|
1104
|
+
result_entries.append(completed_entry)
|
|
1105
|
+
if completed_entry.failure is None:
|
|
1106
|
+
progress_bar.update(1)
|
|
1107
|
+
else:
|
|
1108
|
+
progress_bar.mark_failure(1)
|
|
1109
|
+
progress_bar.update(1)
|
|
1110
|
+
|
|
1111
|
+
result_entries.sort(key=lambda entry: entry.index)
|
|
1112
|
+
return BundledBatchSolveResult(entries=tuple(result_entries))
|
|
1113
|
+
finally:
|
|
1114
|
+
progress_bar.close()
|
|
1115
|
+
|
|
1116
|
+
return BundledBatchSolveSubmission(entries=tuple(entries))._bind_runtime(
|
|
1117
|
+
waiter=backend_daemon_handler,
|
|
1118
|
+
wait_callback=_wait_batch_bundle_submission,
|
|
1119
|
+
)
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def _resolve_bundle_batch_submission_limit(
|
|
1123
|
+
*,
|
|
1124
|
+
daemon_handler: DaemonHandler | None,
|
|
1125
|
+
batch_size: int,
|
|
1126
|
+
) -> int:
|
|
1127
|
+
"""Return the maximum number of bundle submissions to fan out at once."""
|
|
1128
|
+
|
|
1129
|
+
if batch_size <= 0:
|
|
1130
|
+
return 0
|
|
1131
|
+
if daemon_handler is None:
|
|
1132
|
+
return 1
|
|
1133
|
+
resources: tuple[DaemonResource, ...] = daemon_handler.resources.list()
|
|
1134
|
+
if not resources:
|
|
1135
|
+
return 1
|
|
1136
|
+
total_multiplicity: int = sum(resource.multiplicity for resource in resources)
|
|
1137
|
+
return max(1, min(batch_size, total_multiplicity))
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
async def solve_batch_impl(
|
|
1141
|
+
*,
|
|
1142
|
+
project_file: str | Path,
|
|
1143
|
+
caller_file: Path | None,
|
|
1144
|
+
keys: Sequence[Mapping[str, object]],
|
|
1145
|
+
resultbag: Resultbag | None,
|
|
1146
|
+
recompute: bool,
|
|
1147
|
+
max_retries: int,
|
|
1148
|
+
on_error: BatchOnError,
|
|
1149
|
+
context: SolverContext | None,
|
|
1150
|
+
backend: BackendMode | None,
|
|
1151
|
+
working_dir: WorkingDirInput,
|
|
1152
|
+
suffix_pattern: str | None,
|
|
1153
|
+
temporary: bool,
|
|
1154
|
+
dispatch: DispatchMode,
|
|
1155
|
+
wait_break_when: WaitBreakWhen,
|
|
1156
|
+
wait_timeout: float | None,
|
|
1157
|
+
wait_on_timeout: WaitOnTimeout,
|
|
1158
|
+
wait_progress: bool,
|
|
1159
|
+
wait_progress_label: str | None,
|
|
1160
|
+
quiet: bool,
|
|
1161
|
+
mode: SolveMode = "solve",
|
|
1162
|
+
) -> BatchSolveResult | BatchSolveSubmission:
|
|
1163
|
+
"""Process a batch of key dictionaries in stable input order."""
|
|
1164
|
+
|
|
1165
|
+
if on_error not in ("raise", "collect"):
|
|
1166
|
+
raise InvalidConfigurationError(
|
|
1167
|
+
f"Unsupported batch on_error policy: {on_error!r}"
|
|
1168
|
+
)
|
|
1169
|
+
if dispatch not in ("wait", "submit"):
|
|
1170
|
+
raise InvalidConfigurationError(f"Unsupported dispatch policy: {dispatch!r}")
|
|
1171
|
+
if wait_break_when not in ("all", "any"):
|
|
1172
|
+
raise InvalidConfigurationError(
|
|
1173
|
+
f"Unsupported wait_break_when policy: {wait_break_when!r}"
|
|
1174
|
+
)
|
|
1175
|
+
if wait_on_timeout not in ("raise", "return"):
|
|
1176
|
+
raise InvalidConfigurationError(
|
|
1177
|
+
f"Unsupported wait_on_timeout policy: {wait_on_timeout!r}"
|
|
1178
|
+
)
|
|
1179
|
+
if wait_timeout is not None and wait_timeout < 0.0:
|
|
1180
|
+
raise InvalidConfigurationError("wait_timeout must be non-negative.")
|
|
1181
|
+
|
|
1182
|
+
scoped_daemon_handler: DaemonHandler | None = resolve_scoped_handler(
|
|
1183
|
+
context=context
|
|
1184
|
+
)
|
|
1185
|
+
resolved_context: SolverContext
|
|
1186
|
+
if context is not None:
|
|
1187
|
+
resolved_context = resolve_context(context)
|
|
1188
|
+
elif scoped_daemon_handler is not None:
|
|
1189
|
+
resolved_context = scoped_daemon_handler.context
|
|
1190
|
+
else:
|
|
1191
|
+
resolved_context = resolve_context(None)
|
|
1192
|
+
|
|
1193
|
+
effective_backend: BackendMode
|
|
1194
|
+
effective_handler: DaemonHandler | None
|
|
1195
|
+
effective_backend, effective_handler = resolve_backend_mode(
|
|
1196
|
+
backend=backend,
|
|
1197
|
+
context=resolved_context,
|
|
1198
|
+
scoped_daemon_handler=scoped_daemon_handler,
|
|
1199
|
+
)
|
|
1200
|
+
if dispatch == "submit" and effective_backend != "daemon":
|
|
1201
|
+
raise InvalidConfigurationError(
|
|
1202
|
+
"`dispatch='submit'` is only supported for daemon backend solves."
|
|
1203
|
+
)
|
|
1204
|
+
if (
|
|
1205
|
+
effective_backend == "local"
|
|
1206
|
+
and len(keys) > 1
|
|
1207
|
+
and working_dir is None
|
|
1208
|
+
and not temporary
|
|
1209
|
+
):
|
|
1210
|
+
raise InvalidConfigurationError(
|
|
1211
|
+
"Local multi-entry batch solves require isolated output directories. "
|
|
1212
|
+
"Pass one `working_dir` per Project Keys entry or set `temporary=True`."
|
|
1213
|
+
)
|
|
1214
|
+
batch_working_dirs: list[Path | None] = resolve_batch_working_dirs(
|
|
1215
|
+
working_dir=working_dir,
|
|
1216
|
+
batch_size=len(keys),
|
|
1217
|
+
temporary=temporary,
|
|
1218
|
+
temp_dir_root=resolved_context.system.temp_dir_root,
|
|
1219
|
+
)
|
|
1220
|
+
|
|
1221
|
+
entries: list[BatchSolveEntry] = []
|
|
1222
|
+
cached_entries: list[BatchSolveEntry] = []
|
|
1223
|
+
submissions: list[SolveSubmission] = []
|
|
1224
|
+
progress_bar = create_progress_bar(
|
|
1225
|
+
total=len(keys),
|
|
1226
|
+
enabled=wait_progress and dispatch == "wait" and effective_backend != "daemon",
|
|
1227
|
+
description=wait_progress_label or "solve(batch)",
|
|
1228
|
+
)
|
|
1229
|
+
key_index: int
|
|
1230
|
+
key_payload: Mapping[str, object]
|
|
1231
|
+
try:
|
|
1232
|
+
for key_index, key_payload in enumerate(keys):
|
|
1233
|
+
key_dict: dict[str, object] = dict(key_payload)
|
|
1234
|
+
try:
|
|
1235
|
+
entry_dispatch: DispatchMode = (
|
|
1236
|
+
"submit"
|
|
1237
|
+
if (dispatch == "wait" and effective_backend == "daemon")
|
|
1238
|
+
else dispatch
|
|
1239
|
+
)
|
|
1240
|
+
single_result: SolveResult = await solve_single_impl(
|
|
1241
|
+
project_file=project_file,
|
|
1242
|
+
caller_file=caller_file,
|
|
1243
|
+
keys=key_dict,
|
|
1244
|
+
resultbag=resultbag,
|
|
1245
|
+
recompute=recompute,
|
|
1246
|
+
max_retries=max_retries,
|
|
1247
|
+
context=context,
|
|
1248
|
+
backend=backend,
|
|
1249
|
+
working_dir=batch_working_dirs[key_index],
|
|
1250
|
+
suffix_pattern=suffix_pattern,
|
|
1251
|
+
dispatch=entry_dispatch,
|
|
1252
|
+
wait_break_when=wait_break_when,
|
|
1253
|
+
wait_timeout=wait_timeout,
|
|
1254
|
+
wait_on_timeout=wait_on_timeout,
|
|
1255
|
+
wait_progress=wait_progress,
|
|
1256
|
+
wait_progress_label=wait_progress_label,
|
|
1257
|
+
quiet=quiet,
|
|
1258
|
+
mode=mode,
|
|
1259
|
+
)
|
|
1260
|
+
entries.append(
|
|
1261
|
+
BatchSolveEntry(
|
|
1262
|
+
index=key_index,
|
|
1263
|
+
keys=key_dict,
|
|
1264
|
+
result=single_result,
|
|
1265
|
+
failure=None,
|
|
1266
|
+
)
|
|
1267
|
+
)
|
|
1268
|
+
if (
|
|
1269
|
+
single_result.backend_used == "daemon"
|
|
1270
|
+
and single_result.daemon_job_id is not None
|
|
1271
|
+
and _is_pending_daemon_submission(
|
|
1272
|
+
daemon_handler=effective_handler,
|
|
1273
|
+
daemon_job_id=single_result.daemon_job_id,
|
|
1274
|
+
)
|
|
1275
|
+
):
|
|
1276
|
+
source_project_mappings: tuple[tuple[Path, Path], ...] = ()
|
|
1277
|
+
if single_result.project_file is not None:
|
|
1278
|
+
source_project_path: Path = resolve_project_input_path(
|
|
1279
|
+
project_file,
|
|
1280
|
+
caller_file=caller_file,
|
|
1281
|
+
)
|
|
1282
|
+
source_project_mappings = (
|
|
1283
|
+
(single_result.project_file, source_project_path),
|
|
1284
|
+
)
|
|
1285
|
+
submissions.append(
|
|
1286
|
+
SolveSubmission(
|
|
1287
|
+
job_id=single_result.daemon_job_id,
|
|
1288
|
+
result=single_result,
|
|
1289
|
+
keys=dict(key_dict),
|
|
1290
|
+
index=key_index,
|
|
1291
|
+
)._bind_runtime(
|
|
1292
|
+
waiter=effective_handler,
|
|
1293
|
+
resultbag=resultbag,
|
|
1294
|
+
suffix_pattern=suffix_pattern,
|
|
1295
|
+
source_project_mappings=source_project_mappings,
|
|
1296
|
+
)
|
|
1297
|
+
)
|
|
1298
|
+
else:
|
|
1299
|
+
cached_entries.append(
|
|
1300
|
+
BatchSolveEntry(
|
|
1301
|
+
index=key_index,
|
|
1302
|
+
keys=key_dict,
|
|
1303
|
+
result=single_result,
|
|
1304
|
+
failure=None,
|
|
1305
|
+
)
|
|
1306
|
+
)
|
|
1307
|
+
progress_bar.update(1)
|
|
1308
|
+
except Exception as exception:
|
|
1309
|
+
if on_error == "raise":
|
|
1310
|
+
raise
|
|
1311
|
+
partial_result: SolveResult | None = _extract_partial_result(exception)
|
|
1312
|
+
failure: BatchSolveFailure = BatchSolveFailure(
|
|
1313
|
+
index=key_index,
|
|
1314
|
+
keys=key_dict,
|
|
1315
|
+
exception_type=type(exception),
|
|
1316
|
+
message=str(exception),
|
|
1317
|
+
)
|
|
1318
|
+
entries.append(
|
|
1319
|
+
BatchSolveEntry(
|
|
1320
|
+
index=key_index,
|
|
1321
|
+
keys=key_dict,
|
|
1322
|
+
result=None,
|
|
1323
|
+
failure=failure,
|
|
1324
|
+
_log=(None if partial_result is None else partial_result.log),
|
|
1325
|
+
)
|
|
1326
|
+
)
|
|
1327
|
+
progress_bar.mark_failure(1)
|
|
1328
|
+
progress_bar.update(1)
|
|
1329
|
+
|
|
1330
|
+
if dispatch == "submit":
|
|
1331
|
+
resolved_result: BatchSolveResult | None = None
|
|
1332
|
+
if cached_entries:
|
|
1333
|
+
resolved_result = BatchSolveResult(entries=tuple(cached_entries))
|
|
1334
|
+
return BatchSolveSubmission(
|
|
1335
|
+
entries=tuple(submissions),
|
|
1336
|
+
)._bind_runtime(
|
|
1337
|
+
waiter=effective_handler,
|
|
1338
|
+
resolved_result=resolved_result,
|
|
1339
|
+
)
|
|
1340
|
+
|
|
1341
|
+
if submissions:
|
|
1342
|
+
daemon_handler_for_wait: DaemonHandler | None = effective_handler
|
|
1343
|
+
if daemon_handler_for_wait is None:
|
|
1344
|
+
raise BackendUnavailableError(
|
|
1345
|
+
"Daemon handler is not available for batch wait collection."
|
|
1346
|
+
)
|
|
1347
|
+
waited = await daemon_handler_for_wait.wait(
|
|
1348
|
+
BatchSolveSubmission(entries=tuple(submissions)),
|
|
1349
|
+
asynchronous=True,
|
|
1350
|
+
break_when=wait_break_when,
|
|
1351
|
+
timeout=wait_timeout,
|
|
1352
|
+
on_timeout=wait_on_timeout,
|
|
1353
|
+
on_error=on_error,
|
|
1354
|
+
progress=wait_progress,
|
|
1355
|
+
progress_label=wait_progress_label,
|
|
1356
|
+
)
|
|
1357
|
+
if isinstance(waited, BatchSolveResult):
|
|
1358
|
+
waited_indices: set[int] = {entry.index for entry in waited.entries}
|
|
1359
|
+
merged_entries: list[BatchSolveEntry] = [
|
|
1360
|
+
*(entry for entry in entries if entry.index not in waited_indices),
|
|
1361
|
+
*waited.entries,
|
|
1362
|
+
]
|
|
1363
|
+
merged_entries.sort(key=lambda entry: entry.index)
|
|
1364
|
+
return BatchSolveResult(entries=tuple(merged_entries))
|
|
1365
|
+
raise InvalidConfigurationError(
|
|
1366
|
+
"Expected batch wait to return BatchSolveResult."
|
|
1367
|
+
)
|
|
1368
|
+
|
|
1369
|
+
return BatchSolveResult(entries=tuple(entries))
|
|
1370
|
+
finally:
|
|
1371
|
+
progress_bar.close()
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
async def solve_single_impl(
|
|
1375
|
+
*,
|
|
1376
|
+
project_file: str | Path,
|
|
1377
|
+
caller_file: Path | None,
|
|
1378
|
+
keys: Mapping[str, object] | None,
|
|
1379
|
+
resultbag: Resultbag | None,
|
|
1380
|
+
recompute: bool,
|
|
1381
|
+
max_retries: int,
|
|
1382
|
+
context: SolverContext | None,
|
|
1383
|
+
backend: BackendMode | None,
|
|
1384
|
+
working_dir: Path | None,
|
|
1385
|
+
suffix_pattern: str | None,
|
|
1386
|
+
dispatch: DispatchMode,
|
|
1387
|
+
wait_break_when: WaitBreakWhen,
|
|
1388
|
+
wait_timeout: float | None,
|
|
1389
|
+
wait_on_timeout: WaitOnTimeout,
|
|
1390
|
+
wait_progress: bool,
|
|
1391
|
+
wait_progress_label: str | None,
|
|
1392
|
+
quiet: bool,
|
|
1393
|
+
mode: SolveMode = "solve",
|
|
1394
|
+
) -> SolveResult:
|
|
1395
|
+
"""Process one solve item (single key dictionary)."""
|
|
1396
|
+
|
|
1397
|
+
resolved_context: SolverContext
|
|
1398
|
+
resolved_backend: BackendMode
|
|
1399
|
+
backend_daemon_handler: DaemonHandler | None
|
|
1400
|
+
resolved_context, resolved_backend, backend_daemon_handler = (
|
|
1401
|
+
resolve_context_backend(
|
|
1402
|
+
context=context,
|
|
1403
|
+
backend=backend,
|
|
1404
|
+
)
|
|
1405
|
+
)
|
|
1406
|
+
if dispatch == "submit" and resolved_backend != "daemon":
|
|
1407
|
+
raise InvalidConfigurationError(
|
|
1408
|
+
"`dispatch='submit'` is only supported for daemon backend solves."
|
|
1409
|
+
)
|
|
1410
|
+
if max_retries < 0:
|
|
1411
|
+
raise InvalidConfigurationError("max_retries must be >= 0.")
|
|
1412
|
+
|
|
1413
|
+
project_path: Path = resolve_project_input_path(
|
|
1414
|
+
project_file,
|
|
1415
|
+
caller_file=caller_file,
|
|
1416
|
+
)
|
|
1417
|
+
if not project_path.exists():
|
|
1418
|
+
raise InvalidConfigurationError(f"Project file does not exist: {project_path}")
|
|
1419
|
+
|
|
1420
|
+
resolved_keys: Mapping[str, object] = {} if keys is None else keys
|
|
1421
|
+
cached_result = load_cached_result(
|
|
1422
|
+
resultbag=resultbag,
|
|
1423
|
+
project_path=project_path,
|
|
1424
|
+
keys=resolved_keys,
|
|
1425
|
+
recompute=recompute,
|
|
1426
|
+
suffix_pattern=suffix_pattern,
|
|
1427
|
+
)
|
|
1428
|
+
if cached_result is not None:
|
|
1429
|
+
if keys is None:
|
|
1430
|
+
cached_result.keys = None
|
|
1431
|
+
else:
|
|
1432
|
+
cached_result.keys = SolveKeys(
|
|
1433
|
+
dict(resolved_keys), source=cached_result.project_file
|
|
1434
|
+
)
|
|
1435
|
+
return cached_result
|
|
1436
|
+
if resultbag is not None:
|
|
1437
|
+
resultbag.management.clear_attempt_logs(
|
|
1438
|
+
resolved_keys,
|
|
1439
|
+
suffix_pattern=suffix_pattern,
|
|
1440
|
+
)
|
|
1441
|
+
|
|
1442
|
+
prepared_project: Path
|
|
1443
|
+
template_renders: tuple[TemplateRenderInfo, ...]
|
|
1444
|
+
prepared_project, template_renders = prepare_project_inputs(
|
|
1445
|
+
project_path=project_path,
|
|
1446
|
+
working_dir=working_dir,
|
|
1447
|
+
resolved_keys=resolved_keys,
|
|
1448
|
+
suffix_pattern=suffix_pattern,
|
|
1449
|
+
)
|
|
1450
|
+
|
|
1451
|
+
attempt_index: int
|
|
1452
|
+
for attempt_index in range(max_retries + 1):
|
|
1453
|
+
solve_result: SolveResult = SolveResult(
|
|
1454
|
+
project_file=prepared_project,
|
|
1455
|
+
backend_used=resolved_backend,
|
|
1456
|
+
template_renders=template_renders,
|
|
1457
|
+
keys=None,
|
|
1458
|
+
_post_process_only=mode == "postprocess",
|
|
1459
|
+
)
|
|
1460
|
+
if keys is not None:
|
|
1461
|
+
solve_result.keys = SolveKeys(
|
|
1462
|
+
dict(resolved_keys), source=solve_result.project_file
|
|
1463
|
+
)
|
|
1464
|
+
try:
|
|
1465
|
+
solver_log_payload: object
|
|
1466
|
+
if resolved_backend == "local":
|
|
1467
|
+
solver_log_payload = await execute_local_solve(
|
|
1468
|
+
result=solve_result,
|
|
1469
|
+
context=resolved_context,
|
|
1470
|
+
prepared_project=prepared_project,
|
|
1471
|
+
quiet=quiet,
|
|
1472
|
+
mode=mode,
|
|
1473
|
+
)
|
|
1474
|
+
else:
|
|
1475
|
+
solver_log_payload = await submit_or_wait_daemon(
|
|
1476
|
+
result=solve_result,
|
|
1477
|
+
daemon_handler=backend_daemon_handler,
|
|
1478
|
+
prepared_project=prepared_project,
|
|
1479
|
+
source_project=project_path,
|
|
1480
|
+
resolved_keys=resolved_keys,
|
|
1481
|
+
resultbag=resultbag,
|
|
1482
|
+
suffix_pattern=suffix_pattern,
|
|
1483
|
+
dispatch=dispatch,
|
|
1484
|
+
wait_break_when=wait_break_when,
|
|
1485
|
+
wait_timeout=wait_timeout,
|
|
1486
|
+
wait_on_timeout=wait_on_timeout,
|
|
1487
|
+
wait_progress=wait_progress,
|
|
1488
|
+
wait_progress_label=wait_progress_label,
|
|
1489
|
+
mode=mode,
|
|
1490
|
+
)
|
|
1491
|
+
if dispatch == "wait":
|
|
1492
|
+
return cast(SolveResult, solver_log_payload)
|
|
1493
|
+
|
|
1494
|
+
if resultbag is not None and resolved_backend != "daemon":
|
|
1495
|
+
resultbag.management.add(
|
|
1496
|
+
keys=resolved_keys,
|
|
1497
|
+
result=raw_result_from_solve_result(solve_result),
|
|
1498
|
+
log=solver_log_payload,
|
|
1499
|
+
suffix_pattern=suffix_pattern,
|
|
1500
|
+
)
|
|
1501
|
+
return solve_result
|
|
1502
|
+
except Exception as error:
|
|
1503
|
+
_attach_partial_result(error, solve_result)
|
|
1504
|
+
if resultbag is not None:
|
|
1505
|
+
resultbag.management.add_attempt_log(
|
|
1506
|
+
keys=resolved_keys,
|
|
1507
|
+
attempt_index=attempt_index,
|
|
1508
|
+
payload=_build_attempt_log_payload(
|
|
1509
|
+
attempt_index=attempt_index,
|
|
1510
|
+
exception=error,
|
|
1511
|
+
result=solve_result,
|
|
1512
|
+
),
|
|
1513
|
+
suffix_pattern=suffix_pattern,
|
|
1514
|
+
)
|
|
1515
|
+
if attempt_index >= max_retries or not _is_retryable_solve_error(error):
|
|
1516
|
+
raise
|
|
1517
|
+
|
|
1518
|
+
raise RuntimeError("Retry loop terminated unexpectedly.")
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
async def execute_local_solve(
|
|
1522
|
+
*,
|
|
1523
|
+
result: SolveResult,
|
|
1524
|
+
context: SolverContext,
|
|
1525
|
+
prepared_project: Path,
|
|
1526
|
+
quiet: bool,
|
|
1527
|
+
mode: SolveMode = "solve",
|
|
1528
|
+
) -> object:
|
|
1529
|
+
"""Run local JCMsolve and update result/log payload."""
|
|
1530
|
+
|
|
1531
|
+
return await execute_local_solve_runtime(
|
|
1532
|
+
result=result,
|
|
1533
|
+
context=context,
|
|
1534
|
+
prepared_project=prepared_project,
|
|
1535
|
+
quiet=quiet,
|
|
1536
|
+
mode=mode,
|
|
1537
|
+
call_binary=_call_binary,
|
|
1538
|
+
logger=_logger,
|
|
1539
|
+
)
|
|
1540
|
+
|
|
1541
|
+
|
|
1542
|
+
async def submit_or_wait_daemon(
|
|
1543
|
+
*,
|
|
1544
|
+
result: SolveResult,
|
|
1545
|
+
daemon_handler: DaemonHandler | None,
|
|
1546
|
+
prepared_project: Path,
|
|
1547
|
+
source_project: Path,
|
|
1548
|
+
resolved_keys: Mapping[str, object],
|
|
1549
|
+
resultbag: Resultbag | None,
|
|
1550
|
+
suffix_pattern: str | None,
|
|
1551
|
+
dispatch: DispatchMode,
|
|
1552
|
+
wait_break_when: WaitBreakWhen,
|
|
1553
|
+
wait_timeout: float | None,
|
|
1554
|
+
wait_on_timeout: WaitOnTimeout,
|
|
1555
|
+
wait_progress: bool,
|
|
1556
|
+
wait_progress_label: str | None,
|
|
1557
|
+
mode: SolveMode = "solve",
|
|
1558
|
+
) -> object:
|
|
1559
|
+
"""Submit daemon solve and optionally wait for completion."""
|
|
1560
|
+
|
|
1561
|
+
return await submit_or_wait_daemon_runtime(
|
|
1562
|
+
result=result,
|
|
1563
|
+
daemon_handler=daemon_handler,
|
|
1564
|
+
prepared_project=prepared_project,
|
|
1565
|
+
source_project=source_project,
|
|
1566
|
+
resolved_keys=resolved_keys,
|
|
1567
|
+
resultbag=resultbag,
|
|
1568
|
+
suffix_pattern=suffix_pattern,
|
|
1569
|
+
dispatch=dispatch,
|
|
1570
|
+
wait_break_when=wait_break_when,
|
|
1571
|
+
wait_timeout=wait_timeout,
|
|
1572
|
+
wait_on_timeout=wait_on_timeout,
|
|
1573
|
+
wait_progress=wait_progress,
|
|
1574
|
+
wait_progress_label=wait_progress_label,
|
|
1575
|
+
mode=mode,
|
|
1576
|
+
)
|