euler-inference 2.5.0__tar.gz → 2.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_inference-2.5.0 → euler_inference-2.7.0}/PKG-INFO +1 -1
- {euler_inference-2.5.0 → euler_inference-2.7.0}/README.md +29 -2
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/_outputs.py +109 -36
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/inference.py +107 -32
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/models/external_model.py +131 -11
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference.egg-info/PKG-INFO +1 -1
- {euler_inference-2.5.0 → euler_inference-2.7.0}/pyproject.toml +1 -1
- {euler_inference-2.5.0 → euler_inference-2.7.0}/tests/test_model_metadata.py +77 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/tests/test_writer_integration.py +228 -21
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/__init__.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/__main__.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/api.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/config.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/model_card.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference/models/__init__.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference.egg-info/SOURCES.txt +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference.egg-info/dependency_links.txt +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference.egg-info/entry_points.txt +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference.egg-info/requires.txt +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference.egg-info/top_level.txt +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/setup.cfg +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/tests/test_intrinsics_resolution.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/tests/test_model_card.py +0 -0
- {euler_inference-2.5.0 → euler_inference-2.7.0}/tests/test_save_path.py +0 -0
|
@@ -447,7 +447,34 @@ output_base_path/
|
|
|
447
447
|
|
|
448
448
|
If `outputs[].source_modality` is set explicitly but the named modality is missing, or no `euler-loading` writer is available for it, inference fails instead of silently falling back.
|
|
449
449
|
|
|
450
|
-
`points_3d`
|
|
450
|
+
`points_3d` and `intrinsics` are built-in legacy output modalities. Configure them with NumPy output types (`npy` or `npz`) and return float-compatible arrays with shapes `(3, H, W)` and `(3, 3)`, respectively. Their output datasets are indexed with the corresponding functions from the `generic` euler-loading loader, so recent `euler-loading` versions can read them without an explicit loader.
|
|
451
|
+
|
|
452
|
+
For an explicit model card configuration (useful when the metadata must also work with older euler-inference versions), use:
|
|
453
|
+
|
|
454
|
+
```json
|
|
455
|
+
{
|
|
456
|
+
"outputs": [
|
|
457
|
+
{
|
|
458
|
+
"key": "intrinsics",
|
|
459
|
+
"type": "npy",
|
|
460
|
+
"writer": {
|
|
461
|
+
"euler_train": {"used_as": "condition", "modality_type": "intrinsics"},
|
|
462
|
+
"euler_loading": {"loader": "generic", "function": "intrinsics", "version": "2.0.0"},
|
|
463
|
+
"meta": {"shape": [3, 3], "coordinate_frame": "camera"}
|
|
464
|
+
}
|
|
465
|
+
},
|
|
466
|
+
{
|
|
467
|
+
"key": "points_3d",
|
|
468
|
+
"type": "npy",
|
|
469
|
+
"writer": {
|
|
470
|
+
"euler_train": {"used_as": "target", "modality_type": "points_3d"},
|
|
471
|
+
"euler_loading": {"loader": "generic", "function": "points_3d", "version": "2.0.0"},
|
|
472
|
+
"meta": {"shape": [3, "H", "W"], "coordinate_frame": "camera"}
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
]
|
|
476
|
+
}
|
|
477
|
+
```
|
|
451
478
|
|
|
452
479
|
### Pipeline Manifest
|
|
453
480
|
|
|
@@ -481,7 +508,7 @@ Legacy outputs are written via ds-crawler's `DatasetWriter`/`ZipDatasetWriter`,
|
|
|
481
508
|
| `name` | Input dataset's name (from its `output.json`), or the output key if unavailable | `outputs[].writer.name` |
|
|
482
509
|
| `type` | Output key (e.g. `"depth"`) | `outputs[].writer.type` |
|
|
483
510
|
| `euler_train` | `{"used_as": "target", "modality_type": "<key>"}` | `outputs[].writer.euler_train` |
|
|
484
|
-
| `euler_loading` | *(omitted; `points_3d`
|
|
511
|
+
| `euler_loading` | *(omitted; `points_3d` and `intrinsics` default to their matching `generic` loader functions)* | `outputs[].writer.euler_loading` |
|
|
485
512
|
| `separator` | `null` | — |
|
|
486
513
|
| `meta` | *(omitted)* | `outputs[].writer.meta` |
|
|
487
514
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from dataclasses import dataclass
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
4
|
import json
|
|
5
5
|
import logging
|
|
6
6
|
from pathlib import Path
|
|
@@ -20,6 +20,10 @@ PIPELINE_OUTPUTS_MANIFEST_VERSION = 1
|
|
|
20
20
|
|
|
21
21
|
_NUMPY_OUTPUT_TYPES = {"npy", "npz"}
|
|
22
22
|
_LOADABLE_LEGACY_OUTPUTS = {
|
|
23
|
+
"intrinsics": {
|
|
24
|
+
"euler_loading": {"loader": "generic", "function": "intrinsics"},
|
|
25
|
+
"output_types": _NUMPY_OUTPUT_TYPES,
|
|
26
|
+
},
|
|
23
27
|
"points_3d": {
|
|
24
28
|
"euler_loading": {"loader": "generic", "function": "points_3d"},
|
|
25
29
|
"output_types": _NUMPY_OUTPUT_TYPES,
|
|
@@ -164,13 +168,14 @@ def _get_loadable_legacy_output(modality_type: str) -> dict[str, Any] | None:
|
|
|
164
168
|
def _get_writer_type_default(
|
|
165
169
|
output_config: OutputConfig,
|
|
166
170
|
writer_meta: dict[str, Any],
|
|
171
|
+
modality_type: str,
|
|
167
172
|
) -> str:
|
|
168
173
|
euler_train = writer_meta.get("euler_train")
|
|
169
174
|
if isinstance(euler_train, dict):
|
|
170
|
-
|
|
171
|
-
if
|
|
172
|
-
return
|
|
173
|
-
return output_config.key
|
|
175
|
+
euler_train_modality_type = euler_train.get("modality_type")
|
|
176
|
+
if euler_train_modality_type:
|
|
177
|
+
return euler_train_modality_type
|
|
178
|
+
return modality_type or output_config.key
|
|
174
179
|
|
|
175
180
|
|
|
176
181
|
def _validate_loadable_legacy_output(
|
|
@@ -201,16 +206,24 @@ def _to_numpy(data: Any) -> np.ndarray:
|
|
|
201
206
|
|
|
202
207
|
|
|
203
208
|
def _prepare_legacy_output_data(data: Any, output_config: OutputConfig) -> Any:
|
|
209
|
+
arr = _to_numpy(data)
|
|
204
210
|
writer_meta = output_config.writer or {}
|
|
205
211
|
modality_type = _get_writer_modality_type(
|
|
206
212
|
output_config,
|
|
207
213
|
writer_meta,
|
|
208
214
|
strict=True,
|
|
209
215
|
)
|
|
210
|
-
if modality_type
|
|
211
|
-
return
|
|
216
|
+
if modality_type not in {"intrinsics", "points_3d"}:
|
|
217
|
+
return arr
|
|
218
|
+
|
|
219
|
+
if modality_type == "intrinsics":
|
|
220
|
+
if arr.shape != (3, 3):
|
|
221
|
+
raise ValueError(
|
|
222
|
+
"intrinsics output must have shape (3, 3), "
|
|
223
|
+
f"got {arr.shape}"
|
|
224
|
+
)
|
|
225
|
+
return arr.astype(np.float32, copy=False)
|
|
212
226
|
|
|
213
|
-
arr = _to_numpy(data)
|
|
214
227
|
if arr.ndim != 3 or arr.shape[0] != 3:
|
|
215
228
|
raise ValueError(
|
|
216
229
|
"points_3d output must have shape (3, H, W), "
|
|
@@ -237,11 +250,11 @@ def _get_writer_kwargs(
|
|
|
237
250
|
|
|
238
251
|
type_ = writer_meta.get(
|
|
239
252
|
"type",
|
|
240
|
-
_get_writer_type_default(output_config, writer_meta),
|
|
253
|
+
_get_writer_type_default(output_config, writer_meta, modality_type),
|
|
241
254
|
)
|
|
242
255
|
euler_train = writer_meta.get("euler_train", {
|
|
243
256
|
"used_as": "target",
|
|
244
|
-
"modality_type":
|
|
257
|
+
"modality_type": type_,
|
|
245
258
|
})
|
|
246
259
|
kwargs = dict(name=name, type=type_, euler_train=euler_train, separator=None)
|
|
247
260
|
|
|
@@ -279,17 +292,21 @@ def create_writers(
|
|
|
279
292
|
_validate_pipeline_targets(config)
|
|
280
293
|
writers: dict[str, Any] = {}
|
|
281
294
|
selected_outputs = outputs if outputs is not None else config.outputs
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
295
|
+
try:
|
|
296
|
+
for output_config in selected_outputs:
|
|
297
|
+
kwargs = _get_writer_kwargs(
|
|
298
|
+
output_config,
|
|
299
|
+
strict=config.strict,
|
|
300
|
+
dataset_name=dataset_name,
|
|
301
|
+
)
|
|
302
|
+
root, zip_mode = _resolve_output_root(config, output_config)
|
|
303
|
+
if zip_mode:
|
|
304
|
+
writers[output_config.key] = ZipDatasetWriter(root, **kwargs)
|
|
305
|
+
else:
|
|
306
|
+
writers[output_config.key] = DatasetWriter(root, **kwargs)
|
|
307
|
+
except BaseException:
|
|
308
|
+
_close_writers(writers.values())
|
|
309
|
+
raise
|
|
293
310
|
return writers
|
|
294
311
|
|
|
295
312
|
|
|
@@ -487,12 +504,16 @@ def _create_source_backed_writers(
|
|
|
487
504
|
dataset: MultiModalDataset,
|
|
488
505
|
) -> dict[str, Any]:
|
|
489
506
|
writers: dict[str, Any] = {}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
plan.
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
507
|
+
try:
|
|
508
|
+
for plan in plans:
|
|
509
|
+
writers[plan.output_config.key] = dataset.create_output_writer(
|
|
510
|
+
plan.source_modality,
|
|
511
|
+
plan.output_root,
|
|
512
|
+
zip=plan.output_root.suffix.lower() == ".zip",
|
|
513
|
+
)
|
|
514
|
+
except BaseException:
|
|
515
|
+
_close_writers(writers.values())
|
|
516
|
+
raise
|
|
496
517
|
return writers
|
|
497
518
|
|
|
498
519
|
|
|
@@ -511,6 +532,7 @@ class PreparedOutputs:
|
|
|
511
532
|
legacy_writers: dict[str, Any]
|
|
512
533
|
pipeline_manifest_path: Path | None
|
|
513
534
|
pipeline_manifest_targets: list[PipelineOutputTargetConfig]
|
|
535
|
+
_finalized: bool = field(default=False, init=False, repr=False)
|
|
514
536
|
|
|
515
537
|
@classmethod
|
|
516
538
|
def prepare(
|
|
@@ -601,14 +623,65 @@ class PreparedOutputs:
|
|
|
601
623
|
save_output(output_data, path, output_config.type)
|
|
602
624
|
|
|
603
625
|
def finalize(self) -> None:
|
|
604
|
-
|
|
605
|
-
|
|
626
|
+
if self._finalized:
|
|
627
|
+
return
|
|
606
628
|
|
|
607
|
-
|
|
608
|
-
|
|
629
|
+
first_error: tuple[type[BaseException], BaseException, Any] | None = None
|
|
630
|
+
try:
|
|
631
|
+
for writers in (
|
|
632
|
+
self.legacy_writers.values(),
|
|
633
|
+
self.source_backed_writers.values(),
|
|
634
|
+
):
|
|
635
|
+
try:
|
|
636
|
+
_finalize_writers(writers)
|
|
637
|
+
except BaseException as exc:
|
|
638
|
+
if first_error is None:
|
|
639
|
+
first_error = (type(exc), exc, exc.__traceback__)
|
|
640
|
+
|
|
641
|
+
if (
|
|
642
|
+
first_error is None
|
|
643
|
+
and self.pipeline_manifest_path
|
|
644
|
+
and self.pipeline_manifest_targets
|
|
645
|
+
):
|
|
646
|
+
_write_pipeline_outputs_manifest(
|
|
647
|
+
self.pipeline_manifest_path,
|
|
648
|
+
self.pipeline_manifest_targets,
|
|
649
|
+
)
|
|
609
650
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
651
|
+
if first_error is not None:
|
|
652
|
+
_, exc, traceback = first_error
|
|
653
|
+
raise exc.with_traceback(traceback)
|
|
654
|
+
finally:
|
|
655
|
+
self._finalized = True
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def _finalize_writers(writers) -> None:
|
|
659
|
+
first_error: tuple[type[BaseException], BaseException, Any] | None = None
|
|
660
|
+
for writer in writers:
|
|
661
|
+
try:
|
|
662
|
+
writer.save_index()
|
|
663
|
+
except BaseException as exc:
|
|
664
|
+
if first_error is None:
|
|
665
|
+
first_error = (type(exc), exc, exc.__traceback__)
|
|
666
|
+
finally:
|
|
667
|
+
close = getattr(writer, "close", None)
|
|
668
|
+
if callable(close):
|
|
669
|
+
try:
|
|
670
|
+
close()
|
|
671
|
+
except BaseException as exc:
|
|
672
|
+
if first_error is None:
|
|
673
|
+
first_error = (type(exc), exc, exc.__traceback__)
|
|
674
|
+
|
|
675
|
+
if first_error is not None:
|
|
676
|
+
_, exc, traceback = first_error
|
|
677
|
+
raise exc.with_traceback(traceback)
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def _close_writers(writers) -> None:
|
|
681
|
+
for writer in writers:
|
|
682
|
+
close = getattr(writer, "close", None)
|
|
683
|
+
if callable(close):
|
|
684
|
+
try:
|
|
685
|
+
close()
|
|
686
|
+
except Exception:
|
|
687
|
+
pass
|
|
@@ -16,7 +16,7 @@ Usage:
|
|
|
16
16
|
import argparse
|
|
17
17
|
import logging
|
|
18
18
|
from pathlib import Path
|
|
19
|
-
from typing import NamedTuple
|
|
19
|
+
from typing import Any, NamedTuple
|
|
20
20
|
|
|
21
21
|
from euler_loading import Modality, MultiModalDataset
|
|
22
22
|
from tqdm import tqdm
|
|
@@ -253,6 +253,67 @@ def _build_model_inputs(sample: dict[str, object]) -> dict[str, object]:
|
|
|
253
253
|
}
|
|
254
254
|
|
|
255
255
|
|
|
256
|
+
def _close_model(model: Any, logger: logging.Logger) -> None:
|
|
257
|
+
close = getattr(model, "close", None)
|
|
258
|
+
try:
|
|
259
|
+
if callable(close):
|
|
260
|
+
close()
|
|
261
|
+
finally:
|
|
262
|
+
_release_accelerator_cache(logger)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _release_accelerator_cache(logger: logging.Logger) -> None:
|
|
266
|
+
try:
|
|
267
|
+
import torch
|
|
268
|
+
except ImportError:
|
|
269
|
+
return
|
|
270
|
+
|
|
271
|
+
try:
|
|
272
|
+
if torch.cuda.is_available():
|
|
273
|
+
torch.cuda.empty_cache()
|
|
274
|
+
mps = getattr(torch, "mps", None)
|
|
275
|
+
empty_mps_cache = getattr(mps, "empty_cache", None)
|
|
276
|
+
if callable(empty_mps_cache):
|
|
277
|
+
empty_mps_cache()
|
|
278
|
+
except Exception as exc:
|
|
279
|
+
logger.debug("Failed to release accelerator cache: %s", exc)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _cleanup_after_inference(
|
|
283
|
+
*,
|
|
284
|
+
prepared_outputs: PreparedOutputs | None,
|
|
285
|
+
model: Any | None,
|
|
286
|
+
logger: logging.Logger,
|
|
287
|
+
active_error: BaseException | None,
|
|
288
|
+
) -> None:
|
|
289
|
+
cleanup_error: BaseException | None = None
|
|
290
|
+
|
|
291
|
+
if prepared_outputs is not None:
|
|
292
|
+
try:
|
|
293
|
+
prepared_outputs.finalize()
|
|
294
|
+
except BaseException as exc:
|
|
295
|
+
cleanup_error = exc
|
|
296
|
+
|
|
297
|
+
if model is not None:
|
|
298
|
+
try:
|
|
299
|
+
_close_model(model, logger)
|
|
300
|
+
except BaseException as exc:
|
|
301
|
+
if cleanup_error is None:
|
|
302
|
+
cleanup_error = exc
|
|
303
|
+
|
|
304
|
+
if cleanup_error is None:
|
|
305
|
+
return
|
|
306
|
+
|
|
307
|
+
if active_error is not None:
|
|
308
|
+
logger.warning(
|
|
309
|
+
"Inference failed, and cleanup also failed: %s",
|
|
310
|
+
cleanup_error,
|
|
311
|
+
)
|
|
312
|
+
return
|
|
313
|
+
|
|
314
|
+
raise cleanup_error
|
|
315
|
+
|
|
316
|
+
|
|
256
317
|
def _build_arg_parser() -> argparse.ArgumentParser:
|
|
257
318
|
parser = argparse.ArgumentParser(
|
|
258
319
|
description="Run model inference using euler-loading",
|
|
@@ -420,46 +481,60 @@ def run_inference(config: InferenceConfig, verbose: bool = False) -> None:
|
|
|
420
481
|
# Prepare writers (one per output key)
|
|
421
482
|
output_base = Path(config.output_base_path)
|
|
422
483
|
output_base.mkdir(parents=True, exist_ok=True)
|
|
423
|
-
prepared_outputs =
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
484
|
+
prepared_outputs: PreparedOutputs | None = None
|
|
485
|
+
model: Any | None = None
|
|
486
|
+
active_error: BaseException | None = None
|
|
487
|
+
try:
|
|
488
|
+
prepared_outputs = PreparedOutputs.prepare(
|
|
489
|
+
config=config,
|
|
490
|
+
dataset=dataset,
|
|
491
|
+
logger=logger,
|
|
492
|
+
)
|
|
428
493
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
494
|
+
# Load model
|
|
495
|
+
logger.info("Loading model: %s", config.model_name)
|
|
496
|
+
model = get_model_from_config(config)
|
|
497
|
+
device_info = getattr(model, "device", "unknown")
|
|
498
|
+
logger.info("Model loaded on device: %s", device_info)
|
|
434
499
|
|
|
435
|
-
|
|
436
|
-
|
|
500
|
+
# Run inference loop
|
|
501
|
+
logger.info("Starting inference...")
|
|
437
502
|
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
503
|
+
for i, sample in enumerate(tqdm(dataset, total=total_samples, desc="Inference")):
|
|
504
|
+
if config.max_samples and i >= config.max_samples:
|
|
505
|
+
break
|
|
441
506
|
|
|
442
|
-
|
|
507
|
+
full_id = sample["full_id"]
|
|
443
508
|
|
|
444
|
-
|
|
445
|
-
|
|
509
|
+
# Build inputs: pass all loaded modalities (skip euler-loading metadata)
|
|
510
|
+
inputs = _build_model_inputs(sample)
|
|
446
511
|
|
|
447
|
-
|
|
448
|
-
|
|
512
|
+
# Run model inference
|
|
513
|
+
outputs = model.predict(inputs)
|
|
449
514
|
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
515
|
+
prepared_outputs.validate_prediction_keys(outputs)
|
|
516
|
+
prepared_outputs.write_predictions(
|
|
517
|
+
sample_index=i,
|
|
518
|
+
full_id=full_id,
|
|
519
|
+
outputs=outputs,
|
|
520
|
+
)
|
|
521
|
+
del outputs, inputs, sample
|
|
522
|
+
|
|
523
|
+
if verbose and (i + 1) % 100 == 0:
|
|
524
|
+
logger.debug("Processed %d/%d", i + 1, total_samples)
|
|
525
|
+
except BaseException as exc:
|
|
526
|
+
active_error = exc
|
|
527
|
+
raise
|
|
528
|
+
finally:
|
|
529
|
+
# Finalize writers (writes metadata / closes zip archives), release
|
|
530
|
+
# the model, and clear unused accelerator cache for repeated API calls.
|
|
531
|
+
_cleanup_after_inference(
|
|
532
|
+
prepared_outputs=prepared_outputs,
|
|
533
|
+
model=model,
|
|
534
|
+
logger=logger,
|
|
535
|
+
active_error=active_error,
|
|
455
536
|
)
|
|
456
537
|
|
|
457
|
-
if verbose and (i + 1) % 100 == 0:
|
|
458
|
-
logger.debug("Processed %d/%d", i + 1, total_samples)
|
|
459
|
-
|
|
460
|
-
# Finalize writers (writes output.json / closes zip archives)
|
|
461
|
-
prepared_outputs.finalize()
|
|
462
|
-
|
|
463
538
|
logger.info("Inference complete!")
|
|
464
539
|
logger.info("Predictions saved to: %s", output_base)
|
|
465
540
|
|
|
@@ -15,13 +15,23 @@ Models can optionally declare metadata:
|
|
|
15
15
|
- DEFAULT_CONFIG: dict — default model_config values (user values win on merge)
|
|
16
16
|
"""
|
|
17
17
|
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import copy
|
|
21
|
+
import hashlib
|
|
18
22
|
import importlib.util
|
|
23
|
+
import itertools
|
|
19
24
|
import sys
|
|
20
|
-
from
|
|
25
|
+
from contextlib import nullcontext
|
|
26
|
+
from dataclasses import dataclass
|
|
21
27
|
from pathlib import Path
|
|
28
|
+
from types import ModuleType
|
|
22
29
|
from typing import Any, Optional
|
|
23
30
|
|
|
24
31
|
|
|
32
|
+
_MODEL_MODULE_COUNTER = itertools.count()
|
|
33
|
+
|
|
34
|
+
|
|
25
35
|
@dataclass
|
|
26
36
|
class ModelMetadata:
|
|
27
37
|
"""Metadata read from a Model class without instantiating it."""
|
|
@@ -30,12 +40,37 @@ class ModelMetadata:
|
|
|
30
40
|
default_config: Optional[dict] = None
|
|
31
41
|
|
|
32
42
|
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class _LoadedModelModule:
|
|
45
|
+
"""A dynamically loaded external model module."""
|
|
46
|
+
|
|
47
|
+
module: ModuleType
|
|
48
|
+
module_name: str
|
|
49
|
+
|
|
50
|
+
|
|
33
51
|
class ExternalModelError(Exception):
|
|
34
52
|
"""Raised when external model loading or execution fails."""
|
|
35
53
|
|
|
36
54
|
pass
|
|
37
55
|
|
|
38
56
|
|
|
57
|
+
def _copy_metadata(value: Any) -> Any:
|
|
58
|
+
if value is None:
|
|
59
|
+
return None
|
|
60
|
+
try:
|
|
61
|
+
return copy.deepcopy(value)
|
|
62
|
+
except Exception:
|
|
63
|
+
return value
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _torch_no_grad_context():
|
|
67
|
+
try:
|
|
68
|
+
import torch
|
|
69
|
+
except ImportError:
|
|
70
|
+
return nullcontext()
|
|
71
|
+
return torch.no_grad()
|
|
72
|
+
|
|
73
|
+
|
|
39
74
|
class ExternalModel:
|
|
40
75
|
"""
|
|
41
76
|
Wrapper around an external Model class loaded from a .py file.
|
|
@@ -69,9 +104,16 @@ class ExternalModel:
|
|
|
69
104
|
"""
|
|
70
105
|
self._model_path = model_path
|
|
71
106
|
self._device = device
|
|
107
|
+
self._model: Any | None = None
|
|
108
|
+
self._module: ModuleType | None = None
|
|
109
|
+
self._module_name: str | None = None
|
|
110
|
+
self._closed = False
|
|
72
111
|
|
|
73
112
|
# Load the Model class and read metadata
|
|
74
|
-
|
|
113
|
+
loaded_module = self._load_model_module(model_path)
|
|
114
|
+
self._module = loaded_module.module
|
|
115
|
+
self._module_name = loaded_module.module_name
|
|
116
|
+
ModelClass = loaded_module.module.Model
|
|
75
117
|
self.metadata = self._read_model_metadata(ModelClass)
|
|
76
118
|
|
|
77
119
|
# Merge DEFAULT_CONFIG under user config (user values win)
|
|
@@ -84,20 +126,39 @@ class ExternalModel:
|
|
|
84
126
|
try:
|
|
85
127
|
self._model = ModelClass(merged_config, device)
|
|
86
128
|
except Exception as e:
|
|
129
|
+
self.close()
|
|
87
130
|
raise ExternalModelError(
|
|
88
131
|
f"Failed to instantiate Model from {model_path}: {e}"
|
|
89
132
|
) from e
|
|
90
133
|
|
|
91
134
|
@staticmethod
|
|
92
|
-
def
|
|
135
|
+
def _module_name_for_path(path: Path) -> str:
|
|
136
|
+
resolved = str(path.resolve())
|
|
137
|
+
digest = hashlib.sha1(resolved.encode("utf-8")).hexdigest()[:12]
|
|
138
|
+
safe_stem = "".join(
|
|
139
|
+
char if char.isalnum() or char == "_" else "_"
|
|
140
|
+
for char in path.stem
|
|
141
|
+
) or "model"
|
|
142
|
+
return (
|
|
143
|
+
f"_euler_inference_external_model_{safe_stem}_{digest}_"
|
|
144
|
+
f"{next(_MODEL_MODULE_COUNTER)}"
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
@staticmethod
|
|
148
|
+
def _unload_module(module_name: str | None, module: ModuleType | None) -> None:
|
|
149
|
+
if module_name and module is not None and sys.modules.get(module_name) is module:
|
|
150
|
+
del sys.modules[module_name]
|
|
151
|
+
|
|
152
|
+
@staticmethod
|
|
153
|
+
def _load_model_module(model_path: str) -> _LoadedModelModule:
|
|
93
154
|
"""
|
|
94
|
-
Dynamically load the
|
|
155
|
+
Dynamically load the module containing the Model class.
|
|
95
156
|
|
|
96
157
|
Args:
|
|
97
158
|
model_path: Absolute path to the model.py file
|
|
98
159
|
|
|
99
160
|
Returns:
|
|
100
|
-
|
|
161
|
+
Loaded module and the unique name it was registered with
|
|
101
162
|
|
|
102
163
|
Raises:
|
|
103
164
|
ExternalModelError: If the file can't be loaded or has no Model class
|
|
@@ -108,28 +169,49 @@ class ExternalModel:
|
|
|
108
169
|
if not path.suffix == ".py":
|
|
109
170
|
raise ExternalModelError(f"Model file must be .py: {model_path}")
|
|
110
171
|
|
|
111
|
-
|
|
172
|
+
module_name = ExternalModel._module_name_for_path(path)
|
|
173
|
+
spec = importlib.util.spec_from_file_location(module_name, model_path)
|
|
112
174
|
if spec is None or spec.loader is None:
|
|
113
175
|
raise ExternalModelError(
|
|
114
176
|
f"Could not create import spec for: {model_path}"
|
|
115
177
|
)
|
|
116
178
|
|
|
117
179
|
module = importlib.util.module_from_spec(spec)
|
|
118
|
-
sys.modules[
|
|
180
|
+
sys.modules[module_name] = module
|
|
119
181
|
|
|
120
182
|
try:
|
|
121
183
|
spec.loader.exec_module(module)
|
|
122
184
|
except Exception as e:
|
|
185
|
+
ExternalModel._unload_module(module_name, module)
|
|
123
186
|
raise ExternalModelError(
|
|
124
187
|
f"Failed to load model file {model_path}: {e}"
|
|
125
188
|
) from e
|
|
126
189
|
|
|
127
190
|
if not hasattr(module, "Model"):
|
|
191
|
+
ExternalModel._unload_module(module_name, module)
|
|
128
192
|
raise ExternalModelError(
|
|
129
193
|
f"Model file must define a 'Model' class: {model_path}"
|
|
130
194
|
)
|
|
131
195
|
|
|
132
|
-
return module
|
|
196
|
+
return _LoadedModelModule(module=module, module_name=module_name)
|
|
197
|
+
|
|
198
|
+
@staticmethod
|
|
199
|
+
def _load_model_class(model_path: str) -> type:
|
|
200
|
+
"""
|
|
201
|
+
Dynamically load the Model class from a Python file.
|
|
202
|
+
|
|
203
|
+
The loaded module is removed from ``sys.modules`` before returning so
|
|
204
|
+
metadata-only calls do not anchor module globals for the life of the
|
|
205
|
+
process. The returned class still owns its function globals.
|
|
206
|
+
"""
|
|
207
|
+
loaded_module = ExternalModel._load_model_module(model_path)
|
|
208
|
+
try:
|
|
209
|
+
return loaded_module.module.Model
|
|
210
|
+
finally:
|
|
211
|
+
ExternalModel._unload_module(
|
|
212
|
+
loaded_module.module_name,
|
|
213
|
+
loaded_module.module,
|
|
214
|
+
)
|
|
133
215
|
|
|
134
216
|
@staticmethod
|
|
135
217
|
def _read_model_metadata(model_class: type) -> ModelMetadata:
|
|
@@ -142,8 +224,8 @@ class ExternalModel:
|
|
|
142
224
|
Returns:
|
|
143
225
|
ModelMetadata with any declared outputs and default_config
|
|
144
226
|
"""
|
|
145
|
-
outputs = getattr(model_class, "OUTPUTS", None)
|
|
146
|
-
default_config = getattr(model_class, "DEFAULT_CONFIG", None)
|
|
227
|
+
outputs = _copy_metadata(getattr(model_class, "OUTPUTS", None))
|
|
228
|
+
default_config = _copy_metadata(getattr(model_class, "DEFAULT_CONFIG", None))
|
|
147
229
|
return ModelMetadata(outputs=outputs, default_config=default_config)
|
|
148
230
|
|
|
149
231
|
@property
|
|
@@ -151,6 +233,11 @@ class ExternalModel:
|
|
|
151
233
|
"""Return the model name (derived from filename)."""
|
|
152
234
|
return Path(self._model_path).stem
|
|
153
235
|
|
|
236
|
+
@property
|
|
237
|
+
def device(self) -> str | None:
|
|
238
|
+
"""Return the configured model device."""
|
|
239
|
+
return self._device
|
|
240
|
+
|
|
154
241
|
def predict(self, inputs: dict) -> dict:
|
|
155
242
|
"""
|
|
156
243
|
Run model prediction.
|
|
@@ -161,7 +248,40 @@ class ExternalModel:
|
|
|
161
248
|
Returns:
|
|
162
249
|
Dictionary of outputs from the model
|
|
163
250
|
"""
|
|
164
|
-
|
|
251
|
+
if self._model is None or self._closed:
|
|
252
|
+
raise ExternalModelError("Cannot run prediction after model is closed")
|
|
253
|
+
|
|
254
|
+
with _torch_no_grad_context():
|
|
255
|
+
return self._model.predict(inputs)
|
|
256
|
+
|
|
257
|
+
def close(self) -> None:
|
|
258
|
+
"""Release model-owned resources and unload its dynamic module."""
|
|
259
|
+
if self._closed:
|
|
260
|
+
return
|
|
261
|
+
|
|
262
|
+
model = self._model
|
|
263
|
+
try:
|
|
264
|
+
close = getattr(model, "close", None)
|
|
265
|
+
if callable(close):
|
|
266
|
+
close()
|
|
267
|
+
finally:
|
|
268
|
+
self._model = None
|
|
269
|
+
self._unload_module(self._module_name, self._module)
|
|
270
|
+
self._module = None
|
|
271
|
+
self._module_name = None
|
|
272
|
+
self._closed = True
|
|
273
|
+
|
|
274
|
+
def __enter__(self) -> "ExternalModel":
|
|
275
|
+
return self
|
|
276
|
+
|
|
277
|
+
def __exit__(self, exc_type, exc, traceback) -> None:
|
|
278
|
+
self.close()
|
|
279
|
+
|
|
280
|
+
def __del__(self):
|
|
281
|
+
try:
|
|
282
|
+
self.close()
|
|
283
|
+
except Exception:
|
|
284
|
+
pass
|
|
165
285
|
|
|
166
286
|
def to(self, device: str) -> "ExternalModel":
|
|
167
287
|
"""Set the device (no-op, device is set at init)."""
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Tests for model metadata, config resolution, and CLI overrides."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
import sys
|
|
4
5
|
import textwrap
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import Optional
|
|
@@ -164,6 +165,82 @@ class TestDefaultConfigMerge:
|
|
|
164
165
|
assert model._model.config == user_config
|
|
165
166
|
|
|
166
167
|
|
|
168
|
+
class TestExternalModelResourceCleanup:
|
|
169
|
+
"""Tests that external model resources do not stay anchored globally."""
|
|
170
|
+
|
|
171
|
+
def test_metadata_load_does_not_leave_module_in_sys_modules(self, model_with_metadata: Path):
|
|
172
|
+
from euler_inference.models.external_model import load_model_metadata
|
|
173
|
+
|
|
174
|
+
prefix = "_euler_inference_external_model_"
|
|
175
|
+
before = {
|
|
176
|
+
name for name in sys.modules
|
|
177
|
+
if name.startswith(prefix)
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
load_model_metadata(str(model_with_metadata))
|
|
181
|
+
|
|
182
|
+
after = {
|
|
183
|
+
name for name in sys.modules
|
|
184
|
+
if name.startswith(prefix)
|
|
185
|
+
}
|
|
186
|
+
assert after == before
|
|
187
|
+
|
|
188
|
+
def test_close_calls_model_hook_and_unloads_module(self, tmp_path: Path):
|
|
189
|
+
from euler_inference.models.external_model import ExternalModel, ExternalModelError
|
|
190
|
+
|
|
191
|
+
marker = tmp_path / "closed.txt"
|
|
192
|
+
model_file = tmp_path / "model_with_close.py"
|
|
193
|
+
model_file.write_text(textwrap.dedent(f"""\
|
|
194
|
+
class Model:
|
|
195
|
+
def __init__(self, config, device=None):
|
|
196
|
+
self.marker = {str(marker)!r}
|
|
197
|
+
self.device = device
|
|
198
|
+
|
|
199
|
+
def predict(self, inputs):
|
|
200
|
+
return {{"depth": 1}}
|
|
201
|
+
|
|
202
|
+
def close(self):
|
|
203
|
+
with open(self.marker, "w", encoding="utf-8") as handle:
|
|
204
|
+
handle.write("closed")
|
|
205
|
+
"""))
|
|
206
|
+
|
|
207
|
+
model = ExternalModel(str(model_file), model_config={}, device="cpu")
|
|
208
|
+
module_name = model._module_name
|
|
209
|
+
assert module_name in sys.modules
|
|
210
|
+
assert model.device == "cpu"
|
|
211
|
+
|
|
212
|
+
model.close()
|
|
213
|
+
|
|
214
|
+
assert marker.read_text(encoding="utf-8") == "closed"
|
|
215
|
+
assert module_name not in sys.modules
|
|
216
|
+
assert model._model is None
|
|
217
|
+
with pytest.raises(ExternalModelError, match="after model is closed"):
|
|
218
|
+
model.predict({})
|
|
219
|
+
|
|
220
|
+
def test_predict_runs_under_torch_no_grad(self, tmp_path: Path):
|
|
221
|
+
pytest.importorskip("torch")
|
|
222
|
+
|
|
223
|
+
from euler_inference.models.external_model import ExternalModel
|
|
224
|
+
|
|
225
|
+
model_file = tmp_path / "model_checks_grad.py"
|
|
226
|
+
model_file.write_text(textwrap.dedent("""\
|
|
227
|
+
import torch
|
|
228
|
+
|
|
229
|
+
class Model:
|
|
230
|
+
def __init__(self, config, device=None):
|
|
231
|
+
pass
|
|
232
|
+
|
|
233
|
+
def predict(self, inputs):
|
|
234
|
+
return {"grad_enabled": torch.is_grad_enabled()}
|
|
235
|
+
"""))
|
|
236
|
+
|
|
237
|
+
model = ExternalModel(str(model_file), model_config={})
|
|
238
|
+
try:
|
|
239
|
+
assert model.predict({}) == {"grad_enabled": False}
|
|
240
|
+
finally:
|
|
241
|
+
model.close()
|
|
242
|
+
|
|
243
|
+
|
|
167
244
|
# ---------------------------------------------------------------------------
|
|
168
245
|
# Test: resolve_config_with_model
|
|
169
246
|
# ---------------------------------------------------------------------------
|
|
@@ -22,6 +22,11 @@ from euler_inference.inference import (
|
|
|
22
22
|
run_inference,
|
|
23
23
|
save_output,
|
|
24
24
|
)
|
|
25
|
+
from euler_inference._outputs import (
|
|
26
|
+
PreparedOutputs,
|
|
27
|
+
_finalize_writers,
|
|
28
|
+
_prepare_legacy_output_data,
|
|
29
|
+
)
|
|
25
30
|
|
|
26
31
|
|
|
27
32
|
# ---------------------------------------------------------------------------
|
|
@@ -101,8 +106,10 @@ def _read_output_metadata(root: Path) -> dict:
|
|
|
101
106
|
|
|
102
107
|
dataset_head = root / ".ds_crawler" / "dataset-head.json"
|
|
103
108
|
with open(dataset_head) as f:
|
|
104
|
-
|
|
109
|
+
return _metadata_from_dataset_head(json.load(f))
|
|
110
|
+
|
|
105
111
|
|
|
112
|
+
def _metadata_from_dataset_head(head: dict) -> dict:
|
|
106
113
|
modality_key = head["modality"]["key"]
|
|
107
114
|
addons = head.get("addons", {})
|
|
108
115
|
euler_train = dict(addons.get("euler_train", {}))
|
|
@@ -117,6 +124,20 @@ def _read_output_metadata(root: Path) -> dict:
|
|
|
117
124
|
}
|
|
118
125
|
|
|
119
126
|
|
|
127
|
+
def _read_zip_output_metadata(zip_path: Path) -> dict:
|
|
128
|
+
"""Read legacy output.json or newer dataset-head metadata from a zip."""
|
|
129
|
+
with zipfile.ZipFile(zip_path) as zf:
|
|
130
|
+
names = set(zf.namelist())
|
|
131
|
+
if ".ds_crawler/output.json" in names:
|
|
132
|
+
with zf.open(".ds_crawler/output.json") as entry:
|
|
133
|
+
return json.load(io.TextIOWrapper(entry, encoding="utf-8"))
|
|
134
|
+
|
|
135
|
+
with zf.open(".ds_crawler/dataset-head.json") as entry:
|
|
136
|
+
return _metadata_from_dataset_head(
|
|
137
|
+
json.load(io.TextIOWrapper(entry, encoding="utf-8"))
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
120
141
|
# ---------------------------------------------------------------------------
|
|
121
142
|
# _get_writer_kwargs
|
|
122
143
|
# ---------------------------------------------------------------------------
|
|
@@ -211,7 +232,7 @@ class TestGetWriterKwargs:
|
|
|
211
232
|
assert kw["meta"]["scale_to_meters"] == 1.0
|
|
212
233
|
|
|
213
234
|
def test_partial_override_non_strict(self):
|
|
214
|
-
"""Writer dict only overrides specified keys, rest defaults
|
|
235
|
+
"""Writer dict only overrides specified keys, rest defaults to other."""
|
|
215
236
|
oc = OutputConfig(
|
|
216
237
|
key="depth",
|
|
217
238
|
type="npy",
|
|
@@ -219,7 +240,7 @@ class TestGetWriterKwargs:
|
|
|
219
240
|
)
|
|
220
241
|
kw = _get_writer_kwargs(oc, strict=False)
|
|
221
242
|
assert kw["name"] == "custom_name"
|
|
222
|
-
assert kw["type"] == "
|
|
243
|
+
assert kw["type"] == "other"
|
|
223
244
|
assert kw["euler_train"]["modality_type"] == "other"
|
|
224
245
|
|
|
225
246
|
def test_partial_override_strict(self):
|
|
@@ -278,6 +299,26 @@ class TestGetWriterKwargs:
|
|
|
278
299
|
assert kw["euler_train"]["modality_type"] == "points_3d"
|
|
279
300
|
assert kw["euler_loading"]["function"] == "points_3d"
|
|
280
301
|
|
|
302
|
+
def test_intrinsics_defaults_to_generic_loader_metadata(self):
|
|
303
|
+
oc = OutputConfig(key="intrinsics", type="npy")
|
|
304
|
+
kw = _get_writer_kwargs(oc, strict=True)
|
|
305
|
+
assert kw["type"] == "intrinsics"
|
|
306
|
+
assert kw["euler_train"] == {
|
|
307
|
+
"used_as": "target",
|
|
308
|
+
"modality_type": "intrinsics",
|
|
309
|
+
}
|
|
310
|
+
assert kw["euler_loading"] == {
|
|
311
|
+
"used_as": "target",
|
|
312
|
+
"modality_type": "intrinsics",
|
|
313
|
+
"loader": "generic",
|
|
314
|
+
"function": "intrinsics",
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
def test_intrinsics_rejects_non_numpy_output_format(self):
|
|
318
|
+
oc = OutputConfig(key="intrinsics", type="png")
|
|
319
|
+
with pytest.raises(ValueError, match="intrinsics outputs must use"):
|
|
320
|
+
_get_writer_kwargs(oc, strict=True)
|
|
321
|
+
|
|
281
322
|
def test_points_3d_rejects_non_numpy_output_format(self):
|
|
282
323
|
oc = OutputConfig(key="points_3d", type="png")
|
|
283
324
|
with pytest.raises(ValueError, match="points_3d outputs must use"):
|
|
@@ -344,7 +385,7 @@ class TestCreateWriters:
|
|
|
344
385
|
outputs=[OutputConfig(key="depth", type="npy")],
|
|
345
386
|
strict=True,
|
|
346
387
|
)
|
|
347
|
-
with pytest.raises(ValueError, match="meta is required
|
|
388
|
+
with pytest.raises(ValueError, match="meta is required"):
|
|
348
389
|
create_writers(config)
|
|
349
390
|
|
|
350
391
|
def test_strict_with_meta_works(self, tmp_path: Path):
|
|
@@ -526,6 +567,173 @@ class TestSaveOutputFileLike:
|
|
|
526
567
|
np.testing.assert_array_equal(loaded, data)
|
|
527
568
|
|
|
528
569
|
|
|
570
|
+
class TestOutputResourceCleanup:
|
|
571
|
+
"""Tests for output conversion and writer lifetime management."""
|
|
572
|
+
|
|
573
|
+
def test_legacy_output_tensor_is_detached_and_moved_to_numpy(self):
|
|
574
|
+
torch = pytest.importorskip("torch")
|
|
575
|
+
|
|
576
|
+
tensor = (
|
|
577
|
+
torch.arange(6, dtype=torch.float32, requires_grad=True)
|
|
578
|
+
.reshape(2, 3)
|
|
579
|
+
* 2
|
|
580
|
+
)
|
|
581
|
+
|
|
582
|
+
result = _prepare_legacy_output_data(
|
|
583
|
+
tensor,
|
|
584
|
+
OutputConfig(key="depth", type="npy"),
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
assert isinstance(result, np.ndarray)
|
|
588
|
+
np.testing.assert_array_equal(
|
|
589
|
+
result,
|
|
590
|
+
np.array([[0, 2, 4], [6, 8, 10]], dtype=np.float32),
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
def test_intrinsics_output_is_validated_and_converted_to_float32(self):
|
|
594
|
+
result = _prepare_legacy_output_data(
|
|
595
|
+
np.eye(3, dtype=np.float64),
|
|
596
|
+
OutputConfig(key="intrinsics", type="npy"),
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
assert result.shape == (3, 3)
|
|
600
|
+
assert result.dtype == np.float32
|
|
601
|
+
|
|
602
|
+
def test_intrinsics_output_rejects_wrong_shape(self):
|
|
603
|
+
with pytest.raises(ValueError, match=r"shape \(3, 3\)"):
|
|
604
|
+
_prepare_legacy_output_data(
|
|
605
|
+
np.zeros((4, 4), dtype=np.float32),
|
|
606
|
+
OutputConfig(key="intrinsics", type="npy"),
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
def test_finalize_writers_closes_all_writers_when_save_fails(self):
|
|
610
|
+
class Writer:
|
|
611
|
+
def __init__(self, save_error=None):
|
|
612
|
+
self.save_error = save_error
|
|
613
|
+
self.saved = False
|
|
614
|
+
self.closed = False
|
|
615
|
+
|
|
616
|
+
def save_index(self):
|
|
617
|
+
self.saved = True
|
|
618
|
+
if self.save_error:
|
|
619
|
+
raise self.save_error
|
|
620
|
+
|
|
621
|
+
def close(self):
|
|
622
|
+
self.closed = True
|
|
623
|
+
|
|
624
|
+
error = RuntimeError("save failed")
|
|
625
|
+
first = Writer(save_error=error)
|
|
626
|
+
second = Writer()
|
|
627
|
+
|
|
628
|
+
with pytest.raises(RuntimeError, match="save failed"):
|
|
629
|
+
_finalize_writers([first, second])
|
|
630
|
+
|
|
631
|
+
assert first.saved is True
|
|
632
|
+
assert first.closed is True
|
|
633
|
+
assert second.saved is True
|
|
634
|
+
assert second.closed is True
|
|
635
|
+
|
|
636
|
+
def test_prepared_outputs_finalize_is_idempotent(self):
|
|
637
|
+
class Writer:
|
|
638
|
+
def __init__(self):
|
|
639
|
+
self.save_count = 0
|
|
640
|
+
self.close_count = 0
|
|
641
|
+
|
|
642
|
+
def save_index(self):
|
|
643
|
+
self.save_count += 1
|
|
644
|
+
|
|
645
|
+
def close(self):
|
|
646
|
+
self.close_count += 1
|
|
647
|
+
|
|
648
|
+
writer = Writer()
|
|
649
|
+
prepared = PreparedOutputs(
|
|
650
|
+
dataset=None,
|
|
651
|
+
expected_output_keys=("depth",),
|
|
652
|
+
writer_backed_plans=[],
|
|
653
|
+
legacy_outputs=[],
|
|
654
|
+
source_backed_writers={},
|
|
655
|
+
legacy_writers={"depth": writer},
|
|
656
|
+
pipeline_manifest_path=None,
|
|
657
|
+
pipeline_manifest_targets=[],
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
prepared.finalize()
|
|
661
|
+
prepared.finalize()
|
|
662
|
+
|
|
663
|
+
assert writer.save_count == 1
|
|
664
|
+
assert writer.close_count == 1
|
|
665
|
+
|
|
666
|
+
def test_prepared_outputs_finalize_attempts_all_writer_groups_on_error(self):
|
|
667
|
+
class Writer:
|
|
668
|
+
def __init__(self, save_error=None):
|
|
669
|
+
self.save_error = save_error
|
|
670
|
+
self.saved = False
|
|
671
|
+
self.closed = False
|
|
672
|
+
|
|
673
|
+
def save_index(self):
|
|
674
|
+
self.saved = True
|
|
675
|
+
if self.save_error:
|
|
676
|
+
raise self.save_error
|
|
677
|
+
|
|
678
|
+
def close(self):
|
|
679
|
+
self.closed = True
|
|
680
|
+
|
|
681
|
+
legacy_writer = Writer(RuntimeError("legacy failed"))
|
|
682
|
+
source_writer = Writer()
|
|
683
|
+
prepared = PreparedOutputs(
|
|
684
|
+
dataset=None,
|
|
685
|
+
expected_output_keys=("depth", "rgb"),
|
|
686
|
+
writer_backed_plans=[],
|
|
687
|
+
legacy_outputs=[],
|
|
688
|
+
source_backed_writers={"rgb": source_writer},
|
|
689
|
+
legacy_writers={"depth": legacy_writer},
|
|
690
|
+
pipeline_manifest_path=None,
|
|
691
|
+
pipeline_manifest_targets=[],
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
with pytest.raises(RuntimeError, match="legacy failed"):
|
|
695
|
+
prepared.finalize()
|
|
696
|
+
|
|
697
|
+
assert legacy_writer.saved is True
|
|
698
|
+
assert legacy_writer.closed is True
|
|
699
|
+
assert source_writer.saved is True
|
|
700
|
+
assert source_writer.closed is True
|
|
701
|
+
|
|
702
|
+
def test_create_writers_closes_created_writers_when_later_writer_fails(
|
|
703
|
+
self,
|
|
704
|
+
tmp_path: Path,
|
|
705
|
+
monkeypatch,
|
|
706
|
+
):
|
|
707
|
+
import ds_crawler
|
|
708
|
+
|
|
709
|
+
created = []
|
|
710
|
+
|
|
711
|
+
class Writer:
|
|
712
|
+
def __init__(self, root, **kwargs):
|
|
713
|
+
if Path(root).name == "bad":
|
|
714
|
+
raise RuntimeError("writer failed")
|
|
715
|
+
self.closed = False
|
|
716
|
+
created.append(self)
|
|
717
|
+
|
|
718
|
+
def close(self):
|
|
719
|
+
self.closed = True
|
|
720
|
+
|
|
721
|
+
monkeypatch.setattr(ds_crawler, "DatasetWriter", Writer)
|
|
722
|
+
config = _make_config(
|
|
723
|
+
tmp_path,
|
|
724
|
+
outputs=[
|
|
725
|
+
OutputConfig(key="ok", type="npy"),
|
|
726
|
+
OutputConfig(key="bad", type="npy"),
|
|
727
|
+
],
|
|
728
|
+
)
|
|
729
|
+
|
|
730
|
+
with pytest.raises(RuntimeError, match="writer failed"):
|
|
731
|
+
create_writers(config)
|
|
732
|
+
|
|
733
|
+
assert len(created) == 1
|
|
734
|
+
assert created[0].closed is True
|
|
735
|
+
|
|
736
|
+
|
|
529
737
|
# ---------------------------------------------------------------------------
|
|
530
738
|
# Filesystem writer end-to-end
|
|
531
739
|
# ---------------------------------------------------------------------------
|
|
@@ -555,14 +763,10 @@ class TestFilesystemWriterEndToEnd:
|
|
|
555
763
|
|
|
556
764
|
writer.save_index()
|
|
557
765
|
|
|
558
|
-
# Verify
|
|
559
|
-
|
|
560
|
-
assert index_path.exists()
|
|
561
|
-
|
|
562
|
-
with open(index_path) as f:
|
|
563
|
-
index = json.load(f)
|
|
766
|
+
# Verify metadata exists
|
|
767
|
+
index = _read_output_metadata(Path(config.output_base_path) / "depth")
|
|
564
768
|
assert index["name"] == "depth"
|
|
565
|
-
assert index["type"] == "
|
|
769
|
+
assert index["type"] == "other"
|
|
566
770
|
assert index["euler_train"]["modality_type"] == "other"
|
|
567
771
|
|
|
568
772
|
def test_multiple_outputs_separate_dirs(self, tmp_path: Path):
|
|
@@ -590,10 +794,10 @@ class TestFilesystemWriterEndToEnd:
|
|
|
590
794
|
for writer in writers.values():
|
|
591
795
|
writer.save_index()
|
|
592
796
|
|
|
593
|
-
# Both dirs should have
|
|
797
|
+
# Both dirs should have crawler metadata
|
|
594
798
|
for key in ("depth", "confidence"):
|
|
595
|
-
|
|
596
|
-
assert
|
|
799
|
+
index = _read_output_metadata(Path(config.output_base_path) / key)
|
|
800
|
+
assert index["name"], f"Missing metadata for {key}"
|
|
597
801
|
|
|
598
802
|
def test_points_3d_metadata_is_loadable_by_euler_loading(self, tmp_path: Path):
|
|
599
803
|
config = _make_config(
|
|
@@ -651,7 +855,13 @@ class TestZipWriterEndToEnd:
|
|
|
651
855
|
names = zf.namelist()
|
|
652
856
|
# Should have the data file and the index
|
|
653
857
|
assert any("00001_depth.npy" in n for n in names)
|
|
654
|
-
assert any(
|
|
858
|
+
assert any(
|
|
859
|
+
metadata_name in names
|
|
860
|
+
for metadata_name in (
|
|
861
|
+
".ds_crawler/output.json",
|
|
862
|
+
".ds_crawler/dataset-head.json",
|
|
863
|
+
)
|
|
864
|
+
)
|
|
655
865
|
|
|
656
866
|
# Verify the data file is loadable
|
|
657
867
|
data_entry = [n for n in names if "00001_depth.npy" in n][0]
|
|
@@ -702,12 +912,9 @@ class TestZipWriterEndToEnd:
|
|
|
702
912
|
writer.save_index()
|
|
703
913
|
|
|
704
914
|
zip_path = Path(config.output_base_path) / "depth.zip"
|
|
705
|
-
|
|
706
|
-
index_entry = [n for n in zf.namelist() if "output.json" in n][0]
|
|
707
|
-
with zf.open(index_entry) as ef:
|
|
708
|
-
index = json.loads(ef.read())
|
|
915
|
+
index = _read_zip_output_metadata(zip_path)
|
|
709
916
|
assert index["name"] == "depth"
|
|
710
|
-
assert index["type"] == "
|
|
917
|
+
assert index["type"] == "other"
|
|
711
918
|
assert index["euler_train"]["used_as"] == "target"
|
|
712
919
|
|
|
713
920
|
|
|
@@ -1168,7 +1375,7 @@ class TestRunInferenceWriterBackedOutputs:
|
|
|
1168
1375
|
np.load(legacy_path),
|
|
1169
1376
|
np.full((6, 8), 2.5, dtype=np.float32),
|
|
1170
1377
|
)
|
|
1171
|
-
assert (output_root
|
|
1378
|
+
assert _read_output_metadata(output_root)["type"] == "other"
|
|
1172
1379
|
|
|
1173
1380
|
def test_unmapped_points_3d_output_uses_generic_loader_metadata(
|
|
1174
1381
|
self,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{euler_inference-2.5.0 → euler_inference-2.7.0}/euler_inference.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|