quick-spice-manager 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/PKG-INFO +42 -1
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/README.md +41 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/pyproject.toml +1 -1
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/__init__.py +9 -1
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/cli.py +58 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/ftp.py +109 -32
- quick_spice_manager-0.2.4/src/quick_spice_manager/reduce.py +625 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/.gitignore +0 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/LICENSE +0 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/dirs.py +0 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/locking.py +0 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/py.typed +0 -0
- {quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/spice_manager.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: quick-spice-manager
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Project-URL: documentation, https://luca-penasa.github.io/quick-spice-manager
|
|
5
5
|
Project-URL: homepage, https://luca-penasa.github.io/quick-spice-manager
|
|
6
6
|
Project-URL: repository, https://github.com/luca-penasa/quick-spice-manager
|
|
@@ -87,6 +87,8 @@ qsm list -s juice --filter plan # metakernels on the ESA FTP server
|
|
|
87
87
|
qsm download plan -s juice # download the metakernel and its kernels
|
|
88
88
|
qsm resolve plan -s juice # same, then print a furnshable metakernel path
|
|
89
89
|
qsm info plan -s juice # what's cached, without using the network
|
|
90
|
+
qsm reduce plan -w 2032-12-18 2033-01-09 -o test_kernels --drop 'dsk/*'
|
|
91
|
+
# a copy holding only that time window
|
|
90
92
|
qsm cache list # size and location of each cached mission
|
|
91
93
|
qsm cache clear juice # delete a mission's cached kernels
|
|
92
94
|
```
|
|
@@ -222,6 +224,45 @@ SPICE's 80-character limit are split with `+` continuation. When the pool's
|
|
|
222
224
|
metakernel is known, its `SKD_VERSION` / `MK_IDENTIFIER` are copied into the
|
|
223
225
|
snapshot, and the kernels loaded on top of it are listed in its comments.
|
|
224
226
|
|
|
227
|
+
### Reduced kernel sets (test data)
|
|
228
|
+
|
|
229
|
+
`reduce_kernels` writes a copy of a metakernel and its kernels that holds only
|
|
230
|
+
the data inside some time windows, for example test data small enough to commit
|
|
231
|
+
to a repository:
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
from quick_spice_manager import reduce_kernels
|
|
235
|
+
|
|
236
|
+
result = reduce_kernels(
|
|
237
|
+
"5_1_150lb_23_1_a3",
|
|
238
|
+
windows=[("2032-12-18", "2033-01-09"), ("2032-09-01", "2032-10-01")],
|
|
239
|
+
out_dir="tests/data/spice",
|
|
240
|
+
drop=["dsk/*"], # kernels the workload does not need
|
|
241
|
+
)
|
|
242
|
+
print(result.summary()) # what happened to each kernel, and the sizes
|
|
243
|
+
QuickSpiceManager(mk=result.metakernel).load_kernels()
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
The data inside the windows is copied, not resampled, so the reduced set gives
|
|
247
|
+
the same answers there:
|
|
248
|
+
|
|
249
|
+
| kernel | written |
|
|
250
|
+
|--------|---------|
|
|
251
|
+
| SPK, every type | the records covering each window (`spksub`) |
|
|
252
|
+
| CK type 3 | the pointing records covering each window, with their interpolation intervals |
|
|
253
|
+
| binary PCK type 2 | the Chebyshev records covering each window (equal to rounding) |
|
|
254
|
+
| anything else | copied whole: text kernels, DSKs, CKs of other types (spiceypy cannot write type 6), a CK whose SCLK is not defined |
|
|
255
|
+
|
|
256
|
+
Each window is padded by a day (`pad=`, in seconds), so interpolation and
|
|
257
|
+
extrapolation near its edges still work; overlapping windows are merged. A
|
|
258
|
+
binary kernel without data in any window is left out. The reduced metakernel
|
|
259
|
+
(`out_dir/mk/<name>.tm`, `PATH_VALUES = ( '..' )`, the layout of ESA's) and each
|
|
260
|
+
reduced kernel's comment area record the source and the windows; the kernels
|
|
261
|
+
keep their own comments.
|
|
262
|
+
|
|
263
|
+
For JUICE's `juice_crema_5_1_150lb_23_1_a3` (882 MB), a 24-day window without
|
|
264
|
+
the DSKs is 29 MB, 24.5 MB of which is the type 6 attitude CK, copied whole.
|
|
265
|
+
|
|
225
266
|
### Pool management
|
|
226
267
|
|
|
227
268
|
```python
|
|
@@ -51,6 +51,8 @@ qsm list -s juice --filter plan # metakernels on the ESA FTP server
|
|
|
51
51
|
qsm download plan -s juice # download the metakernel and its kernels
|
|
52
52
|
qsm resolve plan -s juice # same, then print a furnshable metakernel path
|
|
53
53
|
qsm info plan -s juice # what's cached, without using the network
|
|
54
|
+
qsm reduce plan -w 2032-12-18 2033-01-09 -o test_kernels --drop 'dsk/*'
|
|
55
|
+
# a copy holding only that time window
|
|
54
56
|
qsm cache list # size and location of each cached mission
|
|
55
57
|
qsm cache clear juice # delete a mission's cached kernels
|
|
56
58
|
```
|
|
@@ -186,6 +188,45 @@ SPICE's 80-character limit are split with `+` continuation. When the pool's
|
|
|
186
188
|
metakernel is known, its `SKD_VERSION` / `MK_IDENTIFIER` are copied into the
|
|
187
189
|
snapshot, and the kernels loaded on top of it are listed in its comments.
|
|
188
190
|
|
|
191
|
+
### Reduced kernel sets (test data)
|
|
192
|
+
|
|
193
|
+
`reduce_kernels` writes a copy of a metakernel and its kernels that holds only
|
|
194
|
+
the data inside some time windows, for example test data small enough to commit
|
|
195
|
+
to a repository:
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from quick_spice_manager import reduce_kernels
|
|
199
|
+
|
|
200
|
+
result = reduce_kernels(
|
|
201
|
+
"5_1_150lb_23_1_a3",
|
|
202
|
+
windows=[("2032-12-18", "2033-01-09"), ("2032-09-01", "2032-10-01")],
|
|
203
|
+
out_dir="tests/data/spice",
|
|
204
|
+
drop=["dsk/*"], # kernels the workload does not need
|
|
205
|
+
)
|
|
206
|
+
print(result.summary()) # what happened to each kernel, and the sizes
|
|
207
|
+
QuickSpiceManager(mk=result.metakernel).load_kernels()
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
The data inside the windows is copied, not resampled, so the reduced set gives
|
|
211
|
+
the same answers there:
|
|
212
|
+
|
|
213
|
+
| kernel | written |
|
|
214
|
+
|--------|---------|
|
|
215
|
+
| SPK, every type | the records covering each window (`spksub`) |
|
|
216
|
+
| CK type 3 | the pointing records covering each window, with their interpolation intervals |
|
|
217
|
+
| binary PCK type 2 | the Chebyshev records covering each window (equal to rounding) |
|
|
218
|
+
| anything else | copied whole: text kernels, DSKs, CKs of other types (spiceypy cannot write type 6), a CK whose SCLK is not defined |
|
|
219
|
+
|
|
220
|
+
Each window is padded by a day (`pad=`, in seconds), so interpolation and
|
|
221
|
+
extrapolation near its edges still work; overlapping windows are merged. A
|
|
222
|
+
binary kernel without data in any window is left out. The reduced metakernel
|
|
223
|
+
(`out_dir/mk/<name>.tm`, `PATH_VALUES = ( '..' )`, the layout of ESA's) and each
|
|
224
|
+
reduced kernel's comment area record the source and the windows; the kernels
|
|
225
|
+
keep their own comments.
|
|
226
|
+
|
|
227
|
+
For JUICE's `juice_crema_5_1_150lb_23_1_a3` (882 MB), a 24-day window without
|
|
228
|
+
the DSKs is 29 MB, 24.5 MB of which is the type 6 attitude CK, copied whole.
|
|
229
|
+
|
|
189
230
|
### Pool management
|
|
190
231
|
|
|
191
232
|
```python
|
|
@@ -31,9 +31,17 @@ def log_disable(mod: str = "quick_spice_manager") -> None:
|
|
|
31
31
|
log.disable(mod)
|
|
32
32
|
|
|
33
33
|
|
|
34
|
+
from .ftp import KernelDownloadError
|
|
35
|
+
from .reduce import reduce_kernels
|
|
34
36
|
from .spice_manager import QuickSpiceManager, snapshot_pool
|
|
35
37
|
|
|
36
|
-
__all__ = [
|
|
38
|
+
__all__ = [
|
|
39
|
+
"KernelDownloadError",
|
|
40
|
+
"QuickSpiceManager",
|
|
41
|
+
"SpiceManager",
|
|
42
|
+
"reduce_kernels",
|
|
43
|
+
"snapshot_pool",
|
|
44
|
+
]
|
|
37
45
|
|
|
38
46
|
|
|
39
47
|
def __getattr__(name: str) -> object:
|
|
@@ -23,6 +23,7 @@ from .ftp import (
|
|
|
23
23
|
_ftp_base,
|
|
24
24
|
list_metakernels_via_ftp,
|
|
25
25
|
)
|
|
26
|
+
from .reduce import DEFAULT_PAD_SECONDS, _reduce_with
|
|
26
27
|
from .spice_manager import (
|
|
27
28
|
QuickSpiceManager,
|
|
28
29
|
_lookup_localized,
|
|
@@ -269,6 +270,63 @@ def info(sm: QuickSpiceManager, *, as_json: bool) -> None:
|
|
|
269
270
|
click.echo(f"{key + ':':<15}{'-' if value is None else value}")
|
|
270
271
|
|
|
271
272
|
|
|
273
|
+
@cli.command()
|
|
274
|
+
@_kernel_options()
|
|
275
|
+
@click.option(
|
|
276
|
+
"-w",
|
|
277
|
+
"--window",
|
|
278
|
+
"windows",
|
|
279
|
+
nargs=2,
|
|
280
|
+
multiple=True,
|
|
281
|
+
required=True,
|
|
282
|
+
metavar="START STOP",
|
|
283
|
+
help="A time window (UTC, anything str2et reads); repeat for more.",
|
|
284
|
+
)
|
|
285
|
+
@click.option(
|
|
286
|
+
"-o",
|
|
287
|
+
"--output",
|
|
288
|
+
"out_dir",
|
|
289
|
+
required=True,
|
|
290
|
+
type=click.Path(file_okay=False, path_type=Path),
|
|
291
|
+
help="Directory for the reduced kernels and mk/<name>.tm.",
|
|
292
|
+
)
|
|
293
|
+
@click.option(
|
|
294
|
+
"--drop",
|
|
295
|
+
multiple=True,
|
|
296
|
+
metavar="GLOB",
|
|
297
|
+
help="Leave out kernels whose relative path matches, e.g. 'dsk/*'; repeat for more.",
|
|
298
|
+
)
|
|
299
|
+
@click.option(
|
|
300
|
+
"--pad",
|
|
301
|
+
type=float,
|
|
302
|
+
default=DEFAULT_PAD_SECONDS,
|
|
303
|
+
show_default=True,
|
|
304
|
+
help="Seconds added before and after each window.",
|
|
305
|
+
)
|
|
306
|
+
@click.option("--name", help="Stem of the reduced metakernel [default: <source>_reduced].")
|
|
307
|
+
def reduce( # noqa: PLR0913 - one option each
|
|
308
|
+
sm: QuickSpiceManager,
|
|
309
|
+
windows: tuple[tuple[str, str], ...],
|
|
310
|
+
out_dir: Path,
|
|
311
|
+
drop: tuple[str, ...],
|
|
312
|
+
pad: float,
|
|
313
|
+
name: str | None,
|
|
314
|
+
) -> None:
|
|
315
|
+
"""Write a copy of a metakernel holding only the data inside time windows.
|
|
316
|
+
|
|
317
|
+
SPKs, type 3 CKs and type 2 binary PCKs keep the records covering the
|
|
318
|
+
windows, so the reduced set gives the same answers there. Other kernels
|
|
319
|
+
are copied whole, binary kernels without data in the windows are left
|
|
320
|
+
out. Prints what happened to each kernel; the metakernel path is the
|
|
321
|
+
last line on stdout.
|
|
322
|
+
|
|
323
|
+
qsm reduce 5_1_150lb_23_1_a3 -w 2032-12-18 2033-01-09 -o tests/data/spice --drop 'dsk/*'
|
|
324
|
+
"""
|
|
325
|
+
result = _reduce_with(sm, windows, out_dir, drop=drop, pad=pad, name=name)
|
|
326
|
+
click.echo(result.summary(), err=True)
|
|
327
|
+
click.echo(result.metakernel.resolve())
|
|
328
|
+
|
|
329
|
+
|
|
272
330
|
@cli.group()
|
|
273
331
|
def cache() -> None:
|
|
274
332
|
"""Inspect or clear the kernels cache."""
|
|
@@ -61,6 +61,31 @@ _SLOT_POLL_INTERVAL_SECONDS = 0.1
|
|
|
61
61
|
# How long to wait for a free connection slot before giving up.
|
|
62
62
|
_CONNECTION_SLOT_TIMEOUT_SECONDS = 300
|
|
63
63
|
|
|
64
|
+
# Kernels whose download failed are tried again this many times in all, with a
|
|
65
|
+
# pause of _RETRY_DELAY_SECONDS * attempt in between (a busy server drops
|
|
66
|
+
# connections; it usually answers again after a short wait).
|
|
67
|
+
_DOWNLOAD_ATTEMPTS = 3
|
|
68
|
+
_RETRY_DELAY_SECONDS = 5.0
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class KernelDownloadError(ConnectionError):
|
|
72
|
+
"""Some kernels of a metakernel could not be downloaded, even after retries.
|
|
73
|
+
|
|
74
|
+
``missing`` holds their paths relative to the kernels directory, and
|
|
75
|
+
``errors`` the last error for each of them.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(self, metakernel: Path, missing: dict[str, BaseException]) -> None:
|
|
79
|
+
self.metakernel = metakernel
|
|
80
|
+
self.missing = sorted(missing)
|
|
81
|
+
self.errors = missing
|
|
82
|
+
listing = "\n".join(f" {rel}: {missing[rel]}" for rel in self.missing)
|
|
83
|
+
super().__init__(
|
|
84
|
+
f"{len(missing)} kernel(s) of {metakernel.name} could not be downloaded "
|
|
85
|
+
f"from {_FTP_HOST} after {_DOWNLOAD_ATTEMPTS} attempts:\n{listing}\n"
|
|
86
|
+
"Try again later: kernels already on disk are kept.",
|
|
87
|
+
)
|
|
88
|
+
|
|
64
89
|
# ---------------------------------------------------------------------------
|
|
65
90
|
# FTP server and mission path constants
|
|
66
91
|
# ---------------------------------------------------------------------------
|
|
@@ -453,8 +478,13 @@ def _ftp_download_file(ftp: ftplib.FTP, remote: str, local: Path) -> None:
|
|
|
453
478
|
a partially-written file: it sees either the previous complete file (if
|
|
454
479
|
any) or the new complete one, never a truncated in-between state. On any
|
|
455
480
|
failure the temp file is removed and the exception re-raised.
|
|
481
|
+
|
|
482
|
+
When the server reports the file size (FTP ``SIZE``), a transfer that
|
|
483
|
+
ends short of it raises :class:`ftplib.error_temp`, so a dropped data
|
|
484
|
+
connection never leaves a truncated kernel in the cache.
|
|
456
485
|
"""
|
|
457
486
|
local.parent.mkdir(parents=True, exist_ok=True)
|
|
487
|
+
expected = _remote_size(ftp, remote)
|
|
458
488
|
tmp_fd, tmp_name = tempfile.mkstemp(
|
|
459
489
|
dir=local.parent, prefix=f"{local.name}.", suffix=".part",
|
|
460
490
|
)
|
|
@@ -462,12 +492,26 @@ def _ftp_download_file(ftp: ftplib.FTP, remote: str, local: Path) -> None:
|
|
|
462
492
|
try:
|
|
463
493
|
with os.fdopen(tmp_fd, "wb") as fh:
|
|
464
494
|
ftp.retrbinary(f"RETR {remote}", fh.write)
|
|
495
|
+
got = tmp_path.stat().st_size
|
|
496
|
+
if expected is not None and got != expected:
|
|
497
|
+
msg = f"transfer of {remote} incomplete: {got} of {expected} bytes"
|
|
498
|
+
raise ftplib.error_temp(msg)
|
|
465
499
|
tmp_path.replace(local) # atomic rename, same filesystem
|
|
466
500
|
except BaseException:
|
|
467
501
|
tmp_path.unlink(missing_ok=True)
|
|
468
502
|
raise
|
|
469
503
|
|
|
470
504
|
|
|
505
|
+
def _remote_size(ftp: ftplib.FTP, remote: str) -> int | None:
|
|
506
|
+
"""Size of *remote* in bytes, or ``None`` when the server does not tell."""
|
|
507
|
+
try:
|
|
508
|
+
ftp.voidcmd("TYPE I") # SIZE is defined for binary transfers
|
|
509
|
+
size = ftp.size(remote)
|
|
510
|
+
except ftplib.error_perm:
|
|
511
|
+
return None
|
|
512
|
+
return size if isinstance(size, int) else None
|
|
513
|
+
|
|
514
|
+
|
|
471
515
|
def _locked_download(
|
|
472
516
|
ftp: ftplib.FTP, remote: str, local: Path, *, force: bool = False,
|
|
473
517
|
) -> bool:
|
|
@@ -608,6 +652,10 @@ def download_kernels_via_ftp(
|
|
|
608
652
|
ConnectionError
|
|
609
653
|
If the ESA FTP server cannot be reached and no usable local cache
|
|
610
654
|
exists for this ``(spacecraft, mk, version)``.
|
|
655
|
+
KernelDownloadError
|
|
656
|
+
If some referenced kernels could not be downloaded even after
|
|
657
|
+
retries (its ``missing`` lists them). Nothing is recorded in the
|
|
658
|
+
resolution cache, so the next call tries them again.
|
|
611
659
|
FileNotFoundError
|
|
612
660
|
If ``download_kernels=False`` and no complete local cache exists.
|
|
613
661
|
"""
|
|
@@ -651,6 +699,8 @@ def download_kernels_via_ftp(
|
|
|
651
699
|
)
|
|
652
700
|
except FileNotFoundError:
|
|
653
701
|
raise # a genuine "no such kernel/version" answer, not connectivity
|
|
702
|
+
except KernelDownloadError:
|
|
703
|
+
raise # the server answered; some kernels did not come through
|
|
654
704
|
except ftplib.all_errors as exc:
|
|
655
705
|
log.warning(f"FTP: could not reach {_FTP_HOST} ({exc})")
|
|
656
706
|
cached_tm = _usable_cached_tm()
|
|
@@ -745,40 +795,67 @@ def _download_kernels_via_ftp_online(
|
|
|
745
795
|
f"{n_skip} already cached — using {n_workers} parallel connections",
|
|
746
796
|
)
|
|
747
797
|
|
|
748
|
-
# --- Parallel download of missing kernels
|
|
749
|
-
n_dl =
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
) as pbar:
|
|
766
|
-
for fut in as_completed(futures):
|
|
767
|
-
local_kernel = futures[fut]
|
|
768
|
-
exc = fut.exception()
|
|
769
|
-
if exc is None:
|
|
770
|
-
log.info(f"FTP: downloaded {local_kernel.name}")
|
|
771
|
-
n_dl += 1
|
|
772
|
-
pbar.set_postfix_str(local_kernel.name, refresh=False)
|
|
773
|
-
else:
|
|
774
|
-
log.warning(
|
|
775
|
-
f"FTP: could not download '{local_kernel.name}': {exc}",
|
|
776
|
-
)
|
|
777
|
-
n_fail += 1
|
|
778
|
-
pbar.update(1)
|
|
798
|
+
# --- Parallel download of missing kernels, retrying the failed ones -----
|
|
799
|
+
n_dl = 0
|
|
800
|
+
failed: dict[str, BaseException] = {}
|
|
801
|
+
pending = to_download
|
|
802
|
+
for attempt in range(1, _DOWNLOAD_ATTEMPTS + 1):
|
|
803
|
+
if not pending:
|
|
804
|
+
break
|
|
805
|
+
if attempt > 1:
|
|
806
|
+
delay = _RETRY_DELAY_SECONDS * (attempt - 1)
|
|
807
|
+
log.warning(
|
|
808
|
+
f"FTP: {len(pending)} kernel(s) failed, retrying in {delay:.0f} s "
|
|
809
|
+
f"(attempt {attempt}/{_DOWNLOAD_ATTEMPTS})",
|
|
810
|
+
)
|
|
811
|
+
time.sleep(delay)
|
|
812
|
+
failed = _download_batch(pending, kernels_dir, n_workers)
|
|
813
|
+
n_dl += len(pending) - len(failed)
|
|
814
|
+
pending = [(r, loc) for r, loc in pending if _rel(loc, kernels_dir) in failed]
|
|
779
815
|
|
|
780
816
|
log.info(
|
|
781
|
-
f"FTP: done - {n_dl} downloaded, {n_skip} cached, {
|
|
817
|
+
f"FTP: done - {n_dl} downloaded, {n_skip} cached, {len(failed)} failed",
|
|
782
818
|
)
|
|
819
|
+
if failed:
|
|
820
|
+
raise KernelDownloadError(local_tm, failed)
|
|
783
821
|
|
|
784
822
|
return local_tm
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def _rel(local: Path, kernels_dir: Path) -> str:
|
|
826
|
+
return local.relative_to(kernels_dir).as_posix()
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def _download_batch(
|
|
830
|
+
to_download: list[tuple[str, Path]], kernels_dir: Path, n_workers: int,
|
|
831
|
+
) -> dict[str, BaseException]:
|
|
832
|
+
"""Download *to_download* in parallel; return the failures by relative path."""
|
|
833
|
+
failed: dict[str, BaseException] = {}
|
|
834
|
+
futures = {}
|
|
835
|
+
with ThreadPoolExecutor(max_workers=n_workers) as pool:
|
|
836
|
+
for remote_kernel, local_kernel in to_download:
|
|
837
|
+
log.debug(f"FTP: queuing {local_kernel.name}")
|
|
838
|
+
fut = pool.submit(
|
|
839
|
+
_download_one_kernel, _FTP_HOST, remote_kernel, local_kernel,
|
|
840
|
+
)
|
|
841
|
+
futures[fut] = local_kernel
|
|
842
|
+
|
|
843
|
+
with tqdm(
|
|
844
|
+
total=len(to_download),
|
|
845
|
+
desc="Downloading kernels",
|
|
846
|
+
unit="file",
|
|
847
|
+
dynamic_ncols=True,
|
|
848
|
+
) as pbar:
|
|
849
|
+
for fut in as_completed(futures):
|
|
850
|
+
local_kernel = futures[fut]
|
|
851
|
+
exc = fut.exception()
|
|
852
|
+
if exc is None:
|
|
853
|
+
log.info(f"FTP: downloaded {local_kernel.name}")
|
|
854
|
+
pbar.set_postfix_str(local_kernel.name, refresh=False)
|
|
855
|
+
else:
|
|
856
|
+
log.warning(
|
|
857
|
+
f"FTP: could not download '{local_kernel.name}': {exc}",
|
|
858
|
+
)
|
|
859
|
+
failed[_rel(local_kernel, kernels_dir)] = exc
|
|
860
|
+
pbar.update(1)
|
|
861
|
+
return failed
|
|
@@ -0,0 +1,625 @@
|
|
|
1
|
+
"""Reduced kernel sets: a metakernel cut down to some time windows.
|
|
2
|
+
|
|
3
|
+
:func:`reduce_kernels` writes a copy of a metakernel and its kernels that
|
|
4
|
+
holds only the data inside the given time windows, and leaves out the
|
|
5
|
+
kernels a workload does not need. The data inside the windows is copied, not
|
|
6
|
+
resampled, so a reduced set gives the same answers as the full set there.
|
|
7
|
+
The typical use is test data small enough to commit to a repository.
|
|
8
|
+
|
|
9
|
+
================= ==========================================================
|
|
10
|
+
kind what is written
|
|
11
|
+
================= ==========================================================
|
|
12
|
+
SPK (all types) the records covering each window (``spksub``)
|
|
13
|
+
CK type 3 the pointing records covering each window, with the
|
|
14
|
+
interpolation intervals they belong to (``ckw03``)
|
|
15
|
+
binary PCK type 2 the Chebyshev records covering each window (``pckw02``)
|
|
16
|
+
anything else copied whole: text kernels, DSKs, CK types other than 3,
|
|
17
|
+
binary kernels with any unsupported segment
|
|
18
|
+
================= ==========================================================
|
|
19
|
+
|
|
20
|
+
A binary kernel without data in any window is left out of the reduced
|
|
21
|
+
metakernel.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import fnmatch
|
|
27
|
+
import shutil
|
|
28
|
+
import textwrap
|
|
29
|
+
from collections.abc import Iterable, Iterator, Sequence
|
|
30
|
+
from contextlib import contextmanager
|
|
31
|
+
from dataclasses import dataclass, field
|
|
32
|
+
from importlib import metadata
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Literal
|
|
35
|
+
|
|
36
|
+
import numpy as np
|
|
37
|
+
import spiceypy
|
|
38
|
+
from loguru import logger as log
|
|
39
|
+
from spiceypy.utils.exceptions import NotFoundError, SpiceyError
|
|
40
|
+
|
|
41
|
+
from .ftp import _parse_mk_kernel_paths
|
|
42
|
+
from .spice_manager import (
|
|
43
|
+
QuickSpiceManager,
|
|
44
|
+
_lookup_localized,
|
|
45
|
+
_pool_lock,
|
|
46
|
+
_quoted_lines,
|
|
47
|
+
_read_mk_keywords,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
#: Seconds added before and after each window by default (one day): the
|
|
51
|
+
#: records next to a window's edges are kept, so are interpolation and a
|
|
52
|
+
#: solver's extrapolation at block edges.
|
|
53
|
+
DEFAULT_PAD_SECONDS = 86400.0
|
|
54
|
+
|
|
55
|
+
Window = tuple["str | float", "str | float"]
|
|
56
|
+
Action = Literal["subset", "copied", "dropped", "no data"]
|
|
57
|
+
|
|
58
|
+
_DAF_EXTENSIONS = {".bsp": "spk", ".bc": "ck", ".bpc": "pck"}
|
|
59
|
+
_TEXT_EXTENSIONS = {".tls", ".tsc", ".tf", ".ti", ".tpc", ".tm", ".txt"}
|
|
60
|
+
_COMMENT_LINE = 79
|
|
61
|
+
_KB_100 = 100_000
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class _CopyWhole(Exception): # noqa: N818 - control flow, never escapes
|
|
65
|
+
"""A kernel that can't be subsetted after all: copy it whole, say why."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class KernelReport:
|
|
70
|
+
"""What :func:`reduce_kernels` did with one kernel."""
|
|
71
|
+
|
|
72
|
+
source: Path
|
|
73
|
+
relative: str
|
|
74
|
+
action: Action
|
|
75
|
+
output: Path | None = None
|
|
76
|
+
note: str = ""
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def size_in(self) -> int:
|
|
80
|
+
return self.source.stat().st_size
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def size_out(self) -> int:
|
|
84
|
+
return self.output.stat().st_size if self.output else 0
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class ReducedKernels:
|
|
89
|
+
"""The reduced metakernel and what happened to each source kernel."""
|
|
90
|
+
|
|
91
|
+
metakernel: Path
|
|
92
|
+
windows_et: list[tuple[float, float]]
|
|
93
|
+
kernels: list[KernelReport] = field(default_factory=list)
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def size_in(self) -> int:
|
|
97
|
+
return sum(k.size_in for k in self.kernels)
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def size_out(self) -> int:
|
|
101
|
+
return sum(k.size_out for k in self.kernels)
|
|
102
|
+
|
|
103
|
+
def summary(self) -> str:
|
|
104
|
+
"""One line per kernel, then the totals."""
|
|
105
|
+
lines = []
|
|
106
|
+
for k in self.kernels:
|
|
107
|
+
sizes = f"{_mb(k.size_in):>9} -> {_mb(k.size_out):>9}"
|
|
108
|
+
note = f" ({k.note})" if k.note else ""
|
|
109
|
+
lines.append(f"{k.action:8} {sizes} {k.relative}{note}")
|
|
110
|
+
lines.append(
|
|
111
|
+
f"{len(self.kernels)} kernels: {_mb(self.size_in)} -> {_mb(self.size_out)}"
|
|
112
|
+
f"; metakernel {self.metakernel}",
|
|
113
|
+
)
|
|
114
|
+
return "\n".join(lines)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def reduce_kernels( # noqa: PLR0913 - keyword options of one operation
|
|
118
|
+
mk: str | Path,
|
|
119
|
+
windows: Iterable[Window],
|
|
120
|
+
out_dir: str | Path,
|
|
121
|
+
*,
|
|
122
|
+
drop: Sequence[str] = (),
|
|
123
|
+
pad: float = DEFAULT_PAD_SECONDS,
|
|
124
|
+
name: str | None = None,
|
|
125
|
+
spacecraft: str = "JUICE",
|
|
126
|
+
version: str = "latest",
|
|
127
|
+
kernels_dir: str | Path | None = None,
|
|
128
|
+
download_kernels: bool = True,
|
|
129
|
+
) -> ReducedKernels:
|
|
130
|
+
"""Write a copy of *mk* that holds only the data inside *windows*.
|
|
131
|
+
|
|
132
|
+
Parameters
|
|
133
|
+
----------
|
|
134
|
+
mk:
|
|
135
|
+
Metakernel name (resolved and downloaded like ``QuickSpiceManager(mk=...)``)
|
|
136
|
+
or path.
|
|
137
|
+
windows:
|
|
138
|
+
``(start, stop)`` pairs, as UTC strings (anything ``str2et`` reads) or
|
|
139
|
+
ephemeris times. Overlapping windows are merged after padding.
|
|
140
|
+
out_dir:
|
|
141
|
+
Where to write. The kernels keep their paths relative to the source
|
|
142
|
+
kernels directory (``spk/...``, ``ck/...``) and the metakernel goes to
|
|
143
|
+
``mk/<name>.tm`` with ``PATH_VALUES = ( '..' )``, the layout of ESA's
|
|
144
|
+
metakernels: ``QuickSpiceManager(mk=<that path>)`` loads it from any
|
|
145
|
+
directory, ``furnsh`` from ``out_dir/mk``.
|
|
146
|
+
drop:
|
|
147
|
+
Glob patterns, matched against each kernel's relative path (e.g.
|
|
148
|
+
``"dsk/*"``), of kernels to leave out.
|
|
149
|
+
pad:
|
|
150
|
+
Seconds added before and after each window.
|
|
151
|
+
name:
|
|
152
|
+
Stem of the reduced metakernel; default ``<source stem>_reduced``.
|
|
153
|
+
spacecraft, version, kernels_dir, download_kernels:
|
|
154
|
+
As for :class:`QuickSpiceManager`.
|
|
155
|
+
|
|
156
|
+
Returns
|
|
157
|
+
-------
|
|
158
|
+
ReducedKernels
|
|
159
|
+
The metakernel written, and a report per source kernel.
|
|
160
|
+
|
|
161
|
+
Raises
|
|
162
|
+
------
|
|
163
|
+
ValueError
|
|
164
|
+
If no window is given, or a window ends before it starts.
|
|
165
|
+
"""
|
|
166
|
+
sm = QuickSpiceManager(
|
|
167
|
+
spacecraft=spacecraft,
|
|
168
|
+
mk=str(mk),
|
|
169
|
+
version=version,
|
|
170
|
+
kernels_dir=kernels_dir,
|
|
171
|
+
download_kernels=download_kernels,
|
|
172
|
+
)
|
|
173
|
+
return _reduce_with(sm, windows, Path(out_dir), drop=drop, pad=pad, name=name)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _reduce_with( # noqa: PLR0913
|
|
177
|
+
sm: QuickSpiceManager,
|
|
178
|
+
windows: Iterable[Window],
|
|
179
|
+
out_dir: Path,
|
|
180
|
+
*,
|
|
181
|
+
drop: Sequence[str],
|
|
182
|
+
pad: float,
|
|
183
|
+
name: str | None,
|
|
184
|
+
) -> ReducedKernels:
|
|
185
|
+
""":func:`reduce_kernels` with the manager already made (the CLI makes its own)."""
|
|
186
|
+
localized = sm.resolved_mk
|
|
187
|
+
info = _lookup_localized(localized)
|
|
188
|
+
if info is None: # pragma: no cover - registered by resolved_mk
|
|
189
|
+
msg = f"could not resolve {localized}"
|
|
190
|
+
raise RuntimeError(msg)
|
|
191
|
+
source_mk, root = info.source, info.kernels_dir
|
|
192
|
+
relatives = _parse_mk_kernel_paths(
|
|
193
|
+
source_mk.read_text(encoding="utf-8", errors="replace"),
|
|
194
|
+
)
|
|
195
|
+
sources = [(rel, _source_path(root, rel)) for rel in relatives]
|
|
196
|
+
|
|
197
|
+
with _pool_lock, _text_kernels_loaded([p for _, p in sources]):
|
|
198
|
+
windows_et = _merge_windows(windows, pad)
|
|
199
|
+
reports = [
|
|
200
|
+
_reduce_one(rel, path, out_dir, windows_et, drop) for rel, path in sources
|
|
201
|
+
]
|
|
202
|
+
result = ReducedKernels(
|
|
203
|
+
metakernel=out_dir / "mk" / f"{name or source_mk.stem + '_reduced'}.tm",
|
|
204
|
+
windows_et=windows_et,
|
|
205
|
+
kernels=reports,
|
|
206
|
+
)
|
|
207
|
+
_write_metakernel(result, source_mk, drop)
|
|
208
|
+
log.info(
|
|
209
|
+
f"reduced {source_mk.name}: {_mb(result.size_in)} -> {_mb(result.size_out)}",
|
|
210
|
+
)
|
|
211
|
+
return result
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
# Windows and the temporary text kernels
|
|
216
|
+
# ---------------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _source_path(root: Path, rel: str) -> Path:
|
|
220
|
+
path = Path(rel)
|
|
221
|
+
return path if path.is_absolute() else root / rel
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _output_relative(rel: str) -> str:
|
|
225
|
+
"""Relative path of a kernel in the output (absolute ones go to ``other/``)."""
|
|
226
|
+
path = Path(rel)
|
|
227
|
+
return f"other/{path.name}" if path.is_absolute() else path.as_posix()
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@contextmanager
|
|
231
|
+
def _text_kernels_loaded(kernels: Sequence[Path]) -> Iterator[None]:
|
|
232
|
+
"""Load the text kernels (LSK, SCLK, FK, ...) needed to convert times and
|
|
233
|
+
name frames; unload afterwards only those that were not loaded before."""
|
|
234
|
+
added = []
|
|
235
|
+
for path in kernels:
|
|
236
|
+
if path.suffix.lower() not in _TEXT_EXTENSIONS or path.suffix.lower() == ".tm":
|
|
237
|
+
continue
|
|
238
|
+
try:
|
|
239
|
+
spiceypy.kinfo(str(path))
|
|
240
|
+
except NotFoundError:
|
|
241
|
+
spiceypy.furnsh(str(path))
|
|
242
|
+
added.append(str(path))
|
|
243
|
+
try:
|
|
244
|
+
yield
|
|
245
|
+
finally:
|
|
246
|
+
for path in reversed(added):
|
|
247
|
+
spiceypy.unload(path)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _to_et(value: str | float) -> float:
|
|
251
|
+
return float(value) if isinstance(value, (int, float)) else spiceypy.str2et(value)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _merge_windows(windows: Iterable[Window], pad: float) -> list[tuple[float, float]]:
|
|
255
|
+
spans = []
|
|
256
|
+
for start, stop in windows:
|
|
257
|
+
lo, hi = _to_et(start), _to_et(stop)
|
|
258
|
+
if hi < lo:
|
|
259
|
+
msg = f"window ends before it starts: {start!r} -> {stop!r}"
|
|
260
|
+
raise ValueError(msg)
|
|
261
|
+
spans.append((lo - pad, hi + pad))
|
|
262
|
+
if not spans:
|
|
263
|
+
msg = "no time window given"
|
|
264
|
+
raise ValueError(msg)
|
|
265
|
+
spans.sort()
|
|
266
|
+
merged = [spans[0]]
|
|
267
|
+
for lo, hi in spans[1:]:
|
|
268
|
+
if lo <= merged[-1][1]:
|
|
269
|
+
merged[-1] = (merged[-1][0], max(merged[-1][1], hi))
|
|
270
|
+
else:
|
|
271
|
+
merged.append((lo, hi))
|
|
272
|
+
return merged
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
# ---------------------------------------------------------------------------
|
|
276
|
+
# One kernel
|
|
277
|
+
# ---------------------------------------------------------------------------
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _reduce_one(
|
|
281
|
+
rel: str,
|
|
282
|
+
source: Path,
|
|
283
|
+
out_dir: Path,
|
|
284
|
+
windows: list[tuple[float, float]],
|
|
285
|
+
drop: Sequence[str],
|
|
286
|
+
) -> KernelReport:
|
|
287
|
+
out_rel = _output_relative(rel)
|
|
288
|
+
if any(fnmatch.fnmatch(out_rel, pattern) for pattern in drop):
|
|
289
|
+
return KernelReport(source, out_rel, "dropped")
|
|
290
|
+
output = out_dir / out_rel
|
|
291
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
292
|
+
output.unlink(missing_ok=True) # DAF writers refuse to overwrite
|
|
293
|
+
|
|
294
|
+
kind = _DAF_EXTENSIONS.get(source.suffix.lower())
|
|
295
|
+
if kind is not None:
|
|
296
|
+
unsupported = _unsupported_segments(source, kind)
|
|
297
|
+
note = f"copied whole: {kind.upper()} segment types {sorted(unsupported)} not subsetted"
|
|
298
|
+
if not unsupported:
|
|
299
|
+
try:
|
|
300
|
+
written = _SUBSETTERS[kind](source, output, windows)
|
|
301
|
+
except _CopyWhole as exc:
|
|
302
|
+
output.unlink(missing_ok=True)
|
|
303
|
+
written, note = -1, f"copied whole: {exc}"
|
|
304
|
+
if written == 0:
|
|
305
|
+
output.unlink(missing_ok=True)
|
|
306
|
+
return KernelReport(
|
|
307
|
+
source,
|
|
308
|
+
out_rel,
|
|
309
|
+
"no data",
|
|
310
|
+
note="no data in the windows",
|
|
311
|
+
)
|
|
312
|
+
if written > 0:
|
|
313
|
+
return KernelReport(
|
|
314
|
+
source,
|
|
315
|
+
out_rel,
|
|
316
|
+
"subset",
|
|
317
|
+
output,
|
|
318
|
+
f"{written} segments",
|
|
319
|
+
)
|
|
320
|
+
else:
|
|
321
|
+
note = ""
|
|
322
|
+
shutil.copyfile(source, output)
|
|
323
|
+
return KernelReport(source, out_rel, "copied", output, note)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _segments(
|
|
327
|
+
handle: int,
|
|
328
|
+
nd: int,
|
|
329
|
+
ni: int,
|
|
330
|
+
) -> Iterator[tuple[np.ndarray, list[float], list[int], str]]:
|
|
331
|
+
spiceypy.dafbfs(handle)
|
|
332
|
+
while spiceypy.daffna():
|
|
333
|
+
descr = spiceypy.dafgs(nd + (ni + 1) // 2)
|
|
334
|
+
dc, ic = spiceypy.dafus(descr, nd, ni)
|
|
335
|
+
yield descr, list(dc), [int(i) for i in ic], spiceypy.dafgn(100).strip()
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
_LAYOUT = {"spk": (2, 6), "ck": (2, 6), "pck": (2, 5)}
|
|
339
|
+
_TYPE_INDEX = {"spk": 3, "ck": 2, "pck": 2}
|
|
340
|
+
_SUPPORTED = {"spk": None, "ck": {3}, "pck": {2}} # None: every type spksub reads
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _unsupported_segments(source: Path, kind: str) -> set[int]:
|
|
344
|
+
nd, ni = _LAYOUT[kind]
|
|
345
|
+
supported = _SUPPORTED[kind]
|
|
346
|
+
if supported is None:
|
|
347
|
+
return set()
|
|
348
|
+
handle = spiceypy.dafopr(str(source))
|
|
349
|
+
try:
|
|
350
|
+
types = {ic[_TYPE_INDEX[kind]] for _, _, ic, _ in _segments(handle, nd, ni)}
|
|
351
|
+
finally:
|
|
352
|
+
spiceypy.dafcls(handle)
|
|
353
|
+
return types - supported
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _overlaps(
|
|
357
|
+
begin: float,
|
|
358
|
+
end: float,
|
|
359
|
+
windows: list[tuple[float, float]],
|
|
360
|
+
) -> Iterator[tuple[float, float]]:
|
|
361
|
+
for lo, hi in windows:
|
|
362
|
+
if lo <= end and hi >= begin:
|
|
363
|
+
yield max(lo, begin), min(hi, end)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _subset_spk(source: Path, output: Path, windows: list[tuple[float, float]]) -> int:
|
|
367
|
+
with _new_daf(source, output, windows, "spk") as (handle, new, written):
|
|
368
|
+
for descr, dc, _ic, name in _segments(handle, 2, 6):
|
|
369
|
+
for lo, hi in _overlaps(dc[0], dc[1], windows):
|
|
370
|
+
spiceypy.spksub(handle, descr, name, lo, hi, new)
|
|
371
|
+
written[0] += 1
|
|
372
|
+
return written[0]
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _subset_ck(source: Path, output: Path, windows: list[tuple[float, float]]) -> int:
|
|
376
|
+
with _new_daf(source, output, windows, "ck") as (handle, new, written):
|
|
377
|
+
for _descr, dc, ic, name in _segments(handle, 2, 6):
|
|
378
|
+
inst, ref, _type, avflag, begin, end = ic
|
|
379
|
+
sclk = spiceypy.ckmeta(inst, "SCLK")
|
|
380
|
+
for lo_et, hi_et in windows:
|
|
381
|
+
try:
|
|
382
|
+
lo, hi = spiceypy.sce2c(sclk, lo_et), spiceypy.sce2c(sclk, hi_et)
|
|
383
|
+
except SpiceyError as exc:
|
|
384
|
+
msg = f"no SCLK {sclk} for CK instrument {inst} ({exc.short})"
|
|
385
|
+
raise _CopyWhole(msg) from exc
|
|
386
|
+
if lo > dc[1] or hi < dc[0]:
|
|
387
|
+
continue
|
|
388
|
+
written[0] += _write_ck03_window(
|
|
389
|
+
handle,
|
|
390
|
+
new,
|
|
391
|
+
(begin, end),
|
|
392
|
+
(dc[0], dc[1]),
|
|
393
|
+
(lo, hi),
|
|
394
|
+
inst=inst,
|
|
395
|
+
ref=spiceypy.frmnam(ref),
|
|
396
|
+
avflag=bool(avflag),
|
|
397
|
+
name=name,
|
|
398
|
+
)
|
|
399
|
+
return written[0]
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _write_ck03_window( # noqa: PLR0913
|
|
403
|
+
handle: int,
|
|
404
|
+
new: int,
|
|
405
|
+
addresses: tuple[int, int],
|
|
406
|
+
coverage: tuple[float, float],
|
|
407
|
+
window: tuple[float, float],
|
|
408
|
+
*,
|
|
409
|
+
inst: int,
|
|
410
|
+
ref: str,
|
|
411
|
+
avflag: bool,
|
|
412
|
+
name: str,
|
|
413
|
+
) -> int:
|
|
414
|
+
"""Copy the type 3 records of one segment that cover *window* (SCLK ticks).
|
|
415
|
+
|
|
416
|
+
Layout (CK Required Reading): N records of 4 (quaternion) or 7 (and angular
|
|
417
|
+
velocity) values, N epochs, the epoch directory, NINT interval starts,
|
|
418
|
+
their directory, NINT, N.
|
|
419
|
+
"""
|
|
420
|
+
begin, end = addresses
|
|
421
|
+
nints, n = (int(v) for v in spiceypy.dafgda(handle, end - 1, end))
|
|
422
|
+
size = 7 if avflag else 4
|
|
423
|
+
epochs = np.asarray(
|
|
424
|
+
spiceypy.dafgda(handle, begin + n * size, begin + n * size + n - 1),
|
|
425
|
+
)
|
|
426
|
+
starts_at = begin + n * size + n + (n - 1) // 100
|
|
427
|
+
starts = np.asarray(spiceypy.dafgda(handle, starts_at, starts_at + nints - 1))
|
|
428
|
+
|
|
429
|
+
lo, hi = window
|
|
430
|
+
i0 = max(int(np.searchsorted(epochs, lo, side="right")) - 1, 0)
|
|
431
|
+
i1 = min(int(np.searchsorted(epochs, hi, side="left")), n - 1)
|
|
432
|
+
if i1 < i0:
|
|
433
|
+
return 0
|
|
434
|
+
records = np.asarray(
|
|
435
|
+
spiceypy.dafgda(handle, begin + i0 * size, begin + (i1 + 1) * size - 1),
|
|
436
|
+
).reshape(-1, size)
|
|
437
|
+
sel = epochs[i0 : i1 + 1]
|
|
438
|
+
# the first record starts an interval; the later starts are kept as they are
|
|
439
|
+
sel_starts = np.concatenate(
|
|
440
|
+
[[sel[0]], starts[(starts > sel[0]) & (starts <= sel[-1])]],
|
|
441
|
+
)
|
|
442
|
+
avvs = records[:, 4:] if avflag else np.zeros((len(sel), 3))
|
|
443
|
+
spiceypy.ckw03(
|
|
444
|
+
new,
|
|
445
|
+
max(coverage[0], min(lo, sel[0])),
|
|
446
|
+
min(coverage[1], max(hi, sel[-1])),
|
|
447
|
+
inst,
|
|
448
|
+
ref,
|
|
449
|
+
avflag,
|
|
450
|
+
name,
|
|
451
|
+
len(sel),
|
|
452
|
+
np.ascontiguousarray(sel),
|
|
453
|
+
np.ascontiguousarray(records[:, :4]),
|
|
454
|
+
np.ascontiguousarray(avvs),
|
|
455
|
+
len(sel_starts),
|
|
456
|
+
np.ascontiguousarray(sel_starts),
|
|
457
|
+
)
|
|
458
|
+
return 1
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _subset_pck(source: Path, output: Path, windows: list[tuple[float, float]]) -> int:
|
|
462
|
+
with _new_daf(source, output, windows, "pck") as (handle, new, written):
|
|
463
|
+
for _descr, dc, ic, name in _segments(handle, 2, 5):
|
|
464
|
+
classid, ref, _type, begin, end = ic
|
|
465
|
+
init, intlen, rsize, n = spiceypy.dafgda(handle, end - 3, end)
|
|
466
|
+
rsize, n = int(rsize), int(n)
|
|
467
|
+
degree = (rsize - 2) // 3 - 1
|
|
468
|
+
for lo, hi in _overlaps(dc[0], dc[1], windows):
|
|
469
|
+
k0 = min(max(int((lo - init) // intlen), 0), n - 1)
|
|
470
|
+
k1 = min(max(int((hi - init) // intlen), 0), n - 1)
|
|
471
|
+
records = np.asarray(
|
|
472
|
+
spiceypy.dafgda(
|
|
473
|
+
handle,
|
|
474
|
+
begin + k0 * rsize,
|
|
475
|
+
begin + (k1 + 1) * rsize - 1,
|
|
476
|
+
),
|
|
477
|
+
).reshape(-1, rsize)
|
|
478
|
+
spiceypy.pckw02(
|
|
479
|
+
new,
|
|
480
|
+
classid,
|
|
481
|
+
spiceypy.frmnam(ref),
|
|
482
|
+
lo,
|
|
483
|
+
hi,
|
|
484
|
+
name,
|
|
485
|
+
intlen,
|
|
486
|
+
k1 - k0 + 1,
|
|
487
|
+
degree,
|
|
488
|
+
np.ascontiguousarray(records[:, 2:]).ravel(),
|
|
489
|
+
init + k0 * intlen,
|
|
490
|
+
)
|
|
491
|
+
written[0] += 1
|
|
492
|
+
return written[0]
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
_SUBSETTERS = {"spk": _subset_spk, "ck": _subset_ck, "pck": _subset_pck}
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
# ---------------------------------------------------------------------------
|
|
499
|
+
# Comments and the metakernel
|
|
500
|
+
# ---------------------------------------------------------------------------
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _windows_utc(windows: list[tuple[float, float]]) -> list[str]:
|
|
504
|
+
return [
|
|
505
|
+
f"{spiceypy.et2utc(lo, 'ISOC', 0)} -> {spiceypy.et2utc(hi, 'ISOC', 0)}"
|
|
506
|
+
for lo, hi in windows
|
|
507
|
+
]
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _provenance(
|
|
511
|
+
source: Path,
|
|
512
|
+
windows: list[tuple[float, float]],
|
|
513
|
+
*,
|
|
514
|
+
more: str = "",
|
|
515
|
+
) -> list[str]:
|
|
516
|
+
version = metadata.version("quick_spice_manager")
|
|
517
|
+
lines = [
|
|
518
|
+
f"Reduced by quick-spice-manager {version} from {source.name}.",
|
|
519
|
+
"It holds the data of the source inside these windows (UTC, padded) only:",
|
|
520
|
+
*[f" {w}" for w in _windows_utc(windows)],
|
|
521
|
+
*([more] if more else []),
|
|
522
|
+
"",
|
|
523
|
+
]
|
|
524
|
+
return [chunk for line in lines for chunk in _wrapped(line)]
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _comment_lines(source: Path, windows: list[tuple[float, float]]) -> list[str]:
|
|
528
|
+
"""Provenance, then the comment area of *source*."""
|
|
529
|
+
lines = _provenance(source, windows, more="The comments of the source follow.")
|
|
530
|
+
handle = spiceypy.dafopr(str(source))
|
|
531
|
+
try:
|
|
532
|
+
done = False
|
|
533
|
+
while not done:
|
|
534
|
+
n, buffer, done = spiceypy.dafec(handle, 200, 1000)
|
|
535
|
+
lines.extend(buffer[:n])
|
|
536
|
+
finally:
|
|
537
|
+
spiceypy.dafcls(handle)
|
|
538
|
+
return [line or " " for line in lines]
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
@contextmanager
|
|
542
|
+
def _new_daf(
|
|
543
|
+
source: Path,
|
|
544
|
+
output: Path,
|
|
545
|
+
windows: list[tuple[float, float]],
|
|
546
|
+
kind: str,
|
|
547
|
+
) -> Iterator[tuple[int, int, list[int]]]:
|
|
548
|
+
"""Open *source* to read and *output* to write, with its comments.
|
|
549
|
+
|
|
550
|
+
Yields the two handles and a one-item counter of the segments written.
|
|
551
|
+
"""
|
|
552
|
+
comments = _comment_lines(source, windows)
|
|
553
|
+
opener, closer = _WRITERS[kind]
|
|
554
|
+
handle = spiceypy.dafopr(str(source))
|
|
555
|
+
new = opener(str(output), "reduced", sum(len(line) + 1 for line in comments))
|
|
556
|
+
written = [0]
|
|
557
|
+
try:
|
|
558
|
+
spiceypy.dafac(new, comments)
|
|
559
|
+
yield handle, new, written
|
|
560
|
+
finally:
|
|
561
|
+
if written[0]:
|
|
562
|
+
closer(new)
|
|
563
|
+
else: # the writers' own close refuses a file without segments
|
|
564
|
+
spiceypy.dafcls(new)
|
|
565
|
+
spiceypy.dafcls(handle)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
_WRITERS = {
|
|
569
|
+
"spk": (spiceypy.spkopn, spiceypy.spkcls),
|
|
570
|
+
"ck": (spiceypy.ckopn, spiceypy.ckcls),
|
|
571
|
+
"pck": (spiceypy.pckopn, spiceypy.pckcls),
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _write_metakernel(
|
|
576
|
+
result: ReducedKernels,
|
|
577
|
+
source_mk: Path,
|
|
578
|
+
drop: Sequence[str],
|
|
579
|
+
) -> None:
|
|
580
|
+
kept = [k for k in result.kernels if k.output is not None]
|
|
581
|
+
skd, identifier = _read_mk_keywords(source_mk)
|
|
582
|
+
keywords = []
|
|
583
|
+
if skd:
|
|
584
|
+
keywords.append(f"SKD_VERSION = '{skd}'")
|
|
585
|
+
if identifier:
|
|
586
|
+
keywords.append(f"MK_IDENTIFIER = '{identifier}_reduced'")
|
|
587
|
+
left_out = [k for k in result.kernels if k.output is None]
|
|
588
|
+
comments = [
|
|
589
|
+
*_provenance(source_mk, result.windows_et),
|
|
590
|
+
*_wrapped(f"Source kernels directory: {source_mk.parent.parent}"),
|
|
591
|
+
f"Dropped by pattern: {', '.join(drop) or 'none'}.",
|
|
592
|
+
f"Left out ({len(left_out)}):",
|
|
593
|
+
*[f" {k.relative} ({k.action})" for k in left_out],
|
|
594
|
+
]
|
|
595
|
+
lines = [
|
|
596
|
+
"KPL/MK",
|
|
597
|
+
"",
|
|
598
|
+
*comments,
|
|
599
|
+
"",
|
|
600
|
+
"\\begindata",
|
|
601
|
+
"",
|
|
602
|
+
"PATH_VALUES = ( '..' )",
|
|
603
|
+
"PATH_SYMBOLS = ( 'KERNELS' )",
|
|
604
|
+
"KERNELS_TO_LOAD = (",
|
|
605
|
+
*[
|
|
606
|
+
line
|
|
607
|
+
for k in kept
|
|
608
|
+
for line in _quoted_lines(f"$KERNELS/{k.relative}", " ")
|
|
609
|
+
],
|
|
610
|
+
")",
|
|
611
|
+
*keywords,
|
|
612
|
+
"",
|
|
613
|
+
"\\begintext",
|
|
614
|
+
"",
|
|
615
|
+
]
|
|
616
|
+
result.metakernel.parent.mkdir(parents=True, exist_ok=True)
|
|
617
|
+
result.metakernel.write_text("\n".join(lines), encoding="utf-8")
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _wrapped(line: str) -> list[str]:
|
|
621
|
+
return textwrap.wrap(line, _COMMENT_LINE, break_on_hyphens=False) or [""]
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def _mb(size: int) -> str:
|
|
625
|
+
return f"{size / 1e6:.2f} MB" if size >= _KB_100 else f"{size / 1e3:.1f} KB"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{quick_spice_manager-0.2.2 → quick_spice_manager-0.2.4}/src/quick_spice_manager/spice_manager.py
RENAMED
|
File without changes
|