quick-spice-manager 0.2.2__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: quick-spice-manager
3
- Version: 0.2.2
3
+ Version: 0.2.4
4
4
  Project-URL: documentation, https://luca-penasa.github.io/quick-spice-manager
5
5
  Project-URL: homepage, https://luca-penasa.github.io/quick-spice-manager
6
6
  Project-URL: repository, https://github.com/luca-penasa/quick-spice-manager
@@ -87,6 +87,8 @@ qsm list -s juice --filter plan # metakernels on the ESA FTP server
87
87
  qsm download plan -s juice # download the metakernel and its kernels
88
88
  qsm resolve plan -s juice # same, then print a furnshable metakernel path
89
89
  qsm info plan -s juice # what's cached, without using the network
90
+ qsm reduce plan -w 2032-12-18 2033-01-09 -o test_kernels --drop 'dsk/*'
91
+ # a copy holding only that time window
90
92
  qsm cache list # size and location of each cached mission
91
93
  qsm cache clear juice # delete a mission's cached kernels
92
94
  ```
@@ -222,6 +224,45 @@ SPICE's 80-character limit are split with `+` continuation. When the pool's
222
224
  metakernel is known, its `SKD_VERSION` / `MK_IDENTIFIER` are copied into the
223
225
  snapshot, and the kernels loaded on top of it are listed in its comments.
224
226
 
227
+ ### Reduced kernel sets (test data)
228
+
229
+ `reduce_kernels` writes a copy of a metakernel and its kernels that holds only
230
+ the data inside some time windows, for example test data small enough to commit
231
+ to a repository:
232
+
233
+ ```python
234
+ from quick_spice_manager import reduce_kernels
235
+
236
+ result = reduce_kernels(
237
+ "5_1_150lb_23_1_a3",
238
+ windows=[("2032-12-18", "2033-01-09"), ("2032-09-01", "2032-10-01")],
239
+ out_dir="tests/data/spice",
240
+ drop=["dsk/*"], # kernels the workload does not need
241
+ )
242
+ print(result.summary()) # what happened to each kernel, and the sizes
243
+ QuickSpiceManager(mk=result.metakernel).load_kernels()
244
+ ```
245
+
246
+ The data inside the windows is copied, not resampled, so the reduced set gives
247
+ the same answers there:
248
+
249
+ | kernel | written |
250
+ |--------|---------|
251
+ | SPK, every type | the records covering each window (`spksub`) |
252
+ | CK type 3 | the pointing records covering each window, with their interpolation intervals |
253
+ | binary PCK type 2 | the Chebyshev records covering each window (equal to rounding) |
254
+ | anything else | copied whole: text kernels, DSKs, CKs of other types (spiceypy cannot write type 6), a CK whose SCLK is not defined |
255
+
256
+ Each window is padded by a day (`pad=`, in seconds), so interpolation and
257
+ extrapolation near its edges still work; overlapping windows are merged. A
258
+ binary kernel without data in any window is left out. The reduced metakernel
259
+ (`out_dir/mk/<name>.tm`, `PATH_VALUES = ( '..' )`, the layout of ESA's) and each
260
+ reduced kernel's comment area record the source and the windows; the kernels
261
+ keep their own comments.
262
+
263
+ For JUICE's `juice_crema_5_1_150lb_23_1_a3` (882 MB), a 24-day window without
264
+ the DSKs is 29 MB, 24.5 MB of which is the type 6 attitude CK, copied whole.
265
+
225
266
  ### Pool management
226
267
 
227
268
  ```python
@@ -51,6 +51,8 @@ qsm list -s juice --filter plan # metakernels on the ESA FTP server
51
51
  qsm download plan -s juice # download the metakernel and its kernels
52
52
  qsm resolve plan -s juice # same, then print a furnshable metakernel path
53
53
  qsm info plan -s juice # what's cached, without using the network
54
+ qsm reduce plan -w 2032-12-18 2033-01-09 -o test_kernels --drop 'dsk/*'
55
+ # a copy holding only that time window
54
56
  qsm cache list # size and location of each cached mission
55
57
  qsm cache clear juice # delete a mission's cached kernels
56
58
  ```
@@ -186,6 +188,45 @@ SPICE's 80-character limit are split with `+` continuation. When the pool's
186
188
  metakernel is known, its `SKD_VERSION` / `MK_IDENTIFIER` are copied into the
187
189
  snapshot, and the kernels loaded on top of it are listed in its comments.
188
190
 
191
+ ### Reduced kernel sets (test data)
192
+
193
+ `reduce_kernels` writes a copy of a metakernel and its kernels that holds only
194
+ the data inside some time windows, for example test data small enough to commit
195
+ to a repository:
196
+
197
+ ```python
198
+ from quick_spice_manager import reduce_kernels
199
+
200
+ result = reduce_kernels(
201
+ "5_1_150lb_23_1_a3",
202
+ windows=[("2032-12-18", "2033-01-09"), ("2032-09-01", "2032-10-01")],
203
+ out_dir="tests/data/spice",
204
+ drop=["dsk/*"], # kernels the workload does not need
205
+ )
206
+ print(result.summary()) # what happened to each kernel, and the sizes
207
+ QuickSpiceManager(mk=result.metakernel).load_kernels()
208
+ ```
209
+
210
+ The data inside the windows is copied, not resampled, so the reduced set gives
211
+ the same answers there:
212
+
213
+ | kernel | written |
214
+ |--------|---------|
215
+ | SPK, every type | the records covering each window (`spksub`) |
216
+ | CK type 3 | the pointing records covering each window, with their interpolation intervals |
217
+ | binary PCK type 2 | the Chebyshev records covering each window (equal to rounding) |
218
+ | anything else | copied whole: text kernels, DSKs, CKs of other types (spiceypy cannot write type 6), a CK whose SCLK is not defined |
219
+
220
+ Each window is padded by a day (`pad=`, in seconds), so interpolation and
221
+ extrapolation near its edges still work; overlapping windows are merged. A
222
+ binary kernel without data in any window is left out. The reduced metakernel
223
+ (`out_dir/mk/<name>.tm`, `PATH_VALUES = ( '..' )`, the layout of ESA's) and each
224
+ reduced kernel's comment area record the source and the windows; the kernels
225
+ keep their own comments.
226
+
227
+ For JUICE's `juice_crema_5_1_150lb_23_1_a3` (882 MB), a 24-day window without
228
+ the DSKs is 29 MB, 24.5 MB of which is the type 6 attitude CK, copied whole.
229
+
189
230
  ### Pool management
190
231
 
191
232
  ```python
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "quick-spice-manager"
3
- version = "0.2.2"
3
+ version = "0.2.4"
4
4
  description = ""
5
5
  authors = [{ name = "Luca Penasa", email = "luca.penasa@gmail.com" }]
6
6
  requires-python = ">=3.10, <4.0"
@@ -31,9 +31,17 @@ def log_disable(mod: str = "quick_spice_manager") -> None:
31
31
  log.disable(mod)
32
32
 
33
33
 
34
+ from .ftp import KernelDownloadError
35
+ from .reduce import reduce_kernels
34
36
  from .spice_manager import QuickSpiceManager, snapshot_pool
35
37
 
36
- __all__ = ["QuickSpiceManager", "SpiceManager", "snapshot_pool"]
38
+ __all__ = [
39
+ "KernelDownloadError",
40
+ "QuickSpiceManager",
41
+ "SpiceManager",
42
+ "reduce_kernels",
43
+ "snapshot_pool",
44
+ ]
37
45
 
38
46
 
39
47
  def __getattr__(name: str) -> object:
@@ -23,6 +23,7 @@ from .ftp import (
23
23
  _ftp_base,
24
24
  list_metakernels_via_ftp,
25
25
  )
26
+ from .reduce import DEFAULT_PAD_SECONDS, _reduce_with
26
27
  from .spice_manager import (
27
28
  QuickSpiceManager,
28
29
  _lookup_localized,
@@ -269,6 +270,63 @@ def info(sm: QuickSpiceManager, *, as_json: bool) -> None:
269
270
  click.echo(f"{key + ':':<15}{'-' if value is None else value}")
270
271
 
271
272
 
273
+ @cli.command()
274
+ @_kernel_options()
275
+ @click.option(
276
+ "-w",
277
+ "--window",
278
+ "windows",
279
+ nargs=2,
280
+ multiple=True,
281
+ required=True,
282
+ metavar="START STOP",
283
+ help="A time window (UTC, anything str2et reads); repeat for more.",
284
+ )
285
+ @click.option(
286
+ "-o",
287
+ "--output",
288
+ "out_dir",
289
+ required=True,
290
+ type=click.Path(file_okay=False, path_type=Path),
291
+ help="Directory for the reduced kernels and mk/<name>.tm.",
292
+ )
293
+ @click.option(
294
+ "--drop",
295
+ multiple=True,
296
+ metavar="GLOB",
297
+ help="Leave out kernels whose relative path matches, e.g. 'dsk/*'; repeat for more.",
298
+ )
299
+ @click.option(
300
+ "--pad",
301
+ type=float,
302
+ default=DEFAULT_PAD_SECONDS,
303
+ show_default=True,
304
+ help="Seconds added before and after each window.",
305
+ )
306
+ @click.option("--name", help="Stem of the reduced metakernel [default: <source>_reduced].")
307
+ def reduce( # noqa: PLR0913 - one option each
308
+ sm: QuickSpiceManager,
309
+ windows: tuple[tuple[str, str], ...],
310
+ out_dir: Path,
311
+ drop: tuple[str, ...],
312
+ pad: float,
313
+ name: str | None,
314
+ ) -> None:
315
+ """Write a copy of a metakernel holding only the data inside time windows.
316
+
317
+ SPKs, type 3 CKs and type 2 binary PCKs keep the records covering the
318
+ windows, so the reduced set gives the same answers there. Other kernels
319
+ are copied whole, binary kernels without data in the windows are left
320
+ out. Prints what happened to each kernel; the metakernel path is the
321
+ last line on stdout.
322
+
323
+ qsm reduce 5_1_150lb_23_1_a3 -w 2032-12-18 2033-01-09 -o tests/data/spice --drop 'dsk/*'
324
+ """
325
+ result = _reduce_with(sm, windows, out_dir, drop=drop, pad=pad, name=name)
326
+ click.echo(result.summary(), err=True)
327
+ click.echo(result.metakernel.resolve())
328
+
329
+
272
330
  @cli.group()
273
331
  def cache() -> None:
274
332
  """Inspect or clear the kernels cache."""
@@ -61,6 +61,31 @@ _SLOT_POLL_INTERVAL_SECONDS = 0.1
61
61
  # How long to wait for a free connection slot before giving up.
62
62
  _CONNECTION_SLOT_TIMEOUT_SECONDS = 300
63
63
 
64
+ # Kernels whose download failed are tried again this many times in all, with a
65
+ # pause of _RETRY_DELAY_SECONDS * attempt in between (a busy server drops
66
+ # connections; it usually answers again after a short wait).
67
+ _DOWNLOAD_ATTEMPTS = 3
68
+ _RETRY_DELAY_SECONDS = 5.0
69
+
70
+
71
+ class KernelDownloadError(ConnectionError):
72
+ """Some kernels of a metakernel could not be downloaded, even after retries.
73
+
74
+ ``missing`` holds their paths relative to the kernels directory, and
75
+ ``errors`` the last error for each of them.
76
+ """
77
+
78
+ def __init__(self, metakernel: Path, missing: dict[str, BaseException]) -> None:
79
+ self.metakernel = metakernel
80
+ self.missing = sorted(missing)
81
+ self.errors = missing
82
+ listing = "\n".join(f" {rel}: {missing[rel]}" for rel in self.missing)
83
+ super().__init__(
84
+ f"{len(missing)} kernel(s) of {metakernel.name} could not be downloaded "
85
+ f"from {_FTP_HOST} after {_DOWNLOAD_ATTEMPTS} attempts:\n{listing}\n"
86
+ "Try again later: kernels already on disk are kept.",
87
+ )
88
+
64
89
  # ---------------------------------------------------------------------------
65
90
  # FTP server and mission path constants
66
91
  # ---------------------------------------------------------------------------
@@ -453,8 +478,13 @@ def _ftp_download_file(ftp: ftplib.FTP, remote: str, local: Path) -> None:
453
478
  a partially-written file: it sees either the previous complete file (if
454
479
  any) or the new complete one, never a truncated in-between state. On any
455
480
  failure the temp file is removed and the exception re-raised.
481
+
482
+ When the server reports the file size (FTP ``SIZE``), a transfer that
483
+ ends short of it raises :class:`ftplib.error_temp`, so a dropped data
484
+ connection never leaves a truncated kernel in the cache.
456
485
  """
457
486
  local.parent.mkdir(parents=True, exist_ok=True)
487
+ expected = _remote_size(ftp, remote)
458
488
  tmp_fd, tmp_name = tempfile.mkstemp(
459
489
  dir=local.parent, prefix=f"{local.name}.", suffix=".part",
460
490
  )
@@ -462,12 +492,26 @@ def _ftp_download_file(ftp: ftplib.FTP, remote: str, local: Path) -> None:
462
492
  try:
463
493
  with os.fdopen(tmp_fd, "wb") as fh:
464
494
  ftp.retrbinary(f"RETR {remote}", fh.write)
495
+ got = tmp_path.stat().st_size
496
+ if expected is not None and got != expected:
497
+ msg = f"transfer of {remote} incomplete: {got} of {expected} bytes"
498
+ raise ftplib.error_temp(msg)
465
499
  tmp_path.replace(local) # atomic rename, same filesystem
466
500
  except BaseException:
467
501
  tmp_path.unlink(missing_ok=True)
468
502
  raise
469
503
 
470
504
 
505
+ def _remote_size(ftp: ftplib.FTP, remote: str) -> int | None:
506
+ """Size of *remote* in bytes, or ``None`` when the server does not tell."""
507
+ try:
508
+ ftp.voidcmd("TYPE I") # SIZE is defined for binary transfers
509
+ size = ftp.size(remote)
510
+ except ftplib.error_perm:
511
+ return None
512
+ return size if isinstance(size, int) else None
513
+
514
+
471
515
  def _locked_download(
472
516
  ftp: ftplib.FTP, remote: str, local: Path, *, force: bool = False,
473
517
  ) -> bool:
@@ -608,6 +652,10 @@ def download_kernels_via_ftp(
608
652
  ConnectionError
609
653
  If the ESA FTP server cannot be reached and no usable local cache
610
654
  exists for this ``(spacecraft, mk, version)``.
655
+ KernelDownloadError
656
+ If some referenced kernels could not be downloaded even after
657
+ retries (its ``missing`` lists them). Nothing is recorded in the
658
+ resolution cache, so the next call tries them again.
611
659
  FileNotFoundError
612
660
  If ``download_kernels=False`` and no complete local cache exists.
613
661
  """
@@ -651,6 +699,8 @@ def download_kernels_via_ftp(
651
699
  )
652
700
  except FileNotFoundError:
653
701
  raise # a genuine "no such kernel/version" answer, not connectivity
702
+ except KernelDownloadError:
703
+ raise # the server answered; some kernels did not come through
654
704
  except ftplib.all_errors as exc:
655
705
  log.warning(f"FTP: could not reach {_FTP_HOST} ({exc})")
656
706
  cached_tm = _usable_cached_tm()
@@ -745,40 +795,67 @@ def _download_kernels_via_ftp_online(
745
795
  f"{n_skip} already cached — using {n_workers} parallel connections",
746
796
  )
747
797
 
748
- # --- Parallel download of missing kernels --------------------------------
749
- n_dl = n_fail = 0
750
- if to_download:
751
- futures = {}
752
- with ThreadPoolExecutor(max_workers=n_workers) as pool:
753
- for remote_kernel, local_kernel in to_download:
754
- log.debug(f"FTP: queuing {local_kernel.name}")
755
- fut = pool.submit(
756
- _download_one_kernel, _FTP_HOST, remote_kernel, local_kernel,
757
- )
758
- futures[fut] = local_kernel
759
-
760
- with tqdm(
761
- total=len(to_download),
762
- desc="Downloading kernels",
763
- unit="file",
764
- dynamic_ncols=True,
765
- ) as pbar:
766
- for fut in as_completed(futures):
767
- local_kernel = futures[fut]
768
- exc = fut.exception()
769
- if exc is None:
770
- log.info(f"FTP: downloaded {local_kernel.name}")
771
- n_dl += 1
772
- pbar.set_postfix_str(local_kernel.name, refresh=False)
773
- else:
774
- log.warning(
775
- f"FTP: could not download '{local_kernel.name}': {exc}",
776
- )
777
- n_fail += 1
778
- pbar.update(1)
798
+ # --- Parallel download of missing kernels, retrying the failed ones -----
799
+ n_dl = 0
800
+ failed: dict[str, BaseException] = {}
801
+ pending = to_download
802
+ for attempt in range(1, _DOWNLOAD_ATTEMPTS + 1):
803
+ if not pending:
804
+ break
805
+ if attempt > 1:
806
+ delay = _RETRY_DELAY_SECONDS * (attempt - 1)
807
+ log.warning(
808
+ f"FTP: {len(pending)} kernel(s) failed, retrying in {delay:.0f} s "
809
+ f"(attempt {attempt}/{_DOWNLOAD_ATTEMPTS})",
810
+ )
811
+ time.sleep(delay)
812
+ failed = _download_batch(pending, kernels_dir, n_workers)
813
+ n_dl += len(pending) - len(failed)
814
+ pending = [(r, loc) for r, loc in pending if _rel(loc, kernels_dir) in failed]
779
815
 
780
816
  log.info(
781
- f"FTP: done - {n_dl} downloaded, {n_skip} cached, {n_fail} failed",
817
+ f"FTP: done - {n_dl} downloaded, {n_skip} cached, {len(failed)} failed",
782
818
  )
819
+ if failed:
820
+ raise KernelDownloadError(local_tm, failed)
783
821
 
784
822
  return local_tm
823
+
824
+
825
+ def _rel(local: Path, kernels_dir: Path) -> str:
826
+ return local.relative_to(kernels_dir).as_posix()
827
+
828
+
829
+ def _download_batch(
830
+ to_download: list[tuple[str, Path]], kernels_dir: Path, n_workers: int,
831
+ ) -> dict[str, BaseException]:
832
+ """Download *to_download* in parallel; return the failures by relative path."""
833
+ failed: dict[str, BaseException] = {}
834
+ futures = {}
835
+ with ThreadPoolExecutor(max_workers=n_workers) as pool:
836
+ for remote_kernel, local_kernel in to_download:
837
+ log.debug(f"FTP: queuing {local_kernel.name}")
838
+ fut = pool.submit(
839
+ _download_one_kernel, _FTP_HOST, remote_kernel, local_kernel,
840
+ )
841
+ futures[fut] = local_kernel
842
+
843
+ with tqdm(
844
+ total=len(to_download),
845
+ desc="Downloading kernels",
846
+ unit="file",
847
+ dynamic_ncols=True,
848
+ ) as pbar:
849
+ for fut in as_completed(futures):
850
+ local_kernel = futures[fut]
851
+ exc = fut.exception()
852
+ if exc is None:
853
+ log.info(f"FTP: downloaded {local_kernel.name}")
854
+ pbar.set_postfix_str(local_kernel.name, refresh=False)
855
+ else:
856
+ log.warning(
857
+ f"FTP: could not download '{local_kernel.name}': {exc}",
858
+ )
859
+ failed[_rel(local_kernel, kernels_dir)] = exc
860
+ pbar.update(1)
861
+ return failed
@@ -0,0 +1,625 @@
1
+ """Reduced kernel sets: a metakernel cut down to some time windows.
2
+
3
+ :func:`reduce_kernels` writes a copy of a metakernel and its kernels that
4
+ holds only the data inside the given time windows, and leaves out the
5
+ kernels a workload does not need. The data inside the windows is copied, not
6
+ resampled, so a reduced set gives the same answers as the full set there.
7
+ The typical use is test data small enough to commit to a repository.
8
+
9
+ ================= ==========================================================
10
+ kind what is written
11
+ ================= ==========================================================
12
+ SPK (all types) the records covering each window (``spksub``)
13
+ CK type 3 the pointing records covering each window, with the
14
+ interpolation intervals they belong to (``ckw03``)
15
+ binary PCK type 2 the Chebyshev records covering each window (``pckw02``)
16
+ anything else copied whole: text kernels, DSKs, CK types other than 3,
17
+ binary kernels with any unsupported segment
18
+ ================= ==========================================================
19
+
20
+ A binary kernel without data in any window is left out of the reduced
21
+ metakernel.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import fnmatch
27
+ import shutil
28
+ import textwrap
29
+ from collections.abc import Iterable, Iterator, Sequence
30
+ from contextlib import contextmanager
31
+ from dataclasses import dataclass, field
32
+ from importlib import metadata
33
+ from pathlib import Path
34
+ from typing import Literal
35
+
36
+ import numpy as np
37
+ import spiceypy
38
+ from loguru import logger as log
39
+ from spiceypy.utils.exceptions import NotFoundError, SpiceyError
40
+
41
+ from .ftp import _parse_mk_kernel_paths
42
+ from .spice_manager import (
43
+ QuickSpiceManager,
44
+ _lookup_localized,
45
+ _pool_lock,
46
+ _quoted_lines,
47
+ _read_mk_keywords,
48
+ )
49
+
50
+ #: Seconds added before and after each window by default (one day): the
51
+ #: records next to a window's edges are kept, so are interpolation and a
52
+ #: solver's extrapolation at block edges.
53
+ DEFAULT_PAD_SECONDS = 86400.0
54
+
55
+ Window = tuple["str | float", "str | float"]
56
+ Action = Literal["subset", "copied", "dropped", "no data"]
57
+
58
+ _DAF_EXTENSIONS = {".bsp": "spk", ".bc": "ck", ".bpc": "pck"}
59
+ _TEXT_EXTENSIONS = {".tls", ".tsc", ".tf", ".ti", ".tpc", ".tm", ".txt"}
60
+ _COMMENT_LINE = 79
61
+ _KB_100 = 100_000
62
+
63
+
64
+ class _CopyWhole(Exception): # noqa: N818 - control flow, never escapes
65
+ """A kernel that can't be subsetted after all: copy it whole, say why."""
66
+
67
+
68
+ @dataclass
69
+ class KernelReport:
70
+ """What :func:`reduce_kernels` did with one kernel."""
71
+
72
+ source: Path
73
+ relative: str
74
+ action: Action
75
+ output: Path | None = None
76
+ note: str = ""
77
+
78
+ @property
79
+ def size_in(self) -> int:
80
+ return self.source.stat().st_size
81
+
82
+ @property
83
+ def size_out(self) -> int:
84
+ return self.output.stat().st_size if self.output else 0
85
+
86
+
87
+ @dataclass
88
+ class ReducedKernels:
89
+ """The reduced metakernel and what happened to each source kernel."""
90
+
91
+ metakernel: Path
92
+ windows_et: list[tuple[float, float]]
93
+ kernels: list[KernelReport] = field(default_factory=list)
94
+
95
+ @property
96
+ def size_in(self) -> int:
97
+ return sum(k.size_in for k in self.kernels)
98
+
99
+ @property
100
+ def size_out(self) -> int:
101
+ return sum(k.size_out for k in self.kernels)
102
+
103
+ def summary(self) -> str:
104
+ """One line per kernel, then the totals."""
105
+ lines = []
106
+ for k in self.kernels:
107
+ sizes = f"{_mb(k.size_in):>9} -> {_mb(k.size_out):>9}"
108
+ note = f" ({k.note})" if k.note else ""
109
+ lines.append(f"{k.action:8} {sizes} {k.relative}{note}")
110
+ lines.append(
111
+ f"{len(self.kernels)} kernels: {_mb(self.size_in)} -> {_mb(self.size_out)}"
112
+ f"; metakernel {self.metakernel}",
113
+ )
114
+ return "\n".join(lines)
115
+
116
+
117
+ def reduce_kernels( # noqa: PLR0913 - keyword options of one operation
118
+ mk: str | Path,
119
+ windows: Iterable[Window],
120
+ out_dir: str | Path,
121
+ *,
122
+ drop: Sequence[str] = (),
123
+ pad: float = DEFAULT_PAD_SECONDS,
124
+ name: str | None = None,
125
+ spacecraft: str = "JUICE",
126
+ version: str = "latest",
127
+ kernels_dir: str | Path | None = None,
128
+ download_kernels: bool = True,
129
+ ) -> ReducedKernels:
130
+ """Write a copy of *mk* that holds only the data inside *windows*.
131
+
132
+ Parameters
133
+ ----------
134
+ mk:
135
+ Metakernel name (resolved and downloaded like ``QuickSpiceManager(mk=...)``)
136
+ or path.
137
+ windows:
138
+ ``(start, stop)`` pairs, as UTC strings (anything ``str2et`` reads) or
139
+ ephemeris times. Overlapping windows are merged after padding.
140
+ out_dir:
141
+ Where to write. The kernels keep their paths relative to the source
142
+ kernels directory (``spk/...``, ``ck/...``) and the metakernel goes to
143
+ ``mk/<name>.tm`` with ``PATH_VALUES = ( '..' )``, the layout of ESA's
144
+ metakernels: ``QuickSpiceManager(mk=<that path>)`` loads it from any
145
+ directory, ``furnsh`` from ``out_dir/mk``.
146
+ drop:
147
+ Glob patterns, matched against each kernel's relative path (e.g.
148
+ ``"dsk/*"``), of kernels to leave out.
149
+ pad:
150
+ Seconds added before and after each window.
151
+ name:
152
+ Stem of the reduced metakernel; default ``<source stem>_reduced``.
153
+ spacecraft, version, kernels_dir, download_kernels:
154
+ As for :class:`QuickSpiceManager`.
155
+
156
+ Returns
157
+ -------
158
+ ReducedKernels
159
+ The metakernel written, and a report per source kernel.
160
+
161
+ Raises
162
+ ------
163
+ ValueError
164
+ If no window is given, or a window ends before it starts.
165
+ """
166
+ sm = QuickSpiceManager(
167
+ spacecraft=spacecraft,
168
+ mk=str(mk),
169
+ version=version,
170
+ kernels_dir=kernels_dir,
171
+ download_kernels=download_kernels,
172
+ )
173
+ return _reduce_with(sm, windows, Path(out_dir), drop=drop, pad=pad, name=name)
174
+
175
+
176
+ def _reduce_with( # noqa: PLR0913
177
+ sm: QuickSpiceManager,
178
+ windows: Iterable[Window],
179
+ out_dir: Path,
180
+ *,
181
+ drop: Sequence[str],
182
+ pad: float,
183
+ name: str | None,
184
+ ) -> ReducedKernels:
185
+ """:func:`reduce_kernels` with the manager already made (the CLI makes its own)."""
186
+ localized = sm.resolved_mk
187
+ info = _lookup_localized(localized)
188
+ if info is None: # pragma: no cover - registered by resolved_mk
189
+ msg = f"could not resolve {localized}"
190
+ raise RuntimeError(msg)
191
+ source_mk, root = info.source, info.kernels_dir
192
+ relatives = _parse_mk_kernel_paths(
193
+ source_mk.read_text(encoding="utf-8", errors="replace"),
194
+ )
195
+ sources = [(rel, _source_path(root, rel)) for rel in relatives]
196
+
197
+ with _pool_lock, _text_kernels_loaded([p for _, p in sources]):
198
+ windows_et = _merge_windows(windows, pad)
199
+ reports = [
200
+ _reduce_one(rel, path, out_dir, windows_et, drop) for rel, path in sources
201
+ ]
202
+ result = ReducedKernels(
203
+ metakernel=out_dir / "mk" / f"{name or source_mk.stem + '_reduced'}.tm",
204
+ windows_et=windows_et,
205
+ kernels=reports,
206
+ )
207
+ _write_metakernel(result, source_mk, drop)
208
+ log.info(
209
+ f"reduced {source_mk.name}: {_mb(result.size_in)} -> {_mb(result.size_out)}",
210
+ )
211
+ return result
212
+
213
+
214
+ # ---------------------------------------------------------------------------
215
+ # Windows and the temporary text kernels
216
+ # ---------------------------------------------------------------------------
217
+
218
+
219
+ def _source_path(root: Path, rel: str) -> Path:
220
+ path = Path(rel)
221
+ return path if path.is_absolute() else root / rel
222
+
223
+
224
+ def _output_relative(rel: str) -> str:
225
+ """Relative path of a kernel in the output (absolute ones go to ``other/``)."""
226
+ path = Path(rel)
227
+ return f"other/{path.name}" if path.is_absolute() else path.as_posix()
228
+
229
+
230
+ @contextmanager
231
+ def _text_kernels_loaded(kernels: Sequence[Path]) -> Iterator[None]:
232
+ """Load the text kernels (LSK, SCLK, FK, ...) needed to convert times and
233
+ name frames; unload afterwards only those that were not loaded before."""
234
+ added = []
235
+ for path in kernels:
236
+ if path.suffix.lower() not in _TEXT_EXTENSIONS or path.suffix.lower() == ".tm":
237
+ continue
238
+ try:
239
+ spiceypy.kinfo(str(path))
240
+ except NotFoundError:
241
+ spiceypy.furnsh(str(path))
242
+ added.append(str(path))
243
+ try:
244
+ yield
245
+ finally:
246
+ for path in reversed(added):
247
+ spiceypy.unload(path)
248
+
249
+
250
+ def _to_et(value: str | float) -> float:
251
+ return float(value) if isinstance(value, (int, float)) else spiceypy.str2et(value)
252
+
253
+
254
+ def _merge_windows(windows: Iterable[Window], pad: float) -> list[tuple[float, float]]:
255
+ spans = []
256
+ for start, stop in windows:
257
+ lo, hi = _to_et(start), _to_et(stop)
258
+ if hi < lo:
259
+ msg = f"window ends before it starts: {start!r} -> {stop!r}"
260
+ raise ValueError(msg)
261
+ spans.append((lo - pad, hi + pad))
262
+ if not spans:
263
+ msg = "no time window given"
264
+ raise ValueError(msg)
265
+ spans.sort()
266
+ merged = [spans[0]]
267
+ for lo, hi in spans[1:]:
268
+ if lo <= merged[-1][1]:
269
+ merged[-1] = (merged[-1][0], max(merged[-1][1], hi))
270
+ else:
271
+ merged.append((lo, hi))
272
+ return merged
273
+
274
+
275
+ # ---------------------------------------------------------------------------
276
+ # One kernel
277
+ # ---------------------------------------------------------------------------
278
+
279
+
280
+ def _reduce_one(
281
+ rel: str,
282
+ source: Path,
283
+ out_dir: Path,
284
+ windows: list[tuple[float, float]],
285
+ drop: Sequence[str],
286
+ ) -> KernelReport:
287
+ out_rel = _output_relative(rel)
288
+ if any(fnmatch.fnmatch(out_rel, pattern) for pattern in drop):
289
+ return KernelReport(source, out_rel, "dropped")
290
+ output = out_dir / out_rel
291
+ output.parent.mkdir(parents=True, exist_ok=True)
292
+ output.unlink(missing_ok=True) # DAF writers refuse to overwrite
293
+
294
+ kind = _DAF_EXTENSIONS.get(source.suffix.lower())
295
+ if kind is not None:
296
+ unsupported = _unsupported_segments(source, kind)
297
+ note = f"copied whole: {kind.upper()} segment types {sorted(unsupported)} not subsetted"
298
+ if not unsupported:
299
+ try:
300
+ written = _SUBSETTERS[kind](source, output, windows)
301
+ except _CopyWhole as exc:
302
+ output.unlink(missing_ok=True)
303
+ written, note = -1, f"copied whole: {exc}"
304
+ if written == 0:
305
+ output.unlink(missing_ok=True)
306
+ return KernelReport(
307
+ source,
308
+ out_rel,
309
+ "no data",
310
+ note="no data in the windows",
311
+ )
312
+ if written > 0:
313
+ return KernelReport(
314
+ source,
315
+ out_rel,
316
+ "subset",
317
+ output,
318
+ f"{written} segments",
319
+ )
320
+ else:
321
+ note = ""
322
+ shutil.copyfile(source, output)
323
+ return KernelReport(source, out_rel, "copied", output, note)
324
+
325
+
326
+ def _segments(
327
+ handle: int,
328
+ nd: int,
329
+ ni: int,
330
+ ) -> Iterator[tuple[np.ndarray, list[float], list[int], str]]:
331
+ spiceypy.dafbfs(handle)
332
+ while spiceypy.daffna():
333
+ descr = spiceypy.dafgs(nd + (ni + 1) // 2)
334
+ dc, ic = spiceypy.dafus(descr, nd, ni)
335
+ yield descr, list(dc), [int(i) for i in ic], spiceypy.dafgn(100).strip()
336
+
337
+
338
+ _LAYOUT = {"spk": (2, 6), "ck": (2, 6), "pck": (2, 5)}
339
+ _TYPE_INDEX = {"spk": 3, "ck": 2, "pck": 2}
340
+ _SUPPORTED = {"spk": None, "ck": {3}, "pck": {2}} # None: every type spksub reads
341
+
342
+
343
+ def _unsupported_segments(source: Path, kind: str) -> set[int]:
344
+ nd, ni = _LAYOUT[kind]
345
+ supported = _SUPPORTED[kind]
346
+ if supported is None:
347
+ return set()
348
+ handle = spiceypy.dafopr(str(source))
349
+ try:
350
+ types = {ic[_TYPE_INDEX[kind]] for _, _, ic, _ in _segments(handle, nd, ni)}
351
+ finally:
352
+ spiceypy.dafcls(handle)
353
+ return types - supported
354
+
355
+
356
+ def _overlaps(
357
+ begin: float,
358
+ end: float,
359
+ windows: list[tuple[float, float]],
360
+ ) -> Iterator[tuple[float, float]]:
361
+ for lo, hi in windows:
362
+ if lo <= end and hi >= begin:
363
+ yield max(lo, begin), min(hi, end)
364
+
365
+
366
+ def _subset_spk(source: Path, output: Path, windows: list[tuple[float, float]]) -> int:
367
+ with _new_daf(source, output, windows, "spk") as (handle, new, written):
368
+ for descr, dc, _ic, name in _segments(handle, 2, 6):
369
+ for lo, hi in _overlaps(dc[0], dc[1], windows):
370
+ spiceypy.spksub(handle, descr, name, lo, hi, new)
371
+ written[0] += 1
372
+ return written[0]
373
+
374
+
375
+ def _subset_ck(source: Path, output: Path, windows: list[tuple[float, float]]) -> int:
376
+ with _new_daf(source, output, windows, "ck") as (handle, new, written):
377
+ for _descr, dc, ic, name in _segments(handle, 2, 6):
378
+ inst, ref, _type, avflag, begin, end = ic
379
+ sclk = spiceypy.ckmeta(inst, "SCLK")
380
+ for lo_et, hi_et in windows:
381
+ try:
382
+ lo, hi = spiceypy.sce2c(sclk, lo_et), spiceypy.sce2c(sclk, hi_et)
383
+ except SpiceyError as exc:
384
+ msg = f"no SCLK {sclk} for CK instrument {inst} ({exc.short})"
385
+ raise _CopyWhole(msg) from exc
386
+ if lo > dc[1] or hi < dc[0]:
387
+ continue
388
+ written[0] += _write_ck03_window(
389
+ handle,
390
+ new,
391
+ (begin, end),
392
+ (dc[0], dc[1]),
393
+ (lo, hi),
394
+ inst=inst,
395
+ ref=spiceypy.frmnam(ref),
396
+ avflag=bool(avflag),
397
+ name=name,
398
+ )
399
+ return written[0]
400
+
401
+
402
+ def _write_ck03_window( # noqa: PLR0913
403
+ handle: int,
404
+ new: int,
405
+ addresses: tuple[int, int],
406
+ coverage: tuple[float, float],
407
+ window: tuple[float, float],
408
+ *,
409
+ inst: int,
410
+ ref: str,
411
+ avflag: bool,
412
+ name: str,
413
+ ) -> int:
414
+ """Copy the type 3 records of one segment that cover *window* (SCLK ticks).
415
+
416
+ Layout (CK Required Reading): N records of 4 (quaternion) or 7 (and angular
417
+ velocity) values, N epochs, the epoch directory, NINT interval starts,
418
+ their directory, NINT, N.
419
+ """
420
+ begin, end = addresses
421
+ nints, n = (int(v) for v in spiceypy.dafgda(handle, end - 1, end))
422
+ size = 7 if avflag else 4
423
+ epochs = np.asarray(
424
+ spiceypy.dafgda(handle, begin + n * size, begin + n * size + n - 1),
425
+ )
426
+ starts_at = begin + n * size + n + (n - 1) // 100
427
+ starts = np.asarray(spiceypy.dafgda(handle, starts_at, starts_at + nints - 1))
428
+
429
+ lo, hi = window
430
+ i0 = max(int(np.searchsorted(epochs, lo, side="right")) - 1, 0)
431
+ i1 = min(int(np.searchsorted(epochs, hi, side="left")), n - 1)
432
+ if i1 < i0:
433
+ return 0
434
+ records = np.asarray(
435
+ spiceypy.dafgda(handle, begin + i0 * size, begin + (i1 + 1) * size - 1),
436
+ ).reshape(-1, size)
437
+ sel = epochs[i0 : i1 + 1]
438
+ # the first record starts an interval; the later starts are kept as they are
439
+ sel_starts = np.concatenate(
440
+ [[sel[0]], starts[(starts > sel[0]) & (starts <= sel[-1])]],
441
+ )
442
+ avvs = records[:, 4:] if avflag else np.zeros((len(sel), 3))
443
+ spiceypy.ckw03(
444
+ new,
445
+ max(coverage[0], min(lo, sel[0])),
446
+ min(coverage[1], max(hi, sel[-1])),
447
+ inst,
448
+ ref,
449
+ avflag,
450
+ name,
451
+ len(sel),
452
+ np.ascontiguousarray(sel),
453
+ np.ascontiguousarray(records[:, :4]),
454
+ np.ascontiguousarray(avvs),
455
+ len(sel_starts),
456
+ np.ascontiguousarray(sel_starts),
457
+ )
458
+ return 1
459
+
460
+
461
+ def _subset_pck(source: Path, output: Path, windows: list[tuple[float, float]]) -> int:
462
+ with _new_daf(source, output, windows, "pck") as (handle, new, written):
463
+ for _descr, dc, ic, name in _segments(handle, 2, 5):
464
+ classid, ref, _type, begin, end = ic
465
+ init, intlen, rsize, n = spiceypy.dafgda(handle, end - 3, end)
466
+ rsize, n = int(rsize), int(n)
467
+ degree = (rsize - 2) // 3 - 1
468
+ for lo, hi in _overlaps(dc[0], dc[1], windows):
469
+ k0 = min(max(int((lo - init) // intlen), 0), n - 1)
470
+ k1 = min(max(int((hi - init) // intlen), 0), n - 1)
471
+ records = np.asarray(
472
+ spiceypy.dafgda(
473
+ handle,
474
+ begin + k0 * rsize,
475
+ begin + (k1 + 1) * rsize - 1,
476
+ ),
477
+ ).reshape(-1, rsize)
478
+ spiceypy.pckw02(
479
+ new,
480
+ classid,
481
+ spiceypy.frmnam(ref),
482
+ lo,
483
+ hi,
484
+ name,
485
+ intlen,
486
+ k1 - k0 + 1,
487
+ degree,
488
+ np.ascontiguousarray(records[:, 2:]).ravel(),
489
+ init + k0 * intlen,
490
+ )
491
+ written[0] += 1
492
+ return written[0]
493
+
494
+
495
+ _SUBSETTERS = {"spk": _subset_spk, "ck": _subset_ck, "pck": _subset_pck}
496
+
497
+
498
+ # ---------------------------------------------------------------------------
499
+ # Comments and the metakernel
500
+ # ---------------------------------------------------------------------------
501
+
502
+
503
+ def _windows_utc(windows: list[tuple[float, float]]) -> list[str]:
504
+ return [
505
+ f"{spiceypy.et2utc(lo, 'ISOC', 0)} -> {spiceypy.et2utc(hi, 'ISOC', 0)}"
506
+ for lo, hi in windows
507
+ ]
508
+
509
+
510
+ def _provenance(
511
+ source: Path,
512
+ windows: list[tuple[float, float]],
513
+ *,
514
+ more: str = "",
515
+ ) -> list[str]:
516
+ version = metadata.version("quick_spice_manager")
517
+ lines = [
518
+ f"Reduced by quick-spice-manager {version} from {source.name}.",
519
+ "It holds the data of the source inside these windows (UTC, padded) only:",
520
+ *[f" {w}" for w in _windows_utc(windows)],
521
+ *([more] if more else []),
522
+ "",
523
+ ]
524
+ return [chunk for line in lines for chunk in _wrapped(line)]
525
+
526
+
527
+ def _comment_lines(source: Path, windows: list[tuple[float, float]]) -> list[str]:
528
+ """Provenance, then the comment area of *source*."""
529
+ lines = _provenance(source, windows, more="The comments of the source follow.")
530
+ handle = spiceypy.dafopr(str(source))
531
+ try:
532
+ done = False
533
+ while not done:
534
+ n, buffer, done = spiceypy.dafec(handle, 200, 1000)
535
+ lines.extend(buffer[:n])
536
+ finally:
537
+ spiceypy.dafcls(handle)
538
+ return [line or " " for line in lines]
539
+
540
+
541
+ @contextmanager
542
+ def _new_daf(
543
+ source: Path,
544
+ output: Path,
545
+ windows: list[tuple[float, float]],
546
+ kind: str,
547
+ ) -> Iterator[tuple[int, int, list[int]]]:
548
+ """Open *source* to read and *output* to write, with its comments.
549
+
550
+ Yields the two handles and a one-item counter of the segments written.
551
+ """
552
+ comments = _comment_lines(source, windows)
553
+ opener, closer = _WRITERS[kind]
554
+ handle = spiceypy.dafopr(str(source))
555
+ new = opener(str(output), "reduced", sum(len(line) + 1 for line in comments))
556
+ written = [0]
557
+ try:
558
+ spiceypy.dafac(new, comments)
559
+ yield handle, new, written
560
+ finally:
561
+ if written[0]:
562
+ closer(new)
563
+ else: # the writers' own close refuses a file without segments
564
+ spiceypy.dafcls(new)
565
+ spiceypy.dafcls(handle)
566
+
567
+
568
+ _WRITERS = {
569
+ "spk": (spiceypy.spkopn, spiceypy.spkcls),
570
+ "ck": (spiceypy.ckopn, spiceypy.ckcls),
571
+ "pck": (spiceypy.pckopn, spiceypy.pckcls),
572
+ }
573
+
574
+
575
+ def _write_metakernel(
576
+ result: ReducedKernels,
577
+ source_mk: Path,
578
+ drop: Sequence[str],
579
+ ) -> None:
580
+ kept = [k for k in result.kernels if k.output is not None]
581
+ skd, identifier = _read_mk_keywords(source_mk)
582
+ keywords = []
583
+ if skd:
584
+ keywords.append(f"SKD_VERSION = '{skd}'")
585
+ if identifier:
586
+ keywords.append(f"MK_IDENTIFIER = '{identifier}_reduced'")
587
+ left_out = [k for k in result.kernels if k.output is None]
588
+ comments = [
589
+ *_provenance(source_mk, result.windows_et),
590
+ *_wrapped(f"Source kernels directory: {source_mk.parent.parent}"),
591
+ f"Dropped by pattern: {', '.join(drop) or 'none'}.",
592
+ f"Left out ({len(left_out)}):",
593
+ *[f" {k.relative} ({k.action})" for k in left_out],
594
+ ]
595
+ lines = [
596
+ "KPL/MK",
597
+ "",
598
+ *comments,
599
+ "",
600
+ "\\begindata",
601
+ "",
602
+ "PATH_VALUES = ( '..' )",
603
+ "PATH_SYMBOLS = ( 'KERNELS' )",
604
+ "KERNELS_TO_LOAD = (",
605
+ *[
606
+ line
607
+ for k in kept
608
+ for line in _quoted_lines(f"$KERNELS/{k.relative}", " ")
609
+ ],
610
+ ")",
611
+ *keywords,
612
+ "",
613
+ "\\begintext",
614
+ "",
615
+ ]
616
+ result.metakernel.parent.mkdir(parents=True, exist_ok=True)
617
+ result.metakernel.write_text("\n".join(lines), encoding="utf-8")
618
+
619
+
620
+ def _wrapped(line: str) -> list[str]:
621
+ return textwrap.wrap(line, _COMMENT_LINE, break_on_hyphens=False) or [""]
622
+
623
+
624
+ def _mb(size: int) -> str:
625
+ return f"{size / 1e6:.2f} MB" if size >= _KB_100 else f"{size / 1e3:.1f} KB"