RDEFpy 1.0.0__cp315-cp315-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rdefpy/__init__.py ADDED
@@ -0,0 +1,85 @@
1
+ # Copyright (c) 2026 Morehead State University
2
+ # Author: Andrea Togni
3
+ # SPDX-License-Identifier: MIT
4
+
5
+ """CCSDS Delta-DOR Raw Data Exchange Format (RDEF) product-file reader and writer.
6
+
7
+ Implements the product files (.prd) of CCSDS 506.1-B-1 (June 2013), §5:
8
+ binary one-second records of complex I/Q with full downconversion
9
+ metadata, and reads the ASCII observation files (§4) that index them.
10
+
11
+ from rdefpy import read_product
12
+
13
+ pf = read_product("scan.prd")
14
+ pf.info()
15
+
16
+ Product files run to gigabytes. ``read_product`` loads records at or below
17
+ 1 kHz (small, the common case) and leaves faster ones on disk: their samples
18
+ are read when accessed and freed when no longer used, so a loop holds one
19
+ record at a time; ``load()`` keeps them, ``release()`` drops them:
20
+
21
+ pf = read_product("scan.prd")
22
+ for rec in pf:
23
+ print(rec.header.timetag, rec.data.power_db)
24
+ pf.load() # keep all in memory
25
+
26
+ Scope
27
+ -----
28
+ This is a format layer: it decodes bytes into samples with correct time
29
+ tags, and encodes them back, and depends on NumPy alone. One estimator is
30
+ included, ``extract_carrier`` (FrFT + SPI open-loop carrier frequency per
31
+ block, see ``rdefpy.carrier``); other spectra, tracking and simulation
32
+ belong to the caller.
33
+
34
+ Low-level decoding is meant to stay as it is: the ``2k+1`` transform
35
+ (§5.3.6), the bit offsets, and the header struct ARE the format.
36
+ ``tests/test_rdefpy.py`` pins all five sample sizes.
37
+ """
38
+
39
+ from importlib.metadata import version as _version
40
+
41
+ from rdefpy._utils import set_threads
42
+ from rdefpy.browse import browse
43
+ from rdefpy.carrier import extract_carrier
44
+ from rdefpy.obs import ObservationFile, ProductEntry, Scan, read_observation
45
+ from rdefpy.prd import (
46
+ DSN_AGENCY_DTYPE,
47
+ ProductFile,
48
+ Record,
49
+ RecordData,
50
+ RecordHeader,
51
+ pack_record_data,
52
+ pack_record_header,
53
+ read_headers,
54
+ read_product,
55
+ scan_product,
56
+ write_product,
57
+ )
58
+
59
+ __version__ = _version("rdefpy")
60
+
61
+ __all__ = [
62
+ # Observation files (.obs, §4)
63
+ "ObservationFile",
64
+ "ProductEntry",
65
+ "Scan",
66
+ "read_observation",
67
+ # Product files (.prd, §5)
68
+ "ProductFile",
69
+ "Record",
70
+ "RecordData",
71
+ "RecordHeader",
72
+ "pack_record_data",
73
+ "pack_record_header",
74
+ "read_headers",
75
+ "read_product",
76
+ "scan_product",
77
+ "write_product",
78
+ "DSN_AGENCY_DTYPE",
79
+ # Carrier extraction (FrFT + SPI)
80
+ "extract_carrier",
81
+ # Record browser (needs matplotlib)
82
+ "browse",
83
+ # Threads for large records
84
+ "set_threads",
85
+ ]
rdefpy/_utils.py ADDED
@@ -0,0 +1,251 @@
1
+ # Copyright (c) 2026 Morehead State University
2
+ # Author: Andrea Togni
3
+ # SPDX-License-Identifier: MIT
4
+
5
+ """Shared helpers for the RDEF reader/writer.
6
+
7
+ Time coercion (CCSDS ASCII Time Code B and ISO-8601), the colon-aligned
8
+ row formatter and number formatter used by every ``__str__``, and the
9
+ progress display, and the thread pool that splits large decodes.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import os
15
+ import re
16
+ import sys
17
+ import threading
18
+ from concurrent.futures import ThreadPoolExecutor
19
+ from datetime import UTC, datetime
20
+
21
+ import numpy as np
22
+
23
+ _CCSDS_B = re.compile(r"(\d{4})-?(\d{3}T\d{2}:\d{2}:\d{2})Z?")
24
+
25
+
26
+ def _parse_ccsds_string(text: str) -> datetime:
27
+ """Parse a CCSDS ASCII Time Code B or ISO-8601 string to datetime.
28
+
29
+ Accepts ``YYYYDDDThh:mm:ss``, ``YYYY-DDDThh:mm:ss``, and standard
30
+ ISO-8601 strings (e.g. ``2026-02-07T12:00:00``). Out-of-range fields
31
+ raise ValueError instead of rolling over.
32
+ """
33
+ m = _CCSDS_B.fullmatch(text)
34
+ if m:
35
+ # Same checks as the observation-file reader; leap second :60 is rejected.
36
+ try:
37
+ dt = datetime.strptime(m[1] + m[2], "%Y%jT%H:%M:%S")
38
+ except ValueError:
39
+ raise ValueError(f"time {text!r}: field out of range") from None
40
+ if dt.year != int(m[1]): # strptime rolls day 366 of a common year over
41
+ raise ValueError(f"time {text!r}: day of year out of range")
42
+ return dt.replace(tzinfo=UTC)
43
+ dt = datetime.fromisoformat(text)
44
+ if dt.tzinfo is None:
45
+ dt = dt.replace(tzinfo=UTC)
46
+ return dt
47
+
48
+
49
+ def _to_datetime(value: datetime | str) -> datetime:
50
+ """Coerce *value* to a timezone-aware ``datetime``.
51
+
52
+ Accepts ``datetime`` or an ISO-8601 / CCSDS
53
+ string (e.g. ``"2026-01-15T12:00:00"``, ``"2026-039T12:00:00"``).
54
+ """
55
+ if isinstance(value, datetime):
56
+ if value.tzinfo is None:
57
+ return value.replace(tzinfo=UTC)
58
+ return value
59
+ if isinstance(value, str):
60
+ return _parse_ccsds_string(value)
61
+ raise TypeError(
62
+ f"Expected datetime or str, got {type(value).__name__}"
63
+ )
64
+
65
+
66
+ def _to_datetime64(
67
+ value: datetime | np.datetime64 | str | None,
68
+ ) -> np.datetime64 | None:
69
+ """Coerce *value* to ``numpy.datetime64[ns]`` for time-window filtering.
70
+
71
+ Returns ``None`` if the input is ``None``.
72
+ """
73
+ if value is None:
74
+ return None
75
+ if isinstance(value, np.datetime64):
76
+ return np.datetime64(value, "ns")
77
+ if isinstance(value, str | datetime):
78
+ # Same rules as _to_datetime (naive = UTC), then an exact conversion:
79
+ # datetime.timestamp() would read a naive value as LOCAL time and
80
+ # round through a float.
81
+ dt = _to_datetime(value).astimezone(UTC).replace(tzinfo=None)
82
+ return np.datetime64(dt, "ns")
83
+ raise TypeError(
84
+ f"Expected datetime, np.datetime64, str, or None, got {type(value).__name__}"
85
+ )
86
+
87
+
88
+ def _fmt_num(x: float, decimals: int = 6) -> str:
89
+ """Thousands separators, at most *decimals* places, trailing zeros trimmed.
90
+
91
+ A nonzero value too small for *decimals* places keeps its significant
92
+ digits (``1e-07``) rather than printing as ``0``.
93
+ """
94
+ if x == 0:
95
+ return "0" # also covers -0.0
96
+ if abs(x) < 10.0**-decimals or abs(x) >= 1e18: # also no 300-digit row
97
+ return f"{x:.6g}"
98
+ return f"{x:,.{decimals}f}".rstrip("0").rstrip(".")
99
+
100
+
101
+ def _align_rows(rows: list[tuple[str, str]], indent: str = "") -> str:
102
+ """Format ``(label, value)`` pairs with the colons lined up.
103
+
104
+ Labels are right-justified to the width of the longest one.
105
+ """
106
+ width = max(len(label) for label, _ in rows)
107
+ return "\n".join(
108
+ f"{indent}{label:>{width}}: {value}".rstrip() for label, value in rows
109
+ )
110
+
111
+
112
+ class _Progress:
113
+ """Progress bar over a byte or record count, or a no-op when disabled.
114
+
115
+ A row of ``*`` on stderr under a ``0% ... 100%`` scale, standard library
116
+ only. ``_Progress(total, enabled=False)`` costs two attribute lookups
117
+ per record and draws nothing.
118
+ """
119
+
120
+ __slots__ = ("_on", "_n", "_seen", "_drawn")
121
+
122
+ _WIDTH = 30
123
+
124
+ def __init__(self, total: float, enabled: bool):
125
+ # total <= 0 happens on an empty file; the bar would divide by zero,
126
+ # so disable rather than special-case every call site.
127
+ self._on = enabled and total > 0
128
+ self._n = 0.0
129
+ self._seen = total
130
+ self._drawn = 0
131
+ if self._on:
132
+ w = self._WIDTH
133
+ sys.stderr.write(" " * 8 + "0%" + " " * (w - 6) + "100%\n" + " " * 8)
134
+
135
+ def update(self, amount: float) -> None:
136
+ """Advance the bar by *amount* (bytes or records)."""
137
+ self._n += amount
138
+ if self._on:
139
+ self._draw(min(self._n, self._seen))
140
+
141
+ def _draw(self, n: float) -> None:
142
+ stars = int(n / self._seen * self._WIDTH)
143
+ if stars > self._drawn:
144
+ sys.stderr.write("*" * (stars - self._drawn))
145
+ sys.stderr.flush()
146
+ self._drawn = stars
147
+
148
+ def finish(self) -> None:
149
+ """Jump the bar to 100%.
150
+
151
+ For an early break -- e.g. ``stop_epoch`` ending a read before EOF --
152
+ so the bar does not stop half-drawn.
153
+ """
154
+ self._n = self._seen
155
+ if self._on:
156
+ self._draw(self._seen)
157
+
158
+ def close(self) -> None:
159
+ """Finish the bar, emitting the trailing newline."""
160
+ if self._on:
161
+ sys.stderr.write("\n")
162
+ sys.stderr.flush()
163
+ self._on = False
164
+
165
+ def __enter__(self) -> _Progress:
166
+ return self
167
+
168
+ def __exit__(self, *exc) -> None:
169
+ self.close()
170
+
171
+
172
+ # Decodes and complex builds of at least this many samples (8 MB of int8
173
+ # I/Q, 64 MB at complex128) run in _THREADS disjoint chunks: a fresh array
174
+ # that size is bound by first-touch page faults, which threads overlap.
175
+ # Measured on one 100 M-sample record, complex128 build: 490 ms on one
176
+ # thread, 112 ms on 8 (Cython kernel; 158 ms with NumPy stores). On 24
177
+ # cores, 16 threads cut a 20 Msps complex build 10-18% further; 24 lost.
178
+ # Smaller machines get 8, or one per core below that.
179
+ _THREADED_MIN_SAMPLES = 4_000_000
180
+ _CPUS = os.cpu_count() or 1
181
+ _DEFAULT_THREADS = 16 if _CPUS >= 24 else min(8, _CPUS)
182
+ _THREADS = _DEFAULT_THREADS
183
+ # ponytail: lazy module-level pools, never shut down. "cpu" decodes, "io"
184
+ # reads, so a read-ahead never queues behind the decode it should overlap.
185
+ _pools: dict[str, ThreadPoolExecutor] = {}
186
+ _pool_lock = threading.Lock() # the read-ahead thread and the caller may race to create one
187
+
188
+
189
+ def _drop_pool() -> None:
190
+ # A forked child inherits the pools but none of their threads: they
191
+ # would queue chunks that nothing runs. Start fresh pools there instead,
192
+ # and a fresh lock: one held by another thread at the fork stays held.
193
+ global _pool_lock
194
+ _pool_lock = threading.Lock()
195
+ _pools.clear()
196
+
197
+
198
+ if hasattr(os, "register_at_fork"):
199
+ os.register_at_fork(after_in_child=_drop_pool)
200
+
201
+
202
+ def set_threads(n: int | None = None) -> None:
203
+ """
204
+ Set how many threads split each large read, decode and complex build.
205
+
206
+ The default is 16 on machines with at least 24 cores, otherwise 8, or
207
+ one per core below 8. It was tuned on a 24-core machine: 16 threads
208
+ built 20 Msps complex samples 10-18% faster than 8, and 24 were slower.
209
+ Machines with more cores may gain from more; 1 gives each pool a single
210
+ worker. Takes effect for calls made after it returns.
211
+
212
+ Parameters
213
+ ----------
214
+ n : int or None
215
+ Threads per pool (positive), or None to restore the default.
216
+
217
+ Raises
218
+ ------
219
+ ValueError
220
+ If *n* is not a positive int.
221
+ """
222
+ global _THREADS
223
+ if n is None:
224
+ n = _DEFAULT_THREADS
225
+ if isinstance(n, bool) or not isinstance(n, int) or n < 1:
226
+ raise ValueError(f"threads must be a positive int, got {n!r}")
227
+ with _pool_lock:
228
+ _THREADS = n
229
+ # Next call starts pools of the new size. No shutdown: a caller may
230
+ # hold an old pool between taking it and mapping on it; dropped pools'
231
+ # threads exit once nothing references them.
232
+ _pools.clear()
233
+
234
+
235
+ def _in_chunks(n: int, per: int, fn, *, pool: str = "cpu") -> None:
236
+ """Call ``fn(a, b)`` over ``[0, n)``: once, or in parallel chunks when large.
237
+
238
+ *n* counts input units of *per* samples each; chunk edges fall on whole
239
+ units. *fn* must release the GIL to gain anything, and write only its
240
+ own ``[a, b)``.
241
+ """
242
+ if n * per < _THREADED_MIN_SAMPLES:
243
+ fn(0, n)
244
+ return
245
+ with _pool_lock:
246
+ ex = _pools.get(pool)
247
+ if ex is None:
248
+ ex = _pools[pool] = ThreadPoolExecutor(_THREADS)
249
+ k = ex._max_workers
250
+ edges = np.linspace(0, n, k + 1).astype(np.int64).tolist()
251
+ list(ex.map(fn, edges[:-1], edges[1:]))