RDEFpy 1.0.0__cp315-cp315-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rdefpy/__init__.py +85 -0
- rdefpy/_utils.py +251 -0
- rdefpy/browse.py +509 -0
- rdefpy/carrier.py +577 -0
- rdefpy/obs/__init__.py +47 -0
- rdefpy/obs/classes.py +191 -0
- rdefpy/obs/reader.py +254 -0
- rdefpy/prd/__init__.py +78 -0
- rdefpy/prd/_binary.py +575 -0
- rdefpy/prd/_display.py +420 -0
- rdefpy/prd/classes.py +2211 -0
- rdefpy/prd/reader.py +852 -0
- rdefpy/prd/unpack_cyth.cp315-win_amd64.pyd +0 -0
- rdefpy/prd/unpack_cyth.pyx +593 -0
- rdefpy/prd/writer.py +813 -0
- rdefpy/py.typed +0 -0
- rdefpy-1.0.0.dist-info/METADATA +226 -0
- rdefpy-1.0.0.dist-info/RECORD +21 -0
- rdefpy-1.0.0.dist-info/WHEEL +5 -0
- rdefpy-1.0.0.dist-info/licenses/LICENSE +21 -0
- rdefpy-1.0.0.dist-info/top_level.txt +1 -0
rdefpy/__init__.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# Copyright (c) 2026 Morehead State University
|
|
2
|
+
# Author: Andrea Togni
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
"""CCSDS Delta-DOR Raw Data Exchange Format (RDEF) product-file reader and writer.
|
|
6
|
+
|
|
7
|
+
Implements the product files (.prd) of CCSDS 506.1-B-1 (June 2013), §5:
|
|
8
|
+
binary one-second records of complex I/Q with full downconversion
|
|
9
|
+
metadata, and reads the ASCII observation files (§4) that index them.
|
|
10
|
+
|
|
11
|
+
from rdefpy import read_product
|
|
12
|
+
|
|
13
|
+
pf = read_product("scan.prd")
|
|
14
|
+
pf.info()
|
|
15
|
+
|
|
16
|
+
Product files run to gigabytes. ``read_product`` loads records at or below
|
|
17
|
+
1 kHz (small, the common case) and leaves faster ones on disk: their samples
|
|
18
|
+
are read when accessed and freed when no longer used, so a loop holds one
|
|
19
|
+
record at a time; ``load()`` keeps them, ``release()`` drops them:
|
|
20
|
+
|
|
21
|
+
pf = read_product("scan.prd")
|
|
22
|
+
for rec in pf:
|
|
23
|
+
print(rec.header.timetag, rec.data.power_db)
|
|
24
|
+
pf.load() # keep all in memory
|
|
25
|
+
|
|
26
|
+
Scope
|
|
27
|
+
-----
|
|
28
|
+
This is a format layer: it decodes bytes into samples with correct time
|
|
29
|
+
tags, and encodes them back, and depends on NumPy alone. One estimator is
|
|
30
|
+
included, ``extract_carrier`` (FrFT + SPI open-loop carrier frequency per
|
|
31
|
+
block, see ``rdefpy.carrier``); other spectra, tracking and simulation
|
|
32
|
+
belong to the caller.
|
|
33
|
+
|
|
34
|
+
Low-level decoding is meant to stay as it is: the ``2k+1`` transform
|
|
35
|
+
(§5.3.6), the bit offsets, and the header struct ARE the format.
|
|
36
|
+
``tests/test_rdefpy.py`` pins all five sample sizes.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from importlib.metadata import version as _version
|
|
40
|
+
|
|
41
|
+
from rdefpy._utils import set_threads
|
|
42
|
+
from rdefpy.browse import browse
|
|
43
|
+
from rdefpy.carrier import extract_carrier
|
|
44
|
+
from rdefpy.obs import ObservationFile, ProductEntry, Scan, read_observation
|
|
45
|
+
from rdefpy.prd import (
|
|
46
|
+
DSN_AGENCY_DTYPE,
|
|
47
|
+
ProductFile,
|
|
48
|
+
Record,
|
|
49
|
+
RecordData,
|
|
50
|
+
RecordHeader,
|
|
51
|
+
pack_record_data,
|
|
52
|
+
pack_record_header,
|
|
53
|
+
read_headers,
|
|
54
|
+
read_product,
|
|
55
|
+
scan_product,
|
|
56
|
+
write_product,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
__version__ = _version("rdefpy")
|
|
60
|
+
|
|
61
|
+
__all__ = [
|
|
62
|
+
# Observation files (.obs, §4)
|
|
63
|
+
"ObservationFile",
|
|
64
|
+
"ProductEntry",
|
|
65
|
+
"Scan",
|
|
66
|
+
"read_observation",
|
|
67
|
+
# Product files (.prd, §5)
|
|
68
|
+
"ProductFile",
|
|
69
|
+
"Record",
|
|
70
|
+
"RecordData",
|
|
71
|
+
"RecordHeader",
|
|
72
|
+
"pack_record_data",
|
|
73
|
+
"pack_record_header",
|
|
74
|
+
"read_headers",
|
|
75
|
+
"read_product",
|
|
76
|
+
"scan_product",
|
|
77
|
+
"write_product",
|
|
78
|
+
"DSN_AGENCY_DTYPE",
|
|
79
|
+
# Carrier extraction (FrFT + SPI)
|
|
80
|
+
"extract_carrier",
|
|
81
|
+
# Record browser (needs matplotlib)
|
|
82
|
+
"browse",
|
|
83
|
+
# Threads for large records
|
|
84
|
+
"set_threads",
|
|
85
|
+
]
|
rdefpy/_utils.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
# Copyright (c) 2026 Morehead State University
|
|
2
|
+
# Author: Andrea Togni
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
|
|
5
|
+
"""Shared helpers for the RDEF reader/writer.
|
|
6
|
+
|
|
7
|
+
Time coercion (CCSDS ASCII Time Code B and ISO-8601), the colon-aligned
|
|
8
|
+
row formatter and number formatter used by every ``__str__``, and the
|
|
9
|
+
progress display, and the thread pool that splits large decodes.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import sys
|
|
17
|
+
import threading
|
|
18
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
19
|
+
from datetime import UTC, datetime
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
|
|
23
|
+
_CCSDS_B = re.compile(r"(\d{4})-?(\d{3}T\d{2}:\d{2}:\d{2})Z?")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _parse_ccsds_string(text: str) -> datetime:
|
|
27
|
+
"""Parse a CCSDS ASCII Time Code B or ISO-8601 string to datetime.
|
|
28
|
+
|
|
29
|
+
Accepts ``YYYYDDDThh:mm:ss``, ``YYYY-DDDThh:mm:ss``, and standard
|
|
30
|
+
ISO-8601 strings (e.g. ``2026-02-07T12:00:00``). Out-of-range fields
|
|
31
|
+
raise ValueError instead of rolling over.
|
|
32
|
+
"""
|
|
33
|
+
m = _CCSDS_B.fullmatch(text)
|
|
34
|
+
if m:
|
|
35
|
+
# Same checks as the observation-file reader; leap second :60 is rejected.
|
|
36
|
+
try:
|
|
37
|
+
dt = datetime.strptime(m[1] + m[2], "%Y%jT%H:%M:%S")
|
|
38
|
+
except ValueError:
|
|
39
|
+
raise ValueError(f"time {text!r}: field out of range") from None
|
|
40
|
+
if dt.year != int(m[1]): # strptime rolls day 366 of a common year over
|
|
41
|
+
raise ValueError(f"time {text!r}: day of year out of range")
|
|
42
|
+
return dt.replace(tzinfo=UTC)
|
|
43
|
+
dt = datetime.fromisoformat(text)
|
|
44
|
+
if dt.tzinfo is None:
|
|
45
|
+
dt = dt.replace(tzinfo=UTC)
|
|
46
|
+
return dt
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _to_datetime(value: datetime | str) -> datetime:
|
|
50
|
+
"""Coerce *value* to a timezone-aware ``datetime``.
|
|
51
|
+
|
|
52
|
+
Accepts ``datetime`` or an ISO-8601 / CCSDS
|
|
53
|
+
string (e.g. ``"2026-01-15T12:00:00"``, ``"2026-039T12:00:00"``).
|
|
54
|
+
"""
|
|
55
|
+
if isinstance(value, datetime):
|
|
56
|
+
if value.tzinfo is None:
|
|
57
|
+
return value.replace(tzinfo=UTC)
|
|
58
|
+
return value
|
|
59
|
+
if isinstance(value, str):
|
|
60
|
+
return _parse_ccsds_string(value)
|
|
61
|
+
raise TypeError(
|
|
62
|
+
f"Expected datetime or str, got {type(value).__name__}"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _to_datetime64(
|
|
67
|
+
value: datetime | np.datetime64 | str | None,
|
|
68
|
+
) -> np.datetime64 | None:
|
|
69
|
+
"""Coerce *value* to ``numpy.datetime64[ns]`` for time-window filtering.
|
|
70
|
+
|
|
71
|
+
Returns ``None`` if the input is ``None``.
|
|
72
|
+
"""
|
|
73
|
+
if value is None:
|
|
74
|
+
return None
|
|
75
|
+
if isinstance(value, np.datetime64):
|
|
76
|
+
return np.datetime64(value, "ns")
|
|
77
|
+
if isinstance(value, str | datetime):
|
|
78
|
+
# Same rules as _to_datetime (naive = UTC), then an exact conversion:
|
|
79
|
+
# datetime.timestamp() would read a naive value as LOCAL time and
|
|
80
|
+
# round through a float.
|
|
81
|
+
dt = _to_datetime(value).astimezone(UTC).replace(tzinfo=None)
|
|
82
|
+
return np.datetime64(dt, "ns")
|
|
83
|
+
raise TypeError(
|
|
84
|
+
f"Expected datetime, np.datetime64, str, or None, got {type(value).__name__}"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _fmt_num(x: float, decimals: int = 6) -> str:
|
|
89
|
+
"""Thousands separators, at most *decimals* places, trailing zeros trimmed.
|
|
90
|
+
|
|
91
|
+
A nonzero value too small for *decimals* places keeps its significant
|
|
92
|
+
digits (``1e-07``) rather than printing as ``0``.
|
|
93
|
+
"""
|
|
94
|
+
if x == 0:
|
|
95
|
+
return "0" # also covers -0.0
|
|
96
|
+
if abs(x) < 10.0**-decimals or abs(x) >= 1e18: # also no 300-digit row
|
|
97
|
+
return f"{x:.6g}"
|
|
98
|
+
return f"{x:,.{decimals}f}".rstrip("0").rstrip(".")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _align_rows(rows: list[tuple[str, str]], indent: str = "") -> str:
|
|
102
|
+
"""Format ``(label, value)`` pairs with the colons lined up.
|
|
103
|
+
|
|
104
|
+
Labels are right-justified to the width of the longest one.
|
|
105
|
+
"""
|
|
106
|
+
width = max(len(label) for label, _ in rows)
|
|
107
|
+
return "\n".join(
|
|
108
|
+
f"{indent}{label:>{width}}: {value}".rstrip() for label, value in rows
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class _Progress:
|
|
113
|
+
"""Progress bar over a byte or record count, or a no-op when disabled.
|
|
114
|
+
|
|
115
|
+
A row of ``*`` on stderr under a ``0% ... 100%`` scale, standard library
|
|
116
|
+
only. ``_Progress(total, enabled=False)`` costs two attribute lookups
|
|
117
|
+
per record and draws nothing.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
__slots__ = ("_on", "_n", "_seen", "_drawn")
|
|
121
|
+
|
|
122
|
+
_WIDTH = 30
|
|
123
|
+
|
|
124
|
+
def __init__(self, total: float, enabled: bool):
|
|
125
|
+
# total <= 0 happens on an empty file; the bar would divide by zero,
|
|
126
|
+
# so disable rather than special-case every call site.
|
|
127
|
+
self._on = enabled and total > 0
|
|
128
|
+
self._n = 0.0
|
|
129
|
+
self._seen = total
|
|
130
|
+
self._drawn = 0
|
|
131
|
+
if self._on:
|
|
132
|
+
w = self._WIDTH
|
|
133
|
+
sys.stderr.write(" " * 8 + "0%" + " " * (w - 6) + "100%\n" + " " * 8)
|
|
134
|
+
|
|
135
|
+
def update(self, amount: float) -> None:
|
|
136
|
+
"""Advance the bar by *amount* (bytes or records)."""
|
|
137
|
+
self._n += amount
|
|
138
|
+
if self._on:
|
|
139
|
+
self._draw(min(self._n, self._seen))
|
|
140
|
+
|
|
141
|
+
def _draw(self, n: float) -> None:
|
|
142
|
+
stars = int(n / self._seen * self._WIDTH)
|
|
143
|
+
if stars > self._drawn:
|
|
144
|
+
sys.stderr.write("*" * (stars - self._drawn))
|
|
145
|
+
sys.stderr.flush()
|
|
146
|
+
self._drawn = stars
|
|
147
|
+
|
|
148
|
+
def finish(self) -> None:
|
|
149
|
+
"""Jump the bar to 100%.
|
|
150
|
+
|
|
151
|
+
For an early break -- e.g. ``stop_epoch`` ending a read before EOF --
|
|
152
|
+
so the bar does not stop half-drawn.
|
|
153
|
+
"""
|
|
154
|
+
self._n = self._seen
|
|
155
|
+
if self._on:
|
|
156
|
+
self._draw(self._seen)
|
|
157
|
+
|
|
158
|
+
def close(self) -> None:
|
|
159
|
+
"""Finish the bar, emitting the trailing newline."""
|
|
160
|
+
if self._on:
|
|
161
|
+
sys.stderr.write("\n")
|
|
162
|
+
sys.stderr.flush()
|
|
163
|
+
self._on = False
|
|
164
|
+
|
|
165
|
+
def __enter__(self) -> _Progress:
|
|
166
|
+
return self
|
|
167
|
+
|
|
168
|
+
def __exit__(self, *exc) -> None:
|
|
169
|
+
self.close()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# Decodes and complex builds of at least this many samples (8 MB of int8
|
|
173
|
+
# I/Q, 64 MB at complex128) run in _THREADS disjoint chunks: a fresh array
|
|
174
|
+
# that size is bound by first-touch page faults, which threads overlap.
|
|
175
|
+
# Measured on one 100 M-sample record, complex128 build: 490 ms on one
|
|
176
|
+
# thread, 112 ms on 8 (Cython kernel; 158 ms with NumPy stores). On 24
|
|
177
|
+
# cores, 16 threads cut a 20 Msps complex build 10-18% further; 24 lost.
|
|
178
|
+
# Smaller machines get 8, or one per core below that.
|
|
179
|
+
_THREADED_MIN_SAMPLES = 4_000_000
|
|
180
|
+
_CPUS = os.cpu_count() or 1
|
|
181
|
+
_DEFAULT_THREADS = 16 if _CPUS >= 24 else min(8, _CPUS)
|
|
182
|
+
_THREADS = _DEFAULT_THREADS
|
|
183
|
+
# ponytail: lazy module-level pools, never shut down. "cpu" decodes, "io"
|
|
184
|
+
# reads, so a read-ahead never queues behind the decode it should overlap.
|
|
185
|
+
_pools: dict[str, ThreadPoolExecutor] = {}
|
|
186
|
+
_pool_lock = threading.Lock() # the read-ahead thread and the caller may race to create one
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _drop_pool() -> None:
|
|
190
|
+
# A forked child inherits the pools but none of their threads: they
|
|
191
|
+
# would queue chunks that nothing runs. Start fresh pools there instead,
|
|
192
|
+
# and a fresh lock: one held by another thread at the fork stays held.
|
|
193
|
+
global _pool_lock
|
|
194
|
+
_pool_lock = threading.Lock()
|
|
195
|
+
_pools.clear()
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
if hasattr(os, "register_at_fork"):
|
|
199
|
+
os.register_at_fork(after_in_child=_drop_pool)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def set_threads(n: int | None = None) -> None:
|
|
203
|
+
"""
|
|
204
|
+
Set how many threads split each large read, decode and complex build.
|
|
205
|
+
|
|
206
|
+
The default is 16 on machines with at least 24 cores, otherwise 8, or
|
|
207
|
+
one per core below 8. It was tuned on a 24-core machine: 16 threads
|
|
208
|
+
built 20 Msps complex samples 10-18% faster than 8, and 24 were slower.
|
|
209
|
+
Machines with more cores may gain from more; 1 gives each pool a single
|
|
210
|
+
worker. Takes effect for calls made after it returns.
|
|
211
|
+
|
|
212
|
+
Parameters
|
|
213
|
+
----------
|
|
214
|
+
n : int or None
|
|
215
|
+
Threads per pool (positive), or None to restore the default.
|
|
216
|
+
|
|
217
|
+
Raises
|
|
218
|
+
------
|
|
219
|
+
ValueError
|
|
220
|
+
If *n* is not a positive int.
|
|
221
|
+
"""
|
|
222
|
+
global _THREADS
|
|
223
|
+
if n is None:
|
|
224
|
+
n = _DEFAULT_THREADS
|
|
225
|
+
if isinstance(n, bool) or not isinstance(n, int) or n < 1:
|
|
226
|
+
raise ValueError(f"threads must be a positive int, got {n!r}")
|
|
227
|
+
with _pool_lock:
|
|
228
|
+
_THREADS = n
|
|
229
|
+
# Next call starts pools of the new size. No shutdown: a caller may
|
|
230
|
+
# hold an old pool between taking it and mapping on it; dropped pools'
|
|
231
|
+
# threads exit once nothing references them.
|
|
232
|
+
_pools.clear()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _in_chunks(n: int, per: int, fn, *, pool: str = "cpu") -> None:
|
|
236
|
+
"""Call ``fn(a, b)`` over ``[0, n)``: once, or in parallel chunks when large.
|
|
237
|
+
|
|
238
|
+
*n* counts input units of *per* samples each; chunk edges fall on whole
|
|
239
|
+
units. *fn* must release the GIL to gain anything, and write only its
|
|
240
|
+
own ``[a, b)``.
|
|
241
|
+
"""
|
|
242
|
+
if n * per < _THREADED_MIN_SAMPLES:
|
|
243
|
+
fn(0, n)
|
|
244
|
+
return
|
|
245
|
+
with _pool_lock:
|
|
246
|
+
ex = _pools.get(pool)
|
|
247
|
+
if ex is None:
|
|
248
|
+
ex = _pools[pool] = ThreadPoolExecutor(_THREADS)
|
|
249
|
+
k = ex._max_workers
|
|
250
|
+
edges = np.linspace(0, n, k + 1).astype(np.int64).tolist()
|
|
251
|
+
list(ex.map(fn, edges[:-1], edges[1:]))
|