RDEFpy 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rdefpy-1.0.0/LICENSE +21 -0
- rdefpy-1.0.0/MANIFEST.in +3 -0
- rdefpy-1.0.0/PKG-INFO +226 -0
- rdefpy-1.0.0/README.md +197 -0
- rdefpy-1.0.0/examples/carrier_synthetic.py +48 -0
- rdefpy-1.0.0/examples/downconversion.py +47 -0
- rdefpy-1.0.0/examples/read_observation.py +19 -0
- rdefpy-1.0.0/examples/read_product_summary.py +19 -0
- rdefpy-1.0.0/examples/samples_and_timetags.py +31 -0
- rdefpy-1.0.0/examples/select_and_write.py +29 -0
- rdefpy-1.0.0/examples/stream_large_file.py +52 -0
- rdefpy-1.0.0/examples/validate_observation.py +26 -0
- rdefpy-1.0.0/pyproject.toml +62 -0
- rdefpy-1.0.0/setup.cfg +4 -0
- rdefpy-1.0.0/setup.py +50 -0
- rdefpy-1.0.0/src/RDEFpy.egg-info/PKG-INFO +226 -0
- rdefpy-1.0.0/src/RDEFpy.egg-info/SOURCES.txt +84 -0
- rdefpy-1.0.0/src/RDEFpy.egg-info/dependency_links.txt +1 -0
- rdefpy-1.0.0/src/RDEFpy.egg-info/requires.txt +4 -0
- rdefpy-1.0.0/src/RDEFpy.egg-info/top_level.txt +1 -0
- rdefpy-1.0.0/src/rdefpy/__init__.py +85 -0
- rdefpy-1.0.0/src/rdefpy/_utils.py +251 -0
- rdefpy-1.0.0/src/rdefpy/browse.py +509 -0
- rdefpy-1.0.0/src/rdefpy/carrier.py +577 -0
- rdefpy-1.0.0/src/rdefpy/obs/__init__.py +47 -0
- rdefpy-1.0.0/src/rdefpy/obs/classes.py +191 -0
- rdefpy-1.0.0/src/rdefpy/obs/reader.py +254 -0
- rdefpy-1.0.0/src/rdefpy/prd/__init__.py +78 -0
- rdefpy-1.0.0/src/rdefpy/prd/_binary.py +575 -0
- rdefpy-1.0.0/src/rdefpy/prd/_display.py +420 -0
- rdefpy-1.0.0/src/rdefpy/prd/classes.py +2211 -0
- rdefpy-1.0.0/src/rdefpy/prd/reader.py +852 -0
- rdefpy-1.0.0/src/rdefpy/prd/unpack_cyth.pyx +593 -0
- rdefpy-1.0.0/src/rdefpy/prd/writer.py +813 -0
- rdefpy-1.0.0/src/rdefpy/py.typed +0 -0
- rdefpy-1.0.0/tests/conftest.py +60 -0
- rdefpy-1.0.0/tests/data/EXMPn000tIsDS24r01c00-26039120000.obs +19 -0
- rdefpy-1.0.0/tests/data/EXMPn001tSsDS24r01c01-26039120000.prd +0 -0
- rdefpy-1.0.0/tests/data/EXMPn001tSsDS24r01c02-26039120000.prd +0 -0
- rdefpy-1.0.0/tests/data/EXMPn001tSsDS24r01c03-26039120000.prd +0 -0
- rdefpy-1.0.0/tests/data/EXMPn001tSsDS24r01c04-26039120000.prd +0 -0
- rdefpy-1.0.0/tests/data/EXMPn001tSsDS24r01c05-26039120000.prd +0 -0
- rdefpy-1.0.0/tests/make_reference.py +212 -0
- rdefpy-1.0.0/tests/rdef_spec.py +336 -0
- rdefpy-1.0.0/tests/reference/README.md +45 -0
- rdefpy-1.0.0/tests/reference/dsn_agency_block.agency +37 -0
- rdefpy-1.0.0/tests/reference/dsn_agency_block.dc +10 -0
- rdefpy-1.0.0/tests/reference/dsn_agency_block.hdr +76 -0
- rdefpy-1.0.0/tests/reference/dsn_agency_block.hex +408 -0
- rdefpy-1.0.0/tests/reference/dsn_agency_block.prd +0 -0
- rdefpy-1.0.0/tests/reference/dsn_agency_block.tab +300 -0
- rdefpy-1.0.0/tests/reference/sample_16bit.hdr +51 -0
- rdefpy-1.0.0/tests/reference/sample_16bit.hex +1022 -0
- rdefpy-1.0.0/tests/reference/sample_16bit.prd +0 -0
- rdefpy-1.0.0/tests/reference/sample_16bit.tab +400 -0
- rdefpy-1.0.0/tests/reference/sample_1bit.hdr +51 -0
- rdefpy-1.0.0/tests/reference/sample_1bit.hex +85 -0
- rdefpy-1.0.0/tests/reference/sample_1bit.prd +0 -0
- rdefpy-1.0.0/tests/reference/sample_1bit.tab +400 -0
- rdefpy-1.0.0/tests/reference/sample_2bit.hdr +51 -0
- rdefpy-1.0.0/tests/reference/sample_2bit.hex +147 -0
- rdefpy-1.0.0/tests/reference/sample_2bit.prd +0 -0
- rdefpy-1.0.0/tests/reference/sample_2bit.tab +400 -0
- rdefpy-1.0.0/tests/reference/sample_4bit.hdr +51 -0
- rdefpy-1.0.0/tests/reference/sample_4bit.hex +272 -0
- rdefpy-1.0.0/tests/reference/sample_4bit.prd +0 -0
- rdefpy-1.0.0/tests/reference/sample_4bit.tab +400 -0
- rdefpy-1.0.0/tests/reference/sample_8bit.hdr +51 -0
- rdefpy-1.0.0/tests/reference/sample_8bit.hex +522 -0
- rdefpy-1.0.0/tests/reference/sample_8bit.prd +0 -0
- rdefpy-1.0.0/tests/reference/sample_8bit.tab +400 -0
- rdefpy-1.0.0/tests/reference/steered_ps0.dc +10 -0
- rdefpy-1.0.0/tests/reference/steered_ps0.hdr +76 -0
- rdefpy-1.0.0/tests/reference/steered_ps0.hex +408 -0
- rdefpy-1.0.0/tests/reference/steered_ps0.prd +0 -0
- rdefpy-1.0.0/tests/reference/steered_ps0.tab +300 -0
- rdefpy-1.0.0/tests/test_browse.py +133 -0
- rdefpy-1.0.0/tests/test_carrier.py +312 -0
- rdefpy-1.0.0/tests/test_examples.py +36 -0
- rdefpy-1.0.0/tests/test_file_samples.py +209 -0
- rdefpy-1.0.0/tests/test_obs.py +189 -0
- rdefpy-1.0.0/tests/test_rdefpy.py +1200 -0
- rdefpy-1.0.0/tests/test_reference.py +155 -0
- rdefpy-1.0.0/tests/test_spec_filenames.py +46 -0
- rdefpy-1.0.0/tests/test_spec_prd_header.py +1234 -0
- rdefpy-1.0.0/tests/test_spec_prd_samples.py +509 -0
rdefpy-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Morehead State University
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
rdefpy-1.0.0/MANIFEST.in
ADDED
rdefpy-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: RDEFpy
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Reader and writer for the CCSDS Delta-DOR Raw Data Exchange Format (RDEF, CCSDS 506.1-B-1).
|
|
5
|
+
Author-email: Andrea Togni <antogni@moreheadstate.edu>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Documentation, https://rdefpy.readthedocs.io/en/latest/
|
|
8
|
+
Project-URL: Repository, https://github.com/atogni/RDEFpy
|
|
9
|
+
Project-URL: Issues, https://github.com/atogni/RDEFpy/issues
|
|
10
|
+
Keywords: CCSDS,RDEF,Delta-DOR,DSN,radio science,open-loop
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
18
|
+
Classifier: Programming Language :: Cython
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Classifier: Intended Audience :: Science/Research
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Astronomy
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: numpy>=1.26
|
|
26
|
+
Provides-Extra: browse
|
|
27
|
+
Requires-Dist: matplotlib>=3.8; extra == "browse"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# RDEFpy
|
|
31
|
+
|
|
32
|
+
[](https://rdefpy.readthedocs.io/en/latest/)
|
|
33
|
+
|
|
34
|
+
Open-source Python reader and writer for the product files of the CCSDS
|
|
35
|
+
Delta-DOR Raw Data Exchange Format (RDEF), CCSDS 506.1-B-1 (June 2013), also
|
|
36
|
+
published as ISO 20208:2015. It covers the standard's §5: binary one-second
|
|
37
|
+
records of complex I/Q with full downconversion metadata, as recorded by
|
|
38
|
+
open-loop receivers for radio science and Delta-DOR. The standard's ASCII
|
|
39
|
+
observation files (§4) are read and validated; writing them, and checks
|
|
40
|
+
across files (§4.2.4 version, §6.2 file names), are not supported.
|
|
41
|
+
|
|
42
|
+
**Documentation: https://rdefpy.readthedocs.io/en/latest/** (user guide,
|
|
43
|
+
examples, API reference).
|
|
44
|
+
|
|
45
|
+
## Introduction
|
|
46
|
+
|
|
47
|
+
Open-loop recordings of spacecraft signals serve radio-science experiments,
|
|
48
|
+
radiometric tracking and Delta-DOR (delta differential one-way ranging), a
|
|
49
|
+
navigation technique derived from very long baseline interferometry (VLBI).
|
|
50
|
+
The Consultative Committee for Space Data Systems (CCSDS) defines a common
|
|
51
|
+
format for these recordings, the Delta-DOR Raw Data Exchange Format (RDEF):
|
|
52
|
+
one-second binary records of complex in-phase/quadrature (I/Q) samples with
|
|
53
|
+
their downconversion metadata, required when stations of different agencies
|
|
54
|
+
share an observation. To our knowledge no licensed open-source library reads
|
|
55
|
+
and writes RDEF, so each station or research group builds and validates its
|
|
56
|
+
own; VLBI stations, which record in the VLBI Data Interchange Format (VDIF),
|
|
57
|
+
face this before joining a navigation campaign. RDEFpy is a Python reader and
|
|
58
|
+
writer for RDEF product files that depends on NumPy alone and decodes every
|
|
59
|
+
sample size the standard defines, with tests pinning each against the
|
|
60
|
+
standard's bit layout. On a laptop workstation it read recordings about
|
|
61
|
+
72,000 times faster than real time for 1-kHz, 16-bit data (6 h), about 3,100
|
|
62
|
+
times for 100-kHz, 16-bit data (2 h, 2.9 GB), and about 9 times for 100-MHz,
|
|
63
|
+
2-bit wideband data (60 s, 3.0 GB). Records above 1 kHz stay on disk until
|
|
64
|
+
accessed, so memory stays bounded on multi-gigabyte files. Stations and
|
|
65
|
+
research groups can produce and consume RDEF without in-house format
|
|
66
|
+
software, and a VLBI station can add it to a VDIF-based workflow without
|
|
67
|
+
changing its recording or correlation pipeline.
|
|
68
|
+
|
|
69
|
+
## Features
|
|
70
|
+
|
|
71
|
+
- Reads and writes RDEF product files (`.prd`).
|
|
72
|
+
- Decodes every record header field and the I/Q samples of all supported sample sizes.
|
|
73
|
+
- Handles multi-gigabyte files: records at or below 1 kHz are loaded;
|
|
74
|
+
faster records stay on disk and are read on access, one at a time.
|
|
75
|
+
- Extracts the carrier frequency before downconversion per integration
|
|
76
|
+
time: fractional Fourier transform plus Successive Parabolic
|
|
77
|
+
Interpolation (Togni et al., 2025, Signal Processing 237, 110094).
|
|
78
|
+
Blocks run in parallel within a memory budget (`memory_limit`, by default
|
|
79
|
+
half of the memory the process can still take), counted at about 400 bytes per block sample.
|
|
80
|
+
- Depends on NumPy alone. An optional Cython extension speeds up I/Q
|
|
81
|
+
unpacking and carrier extraction; without a compiler the package falls
|
|
82
|
+
back to an equivalent NumPy path.
|
|
83
|
+
|
|
84
|
+
## Installation
|
|
85
|
+
|
|
86
|
+
RDEFpy needs Python 3.11 or later and NumPy.
|
|
87
|
+
|
|
88
|
+
From PyPI:
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
pip install rdefpy
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
The prebuilt wheels include the compiled Cython extension. Where no wheel
|
|
95
|
+
matches your platform, pip builds from source as below.
|
|
96
|
+
|
|
97
|
+
The record browser (`rdefpy.browse`) also needs matplotlib:
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
pip install "rdefpy[browse]"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
From GitHub, latest version:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
pip install git+https://github.com/atogni/RDEFpy
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
or from a clone (add `-e` for an editable install when developing):
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
git clone https://github.com/atogni/RDEFpy
|
|
113
|
+
cd RDEFpy
|
|
114
|
+
pip install .
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
A source build compiles the optional Cython extension, which needs a C compiler:
|
|
118
|
+
Microsoft C++ Build Tools on Windows, `gcc` on Linux (e.g. `build-essential`),
|
|
119
|
+
or the Xcode command-line tools on macOS (`xcode-select --install`). Without
|
|
120
|
+
one the build prints a warning and installs the NumPy path only. To check
|
|
121
|
+
that the extension was built:
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
python -c "import rdefpy.prd.unpack_cyth"
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
No error means it is in use.
|
|
128
|
+
|
|
129
|
+
## Quick start
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
from rdefpy import read_product, write_product
|
|
133
|
+
|
|
134
|
+
pf = read_product("scan.prd") # records above 1 kHz stay on disk until accessed
|
|
135
|
+
pf.info() # file summary: time span, rate, bits, band
|
|
136
|
+
print(pf[0]) # first record: header fields
|
|
137
|
+
|
|
138
|
+
for rec in pf: # one one-second record at a time
|
|
139
|
+
x = rec.data.samples # complex I + jQ, NumPy array
|
|
140
|
+
print(rec.header.timetag, rec.data.power_db)
|
|
141
|
+
|
|
142
|
+
buf = None # faster at high rates: decode straight to complex
|
|
143
|
+
for rec in pf: # and refill one array (copy anything you keep)
|
|
144
|
+
buf = rec.read_samples(out=buf)
|
|
145
|
+
|
|
146
|
+
x = pf.samples[:1024] # any sample range, across records
|
|
147
|
+
t = pf.timetags[:1024] # datetime64[ns] of each sample
|
|
148
|
+
f = pf.downconversion_frequency[:1024] # Hz, at each sample (§5.2.6)
|
|
149
|
+
phi = pf.downconversion_phase[:1024] # turns, variable part (§5.2.6.2)
|
|
150
|
+
f0 = pf.record_downconversion_frequency # Hz, at each record's first sample
|
|
151
|
+
f_mean = pf.mean_downconversion_frequency # Hz, mean over each record
|
|
152
|
+
|
|
153
|
+
from rdefpy import extract_carrier # FrFT + SPI, samples untouched
|
|
154
|
+
c = extract_carrier(pf, integration_time=1.0, max_rate_hz_s=10.0)
|
|
155
|
+
c["time"], c["frequency_before_downconversion"] # one row per block; time = block midpoint
|
|
156
|
+
|
|
157
|
+
part = pf.select(start_epoch="2026-02-08T12:00:00", stop_epoch="2026-02-08T12:00:10")
|
|
158
|
+
write_product("part.prd", part)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
An observation file (`.obs`) lists the product files of each scan:
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
from rdefpy import read_observation, read_product
|
|
165
|
+
|
|
166
|
+
for scan in read_observation("session.obs"):
|
|
167
|
+
for entry in scan.products:
|
|
168
|
+
pf = read_product(entry.datafile)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
More in the [quick start](https://rdefpy.readthedocs.io/en/latest/quickstart.html)
|
|
172
|
+
and the [user guide](https://rdefpy.readthedocs.io/en/latest/).
|
|
173
|
+
|
|
174
|
+
## Performance
|
|
175
|
+
|
|
176
|
+
Time to read a file and decode every sample to integer I/Q, by sample size,
|
|
177
|
+
sample rate and recording duration. Median of 10 runs, files in the OS
|
|
178
|
+
cache, 2026-11-06. Intel Core Ultra 9 285HX laptop (24 cores, 64 GB RAM),
|
|
179
|
+
Windows 11, Python 3.15.0, NumPy 2.5.4, RDEFpy at commit c7b5e55 with the
|
|
180
|
+
Cython extension, 16 threads.
|
|
181
|
+
|
|
182
|
+

|
|
183
|
+
|
|
184
|
+
The same measurement for four typical files, with and without the compiled
|
|
185
|
+
Cython extension (without it RDEFpy falls back to pure NumPy). Median of 10
|
|
186
|
+
runs, same machine and versions, RDEFpy at commit 1e7991f.
|
|
187
|
+
|
|
188
|
+

|
|
189
|
+
|
|
190
|
+
The ratios in the introduction come from the IVS GM2026 poster benchmark
|
|
191
|
+
(Cython back-end).
|
|
192
|
+
|
|
193
|
+
## Testing
|
|
194
|
+
|
|
195
|
+
```
|
|
196
|
+
uv run python -m pytest
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
The tests pin the low-level decoding for every sample size against the
|
|
200
|
+
standard's bit layout.
|
|
201
|
+
|
|
202
|
+
`tests/reference/` holds small synthetic product files with their expected
|
|
203
|
+
decoding as text (headers, samples, hex dump, downconversion, DSN 0222
|
|
204
|
+
agency block), written from a plain transcription of the standard, not from
|
|
205
|
+
RDEFpy. Use them to check another decoder: see
|
|
206
|
+
[Checking another decoder](https://rdefpy.readthedocs.io/en/latest/reference_files.html).
|
|
207
|
+
|
|
208
|
+
## Contributing
|
|
209
|
+
|
|
210
|
+
Issues and pull requests are welcome. The low-level decoding (the `2k+1`
|
|
211
|
+
transform of §5.3.6, the bit offsets, the header struct) is the format itself
|
|
212
|
+
and is meant to stay as it is; changes there need a reference to the standard.
|
|
213
|
+
Performance improvements in decoding, reading, or processing data are welcome.
|
|
214
|
+
|
|
215
|
+
## Publications using RDEFpy
|
|
216
|
+
|
|
217
|
+
- Togni, A., Malphrus, B., Arnold, B., Mayo, D., Sanchez-Net, M., & Asmar, S.
|
|
218
|
+
(2026). Extending VLBI Station Capabilities for Delta-DOR and Navigation via
|
|
219
|
+
Open-Source RDEF Software Compliant with CCSDS Standards. Poster, 14th
|
|
220
|
+
General Meeting of the International VLBI Service for Geodesy and Astrometry
|
|
221
|
+
(IVS GM2026), Würzburg.
|
|
222
|
+
https://indico.astro.uni-wuerzburg.de/event/2/contributions/143/
|
|
223
|
+
|
|
224
|
+
## License
|
|
225
|
+
|
|
226
|
+
MIT. See `LICENSE`.
|
rdefpy-1.0.0/README.md
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# RDEFpy
|
|
2
|
+
|
|
3
|
+
[](https://rdefpy.readthedocs.io/en/latest/)
|
|
4
|
+
|
|
5
|
+
Open-source Python reader and writer for the product files of the CCSDS
|
|
6
|
+
Delta-DOR Raw Data Exchange Format (RDEF), CCSDS 506.1-B-1 (June 2013), also
|
|
7
|
+
published as ISO 20208:2015. It covers the standard's §5: binary one-second
|
|
8
|
+
records of complex I/Q with full downconversion metadata, as recorded by
|
|
9
|
+
open-loop receivers for radio science and Delta-DOR. The standard's ASCII
|
|
10
|
+
observation files (§4) are read and validated; writing them, and checks
|
|
11
|
+
across files (§4.2.4 version, §6.2 file names), are not supported.
|
|
12
|
+
|
|
13
|
+
**Documentation: https://rdefpy.readthedocs.io/en/latest/** (user guide,
|
|
14
|
+
examples, API reference).
|
|
15
|
+
|
|
16
|
+
## Introduction
|
|
17
|
+
|
|
18
|
+
Open-loop recordings of spacecraft signals serve radio-science experiments,
|
|
19
|
+
radiometric tracking and Delta-DOR (delta differential one-way ranging), a
|
|
20
|
+
navigation technique derived from very long baseline interferometry (VLBI).
|
|
21
|
+
The Consultative Committee for Space Data Systems (CCSDS) defines a common
|
|
22
|
+
format for these recordings, the Delta-DOR Raw Data Exchange Format (RDEF):
|
|
23
|
+
one-second binary records of complex in-phase/quadrature (I/Q) samples with
|
|
24
|
+
their downconversion metadata, required when stations of different agencies
|
|
25
|
+
share an observation. To our knowledge no licensed open-source library reads
|
|
26
|
+
and writes RDEF, so each station or research group builds and validates its
|
|
27
|
+
own; VLBI stations, which record in the VLBI Data Interchange Format (VDIF),
|
|
28
|
+
face this before joining a navigation campaign. RDEFpy is a Python reader and
|
|
29
|
+
writer for RDEF product files that depends on NumPy alone and decodes every
|
|
30
|
+
sample size the standard defines, with tests pinning each against the
|
|
31
|
+
standard's bit layout. On a laptop workstation it read recordings about
|
|
32
|
+
72,000 times faster than real time for 1-kHz, 16-bit data (6 h), about 3,100
|
|
33
|
+
times for 100-kHz, 16-bit data (2 h, 2.9 GB), and about 9 times for 100-MHz,
|
|
34
|
+
2-bit wideband data (60 s, 3.0 GB). Records above 1 kHz stay on disk until
|
|
35
|
+
accessed, so memory stays bounded on multi-gigabyte files. Stations and
|
|
36
|
+
research groups can produce and consume RDEF without in-house format
|
|
37
|
+
software, and a VLBI station can add it to a VDIF-based workflow without
|
|
38
|
+
changing its recording or correlation pipeline.
|
|
39
|
+
|
|
40
|
+
## Features
|
|
41
|
+
|
|
42
|
+
- Reads and writes RDEF product files (`.prd`).
|
|
43
|
+
- Decodes every record header field and the I/Q samples of all supported sample sizes.
|
|
44
|
+
- Handles multi-gigabyte files: records at or below 1 kHz are loaded;
|
|
45
|
+
faster records stay on disk and are read on access, one at a time.
|
|
46
|
+
- Extracts the carrier frequency before downconversion per integration
|
|
47
|
+
time: fractional Fourier transform plus Successive Parabolic
|
|
48
|
+
Interpolation (Togni et al., 2025, Signal Processing 237, 110094).
|
|
49
|
+
Blocks run in parallel within a memory budget (`memory_limit`, by default
|
|
50
|
+
half of the memory the process can still take), counted at about 400 bytes per block sample.
|
|
51
|
+
- Depends on NumPy alone. An optional Cython extension speeds up I/Q
|
|
52
|
+
unpacking and carrier extraction; without a compiler the package falls
|
|
53
|
+
back to an equivalent NumPy path.
|
|
54
|
+
|
|
55
|
+
## Installation
|
|
56
|
+
|
|
57
|
+
RDEFpy needs Python 3.11 or later and NumPy.
|
|
58
|
+
|
|
59
|
+
From PyPI:
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
pip install rdefpy
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The prebuilt wheels include the compiled Cython extension. Where no wheel
|
|
66
|
+
matches your platform, pip builds from source as below.
|
|
67
|
+
|
|
68
|
+
The record browser (`rdefpy.browse`) also needs matplotlib:
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
pip install "rdefpy[browse]"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
From GitHub, latest version:
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
pip install git+https://github.com/atogni/RDEFpy
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
or from a clone (add `-e` for an editable install when developing):
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
git clone https://github.com/atogni/RDEFpy
|
|
84
|
+
cd RDEFpy
|
|
85
|
+
pip install .
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
A source build compiles the optional Cython extension, which needs a C compiler:
|
|
89
|
+
Microsoft C++ Build Tools on Windows, `gcc` on Linux (e.g. `build-essential`),
|
|
90
|
+
or the Xcode command-line tools on macOS (`xcode-select --install`). Without
|
|
91
|
+
one the build prints a warning and installs the NumPy path only. To check
|
|
92
|
+
that the extension was built:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
python -c "import rdefpy.prd.unpack_cyth"
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
No error means it is in use.
|
|
99
|
+
|
|
100
|
+
## Quick start
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from rdefpy import read_product, write_product
|
|
104
|
+
|
|
105
|
+
pf = read_product("scan.prd") # records above 1 kHz stay on disk until accessed
|
|
106
|
+
pf.info() # file summary: time span, rate, bits, band
|
|
107
|
+
print(pf[0]) # first record: header fields
|
|
108
|
+
|
|
109
|
+
for rec in pf: # one one-second record at a time
|
|
110
|
+
x = rec.data.samples # complex I + jQ, NumPy array
|
|
111
|
+
print(rec.header.timetag, rec.data.power_db)
|
|
112
|
+
|
|
113
|
+
buf = None # faster at high rates: decode straight to complex
|
|
114
|
+
for rec in pf: # and refill one array (copy anything you keep)
|
|
115
|
+
buf = rec.read_samples(out=buf)
|
|
116
|
+
|
|
117
|
+
x = pf.samples[:1024] # any sample range, across records
|
|
118
|
+
t = pf.timetags[:1024] # datetime64[ns] of each sample
|
|
119
|
+
f = pf.downconversion_frequency[:1024] # Hz, at each sample (§5.2.6)
|
|
120
|
+
phi = pf.downconversion_phase[:1024] # turns, variable part (§5.2.6.2)
|
|
121
|
+
f0 = pf.record_downconversion_frequency # Hz, at each record's first sample
|
|
122
|
+
f_mean = pf.mean_downconversion_frequency # Hz, mean over each record
|
|
123
|
+
|
|
124
|
+
from rdefpy import extract_carrier # FrFT + SPI, samples untouched
|
|
125
|
+
c = extract_carrier(pf, integration_time=1.0, max_rate_hz_s=10.0)
|
|
126
|
+
c["time"], c["frequency_before_downconversion"] # one row per block; time = block midpoint
|
|
127
|
+
|
|
128
|
+
part = pf.select(start_epoch="2026-02-08T12:00:00", stop_epoch="2026-02-08T12:00:10")
|
|
129
|
+
write_product("part.prd", part)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
An observation file (`.obs`) lists the product files of each scan:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from rdefpy import read_observation, read_product
|
|
136
|
+
|
|
137
|
+
for scan in read_observation("session.obs"):
|
|
138
|
+
for entry in scan.products:
|
|
139
|
+
pf = read_product(entry.datafile)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
More in the [quick start](https://rdefpy.readthedocs.io/en/latest/quickstart.html)
|
|
143
|
+
and the [user guide](https://rdefpy.readthedocs.io/en/latest/).
|
|
144
|
+
|
|
145
|
+
## Performance
|
|
146
|
+
|
|
147
|
+
Time to read a file and decode every sample to integer I/Q, by sample size,
|
|
148
|
+
sample rate and recording duration. Median of 10 runs, files in the OS
|
|
149
|
+
cache, 2026-11-06. Intel Core Ultra 9 285HX laptop (24 cores, 64 GB RAM),
|
|
150
|
+
Windows 11, Python 3.15.0, NumPy 2.5.4, RDEFpy at commit c7b5e55 with the
|
|
151
|
+
Cython extension, 16 threads.
|
|
152
|
+
|
|
153
|
+

|
|
154
|
+
|
|
155
|
+
The same measurement for four typical files, with and without the compiled
|
|
156
|
+
Cython extension (without it RDEFpy falls back to pure NumPy). Median of 10
|
|
157
|
+
runs, same machine and versions, RDEFpy at commit 1e7991f.
|
|
158
|
+
|
|
159
|
+

|
|
160
|
+
|
|
161
|
+
The ratios in the introduction come from the IVS GM2026 poster benchmark
|
|
162
|
+
(Cython back-end).
|
|
163
|
+
|
|
164
|
+
## Testing
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
uv run python -m pytest
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
The tests pin the low-level decoding for every sample size against the
|
|
171
|
+
standard's bit layout.
|
|
172
|
+
|
|
173
|
+
`tests/reference/` holds small synthetic product files with their expected
|
|
174
|
+
decoding as text (headers, samples, hex dump, downconversion, DSN 0222
|
|
175
|
+
agency block), written from a plain transcription of the standard, not from
|
|
176
|
+
RDEFpy. Use them to check another decoder: see
|
|
177
|
+
[Checking another decoder](https://rdefpy.readthedocs.io/en/latest/reference_files.html).
|
|
178
|
+
|
|
179
|
+
## Contributing
|
|
180
|
+
|
|
181
|
+
Issues and pull requests are welcome. The low-level decoding (the `2k+1`
|
|
182
|
+
transform of §5.3.6, the bit offsets, the header struct) is the format itself
|
|
183
|
+
and is meant to stay as it is; changes there need a reference to the standard.
|
|
184
|
+
Performance improvements in decoding, reading, or processing data are welcome.
|
|
185
|
+
|
|
186
|
+
## Publications using RDEFpy
|
|
187
|
+
|
|
188
|
+
- Togni, A., Malphrus, B., Arnold, B., Mayo, D., Sanchez-Net, M., & Asmar, S.
|
|
189
|
+
(2026). Extending VLBI Station Capabilities for Delta-DOR and Navigation via
|
|
190
|
+
Open-Source RDEF Software Compliant with CCSDS Standards. Poster, 14th
|
|
191
|
+
General Meeting of the International VLBI Service for Geodesy and Astrometry
|
|
192
|
+
(IVS GM2026), Würzburg.
|
|
193
|
+
https://indico.astro.uni-wuerzburg.de/event/2/contributions/143/
|
|
194
|
+
|
|
195
|
+
## License
|
|
196
|
+
|
|
197
|
+
MIT. See `LICENSE`.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Extract the carrier frequency of a synthetic tone file, and compare with the truth.
|
|
2
|
+
|
|
3
|
+
Builds three one-second records of a complex tone with a known baseband frequency,
|
|
4
|
+
quantized to the 2k+1 codes of 16-bit samples (§5.3.6), writes them to a temporary
|
|
5
|
+
product file and runs extract_carrier on it.
|
|
6
|
+
|
|
7
|
+
python examples/carrier_synthetic.py
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import tempfile
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from rdefpy import (
|
|
16
|
+
ProductFile, Record, RecordData, RecordHeader, extract_carrier, read_product, write_product,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
FS = 1000
|
|
20
|
+
F_TONE = 123.4567 # baseband frequency [Hz]
|
|
21
|
+
FIXED = 8.4e9 + 5.0e6 # rf_to_if + if_to_channel downconversion [Hz]
|
|
22
|
+
|
|
23
|
+
pf = ProductFile(name="synthetic")
|
|
24
|
+
for s in range(3):
|
|
25
|
+
t = s + np.arange(FS) / FS
|
|
26
|
+
phase = 2 * np.pi * F_TONE * t
|
|
27
|
+
# 2k+1 values: odd integers, here 2*floor(v/2) + 1
|
|
28
|
+
i, q = (2 * np.floor(12000 * f(phase) / 2).astype(int) + 1 for f in (np.cos, np.sin))
|
|
29
|
+
header = RecordHeader(dict(
|
|
30
|
+
record_version_id=1, station_id=24, spacecraft_id=1,
|
|
31
|
+
sample_size=16, sample_rate=FS, validity_flag=0, agency_flag=0,
|
|
32
|
+
rf_to_if_downconv=8.4e9, if_to_channel_downconv=5.0e6,
|
|
33
|
+
time_tag_year=2026, time_tag_doy=39, time_tag_second_of_day=43200 + s,
|
|
34
|
+
time_tag_picoseconds_of_the_second=0.0, channel_accumulated_phase=0.0,
|
|
35
|
+
channel_phase_polynomial_coefficient_0=0.0, channel_phase_polynomial_coefficient_1=0.0,
|
|
36
|
+
channel_phase_polynomial_coefficient_2=0.0, channel_phase_polynomial_coefficient_3=0.0,
|
|
37
|
+
))
|
|
38
|
+
pf.records.append(Record(header, RecordData(16, FS, i, q)))
|
|
39
|
+
|
|
40
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
41
|
+
path = Path(tmp) / "tone.prd"
|
|
42
|
+
write_product(path, pf)
|
|
43
|
+
c = extract_carrier(read_product(path), integration_time=1.0, max_rate_hz_s=0.0)
|
|
44
|
+
|
|
45
|
+
truth = FIXED + F_TONE
|
|
46
|
+
print(f"truth frequency before downconversion {truth:.6f} Hz (baseband {F_TONE} Hz)")
|
|
47
|
+
for time, f, base in zip(c["time"], c["frequency_before_downconversion"], c["frequency"]):
|
|
48
|
+
print(f"{time} before DC {f:.6f} Hz error {(f - truth) * 1e3:+.3f} mHz baseband {base:.6f} Hz")
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Downconversion frequency and phase of a product file (§5.2.6).
|
|
2
|
+
|
|
3
|
+
python examples/downconversion.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from rdefpy import read_product
|
|
11
|
+
|
|
12
|
+
np.set_printoptions(formatter={"float": "{:.3f}".format})
|
|
13
|
+
data = Path(__file__).parents[1] / "tests" / "data"
|
|
14
|
+
pf = read_product(data / "EXMPn001tSsDS24r01c01-26039120000.prd")
|
|
15
|
+
fs = pf[0].header.sample_rate
|
|
16
|
+
|
|
17
|
+
# The file has a fixed downconversion: all phase coefficients are zero.
|
|
18
|
+
print("per-record [Hz]:", pf.record_downconversion_frequency)
|
|
19
|
+
|
|
20
|
+
# Give the headers one frequency ramp, f_var = C1 + 2*C2*t from the first record's
|
|
21
|
+
# start (headers only, the samples are untouched). Each record continues it from its
|
|
22
|
+
# own start t = n s: c1 is the frequency there, and the phase reached so far is split
|
|
23
|
+
# into whole turns (channel_accumulated_phase) and the fraction c0, so phase and
|
|
24
|
+
# frequency are continuous from record to record (Table 5-1).
|
|
25
|
+
C1, C2 = 100.0, -3.0 # turns/s, turns/s^2
|
|
26
|
+
steered = pf.copy()
|
|
27
|
+
for n, rec in enumerate(steered):
|
|
28
|
+
turns = C1 * n + C2 * n**2 # phase at the start of record n
|
|
29
|
+
rec.correct_header(
|
|
30
|
+
channel_accumulated_phase=float(np.floor(turns)),
|
|
31
|
+
channel_phase_polynomial_coefficient_0=float(turns - np.floor(turns)),
|
|
32
|
+
channel_phase_polynomial_coefficient_1=C1 + 2 * C2 * n,
|
|
33
|
+
channel_phase_polynomial_coefficient_2=C2,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
print("first sample [Hz]:", steered.record_downconversion_frequency)
|
|
37
|
+
print("record mean [Hz]:", steered.mean_downconversion_frequency)
|
|
38
|
+
print("phase at first sample [turns]:", steered.record_downconversion_phase)
|
|
39
|
+
|
|
40
|
+
# Per sample, across the record boundary: continuous.
|
|
41
|
+
print("per-sample [Hz]: ", steered.downconversion_frequency[fs - 1 : fs + 1])
|
|
42
|
+
print("per-sample [turns]:", steered.downconversion_phase[fs - 1 : fs + 1])
|
|
43
|
+
|
|
44
|
+
# Tag each mean at its record midpoint, t0 + T/2 (not the mean of the sample times).
|
|
45
|
+
half = np.timedelta64(500_000_000, "ns") # one-second records
|
|
46
|
+
for rec, f in zip(steered, steered.mean_downconversion_frequency):
|
|
47
|
+
print(f"{rec.header.timetag + half} {f:.3f} Hz")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Read an RDEF observation file and the product files it lists.
|
|
2
|
+
|
|
3
|
+
python examples/read_observation.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from rdefpy import read_observation, read_product
|
|
9
|
+
|
|
10
|
+
data = Path(__file__).parents[1] / "tests" / "data"
|
|
11
|
+
obs = read_observation(data / "EXMPn000tIsDS24r01c00-26039120000.obs")
|
|
12
|
+
print(obs)
|
|
13
|
+
|
|
14
|
+
for scan in obs:
|
|
15
|
+
print()
|
|
16
|
+
print(scan)
|
|
17
|
+
for entry in scan.products:
|
|
18
|
+
print()
|
|
19
|
+
print(read_product(entry.datafile))
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Summarise an RDEF product file: file info, first record header, power per record.
|
|
2
|
+
|
|
3
|
+
python examples/read_product_summary.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from rdefpy import read_product
|
|
9
|
+
|
|
10
|
+
data = Path(__file__).parents[1] / "tests" / "data"
|
|
11
|
+
pf = read_product(data / "EXMPn001tSsDS24r01c01-26039120000.prd")
|
|
12
|
+
|
|
13
|
+
pf.info()
|
|
14
|
+
print()
|
|
15
|
+
print(pf[0].header)
|
|
16
|
+
|
|
17
|
+
for i, rec in enumerate(pf):
|
|
18
|
+
d = rec.data # take it once: each access of an unloaded record may re-read the disk
|
|
19
|
+
print(f"record {i}: {rec.header.timetag} {len(d)} samples {d.power_db:.2f} dB rms {d.rms:.1f}")
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Read samples across records: slices of pf.samples and pf.timetags, and a record loop.
|
|
2
|
+
|
|
3
|
+
python examples/samples_and_timetags.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from rdefpy import read_product
|
|
11
|
+
|
|
12
|
+
data = Path(__file__).parents[1] / "tests" / "data"
|
|
13
|
+
pf = read_product(data / "EXMPn001tSsDS24r01c01-26039120000.prd")
|
|
14
|
+
fs = pf[0].header.sample_rate
|
|
15
|
+
|
|
16
|
+
# A slice that crosses the boundary between record 0 and record 1.
|
|
17
|
+
x = pf.samples[fs - 3 : fs + 3]
|
|
18
|
+
t = pf.timetags[fs - 3 : fs + 3]
|
|
19
|
+
for ti, xi in zip(t, x):
|
|
20
|
+
print(ti, xi)
|
|
21
|
+
|
|
22
|
+
# Record by record, decoding straight into one reused complex buffer.
|
|
23
|
+
buf = None
|
|
24
|
+
total = 0.0
|
|
25
|
+
for rec in pf:
|
|
26
|
+
buf = rec.read_samples(out=buf)
|
|
27
|
+
total += np.sum(np.abs(buf) ** 2)
|
|
28
|
+
print(f"mean power {10 * np.log10(total / (len(pf) * fs)):.2f} dB")
|
|
29
|
+
|
|
30
|
+
whole = pf.samples[:]
|
|
31
|
+
print("loop matches pf.samples:", np.isclose(total, np.sum(np.abs(whole) ** 2)))
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Select a time window of a product file, write it, and read it back.
|
|
2
|
+
|
|
3
|
+
python examples/select_and_write.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import tempfile
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from rdefpy import read_product, write_product
|
|
12
|
+
|
|
13
|
+
data = Path(__file__).parents[1] / "tests" / "data"
|
|
14
|
+
pf = read_product(data / "EXMPn001tSsDS24r01c01-26039120000.prd")
|
|
15
|
+
fs = pf[0].header.sample_rate
|
|
16
|
+
|
|
17
|
+
part = pf.select(start_epoch="2026-02-08T12:00:01", stop_epoch="2026-02-08T12:00:02")
|
|
18
|
+
print(f"{len(pf)} records -> {len(part)} selected, first at {part[0].header.timetag}")
|
|
19
|
+
|
|
20
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
21
|
+
out = Path(tmp) / "part.prd"
|
|
22
|
+
write_product(out, part)
|
|
23
|
+
back = read_product(out)
|
|
24
|
+
print(f"written {out.stat().st_size} bytes, read back {len(back)} record")
|
|
25
|
+
print("samples equal:", np.array_equal(back.samples[:], pf.samples[fs:])) # the second record
|
|
26
|
+
|
|
27
|
+
# A selection of every record is byte-identical to the original file.
|
|
28
|
+
write_product(out, pf.select())
|
|
29
|
+
print("bytes equal:", out.read_bytes() == pf.path.read_bytes())
|