s1-denoise 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- s1_denoise/__init__.py +18 -0
- s1_denoise/__main__.py +36 -0
- s1_denoise/auxiliary_files.py +481 -0
- s1_denoise/baq_decoder.py +203 -0
- s1_denoise/cli.py +118 -0
- s1_denoise/error_calculation.py +262 -0
- s1_denoise/jo_parser.py +295 -0
- s1_denoise/l0_noise_power_calculation.py +399 -0
- s1_denoise/logger.py +81 -0
- s1_denoise/noise_norm_factor.py +291 -0
- s1_denoise/noise_profiles.py +292 -0
- s1_denoise/output.py +551 -0
- s1_denoise/s1safe.py +1415 -0
- s1_denoise/thermal_noise.py +753 -0
- s1_denoise/utils.py +282 -0
- s1_denoise/xml_reader.py +571 -0
- s1_denoise-0.0.0.dev0.dist-info/METADATA +147 -0
- s1_denoise-0.0.0.dev0.dist-info/RECORD +21 -0
- s1_denoise-0.0.0.dev0.dist-info/WHEEL +4 -0
- s1_denoise-0.0.0.dev0.dist-info/entry_points.txt +2 -0
- s1_denoise-0.0.0.dev0.dist-info/licenses/LICENSE +21 -0
s1_denoise/jo_parser.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Aresys S.r.l. <info@aresys.it>
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
Module for parsing and validating Sentinel-1 Job Order XML files.
|
|
6
|
+
|
|
7
|
+
Provides the ``JobOrder`` class, which extracts processor configuration, input product paths, auxiliary file
|
|
8
|
+
references (AUX_PP1, AUX_CAL, AUX_INS) and the output path, with coherence checks against the product manifest.
|
|
9
|
+
Also includes utilities to match L1 products with their corresponding L0N files.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import xmltodict
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
import xml.etree.ElementTree as ET
|
|
16
|
+
|
|
17
|
+
from s1_denoise.logger import denoise_logger
|
|
18
|
+
|
|
19
|
+
FIELDS = ["Processor_Name", "Version", "Stdout_Log_Level", "Stderr_Log_Level", "Test", "Breakpoint_Enable"]
|
|
20
|
+
INPUT_L0N_FILES = ["IW_RAW__0N", "EW_RAW__0N"]
|
|
21
|
+
INPUT_PROCESSING_FILES = ["IW_SLC__1A", "IW_SLC__1S", "EW_SLC__1A", "EW_SLC__1S",
|
|
22
|
+
"IW_GRDH_1A", "IW_GRDH_1S", "IW_GRDM_1A", "IW_GRDM_1S",
|
|
23
|
+
"EW_GRDH_1A", "EW_GRDH_1S", "EW_GRDM_1A", "EW_GRDM_1S"]
|
|
24
|
+
INPUT_AUX_FILES = ["AUX_PP1", "AUX_CAL", "AUX_INS"]
|
|
25
|
+
INPUT_FILE_TYPES = INPUT_L0N_FILES + INPUT_PROCESSING_FILES + INPUT_AUX_FILES
|
|
26
|
+
OUTPUT_FILES_TYPES = ["DENOISE"]
|
|
27
|
+
|
|
28
|
+
def xml_to_dict(xml_path: str):
|
|
29
|
+
"""
|
|
30
|
+
Convert an XML file to a Python dictionary, preserving lists and dropping XML attributes.
|
|
31
|
+
|
|
32
|
+
Elements that may be repeated in a Job Order (``Dynamic_Processing_Parameter``, ``Ipf_Proc``, ``Input``,
|
|
33
|
+
``Output``, ``File_Name``) are always returned as lists, even when they occur once. Attribute keys (those
|
|
34
|
+
starting with ``@``, e.g. ``@count``) are removed recursively.
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
xml_path : str or pathlib.Path
|
|
39
|
+
Path to the XML file.
|
|
40
|
+
|
|
41
|
+
Returns
|
|
42
|
+
-------
|
|
43
|
+
dict
|
|
44
|
+
Nested dictionary representation of the XML document.
|
|
45
|
+
"""
|
|
46
|
+
with open(xml_path, "r", encoding="utf-8") as f:
|
|
47
|
+
xml_content = f.read()
|
|
48
|
+
|
|
49
|
+
# parse XML, enabling list detection
|
|
50
|
+
parsed = xmltodict.parse(xml_content, force_list={
|
|
51
|
+
"Dynamic_Processing_Parameter",
|
|
52
|
+
"Ipf_Proc",
|
|
53
|
+
"Input",
|
|
54
|
+
"Output",
|
|
55
|
+
"File_Name"
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
# remove XML attributes like 'count' if you don’t need them
|
|
59
|
+
def strip_attributes(obj):
|
|
60
|
+
"""Recursively remove keys starting with ``@`` (XML attributes) from dicts and lists."""
|
|
61
|
+
if isinstance(obj, dict):
|
|
62
|
+
return {k: strip_attributes(v) for k, v in obj.items() if not k.startswith("@")}
|
|
63
|
+
elif isinstance(obj, list):
|
|
64
|
+
return [strip_attributes(item) for item in obj]
|
|
65
|
+
else:
|
|
66
|
+
return obj
|
|
67
|
+
|
|
68
|
+
return strip_attributes(parsed)
|
|
69
|
+
|
|
70
|
+
class JobOrder:
|
|
71
|
+
"""
|
|
72
|
+
Parsed content of a Sentinel-1 Job Order XML file.
|
|
73
|
+
|
|
74
|
+
On construction the Job Order is parsed and validated: the AUX_PP1, AUX_CAL and AUX_INS inputs must be present,
|
|
75
|
+
the output path must not lie inside any input path, the L0N and auxiliary inputs must exist on disk and each
|
|
76
|
+
L1 input product must contain a ``manifest.safe``. Any failure is logged and terminates the process with exit
|
|
77
|
+
code 128.
|
|
78
|
+
|
|
79
|
+
Parameters
|
|
80
|
+
----------
|
|
81
|
+
joborder_path : pathlib.Path
|
|
82
|
+
Path to the Job Order XML file.
|
|
83
|
+
|
|
84
|
+
Attributes
|
|
85
|
+
----------
|
|
86
|
+
mission_id : str
|
|
87
|
+
Value of the first dynamic processing parameter (the mission identifier, e.g. ``S1A``).
|
|
88
|
+
log_level : str
|
|
89
|
+
Logging level from ``Stdout_Log_Level``.
|
|
90
|
+
input_files : list of dict
|
|
91
|
+
All ``Input`` entries of the first ``Ipf_Proc``.
|
|
92
|
+
input_files_to_process : list of dict
|
|
93
|
+
``Input`` entries whose ``File_Type`` is an L1 product type (L1A/L1S SLC and GRD in IW/EW modes).
|
|
94
|
+
input_l0n_files : list of pathlib.Path
|
|
95
|
+
Paths of the L0N input products (``IW_RAW__0N``, ``EW_RAW__0N``).
|
|
96
|
+
aux_pp1_filepath : pathlib.Path
|
|
97
|
+
Path of the AUX_PP1 auxiliary file.
|
|
98
|
+
aux_cal_filepath : pathlib.Path
|
|
99
|
+
Path of the AUX_CAL auxiliary file.
|
|
100
|
+
aux_ins_filepath : pathlib.Path
|
|
101
|
+
Path of the AUX_INS auxiliary file.
|
|
102
|
+
output_path : pathlib.Path
|
|
103
|
+
Output directory (first file name of the first ``Output`` entry).
|
|
104
|
+
|
|
105
|
+
Raises
|
|
106
|
+
------
|
|
107
|
+
RuntimeError
|
|
108
|
+
If ``joborder_path`` does not have an ``.xml`` suffix.
|
|
109
|
+
SystemExit
|
|
110
|
+
With code 128 if any of the validation checks fails.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
def __init__(self, joborder_path: Path):
|
|
114
|
+
"""Parse the Job Order file and validate its content."""
|
|
115
|
+
|
|
116
|
+
if joborder_path.suffix != ".xml":
|
|
117
|
+
raise RuntimeError(f"Job order {joborder_path} file is not an .xml file")
|
|
118
|
+
else:
|
|
119
|
+
data = xml_to_dict(joborder_path)
|
|
120
|
+
|
|
121
|
+
# IPF conf
|
|
122
|
+
ipf_conf = data['Ipf_Job_Order']['Ipf_Conf']
|
|
123
|
+
self.mission_id = ipf_conf['List_of_Dynamic_Processing_Parameters']['Dynamic_Processing_Parameter'][0]['Value']
|
|
124
|
+
self.log_level = ipf_conf['Stdout_Log_Level']
|
|
125
|
+
|
|
126
|
+
# IPF input files
|
|
127
|
+
ipf_proc = data['Ipf_Job_Order']['List_of_Ipf_Procs']['Ipf_Proc'][0]
|
|
128
|
+
self.input_files = ipf_proc['List_of_Inputs']['Input']
|
|
129
|
+
self.input_files_to_process = [ elem for elem in self.input_files if elem['File_Type'] in INPUT_PROCESSING_FILES ]
|
|
130
|
+
self.input_l0n_files = [Path(elem['List_of_File_Names']['File_Name'][0]) for elem in self.input_files if elem['File_Type'] in INPUT_L0N_FILES]
|
|
131
|
+
|
|
132
|
+
# auxiliary files
|
|
133
|
+
pp1_list = [elem for elem in self.input_files if elem['File_Type'] == "AUX_PP1"]
|
|
134
|
+
if len(pp1_list) != 0:
|
|
135
|
+
self.aux_pp1_filepath = Path(pp1_list[0]['List_of_File_Names']['File_Name'][0])
|
|
136
|
+
denoise_logger.info(f"AUX_PP1: {self.aux_pp1_filepath}")
|
|
137
|
+
else:
|
|
138
|
+
denoise_logger.error("Missing AUX_PP1 file in Job Order. Aborting...")
|
|
139
|
+
sys.exit(128)
|
|
140
|
+
|
|
141
|
+
cal_list = [elem for elem in self.input_files if elem['File_Type'] == "AUX_CAL"]
|
|
142
|
+
if len(cal_list) != 0:
|
|
143
|
+
self.aux_cal_filepath = Path(cal_list[0]['List_of_File_Names']['File_Name'][0])
|
|
144
|
+
denoise_logger.info(f"AUX_CAL: {self.aux_cal_filepath}")
|
|
145
|
+
else:
|
|
146
|
+
denoise_logger.error("Missing AUX_CAL file in Job Order. Aborting...")
|
|
147
|
+
sys.exit(128)
|
|
148
|
+
|
|
149
|
+
ins_list = [elem for elem in self.input_files if elem['File_Type'] == "AUX_INS"]
|
|
150
|
+
if len(ins_list) != 0:
|
|
151
|
+
self.aux_ins_filepath = Path(ins_list[0]['List_of_File_Names']['File_Name'][0])
|
|
152
|
+
denoise_logger.info(f"AUX_INS: {self.aux_ins_filepath}")
|
|
153
|
+
else:
|
|
154
|
+
denoise_logger.error("Missing AUX_INS file in Job Order. Aborting...")
|
|
155
|
+
sys.exit(128)
|
|
156
|
+
|
|
157
|
+
# output filepath
|
|
158
|
+
self.output_path = Path(ipf_proc['List_of_Outputs']['Output'][0]['File_Name'][0])
|
|
159
|
+
# the processor must not be able to write inside input folders
|
|
160
|
+
input_dirs = [Path(f['List_of_File_Names']['File_Name'][0]) for f in self.input_files]
|
|
161
|
+
if any(
|
|
162
|
+
self.output_path == input_dir or self.output_path.is_relative_to(input_dir)
|
|
163
|
+
for input_dir in input_dirs
|
|
164
|
+
):
|
|
165
|
+
denoise_logger.error("Cannot write results in input folders. Aborting...")
|
|
166
|
+
sys.exit(128)
|
|
167
|
+
|
|
168
|
+
self.check_input_files_exist()
|
|
169
|
+
self.check_manifest_file()
|
|
170
|
+
|
|
171
|
+
def check_input_files_exist(self):
|
|
172
|
+
"""
|
|
173
|
+
Check that the L0N input products and the auxiliary files exist on disk.
|
|
174
|
+
|
|
175
|
+
Every missing file is logged before exiting.
|
|
176
|
+
|
|
177
|
+
Raises
|
|
178
|
+
------
|
|
179
|
+
SystemExit
|
|
180
|
+
With code 128 if at least one file is missing.
|
|
181
|
+
"""
|
|
182
|
+
missing = []
|
|
183
|
+
|
|
184
|
+
for f in self.input_l0n_files:
|
|
185
|
+
if not f.exists():
|
|
186
|
+
missing.append(f)
|
|
187
|
+
|
|
188
|
+
for f in [self.aux_pp1_filepath, self.aux_cal_filepath, self.aux_ins_filepath]:
|
|
189
|
+
if not f.exists():
|
|
190
|
+
missing.append(f)
|
|
191
|
+
|
|
192
|
+
if missing:
|
|
193
|
+
for f in missing:
|
|
194
|
+
denoise_logger.error(f"Input file not found: {f}")
|
|
195
|
+
sys.exit(128)
|
|
196
|
+
|
|
197
|
+
def check_manifest_file(self):
|
|
198
|
+
"""
|
|
199
|
+
Check the L1 input products and the coherence of their auxiliary files with the Job Order.
|
|
200
|
+
|
|
201
|
+
For each product to process, verify that the product directory and its ``manifest.safe`` exist, then compare
|
|
202
|
+
the AUX_INS, AUX_PP1 and AUX_CAL names referenced in the manifest with those given in the Job Order. A mismatch
|
|
203
|
+
only produces a warning.
|
|
204
|
+
|
|
205
|
+
Returns
|
|
206
|
+
-------
|
|
207
|
+
manifest_ins : str
|
|
208
|
+
AUX_INS file name referenced in the manifest of the last checked product.
|
|
209
|
+
manifest_pp1 : str
|
|
210
|
+
AUX_PP1 file name referenced in the manifest of the last checked product.
|
|
211
|
+
manifest_cal : str
|
|
212
|
+
AUX_CAL file name referenced in the manifest of the last checked product.
|
|
213
|
+
|
|
214
|
+
Raises
|
|
215
|
+
------
|
|
216
|
+
SystemExit
|
|
217
|
+
With code 128 if a product directory or its manifest file does not exist.
|
|
218
|
+
"""
|
|
219
|
+
|
|
220
|
+
for prod in self.input_files_to_process:
|
|
221
|
+
prod_name = Path(prod['List_of_File_Names']['File_Name'][0])
|
|
222
|
+
manifest_path = prod_name / "manifest.safe"
|
|
223
|
+
|
|
224
|
+
if not prod_name.exists():
|
|
225
|
+
denoise_logger.error(f"Input product not found: {prod_name}. Aborting...")
|
|
226
|
+
sys.exit(128)
|
|
227
|
+
if not manifest_path.exists():
|
|
228
|
+
denoise_logger.error(f"Manifest file not found: {manifest_path}. Aborting...")
|
|
229
|
+
sys.exit(128)
|
|
230
|
+
|
|
231
|
+
tree = ET.parse(manifest_path)
|
|
232
|
+
root = tree.getroot()
|
|
233
|
+
|
|
234
|
+
ns = {
|
|
235
|
+
"safe": "http://www.esa.int/safe/sentinel-1.0"
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
for res in root.findall(".//safe:resource", ns):
|
|
239
|
+
if res.get("role") == "AUX_INS":
|
|
240
|
+
manifest_ins = Path(res.get("name")).name
|
|
241
|
+
self.check_coherence(manifest_ins, self.aux_ins_filepath.name, prod_name.stem)
|
|
242
|
+
elif res.get("role") == "AUX_PP1":
|
|
243
|
+
manifest_pp1 = Path(res.get("name")).name
|
|
244
|
+
self.check_coherence(manifest_pp1, self.aux_pp1_filepath.name, prod_name.stem)
|
|
245
|
+
elif res.get("role") == "AUX_CAL":
|
|
246
|
+
manifest_cal = Path(res.get("name")).name
|
|
247
|
+
self.check_coherence(manifest_cal, self.aux_cal_filepath.name, prod_name.stem)
|
|
248
|
+
|
|
249
|
+
return manifest_ins, manifest_pp1, manifest_cal
|
|
250
|
+
|
|
251
|
+
def check_coherence(self, manifest, jo_aux_file, prod):
|
|
252
|
+
"""
|
|
253
|
+
Log a warning if the auxiliary file used to generate a product differs from the Job Order one.
|
|
254
|
+
|
|
255
|
+
Parameters
|
|
256
|
+
----------
|
|
257
|
+
manifest : str
|
|
258
|
+
Auxiliary file name referenced in the product manifest.
|
|
259
|
+
jo_aux_file : str
|
|
260
|
+
Auxiliary file name given in the Job Order.
|
|
261
|
+
prod : str
|
|
262
|
+
Product name, used in the warning message.
|
|
263
|
+
"""
|
|
264
|
+
if manifest != jo_aux_file:
|
|
265
|
+
denoise_logger.warning(f"Using {jo_aux_file} for {prod}, while input data was processed with {manifest}. Calculated error could not be precise.")
|
|
266
|
+
|
|
267
|
+
def find_l0n(l1a: Path, l0n_list) -> Path | None:
|
|
268
|
+
"""
|
|
269
|
+
Find the L0N product associated with an L1 product.
|
|
270
|
+
|
|
271
|
+
The absolute orbit number and the datatake ID are taken from the L1 product name (third-last and second-last
|
|
272
|
+
underscore-separated fields) and the first L0N product whose name contains both is returned.
|
|
273
|
+
|
|
274
|
+
Parameters
|
|
275
|
+
----------
|
|
276
|
+
l1a : pathlib.Path
|
|
277
|
+
Path of the L1 SAFE product.
|
|
278
|
+
l0n_list : list of pathlib.Path
|
|
279
|
+
Paths of the candidate L0N products.
|
|
280
|
+
|
|
281
|
+
Returns
|
|
282
|
+
-------
|
|
283
|
+
pathlib.Path or None
|
|
284
|
+
The first matching L0N product, or ``None`` if no match is found.
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
parts = l1a.stem.split('_')
|
|
288
|
+
orbit_number, datatake_id = parts[-3], parts[-2]
|
|
289
|
+
|
|
290
|
+
# look for element in list with same orbit_number and datatake_id
|
|
291
|
+
for item in l0n_list:
|
|
292
|
+
l0n_name = item.stem
|
|
293
|
+
if orbit_number in l0n_name and datatake_id in l0n_name:
|
|
294
|
+
return item # give back first match
|
|
295
|
+
return None
|
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Aresys S.r.l. <info@aresys.it>
|
|
2
|
+
# SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
Extract noise power estimates from Sentinel-1 L0N raw data.
|
|
6
|
+
|
|
7
|
+
Reads the raw packets of an L0N SAFE product directly, producing one noise power sample per instrument burst: noise
|
|
8
|
+
pulse (NOISE signal) packets where present (plain BAQ 5-bit, decoded by `s1_denoise.baq_decoder`), otherwise the
|
|
9
|
+
rank echo packets of the burst (FDBAQ, decoded with ``sentinel1decoder``).
|
|
10
|
+
|
|
11
|
+
The per-burst samples are then filtered according to mission-specific rank echo sequences (IW/EW, S1A/B/C/D).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import sentinel1decoder as s1decoder
|
|
19
|
+
from sentinel1decoder import BaqMode, ECCNumber, RxChannelId, SignalType
|
|
20
|
+
|
|
21
|
+
from perseo_core.timing import PreciseDateTime
|
|
22
|
+
|
|
23
|
+
from s1_denoise.baq_decoder import decode_baq_packet, quads_to_complex
|
|
24
|
+
from s1_denoise.logger import denoise_logger
|
|
25
|
+
|
|
26
|
+
# Seconds between the GPS time epoch (1980-01-06T00:00:00) used by the "Coarse Time"/"Fine Time" packet fields and PreciseDateTime's own epoch (1985-01-01T00:00:00 UTC).
|
|
27
|
+
_GPS_EPOCH_TO_PRECISEDATETIME_EPOCH_SEC = 157420800.0
|
|
28
|
+
# GPS-UTC leap second offset. Constant since the last leap second insertion on 2016-12-31; would need bumping if IERS schedules a new one.
|
|
29
|
+
_GPS_UTC_LEAP_SECONDS = 18
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def iq_bias_correction(raw_data):
|
|
33
|
+
"""
|
|
34
|
+
Apply the I/Q bias correction to complex raw data.
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
raw_data : numpy.ndarray
|
|
39
|
+
Complex raw samples.
|
|
40
|
+
|
|
41
|
+
Returns
|
|
42
|
+
-------
|
|
43
|
+
numpy.ndarray
|
|
44
|
+
Complex samples with zero-mean I and Q components.
|
|
45
|
+
|
|
46
|
+
Notes
|
|
47
|
+
-----
|
|
48
|
+
For Sentinel-1 data the demodulation is performed by the instrument's receive module in the digital domain, so the
|
|
49
|
+
I/Q gain imbalance and non-orthogonality corrections are not necessary: only the constant biases are removed.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
i_values = raw_data.real
|
|
53
|
+
q_values = raw_data.imag
|
|
54
|
+
|
|
55
|
+
# remove constant biases: enforce zero mean
|
|
56
|
+
i_out = i_values - np.mean(i_values)
|
|
57
|
+
q_out = q_values - np.mean(q_values)
|
|
58
|
+
|
|
59
|
+
corrected_data = i_out + 1j * q_out
|
|
60
|
+
|
|
61
|
+
return corrected_data
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _packet_time(coarse_time, fine_time) -> PreciseDateTime:
|
|
65
|
+
"""
|
|
66
|
+
Convert the packet "Coarse Time"/"Fine Time" fields (GPS time) into a UTC PreciseDateTime.
|
|
67
|
+
|
|
68
|
+
Parameters
|
|
69
|
+
----------
|
|
70
|
+
coarse_time : int or float
|
|
71
|
+
Coarse time [s] since the GPS epoch.
|
|
72
|
+
fine_time : float
|
|
73
|
+
Fine time [s].
|
|
74
|
+
|
|
75
|
+
Returns
|
|
76
|
+
-------
|
|
77
|
+
PreciseDateTime
|
|
78
|
+
UTC packet time.
|
|
79
|
+
"""
|
|
80
|
+
sec85 = float(coarse_time) + float(fine_time) - _GPS_EPOCH_TO_PRECISEDATETIME_EPOCH_SEC - _GPS_UTC_LEAP_SECONDS
|
|
81
|
+
return PreciseDateTime.from_sec85(sec85)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _subswath_name(ecc_number, swath_num):
|
|
85
|
+
"""
|
|
86
|
+
Map an (ECC number, swath number) pair to a subswath label, e.g. ``"IW1"`` or ``"EW3"``.
|
|
87
|
+
|
|
88
|
+
Parameters
|
|
89
|
+
----------
|
|
90
|
+
ecc_number : sentinel1decoder.ECCNumber
|
|
91
|
+
ECC number of the packet.
|
|
92
|
+
swath_num : int
|
|
93
|
+
Swath number of the packet.
|
|
94
|
+
|
|
95
|
+
Returns
|
|
96
|
+
-------
|
|
97
|
+
str or None
|
|
98
|
+
Subswath label, or None for modes other than IW and EW.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
if ecc_number == ECCNumber.INTERFEROMETRIC_WIDE_SWATH:
|
|
102
|
+
return f"IW{swath_num - 9}"
|
|
103
|
+
if ecc_number == ECCNumber.EXTRA_WIDE_SWATH:
|
|
104
|
+
return f"EW{swath_num - 19}"
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _polarization_label(polarisation, rx_channel_id):
|
|
109
|
+
"""
|
|
110
|
+
Build the polarization label (e.g. ``"VH"``) from the transmit polarisation and the receive channel.
|
|
111
|
+
|
|
112
|
+
Parameters
|
|
113
|
+
----------
|
|
114
|
+
polarisation : enum
|
|
115
|
+
Packet "Polarisation" field; its name contains ``"TX_H"`` for H transmission.
|
|
116
|
+
rx_channel_id : sentinel1decoder.RxChannelId
|
|
117
|
+
Receive channel identifier.
|
|
118
|
+
|
|
119
|
+
Returns
|
|
120
|
+
-------
|
|
121
|
+
str
|
|
122
|
+
Two-letter polarization label.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
tx = "H" if "TX_H" in polarisation.name else "V"
|
|
126
|
+
rx = "H" if rx_channel_id == RxChannelId.RXH_POL_CHANNEL else "V"
|
|
127
|
+
|
|
128
|
+
return tx + rx
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _channel_lines(dat_path: Path):
|
|
132
|
+
"""
|
|
133
|
+
Decode the noise lines of one L0N measurement data file, grouped per (subswath, polarization) channel.
|
|
134
|
+
|
|
135
|
+
NOISE-signal chunks are fully decoded with the plain BAQ 5-bit decoder; for ECHO chunks, only the first ``Rank``
|
|
136
|
+
lines (rank echoes) are decoded with ``sentinel1decoder``.
|
|
137
|
+
|
|
138
|
+
Parameters
|
|
139
|
+
----------
|
|
140
|
+
dat_path : pathlib.Path
|
|
141
|
+
Path of the ``.dat`` measurement data file.
|
|
142
|
+
|
|
143
|
+
Returns
|
|
144
|
+
-------
|
|
145
|
+
dict
|
|
146
|
+
Dictionary keyed by ``(subswath, pol)``, whose values are dictionaries with:
|
|
147
|
+
|
|
148
|
+
- ``"lines"``: list of complex numpy.ndarray, one per decoded line;
|
|
149
|
+
- ``"times"``: list of PreciseDateTime, one per decoded line;
|
|
150
|
+
- ``"burst_sizes"``: list of int, number of lines of each burst.
|
|
151
|
+
|
|
152
|
+
Raises
|
|
153
|
+
------
|
|
154
|
+
NotImplementedError
|
|
155
|
+
If NOISE packets are not compressed in BAQ 5-bit mode.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
l0 = s1decoder.Level0File(str(dat_path))
|
|
159
|
+
bounds = l0._decoder._user_data_bounds
|
|
160
|
+
|
|
161
|
+
# group chunks per (subswath, pol)
|
|
162
|
+
channels = {}
|
|
163
|
+
with open(dat_path, "rb") as fh:
|
|
164
|
+
for chunk_id in l0.acquisition_chunks:
|
|
165
|
+
const = l0.get_acquisition_chunk_constants(chunk_id)
|
|
166
|
+
if const["signal_type"] not in (SignalType.NOISE, SignalType.ECHO):
|
|
167
|
+
continue
|
|
168
|
+
chunk_meta = l0.get_acquisition_chunk_metadata(chunk_id)
|
|
169
|
+
subswath = _subswath_name(chunk_meta["ECC Number"].iloc[0], const["swath_num"])
|
|
170
|
+
if subswath is None:
|
|
171
|
+
continue
|
|
172
|
+
pol = _polarization_label(chunk_meta["Polarisation"].iloc[0], chunk_meta["Rx Channel ID"].iloc[0])
|
|
173
|
+
key = (subswath, pol)
|
|
174
|
+
channels.setdefault(key, {"lines": [], "times": [], "burst_sizes": [], "_chunks": []})
|
|
175
|
+
channels[key]["_chunks"].append((chunk_id, const, chunk_meta))
|
|
176
|
+
|
|
177
|
+
for key, data in channels.items():
|
|
178
|
+
for chunk_id, const, chunk_meta in sorted(data["_chunks"], key=lambda x: x[0]):
|
|
179
|
+
if const["signal_type"] == SignalType.NOISE:
|
|
180
|
+
if const["baq_mode"] != BaqMode.BAQ_5_BIT_MODE:
|
|
181
|
+
raise NotImplementedError(f"Unsupported BAQ mode for NOISE packets in {key}: {const['baq_mode']}")
|
|
182
|
+
burst_meta = chunk_meta
|
|
183
|
+
burst_lines = []
|
|
184
|
+
for packet_num, num_quads in zip(
|
|
185
|
+
burst_meta.index.get_level_values("Packet Number"), burst_meta["Number of Quads"].astype(int)
|
|
186
|
+
):
|
|
187
|
+
start, length = bounds[packet_num]
|
|
188
|
+
fh.seek(start)
|
|
189
|
+
raw = fh.read(length)
|
|
190
|
+
IE, IO, QE, QO = decode_baq_packet(raw, num_quads, code_bits=5)
|
|
191
|
+
burst_lines.append(quads_to_complex(IE, IO, QE, QO))
|
|
192
|
+
else:
|
|
193
|
+
rank = int(chunk_meta["Rank"].iloc[0])
|
|
194
|
+
burst_meta = chunk_meta.iloc[:rank]
|
|
195
|
+
decoded = l0.get_acquisition_chunk_data(chunk_id, try_load_from_file=False)
|
|
196
|
+
burst_lines = list(decoded[: len(burst_meta)])
|
|
197
|
+
|
|
198
|
+
data["lines"].extend(burst_lines)
|
|
199
|
+
data["times"].extend(_packet_time(c, f) for c, f in zip(burst_meta["Coarse Time"], burst_meta["Fine Time"]))
|
|
200
|
+
data["burst_sizes"].append(len(burst_lines))
|
|
201
|
+
|
|
202
|
+
del data["_chunks"]
|
|
203
|
+
|
|
204
|
+
return channels
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _global_bias_correct_power(lines):
|
|
208
|
+
"""
|
|
209
|
+
Apply the I/Q bias correction globally over all lines of a channel and return the mean power of each line.
|
|
210
|
+
|
|
211
|
+
Parameters
|
|
212
|
+
----------
|
|
213
|
+
lines : list of numpy.ndarray
|
|
214
|
+
Complex lines of one channel.
|
|
215
|
+
|
|
216
|
+
Returns
|
|
217
|
+
-------
|
|
218
|
+
list of float
|
|
219
|
+
Mean power of each bias-corrected line.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
all_real = np.concatenate([line.real for line in lines])
|
|
223
|
+
all_imag = np.concatenate([line.imag for line in lines])
|
|
224
|
+
i_bias = np.mean(all_real)
|
|
225
|
+
q_bias = np.mean(all_imag)
|
|
226
|
+
return [np.mean(np.abs((line.real - i_bias) + 1j * (line.imag - q_bias)) ** 2) for line in lines]
|
|
227
|
+
|
|
228
|
+
def filter_rank_echoes(noise: dict, mission_id: str):
|
|
229
|
+
"""
|
|
230
|
+
Discard biased noise measurements according to mission-specific rank echo sequences.
|
|
231
|
+
|
|
232
|
+
Parameters
|
|
233
|
+
----------
|
|
234
|
+
noise : dict
|
|
235
|
+
Per-burst noise power, keyed by ``(subswath, pol)``, with values ``[az_time, burst_powers]`` (lists of
|
|
236
|
+
PreciseDateTime and of float).
|
|
237
|
+
mission_id : str
|
|
238
|
+
Mission identifier (e.g. ``"s1a"``), case-insensitive.
|
|
239
|
+
|
|
240
|
+
Returns
|
|
241
|
+
-------
|
|
242
|
+
dict
|
|
243
|
+
Filtered noise power, keyed by ``(subswath, pol)``, with values ``[times, powers]``.
|
|
244
|
+
|
|
245
|
+
Notes
|
|
246
|
+
-----
|
|
247
|
+
For S1A/B, the spikes in the first burst of the data-take are removed for IW1 and EW1 (second value), the last EW1
|
|
248
|
+
value is removed (possible nadir returns), and values biased by calibration pulses are discarded in an alternating
|
|
249
|
+
pattern for IW1, IW2 and IW3. For all missions, EW2 values (except the first) are replaced in place with the EW3 ones
|
|
250
|
+
of the same polarization.
|
|
251
|
+
"""
|
|
252
|
+
|
|
253
|
+
filtered = {}
|
|
254
|
+
for (subswath, pol), (az_time, burst_powers) in noise.items():
|
|
255
|
+
|
|
256
|
+
# the rank echo belongs to the first burst of the data take:
|
|
257
|
+
# remove noise spike occurring in S1A/B, IW1 and EW1
|
|
258
|
+
if mission_id.upper() in ["S1A", "S1B"]:
|
|
259
|
+
if subswath == "IW1":
|
|
260
|
+
l0n_noise = [burst_powers[0]] + burst_powers[2:]
|
|
261
|
+
l0n_time = [az_time[0]] + az_time[2:]
|
|
262
|
+
# in EW1 case, also remove last echo (nadir returns might be present)
|
|
263
|
+
elif subswath == "EW1":
|
|
264
|
+
l0n_noise = [burst_powers[0]] + burst_powers[2:-1]
|
|
265
|
+
l0n_time = [az_time[0]] + az_time[2:-1]
|
|
266
|
+
else:
|
|
267
|
+
l0n_noise = burst_powers
|
|
268
|
+
l0n_time = az_time
|
|
269
|
+
else:
|
|
270
|
+
l0n_noise = burst_powers
|
|
271
|
+
l0n_time = az_time
|
|
272
|
+
|
|
273
|
+
# the rank echo belongs to an odd-numbered burst: remove
|
|
274
|
+
if mission_id.upper() in ["S1A", "S1B"]:
|
|
275
|
+
if subswath == "IW1":
|
|
276
|
+
# in this case the first rank of the first burst was already removed
|
|
277
|
+
# (so odd-numbered burst = even-indexed values)
|
|
278
|
+
final_times = np.concatenate(([l0n_time[0]], l0n_time[1::2], [l0n_time[-1]]))
|
|
279
|
+
final_noise = np.concatenate(([l0n_noise[0]], l0n_noise[1::2], [l0n_noise[-1]]))
|
|
280
|
+
elif subswath in ["IW2", "IW3"]:
|
|
281
|
+
final_times = np.concatenate((l0n_time[0::2], [l0n_time[-1]]))
|
|
282
|
+
final_noise = np.concatenate((l0n_noise[0::2], [l0n_noise[-1]]))
|
|
283
|
+
else:
|
|
284
|
+
final_times = l0n_time
|
|
285
|
+
final_noise = l0n_noise
|
|
286
|
+
else:
|
|
287
|
+
final_times = l0n_time
|
|
288
|
+
final_noise = l0n_noise
|
|
289
|
+
|
|
290
|
+
filtered[(subswath, pol)] = [final_times, final_noise]
|
|
291
|
+
|
|
292
|
+
# for EW2, substitute noise values with ones from EW3
|
|
293
|
+
keys_to_replace = [k for k in filtered if k[0] == "EW2"]
|
|
294
|
+
for subswath, pol in keys_to_replace:
|
|
295
|
+
filtered[("EW2", pol)][1][1:] = filtered[("EW3", pol)][1][1:]
|
|
296
|
+
|
|
297
|
+
return filtered
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def calculate_noise_power(l0n_path: Path, mission_id: str, rank_echoes_present: bool):
|
|
301
|
+
"""
|
|
302
|
+
Compute the noise power sequences of each channel from the raw packets of an L0N SAFE product.
|
|
303
|
+
|
|
304
|
+
All ``.dat`` files of the product are decoded; the I/Q bias correction is applied per channel, and the line powers
|
|
305
|
+
are averaged per burst (pre-amble, each rank echo or noise sequence, post-amble), keeping the time of the first
|
|
306
|
+
line. The result is then filtered with `filter_rank_echoes`.
|
|
307
|
+
|
|
308
|
+
Parameters
|
|
309
|
+
----------
|
|
310
|
+
l0n_path : pathlib.Path
|
|
311
|
+
Path of the L0N SAFE directory.
|
|
312
|
+
mission_id : str
|
|
313
|
+
Mission identifier (e.g. ``"s1a"``), case-insensitive.
|
|
314
|
+
rank_echoes_present : bool
|
|
315
|
+
Whether the L0N product contains rank echoes.
|
|
316
|
+
|
|
317
|
+
Returns
|
|
318
|
+
-------
|
|
319
|
+
dict
|
|
320
|
+
Noise power keyed by ``(subswath, pol)``, with values ``[times, powers]`` (azimuth times and mean noise power
|
|
321
|
+
per burst, linear scale).
|
|
322
|
+
|
|
323
|
+
Raises
|
|
324
|
+
------
|
|
325
|
+
SystemExit
|
|
326
|
+
For S1C/S1D products with rank echoes, not supported yet (exit code 128).
|
|
327
|
+
"""
|
|
328
|
+
|
|
329
|
+
if mission_id.upper() in ["S1C", "S1D"] and rank_echoes_present:
|
|
330
|
+
denoise_logger.error(
|
|
331
|
+
"S1C/S1D rank-echo handling not implemented yet. Aborting..."
|
|
332
|
+
)
|
|
333
|
+
sys.exit(128)
|
|
334
|
+
|
|
335
|
+
channels = {}
|
|
336
|
+
for dat_path in sorted(Path(l0n_path).glob("*.dat")):
|
|
337
|
+
denoise_logger.info(f"Decoding raw packets from {dat_path.name}...")
|
|
338
|
+
for key, data in _channel_lines(dat_path).items():
|
|
339
|
+
channels.setdefault(key, {"lines": [], "times": [], "burst_sizes": []})
|
|
340
|
+
channels[key]["lines"].extend(data["lines"])
|
|
341
|
+
channels[key]["times"].extend(data["times"])
|
|
342
|
+
channels[key]["burst_sizes"].extend(data["burst_sizes"])
|
|
343
|
+
|
|
344
|
+
noise = {}
|
|
345
|
+
for key, data in channels.items():
|
|
346
|
+
powers = _global_bias_correct_power(data["lines"])
|
|
347
|
+
times = data["times"]
|
|
348
|
+
burst_sizes = data["burst_sizes"]
|
|
349
|
+
|
|
350
|
+
# reduce each burst's lines to one representative (mean power, first time) sample:
|
|
351
|
+
# burst_sizes = [n_preamble, rank_burst_1, rank_burst_2, ... , n_postamble]
|
|
352
|
+
burst_powers = []
|
|
353
|
+
az_time = []
|
|
354
|
+
idx = 0
|
|
355
|
+
for size in burst_sizes:
|
|
356
|
+
burst_powers.append(np.mean(powers[idx : idx + size]))
|
|
357
|
+
az_time.append(times[idx])
|
|
358
|
+
idx += size
|
|
359
|
+
|
|
360
|
+
noise[key] = [az_time, burst_powers]
|
|
361
|
+
|
|
362
|
+
# rank echoes filtering based on mission ID
|
|
363
|
+
filtered = filter_rank_echoes(noise, mission_id)
|
|
364
|
+
|
|
365
|
+
return filtered
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def get_noise_pwr_from_l0n(l0n_path: Path, aux_ins_path: Path = None):
|
|
369
|
+
"""
|
|
370
|
+
Extract the noise power sequences of each channel from a Sentinel-1 L0N SAFE product.
|
|
371
|
+
|
|
372
|
+
The mission is taken from the product name, and the presence of rank echoes is detected from the manifest.
|
|
373
|
+
|
|
374
|
+
Parameters
|
|
375
|
+
----------
|
|
376
|
+
l0n_path : pathlib.Path
|
|
377
|
+
Path of the L0N SAFE directory.
|
|
378
|
+
aux_ins_path : pathlib.Path, optional
|
|
379
|
+
Path of the AUX_INS file. Currently unused. Default is None.
|
|
380
|
+
|
|
381
|
+
Returns
|
|
382
|
+
-------
|
|
383
|
+
dict
|
|
384
|
+
Noise power keyed by ``(subswath, pol)``, as returned by `calculate_noise_power`.
|
|
385
|
+
"""
|
|
386
|
+
|
|
387
|
+
mission_id = l0n_path.stem[0:3].lower()
|
|
388
|
+
|
|
389
|
+
rank_echoes_present = "echoCompressionType" in Path(l0n_path.resolve() / "manifest.safe").read_text()
|
|
390
|
+
if rank_echoes_present:
|
|
391
|
+
denoise_logger.info("Rank echoes present in L0N file")
|
|
392
|
+
else:
|
|
393
|
+
denoise_logger.info("Rank echoes not present in L0N file")
|
|
394
|
+
|
|
395
|
+
noise = calculate_noise_power(l0n_path, mission_id, rank_echoes_present)
|
|
396
|
+
|
|
397
|
+
denoise_logger.info("Noise power calculation completed successfully")
|
|
398
|
+
|
|
399
|
+
return noise
|