s1-denoise 0.0.0.dev0__py3-none-any.whl

s1_denoise/baq_decoder.py

@@ -0,0 +1,203 @@
+# SPDX-FileCopyrightText: Aresys S.r.l. <info@aresys.it>
+# SPDX-License-Identifier: MIT
+
+"""
+Decode Sentinel-1 plain BAQ NOISE-packet samples (ICD "Data Format C", 3/4/5-bit), which ``sentinel1decoder`` cannot
+decode.
+
+The user data field is made of four 16-bit-aligned sections (IE, IO, QE, QO) of `num_quads` sign-magnitude
+`code_bits`-wide codes (MSB first); QE alone has an extra 8-bit Threshold Index (THIDX) field every 128 samples.
+The raw quantizer code is returned, not the NRL/sigma-corrected amplitude: this is valid only while THIDX stays below
+the ICD reconstruction threshold, which is checked explicitly (see `QE_THIDX_THRESHOLD`).
+"""
+
+import sys
+import numpy as np
+
+from s1_denoise.logger import denoise_logger
+
+QE_BLOCK_LEN = 128
+QE_HEADER_BITS = 8
+
+# THIDX threshold per BAQ mode below which the ICD-decoded sample value equals the raw
+# quantizer code (ICD section 9.1.2.2); at or above it, NRL/sigma-factor reconstruction
+# would be needed instead. Matches AUX_INS's thresholdLutList (thidxThreshold).
+QE_THIDX_THRESHOLD = {3: 3, 4: 5, 5: 10}
+
+
+def _section_align(bit_pos: int) -> int:
+    """
+    Round a bit position up to the next 16-bit boundary, where IE/IO/QE/QO sections start.
+
+    Parameters
+    ----------
+    bit_pos : int
+        Bit position.
+
+    Returns
+    -------
+    int
+        Aligned bit position.
+    """
+
+    rem = bit_pos % 16
+
+    return bit_pos if rem == 0 else bit_pos + (16 - rem)
+
+
+def _decode_plain_section(bits: np.ndarray, offset: int, nbits: int, count: int):
+    """
+    Decode a contiguous run of `count` sign-magnitude `nbits`-wide codes, without headers.
+
+    Parameters
+    ----------
+    bits : numpy.ndarray
+        Unpacked user data bits (0/1).
+    offset : int
+        Bit position of the first code.
+    nbits : int
+        Code width in bits, sign bit included.
+    count : int
+        Number of codes to decode.
+
+    Returns
+    -------
+    values : numpy.ndarray
+        Signed integer quantizer codes.
+    offset : int
+        Bit position following the last decoded code.
+    """
+
+    total = nbits * count
+    chunk = bits[offset : offset + total].reshape(count, nbits)
+    weights = 1 << np.arange(nbits - 1, -1, -1)
+    codes = chunk @ weights
+    sign = (codes >> (nbits - 1)) & 1
+    mag = codes & ((1 << (nbits - 1)) - 1)
+    values = np.where(sign == 1, -mag, mag)
+
+    return values, offset + total
+
+
+def _decode_qe_section(bits: np.ndarray, offset: int, nbits: int, count: int):
+    """
+    Decode the QE section, skipping the 8-bit THIDX field before every 128-sample block.
+
+    Each THIDX is checked to stay below the ICD reconstruction threshold, where the decoded value is the raw quantizer
+    code (see `QE_THIDX_THRESHOLD`).
+
+    Parameters
+    ----------
+    bits : numpy.ndarray
+        Unpacked user data bits (0/1).
+    offset : int
+        Bit position of the first THIDX field.
+    nbits : int
+        Code width in bits, sign bit included.
+    count : int
+        Number of codes to decode.
+
+    Returns
+    -------
+    values : numpy.ndarray
+        Signed integer quantizer codes.
+    offset : int
+        Bit position following the last decoded code.
+
+    Raises
+    ------
+    SystemExit
+        If a THIDX is at or above the reconstruction threshold (exit code 128).
+    """
+
+    threshold = QE_THIDX_THRESHOLD[nbits]
+    values = np.zeros(count, dtype=int)
+    pos = offset
+    idx = 0
+
+    while idx < count:
+        n = min(QE_BLOCK_LEN, count - idx)
+
+        thidx = 0
+        for bit in bits[pos : pos + QE_HEADER_BITS]:
+            thidx = (thidx << 1) | int(bit)
+        if thidx >= threshold:
+            denoise_logger.error(
+                f"QE block THIDX={thidx} is at or above the ICD reconstruction threshold ({threshold}) for BAQ {nbits}-bit mode; the raw quantizer code no longer equals the decoded sample value (NRL/sigma-factor reconstruction would be needed instead). Aborting rather than returning silently wrong noise power values."
+            )
+            sys.exit(128)
+
+        pos += QE_HEADER_BITS
+        block_values, pos = _decode_plain_section(bits, pos, nbits, n)
+        values[idx : idx + n] = block_values
+        idx += n
+
+    return values, pos
+
+
+def decode_baq_packet(raw_user_data: bytes, num_quads: int, code_bits: int = 5):
+    """
+    Decode the user data field of one plain BAQ packet.
+
+    Parameters
+    ----------
+    raw_user_data : bytes
+        Packet user data field.
+    num_quads : int
+        Number of quads, i.e. of codes in each section.
+    code_bits : int, optional
+        BAQ code width in bits. Only 5 (BAQ 5-bit) is supported. Default is 5.
+
+    Returns
+    -------
+    IE, IO, QE, QO : numpy.ndarray
+        Quantizer codes of the even/odd I and even/odd Q samples, one array per section.
+
+    Raises
+    ------
+    SystemExit
+        If `code_bits` is not 5, or a QE THIDX is at or above the reconstruction threshold (exit code 128).
+    """
+
+    if code_bits != 5:
+        denoise_logger.error(
+            f"BAQ {code_bits}-bit decoding is not validated against real data yet (only 5-bit noise packets have been verified); refusing to decode silently to avoid producing unverified noise power values."
+        )
+        sys.exit(128)
+
+    bits = np.unpackbits(np.frombuffer(raw_user_data, dtype=np.uint8)).astype(np.int64)
+
+    IE, pos = _decode_plain_section(bits, 0, code_bits, num_quads)
+    pos = _section_align(pos)
+    IO, pos = _decode_plain_section(bits, pos, code_bits, num_quads)
+    pos = _section_align(pos)
+    QE, pos = _decode_qe_section(bits, pos, code_bits, num_quads)
+    pos = _section_align(pos)
+    QO, _ = _decode_plain_section(bits, pos, code_bits, num_quads)
+
+    return IE, IO, QE, QO
+
+
+def quads_to_complex(IE: np.ndarray, IO: np.ndarray, QE: np.ndarray, QO: np.ndarray) -> np.ndarray:
+    """
+    Interleave the decoded sections into complex samples in the standard order.
+
+    The output is ``IE[0] + 1j*QE[0], IO[0] + 1j*QO[0], IE[1] + 1j*QE[1], ...``.
+
+    Parameters
+    ----------
+    IE, IO, QE, QO : numpy.ndarray
+        Quantizer codes of the even/odd I and even/odd Q samples, as returned by `decode_baq_packet`.
+
+    Returns
+    -------
+    numpy.ndarray
+        Complex64 samples, of length ``2 * len(IE)``.
+    """
+
+    num_quads = len(IE)
+    out = np.empty(num_quads * 2, dtype=np.complex64)
+    out[0::2] = IE + 1j * QE
+    out[1::2] = IO + 1j * QO
+
+    return out

s1_denoise/cli.py

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Aresys S.r.l. <info@aresys.it>
+# SPDX-License-Identifier: MIT
+
+"""
+CLI module for the denoising processor, built with Click.
+
+Defines the ``denoise_cli`` command, which reads a Job Order file to locate input L1 SAFE products and auxiliary
+files, generates noise profiles, writes output XML files, and, when the nominal noise annotations are available
+in the input product, computes the mean and maximum absolute error against them.
+"""
+
+import click
+from pathlib import Path
+import sys
+
+from perseo_core.timing import PreciseDateTime
+
+from s1_denoise.logger import denoise_logger
+from s1_denoise.noise_profiles import generate_noise_profiles, print_exec_time
+from s1_denoise import __version__
+from s1_denoise.jo_parser import JobOrder, find_l0n
+from s1_denoise.auxiliary_files import AuxiliaryFiles
+from s1_denoise.output import write_to_xml
+from s1_denoise.error_calculation import calculate_error
+
+
+@click.command()
+@click.version_option(__version__, prog_name="denoise")
+@click.argument("job_order_path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path)
+)
+def denoise_cli(job_order_path):
+    """
+    Generate noise profiles starting from L1 SAFE files.
+    \f
+    For each L1 product listed in the Job Order, the associated L0N product is looked up, noise vectors are
+    reconstructed and written as XML files in the output directory. If the product contains the corresponding
+    nominal noise annotation, the mean and maximum absolute errors (dB) are logged.
+
+    Parameters
+    ----------
+    job_order_path : pathlib.Path
+        Path to the Job Order XML file.
+
+    Raises
+    ------
+    SystemExit
+        With code 128 on any unexpected error; ``SystemExit`` raised during processing (e.g. by Job Order
+        validation) is propagated. The exit code is always logged before leaving.
+    """
+
+    return_code = 0
+    try:
+        jo = JobOrder(job_order_path)
+
+        # set log level
+        denoise_logger.setLevel(jo.log_level)
+        denoise_logger.debug("Starting noise processor...")
+
+        # read auxiliary files
+        aux_files = AuxiliaryFiles(mission_id = jo.mission_id,
+                                   pp1_path = jo.aux_pp1_filepath,
+                                   cal_path = jo.aux_cal_filepath,
+                                   ins_path = jo.aux_ins_filepath)
+
+        # read file to process from Job Order
+        product_path_list = [
+            Path(file_name)
+            for file_names in jo.input_files_to_process
+            for file_name in file_names['List_of_File_Names']['File_Name']
+        ]
+
+        out_filepath_list = []
+
+        start_time = PreciseDateTime.now()
+
+        for current_prod in product_path_list:
+
+            # respective L0N product
+            associated_l0n = find_l0n(l1a=current_prod, l0n_list=jo.input_l0n_files)
+
+            # denoise product
+            noise_vectors_list = generate_noise_profiles(prod=current_prod, l0n_file=associated_l0n, aux_files=aux_files)
+
+            # generate output files
+            for elem in noise_vectors_list:
+
+                out_filepath = write_to_xml(noise=elem, output_path=jo.output_path, safe_path=current_prod)
+                out_filepath_list.append(out_filepath)
+
+                # if nominal noise files exist, calculate MAE
+                nominal_noise = current_prod / "annotation/calibration" / (out_filepath.stem[2:] + ".xml")
+                if nominal_noise.exists():
+
+                    mae_on_denoise, max_err_on_denoise = calculate_error(nominal_noise_filepath=nominal_noise,
+                                                       denoise_filepath=out_filepath)
+
+                    if abs(mae_on_denoise) < 1.0:
+                        denoise_logger.info(f"Mean absolute error on denoising: {mae_on_denoise:.2e} dB")
+                    else:
+                        denoise_logger.warning(f"High mean absolute error on denoising: {mae_on_denoise:.2e} dB")
+
+                    denoise_logger.info(f"Maximum absolute error on denoising: {max_err_on_denoise:.2e} dB")
+
+        denoise_logger.info("Denoising process terminated successfully")
+
+        print_exec_time(elapsed_seconds=PreciseDateTime.now() - start_time)
+
+    except SystemExit as e:
+        return_code = e.code
+        raise
+    except Exception:
+        # any unhandled error stops the processor with exit code 128
+        denoise_logger.exception("Unexpected error. Aborting...")
+        return_code = 128
+        sys.exit(128)
+    finally:
+        denoise_logger.info(f"Exit code: {return_code}")

s1_denoise/error_calculation.py

@@ -0,0 +1,262 @@
+# SPDX-FileCopyrightText: Aresys S.r.l. <info@aresys.it>
+# SPDX-License-Identifier: MIT
+
+"""
+Compute the error between nominal and reconstructed Sentinel-1 noise vectors.
+
+Provides utilities to read noise XML files, interpolate the noise range vectors onto a common range/azimuth grid, and
+compute the mean and maximum absolute differences in dB between the two, after normalisation by the digital number
+scaling factor of the product.
+"""
+
+import numpy as np
+import xml.etree.ElementTree as ET
+import numpy as np
+from scipy.interpolate import interp1d, RegularGridInterpolator
+
+from perseo_core.timing import PreciseDateTime
+
+from s1_denoise.utils import linear_to_db
+
+DN_VALUES = {
+    ("IW", "SLC"): 237,
+    ("IW", "GRD"): 474,
+    ("EW", "SLC"): 543,
+    ("EW", "GRD"): 1087
+}
+
+def read_noise_xml(path):
+    """
+    Read the noise range vectors from a Sentinel-1 noise XML file (GRD or SLC).
+
+    An XML namespace, if present, is handled.
+
+    Parameters
+    ----------
+    path : str or pathlib.Path
+        Path of the noise XML file.
+
+    Returns
+    -------
+    times : numpy.ndarray
+        Azimuth times of the noise range vectors, as PreciseDateTime objects.
+    pixels_list : list of numpy.ndarray
+        Range pixel indices of each vector.
+    noise_list : list of numpy.ndarray
+        Noise range LUT values of each vector (linear scale).
+    """
+
+    tree = ET.parse(path)
+    root = tree.getroot()
+
+    # Detect namespace (if present)
+    if "}" in root.tag:
+        ns = {"ns": root.tag.split("}")[0].strip("{")}
+        search_path = ".//ns:noiseRangeVector"
+    else:
+        ns = {}
+        search_path = ".//noiseRangeVector"
+
+    times = []
+    pixels_list = []
+    noise_list = []
+
+    for vec in root.findall(search_path, ns):
+
+        def find(tag):
+            """Find a child tag of the current noise range vector, honouring the XML namespace."""
+            if ns:
+                return vec.find(f"ns:{tag}", ns)
+            return vec.find(tag)
+
+        az_time = PreciseDateTime.fromisoformat(find("azimuthTime").text)
+        pixels = np.array(list(map(float, find("pixel").text.split())))
+        noise = np.array(list(map(float, find("noiseRangeLut").text.split())))
+
+        times.append(az_time)
+        pixels_list.append(pixels)
+        noise_list.append(noise)
+
+    return np.array(times), pixels_list, noise_list
+
+
+def interpolate_noise(pixels_list, noise_list, common_pixels):
+    """
+    Linearly interpolate each noise range vector onto a common range pixel axis.
+
+    Parameters
+    ----------
+    pixels_list : list of numpy.ndarray
+        Range pixel indices of each vector.
+    noise_list : list of numpy.ndarray
+        Noise values of each vector.
+    common_pixels : numpy.ndarray
+        Common range pixel axis.
+
+    Returns
+    -------
+    numpy.ndarray
+        Interpolated noise, shape ``(n_vectors, len(common_pixels))``; NaN outside each vector's pixel span.
+    """
+
+    interp_noises = []
+    for pixels, noise in zip(pixels_list, noise_list):
+        f = interp1d(
+            pixels,
+            noise,
+            kind="linear",
+            bounds_error=False,
+            fill_value=np.nan,
+        )
+        interp_noises.append(f(common_pixels))
+    return np.array(interp_noises)
+
+
+def build_common_grid(times1, pixels_list1, times2, pixels_list2):
+    """
+    Compute a common range pixel and azimuth time grid for two sets of noise range vectors.
+
+    Both axes have 1500 points spanning the overlap of the two sets. Azimuth times are expressed in seconds relative to
+    the first vector of each set.
+
+    Parameters
+    ----------
+    times1, times2 : numpy.ndarray
+        Azimuth times of the two sets, as PreciseDateTime objects.
+    pixels_list1, pixels_list2 : list of numpy.ndarray
+        Range pixel indices of each vector of the two sets.
+
+    Returns
+    -------
+    common_pixels : numpy.ndarray
+        Common range pixel axis.
+    times1_sec : numpy.ndarray
+        Azimuth times of the first set [s], relative to its first vector.
+    times2_sec : numpy.ndarray
+        Azimuth times of the second set [s], relative to its first vector.
+    common_times_sec : numpy.ndarray
+        Common azimuth time axis [s].
+    """
+
+    # Common pixel axis
+    min_pixel = max(min(p[0] for p in pixels_list1),
+                    min(p[0] for p in pixels_list2))
+    max_pixel = min(max(p[-1] for p in pixels_list1),
+                    max(p[-1] for p in pixels_list2))
+
+    common_pixels = np.linspace(min_pixel, max_pixel, 1500)
+
+    # Convert azimuth time to seconds
+    times1_sec = np.array([(t - times1[0]) for t in times1])
+    times2_sec = np.array([(t - times2[0]) for t in times2])
+
+    common_times_sec = np.linspace(
+        max(times1_sec[0], times2_sec[0]),
+        min(times1_sec[-1], times2_sec[-1]),
+        1500,
+    )
+
+    return common_pixels, times1_sec, times2_sec, common_times_sec
+
+
+def compare_noise(noise_file, denoise_file, dn):
+    """
+    Compare nominal and reconstructed noise vectors on a common grid.
+
+    Both files are interpolated onto the grid of `build_common_grid` and normalised by ``dn**2``; zero values are set to
+    NaN and values are clipped at 1e-12 before conversion to dB.
+
+    Parameters
+    ----------
+    noise_file : str or pathlib.Path
+        Nominal noise XML file.
+    denoise_file : str or pathlib.Path
+        Reconstructed noise XML file.
+    dn : float
+        Digital number scaling factor of the product (see `DN_VALUES`).
+
+    Returns
+    -------
+    common_pixels : numpy.ndarray
+        Common range pixel axis.
+    common_times_sec : numpy.ndarray
+        Common azimuth time axis [s].
+    diff_db : numpy.ndarray
+        Reconstructed minus nominal noise [dB], shape ``(len(common_times_sec), len(common_pixels))``.
+    f1 : scipy.interpolate.RegularGridInterpolator
+        Interpolator of the nominal noise over (azimuth time [s], range pixel), not normalised by `dn`.
+    f2 : scipy.interpolate.RegularGridInterpolator
+        Interpolator of the reconstructed noise over (azimuth time [s], range pixel), not normalised by `dn`.
+    mae : float
+        Mean absolute difference [dB], ignoring NaN.
+    max_err : float
+        Maximum absolute difference [dB], ignoring NaN.
+    """
+
+    times1, pixels_list1, noise_list = read_noise_xml(noise_file)
+    times2, pixels_list2, denoise_list = read_noise_xml(denoise_file)
+
+    common_pixels, times1_sec, times2_sec, common_times_sec = build_common_grid(times1, pixels_list1, times2, pixels_list2)
+
+    interp_noise = interpolate_noise(pixels_list1, noise_list, common_pixels)
+    interp_denoise = interpolate_noise(pixels_list2, denoise_list, common_pixels)
+
+    f1 = RegularGridInterpolator(
+        (times1_sec, common_pixels),
+        interp_noise,
+        bounds_error=False,
+        fill_value=np.nan,
+    )
+    f2 = RegularGridInterpolator(
+        (times2_sec, common_pixels),
+        interp_denoise,
+        bounds_error=False,
+        fill_value=np.nan,
+    )
+
+    T, P = np.meshgrid(common_times_sec, common_pixels, indexing="ij")
+
+    noise = f1(np.stack([T, P], axis=-1)) / (dn**2)
+    denoise = f2(np.stack([T, P], axis=-1)) / (dn**2)
+
+    noise[noise == 0] = np.nan
+    denoise[denoise == 0] = np.nan
+
+    diff_db = linear_to_db(np.maximum(np.abs(denoise), 1e-12)) - linear_to_db(np.maximum(np.abs(noise), 1e-12))
+
+    mae = np.nanmean(np.abs(diff_db))
+    max_err = np.nanmax(np.abs(diff_db))
+
+    return common_pixels, common_times_sec, diff_db, f1, f2, mae, max_err
+
+
+def calculate_error(nominal_noise_filepath, denoise_filepath):
+    """
+    Compute the mean and maximum absolute error between a nominal and a reconstructed noise XML file.
+
+    The acquisition mode and product type, used to select the digital number in `DN_VALUES`, are derived from the name
+    of the SAFE product containing the nominal noise file (three directory levels up).
+
+    Parameters
+    ----------
+    nominal_noise_filepath : pathlib.Path
+        Nominal noise XML file, inside a SAFE product.
+    denoise_filepath : pathlib.Path
+        Reconstructed noise XML file.
+
+    Returns
+    -------
+    mean_error : float
+        Mean absolute difference [dB].
+    max_error : float
+        Maximum absolute difference [dB].
+    """
+
+    product_name = nominal_noise_filepath.parent.parent.parent.stem
+    mode = product_name[4:6]
+    prod_type = product_name[7:10]
+    dn = DN_VALUES.get((mode, prod_type))
+
+    common_pixels, common_times_sec, diff_db, f1, f2, mean_error, max_error = compare_noise(nominal_noise_filepath, denoise_filepath, dn)
+
+    return mean_error, max_error