cgse 2023.38.0__py3-none-any.whl → 2024.1.3__py3-none-any.whl

scripts/check_hdf5_files.py

@@ -1,192 +0,0 @@
-import glob
-import logging
-from pathlib import Path
-from typing import Optional
-from typing import Union
-
-import click
-import rich
-from h5py import File
-from rich.progress import track
-
-from egse import h5
-from egse.bits import beautify_binary
-from egse.fee import n_fee_mode
-from egse.fee.nfee import HousekeepingData
-from egse.reg import RegisterMap
-from egse.spw import SpaceWirePacket
-
-MODULE_LOGGER = logging.getLogger("egse.check_hdf5_file")
-
-
-def check_if_top_level_groups_complete(hdf5_file: File, indent: str) -> (str, bool):
-
-    msg = ""
-    error = False
-
-    # Check for internal or external sync
-
-    if "/register" in hdf5_file:
-        reg_data = h5.get_data(hdf5_file["/register"])
-        reg_map = RegisterMap("N-FEE", memory_map=reg_data)
-        sync_sel = reg_map["sync_sel"]
-        if sync_sel and ("/1" in hdf5_file or "/2" in hdf5_file or "/3" in hdf5_file):
-            msg += indent + (
-                   "[bold red]ERROR[/]: internal sync contains more than one readout period\n")
-            error = True
-        if not sync_sel and (
-            missing := [idx for idx in range(4) if f"/{idx}" not in hdf5_file]
-        ):
-            msg += indent + f"[bold red]ERROR[/]: missing readouts for external sync {missing}\n"
-            error = True
-
-    return msg, error
-
-
-def check_hdf5_file(filename: Union[str, Path], requested_obsid: Union[int, str], error_flags_only=False, errors_only=False):
-
-    filename = Path(filename)
-
-    if filename.exists():
-        try:
-
-            with h5.get_file(filename, mode='r') as hdf5_file:
-
-                has_error = False
-                indent = "  "
-                full_msg = f"Analysing {filename!s}\n"
-                full_msg += indent + f"Top level groups: {list(hdf5_file.keys())}\n"
-                error_msg = ""
-                obsid = ''
-
-                msg, error = check_if_top_level_groups_complete(hdf5_file, indent)
-                full_msg += msg
-                has_error = error or has_error
-
-                if "/obsid" in hdf5_file:
-                    obsid = h5.get_data(hdf5_file["/obsid"]).item().decode()
-                    if obsid == 'None':
-                        full_msg += indent + "no observation running\n"
-                    else:
-                        full_msg += indent + f"OBSID = {obsid}\n"
-
-                if requested_obsid is not None and str(requested_obsid) not in obsid:
-                    return
-
-                if "/0/timecode" in hdf5_file:
-                    timecode = h5.get_data(hdf5_file["/0/timecode"])
-                    timestamp = h5.get_attribute_value(hdf5_file["/0/timecode"], "timestamp")
-                    full_msg += indent + f"{timecode=!s}, {timestamp=}\n"
-
-                if "/register" in hdf5_file:
-                    reg_data = h5.get_data(hdf5_file["/register"])
-                    reg_map = RegisterMap("N-FEE", memory_map=reg_data)
-                    v_start = reg_map['v_start']
-                    v_end = reg_map['v_end']
-                    digitise_en = reg_map['digitise_en']
-                    dg_en = reg_map['DG_en']
-                    sync_sel = reg_map['sync_sel']
-                    try:
-                        mode = n_fee_mode(reg_map['ccd_mode_config']).name
-                    except ValueError:
-                        mode = f"[bold red]INVALID ({reg_map['ccd_mode_config']})[/]"
-                        has_error = True
-                    full_msg += indent + f"register: {v_start=}, {v_end=}, {digitise_en=}, {dg_en=}, {sync_sel=}, {mode=}\n"
-                else:
-                    full_msg += indent + '[red]no register map in this HDF5 file[/red]\n'
-                    has_error = True
-
-                if "/0/data" in hdf5_file and h5.has_attributes(hdf5_file['/0/data']):
-                    v_start = h5.get_attribute_value(hdf5_file['/0/data'], "v_start")
-                    v_end = h5.get_attribute_value(hdf5_file['/0/data'], "v_end")
-                    full_msg += indent + f"data: {v_start=}, {v_end=}\n"
-
-                for count in range(4):
-                    if f"/{count}/hk" in hdf5_file:
-                        hk_packet = SpaceWirePacket.create_packet(hdf5_file[f"/{count}/hk"][...])
-                        full_msg += indent + f"hk: {hk_packet.type} {beautify_binary(hk_packet.type.value)}\n"
-                        error_flags = HousekeepingData(hk_packet.data)['error_flags']
-                        if error_flags:
-                            msg = indent + (
-                                f"Frame {count}: [red]One or more of the error flags are ON: "
-                                f"{beautify_binary(error_flags, group=4, size=12)}[/red]\n"
-                            )
-                            full_msg += msg
-                            error_msg += msg
-                            has_error = True
-                for count in range(4):
-                    if f"/{count}/commands" in hdf5_file:
-                        for idx in hdf5_file[f"/{count}/commands/"]:
-                            full_msg += indent + f"command in frame {count}: {h5.get_data(hdf5_file[f'/{count}/commands/{idx}'])}\n"
-
-            if error_flags_only and error_msg:
-                rich.print(f"{filename.name=}, size={filename.stat().st_size}")
-                rich.print(error_msg)
-            if not error_flags_only:
-                if errors_only and has_error:
-                    rich.print(full_msg)
-                if not errors_only:
-                    rich.print(full_msg)
-
-        except OSError as exc:
-            rich.print(f"{filename=!s}, {exc}")
-    else:
-        rich.print(f"no such file {filename!s}")
-
-
-def find_hdf5_for_obsid(files: list, requested_obsid: Union[int, str]):
-    new_files = []
-
-    for filename in sorted(files):
-        try:
-            with h5.get_file(filename, mode='r') as hdf5_file:
-                if "/obsid" in hdf5_file:
-                    obsid = h5.get_data(hdf5_file["/obsid"]).item().decode()
-
-                    if str(requested_obsid) in obsid:
-                        new_files.append(filename)
-        except OSError as exc:
-            MODULE_LOGGER.error(f"Couldn't open {filename} ({exc=})")
-    return new_files
-
-
-@click.command()
-@click.argument('root')
-@click.argument('filename_t')
-@click.option("--obsid", help="The full OBSID, e.g. SRON_00058_02380 or only the TEST-ID, e.g. 2380")
-@click.option("--error-flags-only", is_flag=True, help="Print only the filename and error information")
-@click.option("--errors-only", is_flag=True, help="Print only the filename and error information")
-def cli(obsid, error_flags_only, errors_only, root, filename_t):
-    """Script to inspect the HDF5 files generated by the DPU Processor.
-
-        * ROOT - the root folder that contains the HDF5 files to be checked
-
-        * FILENAME_T - a file name template accepted by glob()
-    """
-    root = Path(root)
-
-    files = glob.glob(str(root / filename_t))
-    if obsid is not None:
-        files = find_hdf5_for_obsid(files, obsid)
-    n_files = len(files)
-
-    rich.print(f"Number of files: {n_files}")
-
-    print(" ".join([Path(file).name for file in files]))
-
-    answer = input("So, you want to continue [Y/n]")
-    print(answer)
-    if answer.lower() in ['n', 'no']:
-        return
-
-    for filename in track(sorted(files), description=f"Checking {n_files} files"):
-    # for filename in sorted(files):
-        try:
-            check_hdf5_file(filename, obsid, error_flags_only, errors_only)
-        except Exception as exc:
-            rich.print(f"[red]ERROR[/]: {Path(filename).name}: {exc!s}")
-            MODULE_LOGGER.debug(f"{exc!s}", exc_info=True)
-
-
-if __name__ == "__main__":
-    cli()

scripts/check_register_sync.py

@@ -1,47 +0,0 @@
-"""
-This module contains commands and strategies to test the synchronisation of the Register Map
-between the DPU Processor and the N-FEE.
-
-The reason for this test is that the DPU Processor seems to be out of sync with the N-FEE at some
-points in time:
-
-* When the N-FEE is power-cycled and the DPU CS is not restarted, the register map is not
-  refreshed from the newly pwoered N-FEE
-
-* ...
-
-The DPU CS has a few commands available to fix this:
-
-* dpu.n_fee_sync_register_map() — reads the register map from the N-FEE, updates the DPU internal
-  register map and returns the register map.
-
-* dpu.reset() — sets a special register in the N-FEE ccd_mode_config which commands the N-FEE to
-  resets its register map to its default state.
-
-"""
-
-from egse.dpu import DPUProxy
-from egse.reg import compare_register_maps
-
-dpu_proxy = DPUProxy()
-
-
-reg_1 = dpu_proxy.n_fee_sync_register_map()
-
-# Put the N-FEE in DUMP mode
-
-dpu_proxy.n_fee_set_dump_mode(
-    dict(v_start=0, v_end=2000, ccd_readout_order=0b11111111, n_final_dump=4510))
-
-reg_2 = dpu_proxy.n_fee_sync_register_map()
-
-# Put the N-FEE back into its default mode (same as after a power cycle?)
-
-dpu_proxy.n_fee_reset()
-
-reg_3 = dpu_proxy.n_fee_sync_register_map()
-
-
-compare_register_maps(reg_1, reg_2)
-compare_register_maps(reg_2, reg_3)
-compare_register_maps(reg_1, reg_3)

scripts/create_hdf5_report.py

@@ -1,295 +0,0 @@
-#! /usr/bin/env python3
-
-import glob
-import logging
-import os
-from datetime import datetime
-from enum import IntEnum
-from pathlib import Path
-from typing import List
-
-import click
-import pandas as pd
-from pandas import DataFrame
-from rich import print
-from rich.console import Console
-from rich.progress import track
-from rich.table import Table
-
-from egse import h5
-from egse.setup import load_setup
-from egse.bits import beautify_binary
-from egse.fee import convert_ccd_order_value
-from egse.fee import n_fee_mode
-from egse.fee.nfee import HousekeepingData
-from egse.reg import RegisterMap
-from egse.spw import SpaceWirePacket
-from egse.storage.persistence import CSV2
-
-MODULE_LOGGER = logging.getLogger("egse.create_hdf5_report")
-
-df_awg2: DataFrame
-"""Holds the AEU AWG2 HK data to determine external sync period."""
-
-sync_mode = {0: 'ext', 1: 'int'}
-"""Decodes N-FEE internal or external sync mode."""
-
-reg_map = RegisterMap("N-FEE")
-"""Holds the register map of the N-FEE."""
-
-setup = load_setup()
-sensor_sel = setup.camera.fee.sensor_sel.enum
-"""Decodes N-FEE E-side and F-side information."""
-
-def load_aeu_awg2_data(location: Path):
-    try:
-        awg_fn = glob.glob(str(location / '*_AEU-AWG2.csv'))[0]
-    except IndexError as exc:
-        raise FileNotFoundError(
-            "The AEU AWG2 CSV file is not found, cannot determine correct external sync period."
-        ) from exc
-
-    print(f"Loading {awg_fn}...")
-    converters = {
-         "timestamp": lambda x: datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%f+0000"),
-    }
-    return pd.read_csv(location / awg_fn, index_col="timestamp", converters=converters)
-
-
-def get_external_sync_period(timestamp: str):
-    if timestamp == 'N/A':
-        return float('nan')
-    dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f+0000")
-    try:
-        idx = df_awg2.index.get_loc(dt, method="nearest")
-    except AttributeError:
-        return 0.0
-    return df_awg2.iloc[idx]['GAEU_EXT_CYCLE_TIME']
-
-
-def extract_register_values(filename: Path):
-    with h5.get_file(filename, mode='r', locking=False) as hdf5_file:
-
-        if "/obsid" in hdf5_file:
-            obsid = h5.get_data(hdf5_file["/obsid"]).item().decode()
-        else:
-            obsid = ''
-
-        if "/0/timecode" in hdf5_file:
-            timecode = h5.get_data(hdf5_file["/0/timecode"])
-            timestamp = h5.get_attribute_value(hdf5_file["/0/timecode"], "timestamp")
-        else:
-            timestamp = 'N/A'
-
-        if "/register" not in hdf5_file:
-            raise RuntimeError(f'no register map in this HDF5 file ({filename.name})')
-
-        reg_data = h5.get_data(hdf5_file["/register"])
-        reg_map.set_data(0x0000_0000, bytearray(reg_data.tobytes()), 0x800)
-        v_start = reg_map['v_start']
-        v_end = reg_map['v_end']
-        digitise_en = reg_map['digitise_en']
-        dg_en = reg_map['DG_en']
-        n_final_dump = reg_map['n_final_dump']
-        sync_sel = sync_mode[reg_map['sync_sel']]
-        if sync_sel == 'ext':
-            sync_period = get_external_sync_period(timestamp)
-            n_frames = 4
-        else:
-            sync_period = reg_map['int_sync_period'] / 1000.0
-            n_frames = 1
-        sensor = sensor_sel(reg_map['sensor_sel']).name
-        ccd_readout_order = convert_ccd_order_value(reg_map['ccd_readout_order'])
-
-        charge_injection_en = reg_map['charge_injection_en'] or ''
-        charge_inject_width = reg_map['charge_injection_width'] if charge_injection_en else ''
-        charge_inject_gap = reg_map['charge_injection_gap'] if charge_injection_en else ''
-
-        ccd_vgd_config_19 = reg_map[('reg_19_config', 'ccd_vgd_config')]
-        ccd_vgd_config_20 = reg_map[('reg_20_config', 'ccd_vgd_config')]
-        ccd_vgd_config = (ccd_vgd_config_20 << 4) + ccd_vgd_config_19
-
-        try:
-            mode = n_fee_mode(reg_map['ccd_mode_config']).name
-        except ValueError:
-            mode = f"[bold red]INVALID ({reg_map['ccd_mode_config']})[/]"
-
-        has_error_flags = ''
-        all_error_flags_value = [0, 0, 0, 0]
-        all_error_flags_str = [" "*5, " "*5, " "*5, " "*5]
-
-        for count in range(n_frames):
-            if f"/{count}/hk_data" in hdf5_file:
-                hk_data = hdf5_file[f"/{count}/hk_data"][...]
-            elif f"/{count}/hk" in hdf5_file:
-                hk_packet = SpaceWirePacket.create_packet(hdf5_file[f"/{count}/hk"][...])
-                hk_data = hk_packet.data
-            else:
-                continue
-            if error_flags := HousekeepingData(hk_data)['error_flags']:
-                all_error_flags_value[count] = error_flags
-                # error_flags = beautify_binary(error_flags, group=4, size=12)
-                error_flags = f"0x{error_flags:03X}"
-                all_error_flags_str[count] = error_flags
-                has_error_flags = 'x'
-
-        # Since Rich Table cannot render a List, convert it to a string
-        all_error_flags_str = ", ".join(all_error_flags_str) if has_error_flags else ''
-
-    return (
-            obsid,
-            timestamp[:19],
-            filename.name,
-            str(v_start),
-            str(v_end),
-            str(n_final_dump),
-            str(digitise_en),
-            str(dg_en),
-            sync_sel,
-            f"{sync_period:.3f}",
-            mode,
-            sensor,
-            str(ccd_readout_order),
-            str(charge_injection_en),
-            str(charge_inject_width),
-            str(charge_inject_gap),
-            hex(ccd_vgd_config),
-            str(all_error_flags_value),
-            all_error_flags_str,
-            has_error_flags,
-    )
-
-
-@click.command()
-@click.argument('location')
-@click.option('--output-file', '-o', help="full path name for the report file")
-@click.option('--csv-format', is_flag=True, help="output format in CVS instead of TXT")
-def cli(location, output_file, csv_format):
-    """
-    Script to extract parameter values from the register map in the HDF5 files that are
-    generated by the DPU Processor.
-
-    * location - the folder that contains the HDF5 files to be checked
-
-    """
-    global df_awg2
-
-    location = Path(location)
-    output_file = output_file or ("report.csv" if csv_format else "report.txt")
-
-    files = glob.glob(str(location / '*_SPW_*.hdf5'))
-    n_files = len(files)
-
-    # Read the AEU AWG2 data file to determine the external sync period
-
-    try:
-        df_awg2 = load_aeu_awg2_data(location)
-    except FileNotFoundError:
-        df_awg2 = None
-
-    # Extract all the parameters from the HDF5 register map
-
-    data = []
-    errors = []
-
-    print(f"Number of files: {n_files}")
-
-    for filename in track(sorted(files), description=f"Extracting register from {n_files} files"):
-        # for filename in sorted(files):
-        try:
-            row = extract_register_values(Path(filename))
-            data.append(list(row))
-            if row[-1] == 'x':  # the last element is the has_error_flags, the first is the OBSID
-                errors.append(row[0])
-        except Exception as exc:
-            print(f"[red]ERROR[/]: {Path(filename).name}: {exc!s}")
-            MODULE_LOGGER.error(f"{exc!s}", exc_info=True)
-
-    if csv_format:
-        print_csv(output_file, data, errors)
-    else:
-        print_table(output_file, data, errors)
-
-def exclude_items(*indices):
-    """
-    Returns a function that excludes the items of its operand at the given indices and returns a tuple
-    containing the other elements of the operand using the `__getitem__` method of the operand.
-
-    Args:
-        *indices: indices of items that shall be excluded, can be a negative index also.
-
-    """
-    def func(obj):
-        size = len(obj)
-        pos_indices = [idx if idx > 0 else size + idx for idx in indices]
-        return tuple(obj[idx] for idx in range(size) if idx not in pos_indices)
-    return func
-
-def print_csv(output_file: str, data: List, errors: List):
-    header = [
-        "obsid",
-        "timestamp",
-        "HDF5 filename",
-        "v_start",
-        "v_end",
-        "n_final_dump",
-        "digitise",
-        "DG",
-        "sync",
-        "sync period",
-        "mode",
-        "sensor",
-        "CCD order",
-        "CI",
-        "CI width",
-        "CI gap",
-        "VGD",
-        "error_flags",
-    ]
-
-    prep = {"column_names": header, "mode": 'w', "quote_char": '\"'}
-    with CSV2(output_file, prep=prep) as csv:
-        for row in data:
-            csv.create(row[:-2])  # remove last two values, i.e. all_error_flags_str and has_error_flags
-
-
-def print_table(output_file: str, data: List, errors: List):
-    table = Table(title="HDF5 & Register Map", min_width=180)
-
-    table.add_column("obsid")
-    table.add_column("timestamp", width=20)
-    table.add_column("HDF5 filename", no_wrap=True)
-    table.add_column("v_start", justify="right")
-    table.add_column("v_end", justify="right")
-    table.add_column("n_final_dump", justify="right")
-    table.add_column("digitise", justify="right")
-    table.add_column("DG", justify="right")
-    table.add_column("sync")
-    table.add_column("sync period")
-    table.add_column("mode")
-    table.add_column("sensor")
-    table.add_column("CCD order")
-    table.add_column("CI")
-    table.add_column("CI width")
-    table.add_column("CI gap")
-    table.add_column("VGD")
-    table.add_column("error_flags")
-
-    # We don't need the error_flags-by-value, we need the error_flags-by-str/bin
-    # and, do not use the last element which is the has_error_flags. So, all rows
-    # should exclude element -3 and -1
-    snip = exclude_items(-1, -3)
-
-    for row in data:
-        table.add_row(*snip(row))
-
-    with Console().status("Saving report...", spinner='clock'):
-        with open(output_file, "wt") as report_file:
-            console = Console(file=report_file, width=300)
-            if errors:
-                console.print(f"Observations with N-FEE errors: {', '.join(set(errors))}")
-            console.print(table)
-
-
-if __name__ == "__main__":
-    cli()