cgse 2024.7.0__py3-none-any.whl → 2025.0.2__py3-none-any.whl

scripts/check_hdf5_files.py

@@ -1,192 +0,0 @@
-import glob
-import logging
-from pathlib import Path
-from typing import Optional
-from typing import Union
-
-import click
-import rich
-from h5py import File
-from rich.progress import track
-
-from egse import h5
-from egse.bits import beautify_binary
-from egse.fee import n_fee_mode
-from egse.fee.nfee import HousekeepingData
-from egse.reg import RegisterMap
-from egse.spw import SpaceWirePacket
-
-MODULE_LOGGER = logging.getLogger("egse.check_hdf5_file")
-
-
-def check_if_top_level_groups_complete(hdf5_file: File, indent: str) -> (str, bool):
-
-    msg = ""
-    error = False
-
-    # Check for internal or external sync
-
-    if "/register" in hdf5_file:
-        reg_data = h5.get_data(hdf5_file["/register"])
-        reg_map = RegisterMap("N-FEE", memory_map=reg_data)
-        sync_sel = reg_map["sync_sel"]
-        if sync_sel and ("/1" in hdf5_file or "/2" in hdf5_file or "/3" in hdf5_file):
-            msg += indent + (
-                   "[bold red]ERROR[/]: internal sync contains more than one readout period\n")
-            error = True
-        if not sync_sel and (
-            missing := [idx for idx in range(4) if f"/{idx}" not in hdf5_file]
-        ):
-            msg += indent + f"[bold red]ERROR[/]: missing readouts for external sync {missing}\n"
-            error = True
-
-    return msg, error
-
-
-def check_hdf5_file(filename: Union[str, Path], requested_obsid: Union[int, str], error_flags_only=False, errors_only=False):
-
-    filename = Path(filename)
-
-    if filename.exists():
-        try:
-
-            with h5.get_file(filename, mode='r') as hdf5_file:
-
-                has_error = False
-                indent = "  "
-                full_msg = f"Analysing {filename!s}\n"
-                full_msg += indent + f"Top level groups: {list(hdf5_file.keys())}\n"
-                error_msg = ""
-                obsid = ''
-
-                msg, error = check_if_top_level_groups_complete(hdf5_file, indent)
-                full_msg += msg
-                has_error = error or has_error
-
-                if "/obsid" in hdf5_file:
-                    obsid = h5.get_data(hdf5_file["/obsid"]).item().decode()
-                    if obsid == 'None':
-                        full_msg += indent + "no observation running\n"
-                    else:
-                        full_msg += indent + f"OBSID = {obsid}\n"
-
-                if requested_obsid is not None and str(requested_obsid) not in obsid:
-                    return
-
-                if "/0/timecode" in hdf5_file:
-                    timecode = h5.get_data(hdf5_file["/0/timecode"])
-                    timestamp = h5.get_attribute_value(hdf5_file["/0/timecode"], "timestamp")
-                    full_msg += indent + f"{timecode=!s}, {timestamp=}\n"
-
-                if "/register" in hdf5_file:
-                    reg_data = h5.get_data(hdf5_file["/register"])
-                    reg_map = RegisterMap("N-FEE", memory_map=reg_data)
-                    v_start = reg_map['v_start']
-                    v_end = reg_map['v_end']
-                    digitise_en = reg_map['digitise_en']
-                    dg_en = reg_map['DG_en']
-                    sync_sel = reg_map['sync_sel']
-                    try:
-                        mode = n_fee_mode(reg_map['ccd_mode_config']).name
-                    except ValueError:
-                        mode = f"[bold red]INVALID ({reg_map['ccd_mode_config']})[/]"
-                        has_error = True
-                    full_msg += indent + f"register: {v_start=}, {v_end=}, {digitise_en=}, {dg_en=}, {sync_sel=}, {mode=}\n"
-                else:
-                    full_msg += indent + '[red]no register map in this HDF5 file[/red]\n'
-                    has_error = True
-
-                if "/0/data" in hdf5_file and h5.has_attributes(hdf5_file['/0/data']):
-                    v_start = h5.get_attribute_value(hdf5_file['/0/data'], "v_start")
-                    v_end = h5.get_attribute_value(hdf5_file['/0/data'], "v_end")
-                    full_msg += indent + f"data: {v_start=}, {v_end=}\n"
-
-                for count in range(4):
-                    if f"/{count}/hk" in hdf5_file:
-                        hk_packet = SpaceWirePacket.create_packet(hdf5_file[f"/{count}/hk"][...])
-                        full_msg += indent + f"hk: {hk_packet.type} {beautify_binary(hk_packet.type.value)}\n"
-                        error_flags = HousekeepingData(hk_packet.data)['error_flags']
-                        if error_flags:
-                            msg = indent + (
-                                f"Frame {count}: [red]One or more of the error flags are ON: "
-                                f"{beautify_binary(error_flags, group=4, size=12)}[/red]\n"
-                            )
-                            full_msg += msg
-                            error_msg += msg
-                            has_error = True
-                for count in range(4):
-                    if f"/{count}/commands" in hdf5_file:
-                        for idx in hdf5_file[f"/{count}/commands/"]:
-                            full_msg += indent + f"command in frame {count}: {h5.get_data(hdf5_file[f'/{count}/commands/{idx}'])}\n"
-
-            if error_flags_only and error_msg:
-                rich.print(f"{filename.name=}, size={filename.stat().st_size}")
-                rich.print(error_msg)
-            if not error_flags_only:
-                if errors_only and has_error:
-                    rich.print(full_msg)
-                if not errors_only:
-                    rich.print(full_msg)
-
-        except OSError as exc:
-            rich.print(f"{filename=!s}, {exc}")
-    else:
-        rich.print(f"no such file {filename!s}")
-
-
-def find_hdf5_for_obsid(files: list, requested_obsid: Union[int, str]):
-    new_files = []
-
-    for filename in sorted(files):
-        try:
-            with h5.get_file(filename, mode='r') as hdf5_file:
-                if "/obsid" in hdf5_file:
-                    obsid = h5.get_data(hdf5_file["/obsid"]).item().decode()
-
-                    if str(requested_obsid) in obsid:
-                        new_files.append(filename)
-        except OSError as exc:
-            MODULE_LOGGER.error(f"Couldn't open {filename} ({exc=})")
-    return new_files
-
-
-@click.command()
-@click.argument('root')
-@click.argument('filename_t')
-@click.option("--obsid", help="The full OBSID, e.g. SRON_00058_02380 or only the TEST-ID, e.g. 2380")
-@click.option("--error-flags-only", is_flag=True, help="Print only the filename and error information")
-@click.option("--errors-only", is_flag=True, help="Print only the filename and error information")
-def cli(obsid, error_flags_only, errors_only, root, filename_t):
-    """Script to inspect the HDF5 files generated by the DPU Processor.
-
-        * ROOT - the root folder that contains the HDF5 files to be checked
-
-        * FILENAME_T - a file name template accepted by glob()
-    """
-    root = Path(root)
-
-    files = glob.glob(str(root / filename_t))
-    if obsid is not None:
-        files = find_hdf5_for_obsid(files, obsid)
-    n_files = len(files)
-
-    rich.print(f"Number of files: {n_files}")
-
-    print(" ".join([Path(file).name for file in files]))
-
-    answer = input("So, you want to continue [Y/n]")
-    print(answer)
-    if answer.lower() in ['n', 'no']:
-        return
-
-    for filename in track(sorted(files), description=f"Checking {n_files} files"):
-    # for filename in sorted(files):
-        try:
-            check_hdf5_file(filename, obsid, error_flags_only, errors_only)
-        except Exception as exc:
-            rich.print(f"[red]ERROR[/]: {Path(filename).name}: {exc!s}")
-            MODULE_LOGGER.debug(f"{exc!s}", exc_info=True)
-
-
-if __name__ == "__main__":
-    cli()

scripts/check_register_sync.py

@@ -1,47 +0,0 @@
-"""
-This module contains commands and strategies to test the synchronisation of the Register Map
-between the DPU Processor and the N-FEE.
-
-The reason for this test is that the DPU Processor seems to be out of sync with the N-FEE at some
-points in time:
-
-* When the N-FEE is power-cycled and the DPU CS is not restarted, the register map is not
-  refreshed from the newly pwoered N-FEE
-
-* ...
-
-The DPU CS has a few commands available to fix this:
-
-* dpu.n_fee_sync_register_map() — reads the register map from the N-FEE, updates the DPU internal
-  register map and returns the register map.
-
-* dpu.reset() — sets a special register in the N-FEE ccd_mode_config which commands the N-FEE to
-  resets its register map to its default state.
-
-"""
-
-from egse.dpu import DPUProxy
-from egse.reg import compare_register_maps
-
-dpu_proxy = DPUProxy()
-
-
-reg_1 = dpu_proxy.n_fee_sync_register_map()
-
-# Put the N-FEE in DUMP mode
-
-dpu_proxy.n_fee_set_dump_mode(
-    dict(v_start=0, v_end=2000, ccd_readout_order=0b11111111, n_final_dump=4510))
-
-reg_2 = dpu_proxy.n_fee_sync_register_map()
-
-# Put the N-FEE back into its default mode (same as after a power cycle?)
-
-dpu_proxy.n_fee_reset()
-
-reg_3 = dpu_proxy.n_fee_sync_register_map()
-
-
-compare_register_maps(reg_1, reg_2)
-compare_register_maps(reg_2, reg_3)
-compare_register_maps(reg_1, reg_3)

scripts/check_tcs_calib_coef.py

@@ -1,90 +0,0 @@
-from egse.setup import load_setup
-import numpy as np
-import matplotlib.pyplot as plt
-
-def check_calib_coef(setup_id: int):
-    setup = load_setup(setup_id)
-    print(f"\nThis script checks that TOU RTD calibration coefficients provided by Alter are identical than coefficients"
-          f" set in Setup file for camera {setup.camera.ID} {setup.camera.serial_number}.\n")
-    print('Setup ID loaded: ', setup.get_id())
-
-
-    print("\n\nFirst, we check that serial numbers of TOU RTD set in the setup file match with ABCL.\n")
-    sensors = setup.gse.tcs.calibration.sensors
-    rtd_serial_numbers_in_setup = dict()
-    for tou_rtd in sensors:
-        rtd_serial_numbers_in_setup[sensors[tou_rtd].conversion] = sensors[tou_rtd].serial_number
-    rtd_serial_numbers_in_abcl = dict()
-    for frtd in rtd_serial_numbers_in_setup:
-        rtd_serial_numbers_in_abcl[frtd] = input(f"Serial name of TOU {frtd} (TRP-1#{frtd[-1]}) in ABCL: ")
-
-    err = 0
-    for frtd in rtd_serial_numbers_in_setup:
-        if rtd_serial_numbers_in_setup[frtd] != rtd_serial_numbers_in_abcl[frtd]:
-            print(f"\nERROR in S/N of {frtd}.\n"
-                  f"In Setup file: {rtd_serial_numbers_in_setup[frtd]}\n"
-                  f"In ABCL: {rtd_serial_numbers_in_abcl[frtd]}")
-            err += 1
-    if err == 0:
-        print("\nAll S/N of TOU RTD match: OK")
-    else:
-        print("\nSome S/N are not identical between Setup file and ABCL. See above for detail.")
-        return
-
-    print("\n\nNow we check that TOU RTD calibration coefficients set in the Setup file match with Alter file.")
-    print("Use doucment 'PLATO THERMISTORS CALIBRATION TEST REPORT' (Eclipse ref: 2022013099).\nCopy the entire line of"
-          " the three polynomial equations.\n"
-          "For instance: 'T = -1.636680380E-11R 5 + 5.537426295E-08xR 4 - 7.472300219E-05R 3 + 5.028427373E-02R 2 -"
-          "1.663264016E+01R + 2.013507126E+03'\n")
-
-    rtd_calib_coef_in_setup = dict(setup.gse.tcs.calibration.conversion)
-    rtd_calib_coef_in_alter = dict()
-    for frtd in rtd_serial_numbers_in_setup:
-        list_coef = input(f"Polynomial equation of {frtd} with S/N {rtd_serial_numbers_in_setup[frtd]}: ").strip()
-        while list_coef[-1] not in [str(i) for i in range(10)]:
-            list_coef = list_coef[:-1]
-        list_coef = list_coef.replace(' ', '').replace('T', '').replace('=', '').replace('x', '').replace('X', '').split('R')
-        for i in range(1,5):
-            list_coef[i] = list_coef[i][1:]
-        for i in range(6):
-            list_coef[i] = float(list_coef[i])
-        rtd_calib_coef_in_alter[frtd] = list_coef
-
-    err = 0
-    for frtd in rtd_calib_coef_in_setup:
-        for i in range(6):
-            if rtd_calib_coef_in_setup[frtd][i] != rtd_calib_coef_in_alter[frtd][i]:
-                print(f"\nERROR in {frtd} ({rtd_serial_numbers_in_setup[frtd]}) for coefficient of R{5 - i}.\n"
-                      f"In Setup file: {rtd_calib_coef_in_setup[frtd][i]}\n"
-                      f"In Alter file: {rtd_calib_coef_in_alter[frtd][i]}")
-                err += 1
-
-    if err == 0:
-        print("\nAll coefficients are identical in Setup file and Alter file: OK")
-    else:
-        print(f"\n{err} errors detected in calibration coefficients between Setup file and Alter file. "
-              f"See above for detail.")
-        return
-
-    print("\n\nNow we plot the three relations T(R) defined by the polynomial equations.")
-
-    R = np.arange(520, 1020.5, 0.5)
-    for frtd in rtd_calib_coef_in_setup:
-        list_coef = np.array(rtd_calib_coef_in_setup[frtd])
-        T = np.zeros(np.shape(R))
-        for i in range(6):
-            T += list_coef[i] * R**(5-i)
-        plt.plot(R, T, label=frtd)
-    plt.plot(R, -110*np.ones(np.shape(R)), '--r', linewidth=2)
-    plt.plot(R, -65*np.ones(np.shape(R)), '--r', linewidth=2)
-    plt.legend(loc='best')
-    plt.text(min(R), -109, "-110°C", color='red', fontsize=12)
-    plt.text(min(R), -64, "-65°C", color='red', fontsize=12)
-    plt.title("Thermal calibration curves for TCS using coefficients for cold temperature (below -65°C)")
-    plt.xlabel("Resistance (Ohm)")
-    plt.ylabel("Temperature (°C)")
-    plt.grid()
-    plt.show()
-
-
-check_calib_coef(int(input("Setup file ID to be loaded: ")))

scripts/correct_ccd_cold_temperature_cal.py

@@ -1,157 +0,0 @@
-import os
-import shutil
-from datetime import datetime
-from pathlib import PosixPath
-
-from egse.config import find_file
-from egse.fee.n_fee_hk import counts_to_temperature, ORIGIN
-from egse.obsid import obsid_from_storage
-from egse.setup import NavigableDict
-import pandas as pd
-from pandas.core.frame import DataFrame
-from dateutil.rrule import rrule, DAILY
-
-
-def correct_obsid_range(first_obsid: int, last_obsid: int, sensor_info: NavigableDict, camera_name: str, data_dir: str,
-                        site_id: str):
-    """ Correct the CCD temperature calibration in the cold in the N-FEE HK file for the given obsid range.
-
-    This function does the following:
-
-        * Make a copy of the original N-FEE HK files (incl. "old" in the filename)
-        * In the original files: Apply the correct CCD temperature calibration in the columns for the calibrated CCD
-          temperatures (in the cold)
-
-    Args:
-         - first_obsid: First obsid.
-         - last_obsid: Last obsid.
-         - sensor_info: Calibration information for the CCD_PT1000 sensor type.
-         - data_dir: Full path to the directory in which the data resides.  This is the folder with sub-folders /obs and
-                    /daily, in which the N-FEE HK files are stored.
-        - site_id: Site ID.
-    """
-
-    for obsid in range(first_obsid, last_obsid + 1):
-        correct_obsid(obsid, sensor_info, camera_name, data_dir, site_id)
-
-
-def correct_obsid(obsid: int, sensor_info: NavigableDict, camera_name: str, data_dir: str, site_id: str):
-    """ Correct the CCD temperature calibration in the cold in the N-FEE HK file for the given obsid.
-
-    This function does the following:
-
-        * Make a copy of the original file (incl. "old" in the filename)
-        * In the original files: Apply the correct CCD temperature calibration in the columns for the calibrated CCD
-          temperatures (in the cold)
-
-    Args:
-         - obsid: Observation identifier.
-         - sensor_info: Calibration information for the CCD_PT1000 sensor type.
-         - data_dir: Full path to the directory in which the data resides.  This is the folder with sub-folders /obs and
-                    /daily, in which the N-FEE HK files are stored.
-         - site_id: Site ID.
-    """
-
-    stored_obsid = obsid_from_storage(obsid, data_dir=data_dir, site_id=site_id, camera_name=camera_name)
-    pattern = f"{stored_obsid}_{ORIGIN}_*.csv"
-    nfee_hk_filename = find_file(pattern, root=f"{data_dir}/obs/{stored_obsid}")    # PosixPath
-
-    if nfee_hk_filename is None:
-        print(f"No N-FEE HK available for obsid {stored_obsid}")
-        return
-
-    correct_file(str(nfee_hk_filename), sensor_info)
-
-
-def correct_od_range(first_od: int, last_od: int, sensor_info: NavigableDict, data_dir: str, site_id: str):
-    """ Correct the CCD temperature calibration in the cold in the N-FEE HK file for the given OD range.
-
-    This function does the following:
-
-        * Make a copy of the original N-FEE HK files (incl. "old" in the filename)
-        * In the original files: Apply the correct CCD temperature calibration in the columns for the calibrated CCD
-          temperatures (in the cold)
-
-    Args:
-         - first_od: First observing day [yyyymmdd].
-         - last_od: Last observing day [yyyymmdd].
-         - sensor_info: Calibration information for the CCD_PT1000 sensor type.
-         - data_dir: Full path to the directory in which the data resides.  This is the folder with sub-folders /obs and
-                    /daily, in which the N-FEE HK files are stored.
-        - site_id: Site ID.
-    """
-
-    start = datetime.strptime(str(first_od), "%Y%m%d")
-    end = datetime.strptime(str(last_od), "%Y%m%d")
-
-    for dt in rrule(DAILY, dtstart=start, until=end):
-        correct_od(dt.strftime("%Y%m%d"), sensor_info, data_dir, site_id)
-
-
-def correct_od(od: int, sensor_info: NavigableDict, data_dir: str, site_id: str):
-    """ Correct the CCD temperature calibration in the cold in the N-FEE HK file for the given OD.
-
-     This function does the following:
-
-        * Make a copy of the original N-FEE HK file (incl. "old" in the filename)
-        * In the original files: Apply the correct CCD temperature calibration in the columns for the calibrated CCD
-          temperatures (in the cold)
-
-     Args:
-         - od: Observing day [yyyymmdd].
-         - sensor_info: Calibration information for the CCD_PT1000 sensor type.
-         - data_dir: Full path to the directory in which the data resides.  This is the folder with sub-folders /obs and
-                    /daily, in which the N-FEE HK files are stored.
-         - site_id: Site ID.
-    """
-
-    pattern = f"{od}_{site_id}_{ORIGIN}.csv"
-    nfee_hk_filename = find_file(pattern, root=f"{data_dir}/daily/{od}")    # PosixPath
-
-    if nfee_hk_filename is None:
-        print(f"No N-FEE HK available for OD {od}")
-        return
-
-    correct_file(str(nfee_hk_filename), sensor_info)
-
-
-def correct_file(filename: PosixPath, sensor_info: NavigableDict):
-    """ Correct the CCD temperature calibration in the cold in the N-FEE HK file with the given filename.
-
-     This function does the following:
-
-        * Make a copy of the original file (incl. "old" in the filename)
-        * In the original files: Apply the correct CCD temperature calibration in the columns for the calibrated CCD
-          temperatures (in the cold)
-
-     Args:
-         - filename: Filename of the original N-FEE HK file that needs to be corrected.
-         - sensor_info: Calibration information for the CCD_PT1000 sensor type.
-    """
-
-    filename = str(filename)
-
-    # Make a copy of the original file (incl. "old" in the filename) in the same folder
-
-    copy_filename = f"{filename.split('.')[0]}_old.csv"
-    if not os.path.isfile(copy_filename):
-        shutil.copyfile(filename, copy_filename)
-
-    dataframe: DataFrame = pd.read_csv(filename)
-
-    # Correct the calibration
-
-    for ccd_number in range(1, 5):
-        print(ccd_number)
-
-        name_cal = f"NFEE_T_CCD{ccd_number}"
-        name_raw = name_cal + "_RAW"
-
-        raw_values = dataframe[name_raw].values
-        calibrated_values = [counts_to_temperature(name_raw, raw_value, sensor_info[name_cal])
-                             for raw_value in raw_values]
-        dataframe[name_cal] = calibrated_values
-
-    # Update the changes in the original file (not in the copy that was created at the beginning)
-
-    dataframe.to_csv(filename)