eval-ac 0.3.1__py3-none-any.whl

eval_ac/__init__.py

@@ -0,0 +1,7 @@
+"""
+evaluate the absolute calibration with liquid nitrogen of an
+Radiometer Physics GmbH (RPG) microwave radiometer HATPRO.
+"""
+
+__version__ = "0.3.1"
+__author__ = 'Andreas Foth'

eval_ac/__main__.py

@@ -0,0 +1,6 @@
+"""Allows running eval_ac with ``python -m eval_ac``."""
+import sys
+
+from eval_ac.cli import main
+
+sys.exit(main())

eval_ac/analysis.py

@@ -0,0 +1,326 @@
+"""Selection, quality control and drift analysis of the calibration history.
+
+Typical usage::
+
+    from eval_ac.convert_abscal_his import read_abscal_his
+    from eval_ac.analysis import select_cal_type, calibration_drift, \
+        drift_exceedances
+
+    ds = select_cal_type(read_abscal_his('ABSCAL.HIS'))
+    drift = calibration_drift(ds)
+    for row in drift_exceedances(drift):
+        print(row)
+"""
+
+import warnings
+
+import numpy as np
+import xarray as xr
+
+from eval_ac.convert_abscal_his import TIME_REFERENCE
+
+#: Calibration types stored in ``cal_type_1`` and ``cal_type_2``.
+CAL_TYPE_NONE = 0
+CAL_TYPE_LN2 = 1
+CAL_TYPE_SKY_TIPPING = 2
+
+#: Value of ``calibration_flag`` for channels that were not calibrated.
+FLAG_NOT_CALIBRATED = 0
+
+#: Recommended maximum interval between two absolute calibrations in days:
+#: "every 5 to 6 months of operation or after transportation"
+#: (RPG manual RPG-MWR-STD-SW, section 3.1.3).
+MAX_CALIBRATION_AGE_DAYS = 183
+
+#: Valid range of the non-linearity factor, 0.9 <= alpha < 1
+#: (RPG manual RPG-MWR-STD-SW, section 3.1.3.1, equation 1).
+ALPHA_RANGE = (0.9, 1.0)
+
+#: Variables with one value per calibration entry and channel.
+CHANNEL_VARIABLES = ('gain', 'temp_noise', 'temp_sys', 'alpha')
+
+#: Default number of calibrations before the latest one that form the
+#: reference of the drift analysis.
+DEFAULT_N_REFERENCE = 5
+
+#: Default thresholds of the drift analysis in percent. They are about the
+#: 99th percentile of the deviations found in the example data of a HATPRO
+#: (2018-2026) and should be adapted to the instrument.
+DEFAULT_THRESHOLDS = {
+    'gain': 10.0,
+    'temp_noise': 2.5,
+    'temp_sys': 2.5,
+    'alpha': 0.5,
+}
+
+
+def receiver_masks(data_set: xr.Dataset) -> dict:
+    """Returns a boolean channel mask for each receiver number."""
+    if 'receiver' not in data_set.coords:
+        return {1: np.ones(data_set.sizes['freq'], dtype=bool)}
+    receiver = data_set['receiver'].values
+    return {int(number): receiver == number for number in np.unique(receiver)}
+
+
+def channel_cal_type(data_set: xr.Dataset) -> xr.DataArray:
+    """Returns the calibration type of each entry and channel.
+
+    Channels of receiver 1 get ``cal_type_1``, channels of receiver 2 get
+    ``cal_type_2``.
+    """
+    cal_type = np.empty((data_set.sizes['n_samples'],
+                         data_set.sizes['freq']), dtype=np.int32)
+    for number, mask in receiver_masks(data_set).items():
+        cal_type[:, mask] = data_set[f'cal_type_{number}'].values[:, None]
+    return xr.DataArray(cal_type, dims=('n_samples', 'freq'))
+
+
+def select_cal_type(data_set: xr.Dataset,
+                    cal_type: int = CAL_TYPE_LN2) -> xr.Dataset:
+    """Keeps only the calibrations of the given type.
+
+    Entries in which no receiver has the requested calibration type are
+    removed. If only one receiver of an entry has the requested type, the
+    values of the other receiver are set to NaN.
+
+    Args:
+        data_set: Dataset as returned by
+            :func:`eval_ac.convert_abscal_his.read_abscal_his`.
+        cal_type: Calibration type to keep (default: liquid nitrogen).
+
+    Returns:
+        Dataset with the selected calibrations.
+    """
+    valid = channel_cal_type(data_set) == cal_type
+    keep = valid.any('freq').values
+    selected = data_set.isel(n_samples=keep)
+    valid = valid.isel(n_samples=keep)
+    for var in CHANNEL_VARIABLES:
+        if var in selected:
+            selected[var] = selected[var].where(valid.values,
+                                                drop=False)
+            selected[var].attrs = data_set[var].attrs
+    return selected
+
+
+def not_calibrated(data_set: xr.Dataset) -> np.ndarray:
+    """Returns True for every entry and channel that was not calibrated."""
+    if 'calibration_flag' not in data_set:
+        return np.zeros((data_set.sizes['n_samples'],
+                         data_set.sizes['freq']), dtype=bool)
+    return data_set['calibration_flag'].values == FLAG_NOT_CALIBRATED
+
+
+def valid_samples(values: np.ndarray) -> np.ndarray:
+    """Returns the indices of the entries with at least one value."""
+    return np.flatnonzero(~np.all(np.isnan(values), axis=1))
+
+
+def _receiver_drift(values, flagged, n_reference):
+    """Drift of one variable and one receiver.
+
+    Returns:
+        Tuple ``(reference, deviation, latest, n_used)``.
+    """
+    # the latest entry is defined without the flags, so that a latest
+    # calibration without any calibrated channel is not silently skipped
+    samples = valid_samples(values)
+    values = np.where(flagged, np.nan, values)
+    n_channels = values.shape[1]
+    if samples.size == 0:
+        return (np.full(n_channels, np.nan), np.full(values.shape, np.nan),
+                -1, np.zeros(n_channels, dtype=np.int32))
+    latest = samples[-1]
+    reference_values = values[samples[-1 - n_reference:-1]]
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', RuntimeWarning)  # all-NaN channels
+        reference = np.nanmedian(reference_values, axis=0) \
+            if reference_values.size else np.full(n_channels, np.nan)
+    deviation = 100 * (values - reference) / reference
+    n_used = np.sum(~np.isnan(reference_values), axis=0).astype(np.int32)
+    return reference, deviation, latest, n_used
+
+
+def calibration_drift(data_set: xr.Dataset,
+                      variables=CHANNEL_VARIABLES,
+                      n_reference: int = DEFAULT_N_REFERENCE) -> xr.Dataset:
+    """Compares each calibration with the median of the previous ones.
+
+    For each receiver, the reference is the median of the ``n_reference``
+    calibrations before the latest one. Channels that were not calibrated
+    (``calibration_flag == 0``) are neither used for the reference nor
+    evaluated.
+
+    Args:
+        data_set: Dataset, typically after :func:`select_cal_type`.
+        variables: Variables to analyse.
+        n_reference: Number of calibrations forming the reference.
+
+    Returns:
+        :class:`xarray.Dataset` with, for each analysed variable ``var``,
+        ``var_reference`` (freq) with the reference value and
+        ``var_deviation`` (n_samples, freq) with the deviation of every entry
+        from the reference in percent. In addition, ``latest_sample`` (freq)
+        holds the position of the latest entry of each channel and
+        ``n_reference_used`` (freq) the number of values in the reference.
+    """
+    flagged = not_calibrated(data_set)
+    n_samples, n_freq = data_set.sizes['n_samples'], data_set.sizes['freq']
+    latest = np.full(n_freq, -1, dtype=np.int32)
+    n_used = np.zeros(n_freq, dtype=np.int32)
+    drift = xr.Dataset(coords={name: data_set.coords[name]
+                               for name in data_set.coords})
+
+    for var in variables:
+        reference = np.full(n_freq, np.nan)
+        deviation = np.full((n_samples, n_freq), np.nan)
+        for mask in receiver_masks(data_set).values():
+            (reference[mask], deviation[:, mask], latest[mask],
+             n_used[mask]) = _receiver_drift(
+                 data_set[var].values[:, mask].astype(float),
+                 flagged[:, mask], n_reference)
+        long_name = data_set[var].attrs.get('long_name', var)
+        drift[f'{var}_reference'] = ('freq', reference, {
+            'long_name': f'{long_name}, median of {n_reference} previous '
+                         'calibrations',
+            'units': data_set[var].attrs.get('units', '')})
+        drift[f'{var}_deviation'] = (('n_samples', 'freq'), deviation, {
+            'long_name': f'deviation of {long_name} from reference',
+            'units': '%'})
+
+    drift['latest_sample'] = ('freq', latest, {
+        'long_name': 'position of the latest calibration along n_samples'})
+    drift['n_reference_used'] = ('freq', n_used, {
+        'long_name': 'number of calibrations in the reference'})
+    drift.attrs['n_reference'] = n_reference
+    return drift
+
+
+def _latest_per_receiver(data_set: xr.Dataset, variable='gain'):
+    """Yields ``(receiver number, channel mask, position of latest entry)``.
+
+    Receivers without any entry are skipped.
+    """
+    for number, mask in receiver_masks(data_set).items():
+        samples = valid_samples(data_set[variable].values[:, mask]
+                                .astype(float))
+        if samples.size:
+            yield number, mask, samples[-1]
+
+
+def latest_not_calibrated(data_set: xr.Dataset, variable='gain') -> list:
+    """Lists the channels of the latest calibration with flag 0.
+
+    The latest calibration is determined per receiver.
+
+    Returns:
+        List of dictionaries with the keys ``freq`` and ``receiver``.
+    """
+    flagged = not_calibrated(data_set)
+    rows = []
+    for number, mask, latest in _latest_per_receiver(data_set, variable):
+        for freq in data_set['freq'].values[mask][flagged[latest, mask]]:
+            rows.append({'freq': float(freq), 'receiver': number})
+    return rows
+
+
+def receiver_time(data_set: xr.Dataset, number: int) -> np.ndarray:
+    """Returns the calibration times of receiver ``number``.
+
+    Uses ``time_of_rec_<number>`` (seconds since 2001-01-01, or already
+    decoded when read from a NetCDF file) and falls back to the coordinate
+    ``time``.
+    """
+    name = f'time_of_rec_{number}'
+    if name not in data_set:
+        return data_set['time'].values
+    values = data_set[name].values
+    if np.issubdtype(values.dtype, np.datetime64):
+        return values
+    return TIME_REFERENCE + values.astype('timedelta64[s]')
+
+
+def calibration_age(data_set: xr.Dataset, now=None) -> list:
+    """Returns the age of the latest calibration of each receiver.
+
+    Args:
+        data_set: Dataset, typically after :func:`select_cal_type`.
+        now: Reference time (anything accepted by ``numpy.datetime64``,
+            default: current UTC time).
+
+    Returns:
+        List of dictionaries with the keys ``receiver``, ``time``
+        (``numpy.datetime64``) and ``age_days``.
+    """
+    now = np.datetime64('now' if now is None else now, 's')
+    rows = []
+    for number, _, latest in _latest_per_receiver(data_set):
+        time = receiver_time(data_set, number)[latest].astype(
+            'datetime64[s]')
+        rows.append({'receiver': number, 'time': time,
+                     'age_days': float((now - time) / np.timedelta64(1, 'D'))})
+    return rows
+
+
+def alpha_out_of_range(data_set: xr.Dataset, valid_range=ALPHA_RANGE) -> list:
+    """Lists channels of the latest calibration with alpha out of range.
+
+    Channels that were not calibrated (flag 0) are not checked.
+
+    Args:
+        data_set: Dataset, typically after :func:`select_cal_type`.
+        valid_range: Tuple ``(low, high)``; valid is ``low <= alpha < high``.
+
+    Returns:
+        List of dictionaries with the keys ``freq``, ``receiver`` and
+        ``alpha``.
+    """
+    low, high = valid_range
+    flagged = not_calibrated(data_set)
+    rows = []
+    for number, mask, latest in _latest_per_receiver(data_set, 'alpha'):
+        alpha = data_set['alpha'].values[latest, mask].astype(float)
+        with np.errstate(invalid='ignore'):
+            bad = ~((alpha >= low) & (alpha < high))
+        bad &= ~flagged[latest, mask] & ~np.isnan(alpha)
+        for freq, value in zip(data_set['freq'].values[mask][bad],
+                               alpha[bad]):
+            rows.append({'freq': float(freq), 'receiver': number,
+                         'alpha': float(value)})
+    return rows
+
+
+def drift_exceedances(drift: xr.Dataset, thresholds=None) -> list:
+    """Lists the channels of the latest calibration exceeding a threshold.
+
+    Args:
+        drift: Dataset as returned by :func:`calibration_drift`.
+        thresholds: Dictionary ``{variable: threshold in percent}``. Missing
+            variables use :data:`DEFAULT_THRESHOLDS`.
+
+    Returns:
+        List of dictionaries with the keys ``variable``, ``freq``,
+        ``receiver``, ``deviation`` and ``threshold``, sorted by variable
+        and frequency.
+    """
+    thresholds = {**DEFAULT_THRESHOLDS, **(thresholds or {})}
+    latest = drift['latest_sample'].values
+    rows = []
+    for var, threshold in thresholds.items():
+        if f'{var}_deviation' not in drift:
+            continue
+        deviation = drift[f'{var}_deviation'].values
+        for channel, sample in enumerate(latest):
+            if sample < 0:
+                continue
+            value = deviation[sample, channel]
+            if abs(value) > threshold:  # NaN never exceeds
+                rows.append({
+                    'variable': var,
+                    'freq': float(drift['freq'].values[channel]),
+                    'receiver': int(drift['receiver'].values[channel])
+                    if 'receiver' in drift.coords else 1,
+                    'deviation': float(value),
+                    'threshold': float(threshold),
+                })
+    return rows

eval_ac/cli.py

@@ -0,0 +1,234 @@
+"""Command line interface of eval_ac.
+
+Example::
+
+    eval-ac ABSCAL.HIS --netcdf abscal.nc --plot results_ln2_cal.png
+"""
+
+import argparse
+import sys
+
+from eval_ac import __version__
+
+#: Exit code if the quality report contains a warning and
+#: ``--fail-on-warning`` is given.
+EXIT_WARNING = 2
+
+
+def _threshold(text: str) -> tuple:
+    """Parses 'VARIABLE=PERCENT'."""
+    # pylint: disable=import-outside-toplevel
+    from eval_ac.analysis import DEFAULT_THRESHOLDS
+    try:
+        name, value = text.split('=')
+        value = float(value)
+    except ValueError as error:
+        raise argparse.ArgumentTypeError(
+            f'expected VARIABLE=PERCENT, got {text!r}') from error
+    if name not in DEFAULT_THRESHOLDS:
+        raise argparse.ArgumentTypeError(
+            f'unknown variable {name!r}, choose from '
+            f'{", ".join(DEFAULT_THRESHOLDS)}')
+    return name, value
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Creates the argument parser."""
+    parser = argparse.ArgumentParser(
+        prog='eval-ac',
+        description='Read the ABSCAL.HIS file of an RPG microwave radiometer '
+                    '(e.g. HATPRO) and plot the history of the absolute '
+                    'calibration with liquid nitrogen.')
+    parser.add_argument('input', help='path of the ABSCAL.HIS file')
+    parser.add_argument('-n', '--netcdf', metavar='FILE',
+                        help='write the converted data (all calibration '
+                             'types) to this NetCDF file')
+    parser.add_argument('-p', '--plot', metavar='FILE',
+                        default='results_ln2_cal.png',
+                        help='save the history plot to this file '
+                             '(default: %(default)s)')
+    parser.add_argument('-d', '--drift-plot', metavar='FILE',
+                        default='results_ln2_drift.png',
+                        help='save the drift plot to this file '
+                             '(default: %(default)s)')
+    parser.add_argument('--no-plot', action='store_true',
+                        help='do not create any plot')
+    parser.add_argument('--show', action='store_true',
+                        help='show the plots in a window')
+    parser.add_argument('--all-cal-types', action='store_true',
+                        help='use all calibrations instead of only those '
+                             'with liquid nitrogen')
+
+    drift = parser.add_argument_group('drift analysis')
+    drift.add_argument('--n-reference', type=int, default=5, metavar='N',
+                       help='number of calibrations before the latest one '
+                            'forming the reference (default: %(default)s)')
+    drift.add_argument('-t', '--threshold', type=_threshold, action='append',
+                       default=[], metavar='VARIABLE=PERCENT',
+                       help='drift threshold in percent, e.g. gain=8; can be '
+                            'given several times (defaults: gain=10, '
+                            'temp_noise=2.5, temp_sys=2.5, alpha=0.5)')
+
+    checks = parser.add_argument_group('quality checks')
+    checks.add_argument('--max-age', type=float, default=183, metavar='DAYS',
+                        help='warn if the latest calibration is older '
+                             '(default: %(default)s days; RPG recommends an '
+                             'absolute calibration every 5 to 6 months)')
+    checks.add_argument('--fail-on-warning', action='store_true',
+                        help=f'exit with code {EXIT_WARNING} if the report '
+                             'contains a warning (drift, age, flag 0, alpha '
+                             'out of range)')
+
+    parser.add_argument('--version', action='version',
+                        version=f'%(prog)s {__version__}')
+    return parser
+
+
+def _report_age(data_set, max_age):
+    """Prints the date of the latest calibration; returns True on warning."""
+    # pylint: disable=import-outside-toplevel
+    from eval_ac.analysis import calibration_age
+    warned = False
+    for row in calibration_age(data_set):
+        date = str(row['time'].astype('datetime64[D]'))
+        line = (f'receiver {row["receiver"]}: latest calibration {date} '
+                f'({row["age_days"]:.0f} days ago)')
+        if row['age_days'] > max_age:
+            print(f'warning: {line}, older than {max_age:g} days')
+            warned = True
+        else:
+            print(line)
+    return warned
+
+
+def _report_channels(data_set, rows, message, value=None):
+    """Prints a list of channels; returns True if there are any."""
+    if not rows:
+        return False
+    units = data_set['freq'].attrs.get('units', '')
+    print(f'warning: {len(rows)} channel(s) of the latest calibration '
+          f'{message}:')
+    for row in rows:
+        extra = f'  {value(row)}' if value else ''
+        print(f'  {row["freq"]:7.2f} {units}  receiver {row["receiver"]}'
+              f'{extra}')
+    return True
+
+
+def _report(data_set, drift, args) -> bool:
+    """Prints the quality report of the latest calibration.
+
+    Returns:
+        True if the report contains a warning.
+    """
+    # pylint: disable=import-outside-toplevel
+    from eval_ac import analysis
+
+    warnings = [
+        _report_age(data_set, args.max_age),
+        _report_channels(data_set, analysis.latest_not_calibrated(data_set),
+                         'were not calibrated (flag 0)'),
+        _report_channels(
+            data_set, analysis.alpha_out_of_range(data_set),
+            'have a non-linearity factor outside 0.9 <= alpha < 1',
+            lambda row: f'alpha = {row["alpha"]:.4f}'),
+    ]
+    exceedances = analysis.drift_exceedances(drift, dict(args.threshold))
+    if exceedances:
+        warnings.append(_report_channels(
+            data_set, exceedances, 'exceed the drift threshold',
+            lambda row: f'{row["variable"]:<11} {row["deviation"]:+7.2f} % '
+                        f'(threshold ±{row["threshold"]:g} %)'))
+    else:
+        print('drift: all channels of the latest calibration are within '
+              'the thresholds')
+    return any(warnings)
+
+
+def _load(args):
+    """Reads the file and selects the calibration type.
+
+    Returns:
+        Dataset, or None after printing an error message.
+    """
+    # pylint: disable=import-outside-toplevel
+    from eval_ac import analysis
+    from eval_ac.convert_abscal_his import read_abscal_his, write_netcdf
+
+    try:
+        data_set = read_abscal_his(args.input)
+    except (OSError, ValueError) as error:
+        print(f'eval-ac: error: {error}', file=sys.stderr)
+        return None
+    print(f'read {data_set.sizes["n_samples"]} calibration entries '
+          f'from {args.input}')
+
+    if args.netcdf:
+        write_netcdf(data_set, args.netcdf)
+        print(f'wrote {args.netcdf}')
+
+    if not args.all_cal_types:
+        data_set = analysis.select_cal_type(data_set, analysis.CAL_TYPE_LN2)
+        print(f'{data_set.sizes["n_samples"]} of them are calibrations with '
+              'liquid nitrogen')
+        if data_set.sizes['n_samples'] == 0:
+            print('eval-ac: error: no calibrations with liquid nitrogen '
+                  'found (use --all-cal-types)', file=sys.stderr)
+            return None
+    return data_set
+
+
+def _save_plots(args, data_set, drift, thresholds):
+    """Creates, saves and optionally shows the plots."""
+    # pylint: disable=import-outside-toplevel
+    import matplotlib.pyplot as plt
+    from eval_ac.plotting import plot_calibration_history, plot_drift
+
+    figures = [
+        (plot_calibration_history(data_set), args.plot),
+        (plot_drift(drift, thresholds=thresholds), args.drift_plot),
+    ]
+    for figure, filename in figures:
+        figure.savefig(filename)
+        print(f'wrote {filename}')
+    if args.show:
+        plt.show()
+    for figure, _ in figures:
+        plt.close(figure)
+
+
+def main(argv=None) -> int:
+    """Runs the command line interface.
+
+    Args:
+        argv: Command line arguments (default: ``sys.argv[1:]``).
+
+    Returns:
+        Exit code (0 on success, 1 on error, 2 on a warning with
+        ``--fail-on-warning``).
+    """
+    args = build_parser().parse_args(argv)
+
+    # pylint: disable=import-outside-toplevel
+    import matplotlib
+    if not args.show:
+        matplotlib.use('Agg')  # no display needed, e.g. on servers
+    from eval_ac import analysis
+
+    data_set = _load(args)
+    if data_set is None:
+        return 1
+
+    drift = analysis.calibration_drift(data_set, n_reference=args.n_reference)
+    warned = _report(data_set, drift, args)
+
+    if not args.no_plot:
+        _save_plots(args, data_set, drift, dict(args.threshold))
+
+    if args.fail_on_warning and warned:
+        return EXIT_WARNING
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())