cloudnetpy 1.66.5__py3-none-any.whl → 1.66.6__py3-none-any.whl

cloudnetpy/cli.py

@@ -0,0 +1,605 @@
+import argparse
+import gzip
+import importlib
+import logging
+import os
+import re
+import shutil
+from collections.abc import Generator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Final
+
+import requests
+
+from cloudnetpy import concat_lib, instruments
+from cloudnetpy.categorize import generate_categorize
+from cloudnetpy.exceptions import PlottingError
+from cloudnetpy.plotting import generate_figure
+from cloudnetpy.utils import md5sum
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+cloudnet_api_url: Final = "https://cloudnet.fmi.fi/api/"
+
+
+@dataclass
+class Instrument:
+    id: str
+    pid: str
+    name: str
+
+
+def run(args: argparse.Namespace, tmpdir: str):
+    cat_files = {}
+
+    # Instrument based products
+    if source_instruments := _get_source_instruments(args.products):
+        for product, possible_instruments in source_instruments.items():
+            meta = _fetch_raw_meta(possible_instruments, args)
+            instrument = _select_instrument(meta, product)
+            if not instrument:
+                logging.info("No instrument found for %s", product)
+                continue
+            meta = _filter_by_instrument(meta, instrument)
+            meta = _filter_by_suffix(meta, product)
+            if not meta:
+                logging.info("No suitable data available for %s", product)
+                continue
+            output_filepath = _process_instrument_product(
+                product, meta, instrument, tmpdir, args
+            )
+            _plot(output_filepath, product, args)
+            cat_files[product] = output_filepath
+
+    prod_sources = _get_product_sources(args.products)
+
+    # Categorize based products
+    if "categorize" in args.products:
+        cat_filepath = _process_categorize(cat_files, args)
+        _plot(cat_filepath, "categorize", args)
+    else:
+        cat_filepath = None
+    cat_products = [p for p in prod_sources if "categorize" in prod_sources[p]]
+    for product in cat_products:
+        if cat_filepath is None:
+            cat_filepath = _fetch_product(args, "categorize")
+        if cat_filepath is None:
+            logging.info("No categorize data available for {}")
+            break
+        l2_filename = _process_cat_product(product, cat_filepath)
+        _plot(l2_filename, product, args)
+
+    # MWR-L1c based products
+    mwrpy_products = [p for p in prod_sources if "mwr-l1c" in prod_sources[p]]
+    for product in mwrpy_products:
+        if "mwr-l1c" in cat_files:
+            mwrpy_filepath = cat_files.get("mwr-l1c")
+        else:
+            mwrpy_filepath = _fetch_product(args, "mwr-l1c")
+        if mwrpy_filepath is None:
+            logging.info("No MWR-L1c data available for %s", product)
+            break
+        l2_filename = _process_mwrpy_product(product, mwrpy_filepath, args)
+        _plot(l2_filename, product, args)
+
+
+def _process_categorize(input_files: dict, args: argparse.Namespace) -> str | None:
+    cat_filepath = _create_categorize_filepath(args)
+
+    input_files["model"] = _fetch_model(args)
+    if input_files["model"] is None:
+        logging.info("No model data available for this date.")
+        return None
+
+    for product in ("radar", "lidar", "disdrometer"):
+        if product not in input_files and (filepath := _fetch_product(args, product)):
+            input_files[product] = filepath
+
+    if mwr := _fetch_mwr(args):
+        input_files["mwr"] = mwr
+
+    try:
+        logging.info("Processing categorize...")
+        generate_categorize(input_files, cat_filepath)
+        logging.info("Processed categorize to %s", cat_filepath)
+    except NameError:
+        logging.info("No data available for this date.")
+        return None
+    return cat_filepath
+
+
+def _fetch_mwr(args: argparse.Namespace) -> str | None:
+    mwr_sources = [
+        ("mwr-single", None),
+        ("mwr", None),
+        ("radar", "rpg-fmcw-35"),
+        ("radar", "rpg-fmcw-94"),
+    ]
+    for product, source in mwr_sources:
+        mwr = _fetch_product(args, product, source=source)
+        if mwr:
+            return mwr
+    return None
+
+
+def _process_instrument_product(
+    product: str,
+    meta: list[dict],
+    instrument: Instrument,
+    tmpdir: str,
+    args: argparse.Namespace,
+) -> str | None:
+    output_filepath = _create_instrument_filepath(instrument, args)
+    site_meta = _read_site_meta(meta)
+    input_files: list[str] | str
+    input_files = _fetch_raw(meta, args)
+    if args.dl:
+        return None
+    input_folder = str(Path(input_files[0]).parent)
+    calibration = _get_calibration(instrument, args)
+    fun: Callable
+    match (product, instrument.id):
+        case ("radar", _id) if "mira" in _id:
+            fun = instruments.mira2nc
+        case ("radar", _id) if "rpg" in _id:
+            fun = instruments.rpg2nc
+            input_files = input_folder
+        case ("radar", _id) if "basta" in _id:
+            fun = instruments.basta2nc
+            _check_input(input_files)
+            input_files = input_files[0]
+        case ("radar", _id) if "copernicus" in _id:
+            fun = instruments.copernicus2nc
+        case ("radar", _id) if "galileo" in _id:
+            fun = instruments.galileo2nc
+        case ("disdrometer", _id) if "parsivel" in _id:
+            fun = instruments.parsivel2nc
+        case ("disdrometer", _id) if "thies" in _id:
+            fun = instruments.thies2nc
+            input_files = _concatenate_(input_files, tmpdir)
+        case ("lidar", _id) if "pollyxt" in _id:
+            fun = instruments.pollyxt2nc
+        case ("lidar", _id) if _id == "cl61d":
+            fun = instruments.ceilo2nc
+            variables = ["x_pol", "p_pol", "beta_att", "time", "tilt_angle"]
+            concat_file = str(Path(tmpdir) / "tmp.nc")
+            concat_lib.bundle_netcdf_files(
+                input_files,
+                args.date,
+                concat_file,
+                variables=variables,
+            )
+            input_files = concat_file
+            site_meta["model"] = instrument.id
+        case ("lidar", _id):
+            fun = instruments.ceilo2nc
+            input_files = _concatenate_(input_files, tmpdir)
+            site_meta["model"] = instrument.id
+            if factor := calibration.get("calibration_factor"):
+                site_meta["calibration_factor"] = factor
+        case ("mwr", _id):
+            fun = instruments.hatpro2nc
+            input_files = input_folder
+        case ("mwr-l1c", _id):
+            fun = instruments.hatpro2l1c
+            coefficients = _fetch_coefficient_files(calibration, tmpdir)
+            site_meta = {**site_meta, **calibration}
+            site_meta["coefficientFiles"] = coefficients
+            input_files = input_folder
+        case ("mrr", _id):
+            fun = instruments.mrr2nc
+        case ("weather-station", _id):
+            fun = instruments.ws2nc
+    logging.info("Processing %s...", product)
+    fun(input_files, output_filepath, site_meta, date=args.date)
+    logging.info("Processed %s: %s", product, output_filepath)
+    return output_filepath
+
+
+def _concatenate_(input_files: list[str], tmpdir: str) -> str:
+    if len(input_files) > 1:
+        concat_file = str(Path(tmpdir) / "tmp.nc")
+        try:
+            concat_lib.concatenate_files(input_files, concat_file)
+        except OSError:
+            concat_lib.concatenate_text_files(input_files, concat_file)
+        return concat_file
+    return input_files[0]
+
+
+def _fetch_coefficient_files(calibration: dict, tmpdir: str) -> list:
+    if not (links := calibration.get("coefficientLinks")):
+        msg = "No calibration coefficients found"
+        raise ValueError(msg)
+    coefficient_paths = []
+    for filename in links:
+        res = requests.get(filename, timeout=60)
+        res.raise_for_status()
+        filepath = Path(tmpdir) / Path(filename).name
+        filepath.write_bytes(res.content)
+        coefficient_paths.append(str(filepath))
+    return coefficient_paths
+
+
+def _get_calibration(instrument: Instrument, args) -> dict:
+    params = {
+        "date": args.date,
+        "instrumentPid": instrument.pid,
+    }
+    res = requests.get(
+        f"{cloudnet_api_url}calibration",
+        params=params,
+        timeout=60,
+    )
+    if res.status_code == 404:
+        return {}
+    return res.json().get("data", {})
+
+
+def _create_instrument_filepath(
+    instrument: Instrument, args: argparse.Namespace
+) -> str:
+    folder = _create_output_folder("instrument", args)
+    pid = _shorten_pid(instrument.pid)
+    filename = f"{args.date.replace('-', '')}_{args.site}_{instrument.id}_{pid}.nc"
+    return str(folder / filename)
+
+
+def _create_categorize_filepath(args: argparse.Namespace) -> str:
+    folder = _create_output_folder("geophysical", args)
+    filename = f"{args.date.replace('-', '')}_{args.site}_categorize.nc"
+    return str(folder / filename)
+
+
+def _create_input_folder(end_point: str, args: argparse.Namespace) -> Path:
+    folder = args.input / args.site / args.date / end_point
+    folder.mkdir(parents=True, exist_ok=True)
+    return folder
+
+
+def _create_output_folder(end_point: str, args: argparse.Namespace) -> Path:
+    folder = args.output / args.site / args.date / end_point
+    folder.mkdir(parents=True, exist_ok=True)
+    return folder
+
+
+def _fetch_raw_meta(instruments: list[str], args: argparse.Namespace) -> list[dict]:
+    res = requests.get(
+        f"{cloudnet_api_url}raw-files/",
+        params={
+            "site": args.site,
+            "date": args.date,
+            "instrument": instruments,
+            "status": ["uploaded", "processed"],
+        },
+        timeout=60,
+    )
+    res.raise_for_status()
+    return res.json()
+
+
+def _filter_by_instrument(meta: list[dict], instrument: Instrument) -> list[dict]:
+    return [m for m in meta if m["instrumentInfo"]["pid"] == instrument.pid]
+
+
+def _filter_by_suffix(meta: list[dict], product: str) -> list[dict]:
+    if product == "radar":
+        meta = [m for m in meta if not m["filename"].lower().endswith(".lv0")]
+    elif product == "mwr":
+        meta = [
+            m for m in meta if re.search(r"\.(lwp|iwv)", m["filename"], re.IGNORECASE)
+        ]
+    elif product == "mwr-l1c":
+        meta = [m for m in meta if not m["filename"].lower().endswith(".nc")]
+    return meta
+
+
+def _get_source_instruments(products: list[str]) -> dict[str, list[str]]:
+    source_instruments = {}
+    for product in products:
+        prod, model = _parse_instrument(product)
+        res = requests.get(f"{cloudnet_api_url}products/{prod}", timeout=60)
+        res.raise_for_status()
+        if sources := res.json().get("sourceInstruments", []):
+            source_instruments[prod] = [i["id"] for i in sources]
+            if match := [i for i in source_instruments[prod] if i == model]:
+                source_instruments[prod] = match
+    return source_instruments
+
+
+def _get_product_sources(products: list[str]) -> dict[str, list[str]]:
+    source_products = {}
+    for product in products:
+        prod, _ = _parse_instrument(product)
+        res = requests.get(f"{cloudnet_api_url}products/{prod}", timeout=60)
+        res.raise_for_status()
+        if sources := res.json().get("sourceProducts", []):
+            source_products[prod] = [i["id"] for i in sources]
+    return source_products
+
+
+def _parse_instrument(s: str) -> tuple[str, str | None]:
+    if "[" in s and s.endswith("]"):
+        name = s[: s.index("[")]
+        value = s[s.index("[") + 1 : -1]
+    else:
+        name = s
+        value = None
+    return name, value
+
+
+def _select_instrument(meta: list[dict], product: str) -> Instrument | None:
+    instruments = _get_unique_instruments(meta)
+    if len(instruments) == 0:
+        logging.info("No instruments found")
+        return None
+    if len(instruments) > 1:
+        logging.info("Multiple instruments found for %s", product)
+        logging.info("Please specify which one to use")
+        for i, instrument in enumerate(instruments):
+            logging.info("%d: %s", i + 1, instrument.name)
+        ind = int(input("Select: ")) - 1
+        selected_instrument = instruments[ind]
+    else:
+        selected_instrument = instruments[0]
+        logging.info("Single instrument found: %s", selected_instrument.name)
+    return selected_instrument
+
+
+def _get_unique_instruments(meta: list[dict]) -> list[Instrument]:
+    unique_pids = {m["instrumentInfo"]["pid"] for m in meta}
+    unique_instruments = []
+    for pid in unique_pids:
+        for m in meta:
+            if m["instrumentInfo"]["pid"] == pid:
+                i = m["instrumentInfo"]
+                unique_instruments.append(
+                    Instrument(i["instrumentId"], i["pid"], i["name"])
+                )
+                break
+    return sorted(unique_instruments, key=lambda x: x.name)
+
+
+def _fetch_product(
+    args: argparse.Namespace, product: str, source: str | None = None
+) -> str | None:
+    payload = {
+        "date": args.date,
+        "site": args.site,
+        "product": product,
+    }
+    url = f"{cloudnet_api_url}files"
+    res = requests.get(url, payload, timeout=60)
+    res.raise_for_status()
+    meta = res.json()
+    if source:
+        meta = [
+            m for m in meta if "instrument" in m and m["instrument"]["id"] == source
+        ]
+    if not meta:
+        logging.info("No data available for %s", product)
+        return None
+    if len(meta) > 1:
+        logging.info(
+            "Multiple files for %s ... taking the first but some logic needed", product
+        )
+    meta = meta[0]
+    suffix = "geophysical" if "geophysical" in meta["product"]["type"] else "instrument"
+    folder = _create_output_folder(suffix, args)
+    return _download_product_file(meta, folder)
+
+
+def _fetch_model(args: argparse.Namespace) -> str | None:
+    payload = {
+        "date": args.date,
+        "site": args.site,
+    }
+    url = f"{cloudnet_api_url}model-files"
+    res = requests.get(url, payload, timeout=60)
+    res.raise_for_status()
+    meta = res.json()
+    if not meta:
+        logging.info("No model data available for this date")
+        return None
+    meta = meta[0]
+    folder = _create_output_folder("instrument", args)
+    return _download_product_file(meta, folder)
+
+
+def _fetch_raw(metadata: list[dict], args: argparse.Namespace) -> list[str]:
+    pid = _shorten_pid(metadata[0]["instrumentInfo"]["pid"])
+    instrument = f"{metadata[0]['instrumentInfo']['instrumentId']}_{pid}"
+    folder = _create_input_folder(instrument, args)
+    filepaths = []
+    with ThreadPoolExecutor() as executor:
+        futures = [
+            executor.submit(_download_raw_file, meta, folder) for meta in metadata
+        ]
+        for future in as_completed(futures):
+            filepaths.append(future.result())
+    return filepaths
+
+
+def _download_raw_file(meta: dict, folder: Path) -> str:
+    filepath = folder / meta["filename"]
+    possible_filepaths = [filepath]
+    if filepath.suffix == ".gz":
+        possible_filepaths.append(filepath.with_suffix(""))
+    for path in possible_filepaths:
+        if path.exists() and md5sum(path) == meta["checksum"]:
+            logging.info("Existing file found: %s", path)
+            return str(path)
+    logging.info("Downloading file: %s", filepath)
+    res = requests.get(meta["downloadUrl"], timeout=60)
+    res.raise_for_status()
+    filepath.write_bytes(res.content)
+    if filepath.suffix == ".gz":
+        filepath = _unzip_gz_file(filepath)
+    return str(filepath)
+
+
+def _download_product_file(meta: dict, folder: Path) -> str:
+    filepath = folder / meta["filename"]
+    if filepath.exists():
+        logging.info("Existing file found: %s", filepath)
+        return str(filepath)
+    logging.info("Downloading file: %s", filepath)
+    res = requests.get(meta["downloadUrl"], timeout=60)
+    res.raise_for_status()
+    filepath.write_bytes(res.content)
+    return str(filepath)
+
+
+def _unzip_gz_file(path_in: Path) -> Path:
+    if path_in.suffix != ".gz":
+        return path_in
+    path_out = path_in.with_suffix("")
+    logging.debug("Decompressing %s to %s", path_in, path_out)
+    with gzip.open(path_in, "rb") as file_in, open(path_out, "wb") as file_out:
+        shutil.copyfileobj(file_in, file_out)
+    path_in.unlink()
+    return path_out
+
+
+def _read_site_meta(meta: list[dict]) -> dict:
+    return {
+        "latitude": meta[0]["site"]["latitude"],
+        "longitude": meta[0]["site"]["longitude"],
+        "altitude": meta[0]["site"]["altitude"],
+        "name": meta[0]["site"]["humanReadableName"],
+    }
+
+
+def _shorten_pid(pid: str) -> str:
+    return pid.split(".")[-1][:8]
+
+
+def _check_input(files: list) -> None:
+    if len(files) > 1:
+        msg = "Multiple input files found"
+        raise ValueError(msg)
+
+
+def _plot(filepath: os.PathLike | str | None, product: str, args: argparse.Namespace):
+    if filepath is None or (not args.plot and not args.show):
+        return
+    res = requests.get(f"{cloudnet_api_url}products/variables", timeout=60)
+    res.raise_for_status()
+    variables = next(var["variables"] for var in res.json() if var["id"] == product)
+    variables = [var["id"].split("-")[-1] for var in variables]
+    image_name = str(filepath).replace(".nc", ".png") if args.plot else None
+    try:
+        generate_figure(
+            filepath,
+            variables,
+            show=args.show,
+            output_filename=image_name,
+        )
+    except PlottingError as e:
+        logging.info("Failed to plot %s: %s", product, e)
+    if args.plot:
+        logging.info("Plotted %s: %s", product, image_name)
+
+
+def _process_cat_product(product: str, categorize_file: str) -> str:
+    output_file = categorize_file.replace("categorize", product)
+    module = importlib.import_module("cloudnetpy.products")
+    getattr(module, f"generate_{product}")(categorize_file, output_file)
+    logging.info("Processed %s: %s", product, output_file)
+    return output_file
+
+
+def _process_mwrpy_product(
+    product: str, mwr_l1c_file: str, args: argparse.Namespace
+) -> str:
+    filename = f"{args.date}_{args.site}_{product}.nc"
+    output_file = _create_output_folder("geophysical", args) / filename
+    module = importlib.import_module("cloudnetpy.products")
+    getattr(module, f"generate_{product.replace('-','_')}")(mwr_l1c_file, output_file)
+    logging.info("Processed %s: %s", product, output_file)
+    return str(output_file)
+
+
+def _fetch_cloudnet_sites() -> list[str]:
+    res = requests.get(f"{cloudnet_api_url}sites", timeout=60)
+    res.raise_for_status()
+    return [site["id"] for site in res.json()]
+
+
+def _parse_products(product_argument: str) -> Generator:
+    products = product_argument.split(",")
+    res = requests.get(f"{cloudnet_api_url}products", timeout=60)
+    res.raise_for_status()
+    valid_options = [p["id"] for p in res.json()]
+    for product in products:
+        prod, _ = _parse_instrument(product)
+        if prod in valid_options:
+            yield product
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Command line interface for running CloudnetPy."
+    )
+    parser.add_argument(
+        "-s",
+        "--site",
+        type=str,
+        help="Site",
+        required=True,
+        choices=_fetch_cloudnet_sites(),
+    )
+    parser.add_argument(
+        "-d", "--date", type=str, help="Date in YYYY-MM-DD", required=True
+    )
+    parser.add_argument(
+        "-p",
+        "--products",
+        type=_parse_products,
+        help=(
+            "Products to process, e.g. 'radar' or 'classification'. If the site "
+            "has many instruments, you can specify the instrument in brackets, "
+            "e.g. radar[mira-35]."
+        ),
+        required=True,
+    )
+    parser.add_argument("--input", type=Path, help="Input path", default="input/")
+    parser.add_argument("--output", type=Path, help="Output path", default="output/")
+    parser.add_argument(
+        "--plot",
+        help="Plot the processed data",
+        default=False,
+        action=argparse.BooleanOptionalAction,
+    )
+    parser.add_argument(
+        "--show",
+        help="Show plotted image",
+        default=False,
+        action=argparse.BooleanOptionalAction,
+    )
+    parser.add_argument(
+        "--dl",
+        help="Download raw data only",
+        default=False,
+        action=argparse.BooleanOptionalAction,
+    )
+    args = parser.parse_args()
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter("%(levelname)s: %(message)s")
+    handler.setFormatter(formatter)
+    logger.handlers = [handler]
+
+    with TemporaryDirectory() as tmpdir:
+        run(args, tmpdir)
+
+
+if __name__ == "__main__":
+    main()

cloudnetpy/concat_lib.py

@@ -1,8 +1,12 @@
 """Module for concatenating netCDF files."""
 
+import shutil
+from os import PathLike
+
 import netCDF4
 import numpy as np
 
+from cloudnetpy import utils
 from cloudnetpy.exceptions import InconsistentDataError
 
 
@@ -264,3 +268,63 @@ def _update_fields(
                 nc_old.variables[field][idx, :] = nc_new.variables[field][valid_ind, :]
             elif len(dimensions) == 2 and concat_ind == 1:
                 nc_old.variables[field][:, idx] = nc_new.variables[field][:, valid_ind]
+
+
+def concatenate_text_files(filenames: list, output_filename: str | PathLike) -> None:
+    """Concatenates text files."""
+    with open(output_filename, "wb") as target:
+        for filename in filenames:
+            with open(filename, "rb") as source:
+                shutil.copyfileobj(source, target)
+
+
+def bundle_netcdf_files(
+    files: list,
+    date: str,
+    output_file: str,
+    concat_dimensions: tuple[str, ...] = ("time", "profile"),
+    variables: list | None = None,
+) -> list:
+    """Concatenates several netcdf files into daily file with
+    some extra data manipulation.
+    """
+    with netCDF4.Dataset(files[0]) as nc:
+        concat_dimension = None
+        for key in concat_dimensions:
+            if key in nc.dimensions:
+                concat_dimension = key
+                break
+        if concat_dimension is None:
+            msg = f"Dimension '{concat_dimensions}' not found in the files."
+            raise KeyError(msg)
+    if len(files) == 1:
+        shutil.copy(files[0], output_file)
+        return files
+    valid_files = []
+    for file in files:
+        try:
+            with netCDF4.Dataset(file) as nc:
+                time = nc.variables["time"]
+                time_array = time[:]
+                time_units = time.units
+        except OSError:
+            continue
+        epoch = utils.get_epoch(time_units)
+        for timestamp in time_array:
+            if utils.seconds2date(timestamp, epoch)[:3] == date.split("-"):
+                valid_files.append(file)
+                break
+    concatenate_files(
+        valid_files,
+        output_file,
+        concat_dimension=concat_dimension,
+        variables=variables,
+        ignore=[
+            "minimum",
+            "maximum",
+            "number_integrated_samples",
+            "Min_LWP",
+            "Max_LWP",
+        ],
+    )
+    return valid_files

cloudnetpy/plotting/plotting.py

@@ -43,6 +43,8 @@ class PlotParameters:
             instruments and model).
         footer_text: The text to display in the footer of the plot.
         plot_meta: Additional metadata for the plot.
+        raise_on_empty: Whether to raise an error if no data is found for a
+            plotted variable.
     """
 
     dpi: float = 120
@@ -55,6 +57,7 @@ class PlotParameters:
     show_sources: bool = False
     footer_text: str | None = None
     plot_meta: PlotMeta | None = None
+    raise_on_empty: bool = False
 
 
 class Dimensions:
@@ -492,7 +495,7 @@ class Plot2D(Plot):
             smoothed_data = uniform_filter(self._data[valid_time_ind, :], sigma_units)
             self._data[valid_time_ind, :] = smoothed_data
 
-        if self._data.mask.all():
+        if self._data.mask.all() and figure_data.options.raise_on_empty:
             msg = "All data is masked"
             raise PlottingError(msg)
 
@@ -603,7 +606,7 @@ class Plot1D(Plot):
             raise PlottingError(msg)
         self._data = self._data[:, freq_ind]
         self._data[np.isnan(self._data)] = ma.masked
-        if self._data.mask.all():
+        if self._data.mask.all() and figure_data.options.raise_on_empty:
             msg = "All data is masked"
             raise PlottingError(msg)
         self._data_orig = self._data_orig[:, freq_ind]

cloudnetpy/utils.py

@@ -1,6 +1,8 @@
 """This module contains general helper functions."""
 
+import base64
 import datetime
+import hashlib
 import logging
 import os
 import re
@@ -140,7 +142,7 @@ def rebin_2d(
     x_in: np.ndarray,
     array: ma.MaskedArray,
     x_new: np.ndarray,
-    statistic: str = "mean",
+    statistic: Literal["mean", "std"] = "mean",
     n_min: int = 1,
     *,
     mask_zeros: bool = True,
@@ -178,14 +180,12 @@ def rebin_2d(
         masked_result = ma.array(result)
 
     # Fill bins with not enough profiles
-    empty_indices = []
-    for ind in range(len(edges) - 1):
-        is_data = np.where((x_in > edges[ind]) & (x_in <= edges[ind + 1]))[0]
-        if len(is_data) < n_min:
-            masked_result[ind, :] = ma.masked
-            empty_indices.append(ind)
+    x_hist, _ = np.histogram(x_in, bins=edges)
+    empty_mask = x_hist < n_min
+    masked_result[empty_mask, :] = ma.masked
+    empty_indices = list(np.nonzero(empty_mask)[0])
     if len(empty_indices) > 0:
-        logging.debug("No radar data in %s bins", len(empty_indices))
+        logging.debug("No data in %s bins", len(empty_indices))
 
     return masked_result, empty_indices
 
@@ -1028,3 +1028,23 @@ def remove_masked_blocks(array: ma.MaskedArray, limit: int = 50) -> np.ndarray:
     mask = np.bincount(labeled_array) < limit
     mask[0] = True
     return mask[labeled_array]
+
+
+def sha256sum(filename: str | os.PathLike) -> str:
+    """Calculates hash of file using sha-256."""
+    return _calc_hash_sum(filename, "sha256", is_base64=False)
+
+
+def md5sum(filename: str | os.PathLike, *, is_base64: bool = False) -> str:
+    """Calculates hash of file using md5."""
+    return _calc_hash_sum(filename, "md5", is_base64=is_base64)
+
+
+def _calc_hash_sum(filename, method, *, is_base64: bool) -> str:
+    hash_sum = getattr(hashlib, method)()
+    with open(filename, "rb") as f:
+        for byte_block in iter(lambda: f.read(4096), b""):
+            hash_sum.update(byte_block)
+    if is_base64:
+        return base64.encodebytes(hash_sum.digest()).decode("utf-8").strip()
+    return hash_sum.hexdigest()

cloudnetpy/version.py

@@ -1,4 +1,4 @@
 MAJOR = 1
 MINOR = 66
-PATCH = 5
+PATCH = 6
 __version__ = f"{MAJOR}.{MINOR}.{PATCH}"

{cloudnetpy-1.66.5.dist-info → cloudnetpy-1.66.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloudnetpy
-Version: 1.66.5
+Version: 1.66.6
 Summary: Python package for Cloudnet processing
 Author: Simo Tukiainen
 License: MIT License

{cloudnetpy-1.66.5.dist-info → cloudnetpy-1.66.6.dist-info}/RECORD

@@ -1,14 +1,15 @@
 cloudnetpy/__init__.py,sha256=X_FqY-4yg5GUj5Edo14SToLEos6JIsC3fN-v1FUgQoA,43
+cloudnetpy/cli.py,sha256=KXFguR4nnVXRr4VV8htI2IKcqqQ7LZ7eZwSHj-uG_3w,20729
 cloudnetpy/cloudnetarray.py,sha256=Ol1ha4RPAmFZANL__U5CaMKX4oYMXYR6OnjoCZ9w3eo,7077
-cloudnetpy/concat_lib.py,sha256=8Ek059RMLAfbbXCkX90cgnhw_8ZpcDrxw1yPvwtuitU,9846
+cloudnetpy/concat_lib.py,sha256=QxGWGsA_6el3Ma5-1y2MtrgFCC9Ohpe3yo6EzrPAiRI,11773
 cloudnetpy/constants.py,sha256=RDB9aqpBRztk3QVCFgsmi9fwhtLuit_0WJrt0D6sDcc,736
 cloudnetpy/datasource.py,sha256=j7N4g59HPvOBWle-W9bOUF0BfRLgvR4zwOi_B50cI7Q,7921
 cloudnetpy/exceptions.py,sha256=ns48useL9RN3mPh7CqIiLA19VI9OmVbyRsKTkwbThF8,1760
 cloudnetpy/metadata.py,sha256=v_VDo2vbdTxB0zIsfP69IcrwSKiRlLpsGdq6JPI4CoA,5306
 cloudnetpy/output.py,sha256=YrWRBEZg0QNZRVnd9ziAziH-eJSh7O5JuWiH4ZxM0_s,15584
 cloudnetpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cloudnetpy/utils.py,sha256=RIqxZoB62JmMOOfYH6_fqpaudH8wqS9WDbpnR2HCcTM,29040
-cloudnetpy/version.py,sha256=RRwzP1UIktb7j48KlcmVNqYg-tOvF82DH-sZMh_TCHI,72
+cloudnetpy/utils.py,sha256=uhSdfx1ha6AqYl-Rlvlf053lIF-UnzcIAjjnh3fbS6U,29725
+cloudnetpy/version.py,sha256=FiFGNpvgmpe5OMB-IflyE51XCkEqV1mD8JLRmF514Hg,72
 cloudnetpy/categorize/__init__.py,sha256=s-SJaysvVpVVo5kidiruWQO6p3gv2TXwY1wEHYO5D6I,44
 cloudnetpy/categorize/atmos_utils.py,sha256=9-ymI6i1xASf-XAFyO87FaTfvq6bF89N1i_27OkUp-M,10104
 cloudnetpy/categorize/attenuation.py,sha256=Y_-fzmQTltWTqIZTulJhovC7a6ifpMcaAazDJcnMIOc,990
@@ -100,7 +101,7 @@ cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py,sha256=Ra3r4V
 cloudnetpy/model_evaluation/tests/unit/test_tools.py,sha256=Ia_VrLdV2NstX5gbx_3AZTOAlrgLAy_xFZ8fHYVX0xI,3817
 cloudnetpy/plotting/__init__.py,sha256=lg9Smn4BI0dVBgnDLC3JVJ4GmwoSnO-qoSd4ApvwV6Y,107
 cloudnetpy/plotting/plot_meta.py,sha256=ZvaKU3eXy1KFxQomnsEu3mCYpwwBYKAYk7oAwOzAGSg,16143
-cloudnetpy/plotting/plotting.py,sha256=5mLDRZKcpgO0V9fUdk_Xf1E8BVQGrVihiBdfifdWWPk,35208
+cloudnetpy/plotting/plotting.py,sha256=RGtRMfrZ6wFsFCXZ540I18a7p_O0x3S7FGIAoyuyOxw,35425
 cloudnetpy/products/__init__.py,sha256=2hRb5HG9hNrxH1if5laJkLeFeaZCd5W1q3hh4ewsX0E,273
 cloudnetpy/products/classification.py,sha256=AKb9GCatvhS5KR0c9LfN96nUvzi02175ZCQlvMH1Dws,8077
 cloudnetpy/products/der.py,sha256=soypE7uSEP4uHUCCQVEhyXsKY6e9mzV9B_2S5GUizqk,12729
@@ -114,8 +115,9 @@ cloudnetpy/products/mie_lu_tables.nc,sha256=It4fYpqJXlqOgL8jeZ-PxGzP08PMrELIDVe5
 cloudnetpy/products/mwr_tools.py,sha256=rd7UC67O4fsIE5SaHVZ4qWvUJTj41ZGwgQWPwZzOM14,5377
 cloudnetpy/products/product_tools.py,sha256=01Zc6xV8CSuYcIcLpchFf5POL3_c629-YMNDZJ51udA,10853
 docs/source/conf.py,sha256=IKiFWw6xhUd8NrCg0q7l596Ck1d61XWeVjIFHVSG9Og,1490
-cloudnetpy-1.66.5.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
-cloudnetpy-1.66.5.dist-info/METADATA,sha256=Fn9VQMj4YeMVPSsIKg2zuBzgrU4E_YDzXh3pcW6R4k0,5784
-cloudnetpy-1.66.5.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-cloudnetpy-1.66.5.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
-cloudnetpy-1.66.5.dist-info/RECORD,,
+cloudnetpy-1.66.6.dist-info/LICENSE,sha256=wcZF72bdaoG9XugpyE95Juo7lBQOwLuTKBOhhtANZMM,1094
+cloudnetpy-1.66.6.dist-info/METADATA,sha256=20TMauFPuw3zoVxRuuqG5sjtL5Sh1VUiO96s_4M4bro,5784
+cloudnetpy-1.66.6.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+cloudnetpy-1.66.6.dist-info/entry_points.txt,sha256=HhY7LwCFk4qFgDlXx_Fy983ZTd831WlhtdPIzV-Y3dY,51
+cloudnetpy-1.66.6.dist-info/top_level.txt,sha256=ibSPWRr6ojS1i11rtBFz2_gkIe68mggj7aeswYfaOo0,16
+cloudnetpy-1.66.6.dist-info/RECORD,,

{cloudnetpy-1.66.5.dist-info → cloudnetpy-1.66.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

cloudnetpy-1.66.6.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cloudnetpy = cloudnetpy.cli:main