cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

cloudnetpy/output.py

@@ -1,49 +1,57 @@
-""" Functions for file writing."""
+"""Functions for file writing."""
+
 import datetime
 import logging
+from dataclasses import fields
+from os import PathLike
+from uuid import UUID
 
 import netCDF4
-import numpy as np
 from numpy import ma
 
 from cloudnetpy import utils, version
+from cloudnetpy.categorize.containers import Observations
+from cloudnetpy.categorize.model import Model
+from cloudnetpy.datasource import DataSource
 from cloudnetpy.instruments.instruments import Instrument
-from cloudnetpy.metadata import COMMON_ATTRIBUTES, MetaData
+from cloudnetpy.metadata import COMMON_ATTRIBUTES
 
 
-def save_level1b(obj, output_file: str, uuid: str | None = None) -> str:
+def save_level1b(
+    obj,  # noqa: ANN001
+    output_file: PathLike | str,
+    uuid: UUID,
+) -> None:
     """Saves Cloudnet Level 1b file."""
     dimensions = _get_netcdf_dimensions(obj)
     with init_file(output_file, dimensions, obj.data, uuid) as nc:
-        file_uuid = nc.file_uuid
         fix_attribute_name(nc)
         location = obj.site_meta["name"]
         nc.cloudnet_file_type = obj.instrument.domain
         nc.title = get_l1b_title(obj.instrument, location)
-        if isinstance(obj.date, list):
-            nc.year, nc.month, nc.day = obj.date
-        elif isinstance(obj.date, datetime.date):
-            nc.year = str(obj.date.year)
-            nc.month = str(obj.date.month).zfill(2)
-            nc.day = str(obj.date.day).zfill(2)
-        else:
-            raise TypeError
+        nc.year = str(obj.date.year)
+        nc.month = str(obj.date.month).zfill(2)
+        nc.day = str(obj.date.day).zfill(2)
         nc.location = location
         nc.history = get_l1b_history(obj.instrument)
         nc.source = get_l1b_source(obj.instrument)
         if hasattr(obj, "serial_number") and obj.serial_number is not None:
             nc.serial_number = obj.serial_number
+        if hasattr(obj, "software"):
+            for software, version in obj.software.items():
+                nc.setncattr(f"{software}_version", version)
         nc.references = get_references()
-    return file_uuid
 
 
-def _get_netcdf_dimensions(obj) -> dict:
+def _get_netcdf_dimensions(obj) -> dict:  # noqa: ANN001
     dimensions = {
         key: len(obj.data[key][:]) for key in ("time", "range") if key in obj.data
     }
     # RPG cloud radar
     if "chirp_start_indices" in obj.data:
-        dimensions["chirp_sequence"] = len(obj.data["chirp_start_indices"][:])
+        ind = obj.data["chirp_start_indices"][:]
+        dimensions["chirp_sequence"] = ind.shape[1] if ind.ndim > 1 else len(ind)
+
     # disdrometer
     if hasattr(obj, "n_diameter") and hasattr(obj, "n_velocity"):
         dimensions["diameter"] = obj.n_diameter
@@ -62,11 +70,11 @@ def _get_netcdf_dimensions(obj) -> dict:
 
 def save_product_file(
     short_id: str,
-    obj,
-    file_name: str,
-    uuid: str | None = None,
+    obj: DataSource,
+    file_name: str | PathLike,
+    uuid: UUID,
     copy_from_cat: tuple = (),
-) -> str:
+) -> None:
     """Saves a standard Cloudnet product file.
 
     Args:
@@ -83,7 +91,6 @@ def save_product_file(
         "height": len(obj.dataset.variables["height"]),
     }
     with init_file(file_name, dimensions, obj.data, uuid) as nc:
-        file_uuid = nc.file_uuid
         nc.cloudnet_file_type = short_id
         vars_from_source = (
             "altitude",
@@ -91,24 +98,37 @@ def save_product_file(
             "longitude",
             "time",
             "height",
-        ) + copy_from_cat
+            *copy_from_cat,
+        )
         copy_variables(obj.dataset, nc, vars_from_source)
         nc.title = (
             f"{human_readable_file_type.capitalize()} products from"
             f" {obj.dataset.location}"
         )
-        nc.source_file_uuids = get_source_uuids(nc, obj)
-        copy_global(obj.dataset, nc, ("location", "day", "month", "year", "source"))
-        merge_history(nc, human_readable_file_type, {"categorize": obj})
+        nc.source_file_uuids = get_source_uuids([nc, obj])
+        copy_global(
+            obj.dataset,
+            nc,
+            (
+                "location",
+                "day",
+                "month",
+                "year",
+                "source",
+                "source_instrument_pids",
+                "voodoonet_version",
+            ),
+        )
+        merge_history(nc, human_readable_file_type, obj)
         nc.references = get_references(short_id)
-    return file_uuid
 
 
 def get_l1b_source(instrument: Instrument) -> str:
     """Returns level 1b file source."""
-    return " ".join(
+    parts = [
         item for item in [instrument.manufacturer, instrument.model] if item is not None
-    )
+    ]
+    return " ".join(parts) if parts else instrument.category.capitalize()
 
 
 def get_l1b_history(instrument: Instrument) -> str:
@@ -128,10 +148,11 @@ def get_l1b_title(instrument: Instrument, location: str) -> str:
 
 
 def get_references(identifier: str | None = None, extra: list | None = None) -> str:
-    """ "Returns references.
+    """Returns references.
 
     Args:
         identifier: Cloudnet file type, e.g., 'iwc'.
+        extra: List of additional references to include
 
     """
     references = "https://doi.org/10.21105/joss.02123"
@@ -158,63 +179,93 @@ def get_references(identifier: str | None = None, extra: list | None = None) ->
     return references
 
 
-def get_source_uuids(*sources) -> str:
+def get_source_uuids(data: Observations | list[netCDF4.Dataset | DataSource]) -> str:
     """Returns file_uuid attributes of objects.
 
     Args:
-        *sources: Objects whose file_uuid attributes are read (if exist).
+        data: Observations instance.
 
     Returns:
         str: UUIDs separated by comma.
 
     """
+    if isinstance(data, Observations):
+        obs = [getattr(data, field.name) for field in fields(data)]
+    elif isinstance(data, list):
+        obs = data
     uuids = [
-        source.dataset.file_uuid
-        for source in sources
-        if hasattr(source, "dataset") and hasattr(source.dataset, "file_uuid")
+        obj.dataset.file_uuid
+        for obj in obs
+        if hasattr(obj, "dataset") and hasattr(obj.dataset, "file_uuid")
     ]
-    unique_uuids = list(set(uuids))
+    unique_uuids = sorted(set(uuids))
     return ", ".join(unique_uuids)
 
 
-def merge_history(nc: netCDF4.Dataset, file_type: str, data: dict) -> None:
-    """Merges history fields from one or several files and creates a new record.
+def merge_history(
+    nc: netCDF4.Dataset, file_type: str, data: Observations | DataSource
+) -> None:
+    """Merges history fields from one or several files and creates a new record."""
 
-    Args:
-        nc: The netCDF Dataset instance.
-        file_type: Long description of the file.
-        data: Dictionary of objects with history attribute.
+    def extract_history(obj: DataSource | Observations) -> list[str]:
+        if hasattr(obj, "dataset") and hasattr(obj.dataset, "history"):
+            history = obj.dataset.history
+            if isinstance(obj, Model):
+                return [history.split("\n")[-1]]
+            return history.split("\n")
+        return []
+
+    histories: list[str] = []
+    if isinstance(data, DataSource):
+        histories.extend(extract_history(data))
+    elif isinstance(data, Observations):
+        for field in fields(data):
+            histories.extend(extract_history(getattr(data, field.name)))
+
+    # Remove duplicates
+    histories = list(dict.fromkeys(histories))
+
+    def parse_time(line: str) -> datetime.datetime:
+        try:
+            return datetime.datetime.strptime(
+                line.split(" - ")[0].strip(), "%Y-%m-%d %H:%M:%S %z"
+            )
+        except ValueError:
+            return datetime.datetime.min.replace(
+                tzinfo=datetime.timezone.utc
+            )  # malformed lines to bottom
 
-    """
+    histories.sort(key=parse_time, reverse=True)
     new_record = f"{utils.get_time()} - {file_type} file created"
-    histories = []
-    for key, obj in data.items():
-        if (
-            not isinstance(obj, (str, list))
-            and obj is not None
-            and hasattr(obj.dataset, "history")
-        ):
-            history = obj.dataset.history
-            history = history.split("\n")[-1] if key == "model" else history
-            histories.append(history)
-    histories.sort(reverse=True)
-    old_history = [f"\n{history}" for history in histories]
-    old_history_str = "".join(old_history)
-    nc.history = f"{new_record}{old_history_str}"
+    nc.history = new_record + "".join(f"\n{h}" for h in histories)
 
 
-def add_source_instruments(nc: netCDF4.Dataset, data: dict) -> None:
+def add_source_instruments(nc: netCDF4.Dataset, data: Observations) -> None:
     """Adds source attribute to categorize file."""
-    sources = [obj.source for obj in data.values() if hasattr(obj, "source")]
-    sources = [sources[0]] + [f"\n{source}" for source in sources[1:]]
-    nc.source = "".join(sources)
+    sources = {
+        src
+        for field in fields(data)
+        for obj in [getattr(data, field.name)]
+        if hasattr(obj, "source")
+        for src in obj.source.split("\n")
+    }
+    if sources:
+        nc.source = "\n".join(sorted(sources))
+    source_pids = {
+        obj.instrument_pid
+        for field in fields(data)
+        for obj in [getattr(data, field.name)]
+        if getattr(obj, "instrument_pid", "")
+    }
+    if source_pids:
+        nc.source_instrument_pids = "\n".join(sorted(source_pids))
 
 
 def init_file(
-    file_name: str,
+    file_name: PathLike | str,
     dimensions: dict,
     cloudnet_arrays: dict,
-    uuid: str | None = None,
+    uuid: UUID,
 ) -> netCDF4.Dataset:
     """Initializes a Cloudnet file for writing.
 
@@ -234,7 +285,9 @@ def init_file(
 
 
 def copy_variables(
-    source: netCDF4.Dataset, target: netCDF4.Dataset, keys: tuple
+    source: netCDF4.Dataset,
+    target: netCDF4.Dataset,
+    keys: tuple,
 ) -> None:
     """Copies variables (and their attributes) from one file to another.
 
@@ -252,6 +305,7 @@ def copy_variables(
                 key,
                 variable.datatype,
                 variable.dimensions,
+                zlib=True,
                 fill_value=fill_value,
             )
             var_out.setncatts(
@@ -259,13 +313,15 @@ def copy_variables(
                     k: variable.getncattr(k)
                     for k in variable.ncattrs()
                     if k != "_FillValue"
-                }
+                },
             )
             var_out[:] = variable[:]
 
 
 def copy_global(
-    source: netCDF4.Dataset, target: netCDF4.Dataset, attributes: tuple
+    source: netCDF4.Dataset,
+    target: netCDF4.Dataset,
+    attributes: tuple,
 ) -> None:
     """Copies global attributes from one file to another.
 
@@ -282,25 +338,21 @@ def copy_global(
 
 
 def add_time_attribute(
-    attributes: dict, date: list[str] | datetime.date, key: str = "time"
+    attributes: dict,
+    date: datetime.date,
+    key: str = "time",
 ) -> dict:
     """Adds time attribute with correct units."""
-    if isinstance(date, list):
-        date_str = "-".join(date)
-    elif isinstance(date, datetime.date):
-        date_str = date.isoformat()
-    else:
-        raise TypeError
+    date_str = date.isoformat()
     units = f"hours since {date_str} 00:00:00 +00:00"
     if key not in attributes:
-        attributes[key] = MetaData(units=units)
-    else:
-        attributes[key] = attributes[key]._replace(units=units)
+        attributes[key] = COMMON_ATTRIBUTES[key]
+    attributes[key] = attributes[key]._replace(units=units)
     return attributes
 
 
-def add_source_attribute(attributes: dict, data: dict):
-    """Adds source attribute."""
+def add_source_attribute(attributes: dict, data: Observations) -> dict:
+    """Adds source attribute to variables."""
     variables = {
         "radar": (
             "v",
@@ -317,14 +369,19 @@ def add_source_attribute(attributes: dict, data: dict):
         "lidar": ("beta", "lidar_wavelength"),
         "mwr": ("lwp",),
         "model": ("uwind", "vwind", "Tw", "q", "pressure", "temperature"),
+        "disdrometer": ("rainfall_rate",),
     }
     for instrument, keys in variables.items():
-        source = data[instrument].dataset.source
+        if getattr(data, instrument) is None:
+            continue
+        source = getattr(data, instrument).dataset.source
+        source_pid = getattr(getattr(data, instrument).dataset, "instrument_pid", None)
         for key in keys:
-            if key in attributes:
-                attributes[key] = attributes[key]._replace(source=source)
-            else:
-                attributes[key] = MetaData(source=source)
+            if key not in attributes:
+                attributes[key] = COMMON_ATTRIBUTES[key]
+            attributes[key] = attributes[key]._replace(
+                source=source, source_instrument_pid=source_pid
+            )
     return attributes
 
 
@@ -353,34 +410,36 @@ def _write_vars2nc(nc: netCDF4.Dataset, cloudnet_variables: dict) -> None:
             fill_value = netCDF4.default_fillvals[obj.data_type]
         else:
             fill_value = False
-
-        size = obj.dimensions or _get_dimensions(nc, obj.data)
-
+        size = obj.dimensions if obj.dimensions is not None else ()
         nc_variable = nc.createVariable(
-            obj.name, obj.data_type, size, zlib=True, fill_value=fill_value
+            obj.name,
+            obj.data_type,
+            size,
+            zlib=True,
+            fill_value=fill_value,
         )
-        nc_variable[:] = obj.data
+        try:
+            nc_variable[:] = obj.data
+        except IndexError as err:
+            msg = f"Unable to write variable {obj.name} to file: {err}"
+            raise IndexError(msg) from err
         for attr in obj.fetch_attributes():
             setattr(nc_variable, attr, getattr(obj, attr))
 
 
-def _get_dimensions(nc: netCDF4.Dataset, data: np.ndarray) -> tuple:
-    """Finds correct dimensions for a variable."""
-    if utils.isscalar(data):
-        return ()
-    variable_size: list = []
-    file_dims = nc.dimensions
-    array_dims = data.shape
-    for length in array_dims:
-        dim = [key for key in file_dims.keys() if file_dims[key].size == length][0]
-        variable_size = variable_size + [dim]
-    return tuple(variable_size)
-
-
 def _get_identifier(short_id: str) -> str:
-    valid_ids = ("lwc", "iwc", "drizzle", "classification", "der", "ier")
+    valid_ids = (
+        "lwc",
+        "iwc",
+        "drizzle",
+        "classification",
+        "der",
+        "ier",
+        "classification-voodoo",
+    )
     if short_id not in valid_ids:
-        raise ValueError("Invalid product id.")
+        msg = f"Invalid file identifier: {short_id}"
+        raise ValueError(msg)
     if short_id == "iwc":
         return "ice water content"
     if short_id == "lwc":
@@ -392,12 +451,10 @@ def _get_identifier(short_id: str) -> str:
     return short_id
 
 
-def add_standard_global_attributes(
-    nc: netCDF4.Dataset, uuid: str | None = None
-) -> None:
+def add_standard_global_attributes(nc: netCDF4.Dataset, uuid: UUID) -> None:
     nc.Conventions = "CF-1.8"
     nc.cloudnetpy_version = version.__version__
-    nc.file_uuid = uuid or utils.get_uuid()
+    nc.file_uuid = str(uuid)
 
 
 def fix_attribute_name(nc: netCDF4.Dataset) -> None:
@@ -417,14 +474,16 @@ def fix_time_attributes(nc: netCDF4.Dataset) -> None:
     nc.variables["time"].standard_name = "time"
     nc.variables["time"].long_name = "Time UTC"
     nc.variables["time"].calendar = "standard"
-    nc.variables["time"].units = (
-        f"hours since " f"{nc.year}-{nc.month}-{nc.day} " f"00:00:00 +00:00"
-    )
+    nc.variables[
+        "time"
+    ].units = f"hours since {nc.year}-{nc.month}-{nc.day} 00:00:00 +00:00"
 
 
 def replace_attribute_with_standard_value(
-    nc: netCDF4.Dataset, variables: tuple, attributes: tuple
-):
+    nc: netCDF4.Dataset,
+    variables: tuple,
+    attributes: tuple,
+) -> None:
     for key in variables:
         if key in COMMON_ATTRIBUTES and key in nc.variables:
             for attr in attributes:

cloudnetpy/plotting/__init__.py

@@ -1 +1,2 @@
-from .plotting import compare_files, generate_figure, plot_2d
+from .plot_meta import PlotMeta
+from .plotting import Dimensions, PlotParameters, generate_figure, plot_2d