licelformat 0.1.0__py3-none-any.whl

licelformat/__init__.py

@@ -0,0 +1,29 @@
+"""
+LicelFormat — Python port of the Go licelformat package.
+
+Provides utilities for parsing and processing Licel format data files.
+Supports reading, extracting metadata, and converting binary data into
+usable formats. Works with Licel files containing measurement profiles
+and other associated data.
+"""
+
+from .licelfile import (
+    LicelFile,
+    LicelProfilesList,
+    LoadLicelFile,
+    LoadLicelFileFromReader,
+)
+from .licelpack import LicelPack, NewLicelPack, NewLicelPackFromZip
+from .licelprofile import LICEL_MAX_RESERVED, LicelProfile
+
+__all__ = [
+    "LicelProfile",
+    "LICEL_MAX_RESERVED",
+    "LicelFile",
+    "LicelProfilesList",
+    "LoadLicelFile",
+    "LoadLicelFileFromReader",
+    "LicelPack",
+    "NewLicelPack",
+    "NewLicelPackFromZip",
+]

licelformat/licelfile.py

@@ -0,0 +1,264 @@
+"""
+LicelFile — structure representing a single Licel measurement.
+
+Provides loading, saving, and querying functionality for Licel format files.
+"""
+
+import os
+import struct
+from datetime import datetime
+from io import BufferedReader, RawIOBase
+from typing import IO, List, Optional
+
+import numpy as np
+
+from .licelprofile import LICEL_MAX_RESERVED, LicelProfile, _str2float, _str2int
+
+LicelProfilesList = List[LicelProfile]
+
+
+class LicelFile:
+    """Structure representing a single Licel measurement."""
+
+    __slots__ = (
+        "MeasurementSite",
+        "MeasurementStartTime",
+        "MeasurementStopTime",
+        "AltitudeAboveSeaLevel",
+        "Longitude",
+        "Latitude",
+        "Zenith",
+        "Laser1NShots",
+        "Laser1Freq",
+        "Laser2NShots",
+        "Laser2Freq",
+        "NDatasets",
+        "Laser3NShots",
+        "Laser3Freq",
+        "FileLoaded",
+        "Profiles",
+    )
+
+    def __init__(self):
+        self.MeasurementSite: str = ""
+        self.MeasurementStartTime: Optional[datetime] = None
+        self.MeasurementStopTime: Optional[datetime] = None
+        self.AltitudeAboveSeaLevel: float = 0.0
+        self.Longitude: float = 0.0
+        self.Latitude: float = 0.0
+        self.Zenith: float = 0.0
+        self.Laser1NShots: int = 0
+        self.Laser1Freq: int = 0
+        self.Laser2NShots: int = 0
+        self.Laser2Freq: int = 0
+        self.NDatasets: int = 0
+        self.Laser3NShots: int = 0
+        self.Laser3Freq: int = 0
+        self.FileLoaded: bool = False
+        self.Profiles: LicelProfilesList = []
+
+    def select_certain_wavelength(
+        self, is_photon: bool, wavelength: float
+    ) -> LicelProfile:
+        """Select a profile by its wavelength and type."""
+        for profile in self.Profiles:
+            if profile.Photon == is_photon and profile.Wavelength == wavelength:
+                return profile
+        return LicelProfile()
+
+    def save(self, fname: str) -> None:
+        """Save the Licel file to disk."""
+        with open(fname + "1", "wb") as f:
+            f.write(self._format_first_line(fname).encode("latin-1"))
+            f.write(self._format_second_line().encode("latin-1"))
+            f.write(self._format_third_line().encode("latin-1"))
+            for profile in self.Profiles:
+                f.write(profile.metadata().encode("latin-1"))
+            f.write(b"\r\n")
+            for profile in self.Profiles:
+                f.write(profile.profile())
+
+    def _format_first_line(self, fname: str) -> str:
+        """Return the first line of a LICEL file."""
+        return f" {fname:<77s}\r\n"
+
+    def _format_second_line(self) -> str:
+        """Return the second line of a LICEL file."""
+        s = (
+            f" {self.MeasurementSite} "
+            f"{self.MeasurementStartTime.strftime('%d/%m/%Y')} "
+            f"{self.MeasurementStartTime.strftime('%H:%M:%S')} "
+            f"{self.MeasurementStopTime.strftime('%d/%m/%Y')} "
+            f"{self.MeasurementStopTime.strftime('%H:%M:%S')} "
+            f"{self.AltitudeAboveSeaLevel:04.0f} "
+            f"{self.Longitude:06.1f} "
+            f"{self.Latitude:06.1f} "
+            f"{self.Zenith:02.0f}"
+        )
+        return f"{s:<78s}\r\n"
+
+    def _format_third_line(self) -> str:
+        """Return the third line of a LICEL file."""
+        s = (
+            f" {self.Laser1NShots:07d} {self.Laser1Freq:04d} "
+            f"{self.Laser2NShots:07d} {self.Laser2Freq:04d} "
+            f"{self.NDatasets:02d} {self.Laser3NShots:07d} "
+            f"{self.Laser3Freq:04d}"
+        )
+        return f"{s:<78s}\r\n"
+
+    def to_dict(self) -> dict:
+        """Convert LicelFile to a dictionary (for JSON serialization)."""
+        return {
+            "location": self.MeasurementSite,
+            "start_time": (
+                self.MeasurementStartTime.isoformat()
+                if self.MeasurementStartTime
+                else None
+            ),
+            "stop_time": (
+                self.MeasurementStopTime.isoformat()
+                if self.MeasurementStopTime
+                else None
+            ),
+            "lidar_altitude": self.AltitudeAboveSeaLevel,
+            "longitude": self.Longitude,
+            "latitude": self.Latitude,
+            "zenith": self.Zenith,
+            "laser1_nshots": self.Laser1NShots,
+            "laser1_freq": self.Laser1Freq,
+            "laser2_nshots": self.Laser2NShots,
+            "laser2_freq": self.Laser2Freq,
+            "dataset_count": self.NDatasets,
+            "laser3_nshots": self.Laser3NShots,
+            "laser3_freq": self.Laser3Freq,
+            "datasets": [p.to_dict() for p in self.Profiles],
+        }
+
+    def __repr__(self) -> str:
+        return (
+            f"LicelFile(site={self.MeasurementSite}, "
+            f"start={self.MeasurementStartTime}, "
+            f"profiles={len(self.Profiles)})"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Helper functions
+# ---------------------------------------------------------------------------
+
+
+def _parse_time(s: str) -> datetime:
+    """Parse datetime string in format 'dd/mm/yyyy hh:mm:ss'."""
+    return datetime.strptime(s, "%d/%m/%Y %H:%M:%S")
+
+
+def _read_and_trim_line(r: BufferedReader) -> str:
+    """Read a line from reader and trim right-side whitespace."""
+    line = r.readline()
+    if not line:
+        raise EOFError("Unexpected end of file while reading header line")
+    return line.decode("latin-1").rstrip("\t\r\n ")
+
+
+def _skip_crlf(r: BufferedReader) -> None:
+    """Skip CR+LF (2 bytes) from reader."""
+    crlf = r.read(2)
+    if len(crlf) < 2:
+        raise EOFError("Unexpected end of file while skipping CRLF")
+
+
+def _bytes_to_float64_array(b: bytes) -> np.ndarray:
+    """Convert raw bytes to numpy float64 array (little-endian int32 → float64)."""
+    # Interpret bytes as little-endian int32, then cast to float64
+    arr = np.frombuffer(b, dtype=np.int32).astype(np.float64)
+    return arr
+
+
+# ---------------------------------------------------------------------------
+# Main loading functions
+# ---------------------------------------------------------------------------
+
+
+def LoadLicelFile(fname: str) -> LicelFile:
+    """Load a LicelFile from the specified file path."""
+    with open(fname, "rb") as f:
+        return _load_licel_file_from_buffered_reader(f)
+
+
+def LoadLicelFileFromReader(stream: IO[bytes], size: int = 0) -> LicelFile:
+    """Load a LicelFile from a binary reader/stream."""
+    # Wrap in BufferedReader if needed
+    if isinstance(stream, BufferedReader):
+        r = stream
+    else:
+        r = BufferedReader(stream)  # type: ignore[arg-type]
+    return _load_licel_file_from_buffered_reader(r)
+
+
+def _load_licel_file_from_buffered_reader(r: BufferedReader) -> LicelFile:
+    """Core loading logic from a buffered binary reader."""
+    licf = LicelFile()
+
+    # Skip first line (contains filename or is empty)
+    _read_and_trim_line(r)
+
+    # Second line: basic information
+    header = _read_and_trim_line(r)
+    tmp = header.split()
+
+    licf.MeasurementSite = tmp[0]
+    licf.MeasurementStartTime = _parse_time(tmp[1] + " " + tmp[2])
+    licf.MeasurementStopTime = _parse_time(tmp[3] + " " + tmp[4])
+    licf.AltitudeAboveSeaLevel = _str2float(tmp[5])
+    licf.Longitude = _str2float(tmp[6])
+    licf.Latitude = _str2float(tmp[7])
+    licf.Zenith = _str2float(tmp[8])
+
+    # Third line: laser parameters
+    header = _read_and_trim_line(r)
+    tmp = header.split()
+    licf.Laser1NShots = _str2int(tmp[0])
+    licf.Laser1Freq = _str2int(tmp[1])
+    licf.Laser2NShots = _str2int(tmp[2])
+    licf.Laser2Freq = _str2int(tmp[3])
+    licf.NDatasets = _str2int(tmp[4])
+    licf.Laser3NShots = _str2int(tmp[5])
+    licf.Laser3Freq = _str2int(tmp[6])
+
+    # Profiles (headers)
+    licf.Profiles = []
+    for _ in range(licf.NDatasets):
+        header = _read_and_trim_line(r)
+        licf.Profiles.append(LicelProfile(header))
+
+    # After headers — binary data
+    _skip_crlf(r)
+
+    for i in range(licf.NDatasets):
+        n_bytes = licf.Profiles[i].NDataPoints * 4
+        pr_tmp = r.read(n_bytes)
+        if len(pr_tmp) < n_bytes:
+            raise EOFError("Error reading binary data: unexpected end of file")
+
+        licf.Profiles[i].Data = _bytes_to_float64_array(pr_tmp)
+
+        # Apply scaling
+        if not licf.Profiles[i].Photon:
+            # Analog channel
+            adc_scale = 1 << licf.Profiles[i].AdcBits
+            scale = (
+                licf.Profiles[i].DiscrLevel
+                * 1000.0
+                / float(adc_scale * licf.Profiles[i].NShots)
+            )
+        else:
+            # Photon counting channel
+            scale = 1.0 / (float(licf.Profiles[i].NShots) * 0.05)
+
+        licf.Profiles[i].Data *= scale
+
+        _skip_crlf(r)
+
+    licf.FileLoaded = True
+    return licf

licelformat/licelpack.py

@@ -0,0 +1,100 @@
+"""
+LicelPack — collection of Licel measurements loaded by file mask or from ZIP.
+
+Provides loading and querying functionality for multiple Licel files.
+"""
+
+import glob
+import io
+import re
+import zipfile
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from .licelfile import (
+    LicelFile,
+    LicelProfilesList,
+    LoadLicelFile,
+    LoadLicelFileFromReader,
+)
+
+
+def _is_valid_filename(filename: str) -> bool:
+    """Check if filename matches the pattern 'b*.*'."""
+    return bool(re.match(r"^b.*\..+", filename))
+
+
+class LicelPack:
+    """Collection of Licel measurements."""
+
+    __slots__ = ("StartTime", "Data")
+
+    def __init__(self):
+        self.StartTime: Optional[datetime] = None
+        self.Data: Dict[str, LicelFile] = {}
+
+    def select_certain_wavelength(
+        self, is_photon: bool, wavelength: float
+    ) -> LicelProfilesList:
+        """Select profiles by wavelength and type from all files in the pack."""
+        result: LicelProfilesList = []
+        for licf in self.Data.values():
+            profile = licf.select_certain_wavelength(is_photon, wavelength)
+            if profile.Wavelength != 0:
+                result.append(profile)
+        return result
+
+    def save(self) -> None:
+        """Save all files in the pack to disk."""
+        for fname, licf in self.Data.items():
+            licf.save(fname)
+
+    def __repr__(self) -> str:
+        return f"LicelPack(start={self.StartTime}, files={len(self.Data)})"
+
+    def to_dict(self) -> dict:
+        """Convert LicelPack to a dictionary (for JSON serialization)."""
+        return {
+            "start_time": (self.StartTime.isoformat() if self.StartTime else None),
+            "data": {name: lf.to_dict() for name, lf in self.Data.items()},
+        }
+
+
+def NewLicelPack(mask: str) -> LicelPack:
+    """Load files according to a glob mask."""
+    pack = LicelPack()
+    files = glob.glob(mask)
+    if not files:
+        raise FileNotFoundError(f"No files found matching mask: {mask}")
+
+    for i, fname in enumerate(files):
+        pack.Data[fname] = LoadLicelFile(fname)
+        if i == 0:
+            pack.StartTime = pack.Data[fname].MeasurementStartTime
+
+    return pack
+
+
+def NewLicelPackFromZip(zip_path: str) -> LicelPack:
+    """Load files from a ZIP archive."""
+    pack = LicelPack()
+
+    with zipfile.ZipFile(zip_path, "r") as zr:
+        for info in zr.infolist():
+            fname = info.filename
+            if not _is_valid_filename(fname):
+                continue
+
+            # Read file contents into memory
+            file_bytes = zr.read(fname)
+
+            # Load LicelFile from bytes
+            licf = LoadLicelFileFromReader(io.BytesIO(file_bytes), len(file_bytes))
+
+            full_path = "/" + fname
+            pack.Data[full_path] = licf
+
+            if len(pack.Data) == 1:
+                pack.StartTime = licf.MeasurementStartTime
+
+    return pack

licelformat/licelprofile.py

@@ -0,0 +1,177 @@
+"""
+LicelProfile — measurement channel structure.
+
+Represents a single measurement channel (profile) in a Licel file.
+"""
+
+import struct
+from typing import List
+
+LICEL_MAX_RESERVED = 3
+
+
+def _str2bool(s: str) -> bool:
+    """Convert string to boolean."""
+    return s.lower() in ("1", "true", "yes")
+
+
+def _str2int(s: str) -> int:
+    """Convert string to integer."""
+    return int(s)
+
+
+def _str2float(s: str) -> float:
+    """Convert string to float."""
+    return float(s)
+
+
+def _btoi(b: bool) -> int:
+    """Convert boolean to integer (0/1)."""
+    return 1 if b else 0
+
+
+class LicelProfile:
+    """Represents a single measurement channel in a Licel file."""
+
+    __slots__ = (
+        "Active",
+        "Photon",
+        "LaserType",
+        "NDataPoints",
+        "Reserved",
+        "HighVoltage",
+        "BinWidth",
+        "Wavelength",
+        "Polarization",
+        "BinShift",
+        "DecBinShift",
+        "AdcBits",
+        "NShots",
+        "DiscrLevel",
+        "DeviceID",
+        "NCrate",
+        "Data",
+    )
+
+    def __init__(self, line: str = None):
+        self.Active: bool = False
+        self.Photon: bool = False
+        self.LaserType: int = 0
+        self.NDataPoints: int = 0
+        self.Reserved: List[int] = [0, 0, 0]
+        self.HighVoltage: int = 0
+        self.BinWidth: float = 0.0
+        self.Wavelength: float = 0.0
+        self.Polarization: str = ""
+        self.BinShift: int = 0
+        self.DecBinShift: int = 0
+        self.AdcBits: int = 0
+        self.NShots: int = 0
+        self.DiscrLevel: float = 0.0
+        self.DeviceID: str = ""
+        self.NCrate: int = 0
+        self.Data: List[float] = []
+
+        if line is not None:
+            self._parse(line)
+
+    def _parse(self, line: str) -> None:
+        """Parse a string line into LicelProfile."""
+        items = line.split()
+        wvlpol = items[7].split(".", 1)
+
+        self.Active = _str2bool(items[0])
+        self.Photon = _str2bool(items[1])
+        self.LaserType = _str2int(items[2])
+        self.NDataPoints = _str2int(items[3])
+        self.Reserved = [
+            _str2int(items[4]),
+            _str2int(items[8]),
+            _str2int(items[9]),
+        ]
+        self.HighVoltage = _str2int(items[5])
+        self.BinWidth = _str2float(items[6])
+        self.Wavelength = _str2float(wvlpol[0])
+        self.Polarization = wvlpol[1] if len(wvlpol) > 1 else ""
+        self.BinShift = _str2int(items[10])
+        self.DecBinShift = _str2int(items[11])
+        self.AdcBits = _str2int(items[12])
+        self.NShots = _str2int(items[13])
+        self.DiscrLevel = _str2float(items[14])
+        self.DeviceID = items[15][:2]
+        self.NCrate = _str2int(items[15][2:])
+
+    def metadata(self) -> str:
+        """Return the metadata string for this profile."""
+        if self.Photon:
+            s = (
+                f" {_btoi(self.Active):1d} {_btoi(self.Photon):1d} {self.LaserType:1d} "
+                f"{self.NDataPoints:05d} {self.Reserved[0]:1d} {self.HighVoltage:04d} "
+                f"{self.BinWidth:04.2f} {int(self.Wavelength):05d}.{self.Polarization:<1s} "
+                f"{0:1d} {0:1d} {self.BinShift:02d} {self.DecBinShift:03d} "
+                f"{self.AdcBits:02d} {self.NShots:06d} {self.DiscrLevel:05.4f} "
+                f"{self.DeviceID:2s}{self.NCrate:01d}"
+            )
+        else:
+            s = (
+                f" {_btoi(self.Active):1d} {_btoi(self.Photon):1d} {self.LaserType:1d} "
+                f"{self.NDataPoints:05d} {self.Reserved[0]:1d} {self.HighVoltage:04d} "
+                f"{self.BinWidth:04.2f} {int(self.Wavelength):05d}.{self.Polarization:<1s} "
+                f"{0:1d} {0:1d} {self.BinShift:02d} {self.DecBinShift:03d} "
+                f"{self.AdcBits:02d} {self.NShots:06d} {self.DiscrLevel:05.3f} "
+                f"{self.DeviceID:2s}{self.NCrate:01d}"
+            )
+        return f"{s:<78s}\r\n"
+
+    def scale_factor(self) -> float:
+        """Return the scaling factor that was applied during loading.
+
+        Analog channels: scale = DiscrLevel * 1000 / (2^AdcBits * NShots)
+        Photon channels: scale = 1 / (NShots * 0.05)
+        """
+        if not self.Photon:
+            adc_scale = 1 << self.AdcBits
+            return self.DiscrLevel * 1000.0 / float(adc_scale * self.NShots)
+        else:
+            return 1.0 / (float(self.NShots) * 0.05)
+
+    def profile(self) -> bytes:
+        """Convert profile data to binary bytes (little-endian int32).
+
+        Applies unscale first: raw = round(scaled_value / scale_factor),
+        then packs as little-endian int32, matching the original file format.
+        """
+        inv_scale = 1.0 / self.scale_factor() if self.scale_factor() != 0.0 else 1.0
+        buf = b""
+        for num in self.Data:
+            buf += struct.pack("<i", round(num * inv_scale))
+        return buf + b"\r\n"
+
+    def __repr__(self) -> str:
+        return (
+            f"LicelProfile(Active={self.Active}, Photon={self.Photon}, "
+            f"Wavelength={self.Wavelength}, NDataPoints={self.NDataPoints}, "
+            f"NShots={self.NShots})"
+        )
+
+    def to_dict(self) -> dict:
+        """Convert profile to a dictionary (for JSON serialization)."""
+        return {
+            "is_active": self.Active,
+            "is_photon": self.Photon,
+            "laser_type": self.LaserType,
+            "data_points": self.NDataPoints,
+            "reserved": self.Reserved,
+            "high_voltage": self.HighVoltage,
+            "bin_width": self.BinWidth,
+            "wavelength": self.Wavelength,
+            "polarization": self.Polarization,
+            "bin_shift": self.BinShift,
+            "dec_bin_shift": self.DecBinShift,
+            "adc_bits": self.AdcBits,
+            "n_shots": self.NShots,
+            "discr_level": self.DiscrLevel,
+            "device_id": self.DeviceID,
+            "n_crate": self.NCrate,
+            "data": self.Data,
+        }