quantized-lab 0.8.0__py3-none-any.whl

quantized/io/delimited.py

@@ -0,0 +1,222 @@
+"""Generic delimited-text (CSV/TSV) parser. Port of MATLAB parser.importCSV.
+
+Auto-detects delimiter, comment lines, the header row, an optional units row,
+and the data start. By default the first column is the x-axis (time) and the
+remaining numeric columns are values. Named ``delimited`` to avoid shadowing
+the stdlib ``csv`` module.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from quantized.datastruct import DataStruct
+from quantized.io.base import resolve_column
+
+__all__ = ["import_csv"]
+
+_COMMENT_CHARS = "#%"
+_NA_TOKENS = {"", "nan", "na", "-", "n/a"}
+_DELIM_CANDIDATES = (",", "\t", ";", " ")
+
+
+def _is_numeric(token: str) -> bool:
+    """True if token parses to a number; NaN counts as non-numeric (str2double parity)."""
+    try:
+        value = float(token)
+    except ValueError:
+        return False
+    return not math.isnan(value)
+
+
+def _to_float(token: str) -> float:
+    stripped = token.strip()
+    if stripped.lower() in _NA_TOKENS:
+        return float("nan")
+    try:
+        return float(stripped)
+    except ValueError:
+        return float("nan")
+
+
+def _read_raw_lines(text: str) -> list[str]:
+    out: list[str] = []
+    for raw in text.splitlines():
+        stripped = raw.strip()
+        if not stripped or stripped[0] in _COMMENT_CHARS:
+            continue
+        out.append(stripped)
+    return out
+
+
+def _detect_delimiter(raw_lines: Sequence[str]) -> str:
+    test = raw_lines[:10]
+    best_delim = ","
+    best_score = 0.0
+    for ch in _DELIM_CANDIDATES:
+        counts = [line.count(ch) for line in test]
+        if counts and all(c > 0 for c in counts):
+            mean = sum(counts) / len(counts)
+            std = (sum((c - mean) ** 2 for c in counts) / len(counts)) ** 0.5
+            if std < mean * 0.5 and mean > best_score:
+                best_score = mean
+                best_delim = ch
+    return best_delim
+
+
+def _extract_units(header: str) -> tuple[str, str]:
+    """``'Temp (C)'`` -> ``('C', 'Temp')`` (also ``[...]``). Returns (unit, label)."""
+    paren = re.match(r"(.+?)\s*\(([^)]+)\)\s*$", header)
+    if paren:
+        return paren.group(2).strip(), paren.group(1).strip()
+    brack = re.match(r"(.+?)\s*\[([^\]]+)\]\s*$", header)
+    if brack:
+        return brack.group(2).strip(), brack.group(1).strip()
+    return "", header
+
+
+def _numeric_score(row: Sequence[str]) -> float:
+    if not row:
+        return 0.0
+    return sum(1 for t in row if _is_numeric(t.strip())) / len(row)
+
+
+def _looks_like_units_row(row: Sequence[str], n_data_cols: int) -> bool:
+    n = len(row)
+    if n < max(n_data_cols * 0.5, 2):
+        return False
+    n_unit_like = 0
+    n_non_empty = 0
+    for cell in row:
+        token = cell.strip()
+        if not token:
+            n_unit_like += 1
+            continue
+        n_non_empty += 1
+        if re.match(r"^[(\[{].*[)\]}]$", token):
+            n_unit_like += 1
+        elif " " not in token and not _is_numeric(token):
+            has_non_alpha = re.search(r"[^a-zA-Z]", token) is not None
+            if (has_non_alpha and len(token) <= 10) or (not has_non_alpha and len(token) <= 4):
+                n_unit_like += 1
+    return n_non_empty > 0 and (n_unit_like / max(n, 1)) >= 0.6
+
+
+def _detect_layout(tokens: Sequence[Sequence[str]]) -> tuple[int, int, int]:
+    """Return 0-based (header_row, data_start, units_row); -1 when absent."""
+    scores = [_numeric_score(row) for row in tokens]
+    first_data = next((i for i, s in enumerate(scores) if s > 0.5), 0)
+    header_row = -1
+    units_row = -1
+    if (
+        first_data >= 2
+        and scores[first_data - 1] < 0.5
+        and scores[first_data - 2] < 0.5
+        and _looks_like_units_row(tokens[first_data - 1], len(tokens[first_data]))
+    ):
+        units_row = first_data - 1
+        header_row = first_data - 2
+    elif first_data >= 1 and scores[first_data - 1] < 0.5:
+        header_row = first_data - 1
+    return header_row, first_data, units_row
+
+
+def import_csv(
+    filepath: str | Path,
+    *,
+    time_column: int | str = 0,
+    data_columns: Sequence[int | str] | None = None,
+) -> DataStruct:
+    """Import a generic delimited text file (first column = x-axis by default)."""
+    path = Path(filepath)
+    raw_lines = _read_raw_lines(path.read_text(encoding="latin-1"))
+    if not raw_lines:
+        raise ValueError(f"file empty or only comments: {path.name}")
+    delim = _detect_delimiter(raw_lines)
+    tokens = [line.split(delim) for line in raw_lines]
+
+    header_row, data_start, units_row = _detect_layout(tokens)
+    n_data_cols = len(tokens[data_start])
+    if header_row >= 0:
+        col_headers = [c.strip() for c in tokens[header_row]]
+    else:
+        col_headers = [f"Col{k + 1}" for k in range(n_data_cols)]
+    if len(col_headers) < n_data_cols:
+        col_headers += [f"Col{k + 1}" for k in range(len(col_headers), n_data_cols)]
+    elif len(col_headers) > n_data_cols:
+        col_headers = col_headers[:n_data_cols]
+    col_headers = [h if h.strip() else f"Col{k + 1}" for k, h in enumerate(col_headers)]
+    n_cols = len(col_headers)
+
+    row_units: list[str] = []
+    if units_row >= 0:
+        utok = [u.strip() for u in tokens[units_row]]
+        for k in range(n_cols):
+            cell = utok[k] if k < len(utok) else ""
+            row_units.append(re.sub(r"^\s*[(\[](.*?)[)\]]\s*$", r"\1", cell))
+
+    rows: list[list[float]] = []
+    for row in tokens[data_start:]:
+        vals = [float("nan")] * n_cols
+        for c in range(min(len(row), n_cols)):
+            vals[c] = _to_float(row[c])
+        rows.append(vals)
+    matrix = np.asarray(rows, dtype=float)
+    n_rows = matrix.shape[0]
+
+    if isinstance(time_column, int) and time_column < 0:
+        time_idx = -1
+    else:
+        time_idx = resolve_column(time_column, col_headers)
+
+    time_vec = np.arange(1, n_rows + 1, dtype=float) if time_idx < 0 else matrix[:, time_idx]
+
+    if data_columns is None:
+        candidates = [c for c in range(n_cols) if c != time_idx]
+        data_idx = [
+            c
+            for c in candidates
+            if (np.count_nonzero(~np.isnan(matrix[:, c])) / n_rows) > 0.1
+        ]
+    else:
+        data_idx = [resolve_column(s, col_headers) for s in data_columns]
+    if not data_idx:
+        raise ValueError(f"no valid data columns in {path.name}")
+
+    labels: list[str] = []
+    units: list[str] = []
+    for c in data_idx:
+        unit, label = _extract_units(col_headers[c])
+        labels.append(label)
+        units.append(unit)
+    if row_units:
+        for i, c in enumerate(data_idx):
+            if c < len(row_units) and row_units[c]:
+                units[i] = row_units[c]
+
+    if time_idx >= 0:
+        x_unit, x_name = _extract_units(col_headers[time_idx])
+        if not x_name:
+            x_name = col_headers[time_idx]
+        if row_units and time_idx < len(row_units) and row_units[time_idx]:
+            x_unit = row_units[time_idx]
+    else:
+        x_name, x_unit = "Sample Index", ""
+
+    metadata: dict[str, Any] = {
+        "source": str(path),
+        "parser_name": "import_csv",
+        "x_column_name": x_name,
+        "x_column_unit": x_unit,
+        "delimiter": delim,
+        "all_column_names": col_headers,
+    }
+    return DataStruct.create(
+        time_vec, matrix[:, data_idx], labels=labels, units=units, metadata=metadata
+    )

quantized/io/excel.py

@@ -0,0 +1,135 @@
+"""Excel ``.xlsx`` parser. Port of MATLAB parser.importExcel (via openpyxl).
+
+Reads a sheet's cell grid, detects the header + data-start rows from a numeric
+shadow matrix, and (by default) takes the first column as the x-axis. Mirrors
+importExcel's logic; the cell grid replaces MATLAB's ``readcell``.
+"""
+
+from __future__ import annotations
+
+import zipfile
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import openpyxl
+from openpyxl.utils.exceptions import InvalidFileException
+
+from quantized.datastruct import DataStruct
+from quantized.io.base import resolve_column
+from quantized.io.delimited import _extract_units
+
+__all__ = ["import_excel"]
+
+
+def _cell_to_float(value: Any) -> float:
+    if isinstance(value, bool):  # bool is an int subclass — not data
+        return float("nan")
+    if isinstance(value, (int, float)):
+        return float(value)
+    return float("nan")
+
+
+def _header_str(value: Any, col: int) -> str:
+    if isinstance(value, str):
+        return value.strip()
+    if not isinstance(value, bool) and isinstance(value, (int, float)):
+        return str(value)
+    return f"Col{col + 1}"
+
+
+def import_excel(
+    filepath: str | Path,
+    *,
+    sheet: int | str = 0,
+    time_column: int | str = 0,
+    data_columns: Sequence[int | str] | None = None,
+) -> DataStruct:
+    """Import an ``.xlsx`` sheet (first column = x-axis by default)."""
+    path = Path(filepath)
+    try:
+        workbook = openpyxl.load_workbook(path, data_only=True, read_only=True)
+    except (zipfile.BadZipFile, InvalidFileException, OSError) as exc:
+        # An empty / non-ZIP / truncated .xlsx raises BadZipFile or
+        # InvalidFileException (neither a ValueError) -> would 500 the import
+        # route. Reject cleanly instead.
+        raise ValueError(f"{path.name} is not a readable .xlsx workbook: {exc}") from exc
+    try:
+        worksheet = workbook[sheet] if isinstance(sheet, str) else workbook.worksheets[sheet]
+        sheet_name = worksheet.title
+        grid: list[list[Any]] = [list(row) for row in worksheet.iter_rows(values_only=True)]
+    finally:
+        workbook.close()
+
+    while grid and all(v is None for v in grid[-1]):
+        grid.pop()
+    if not grid:
+        raise ValueError(f"sheet has no data: {path.name}")
+    n_cols = max(len(r) for r in grid)
+    grid = [r + [None] * (n_cols - len(r)) for r in grid]
+    while n_cols > 0 and all(row[n_cols - 1] is None for row in grid):
+        n_cols -= 1
+        grid = [row[:n_cols] for row in grid]
+
+    num_mat = np.array([[_cell_to_float(v) for v in row] for row in grid], dtype=float)
+    scores = [
+        (float(np.count_nonzero(~np.isnan(num_mat[i]))) / n_cols) if n_cols else 0.0
+        for i in range(len(grid))
+    ]
+    first_data = next((i for i, s in enumerate(scores) if s > 0.5), 0)
+    header_row = first_data - 1 if first_data >= 1 and scores[first_data - 1] < 0.5 else -1
+
+    if header_row >= 0:
+        col_headers = [_header_str(grid[header_row][c], c) for c in range(n_cols)]
+    else:
+        col_headers = [f"Col{c + 1}" for c in range(n_cols)]
+
+    data = num_mat[first_data:]
+    keep = [i for i in range(data.shape[0]) if not np.all(np.isnan(data[i]))]
+    data = data[keep]
+    if data.shape[0] == 0:
+        raise ValueError(f"no numeric data rows in {path.name}")
+    n_rows = data.shape[0]
+
+    if isinstance(time_column, int) and time_column < 0:
+        time_idx = -1
+    else:
+        time_idx = resolve_column(time_column, col_headers)
+    time_vec = np.arange(1, n_rows + 1, dtype=float) if time_idx < 0 else data[:, time_idx]
+
+    if data_columns is None:
+        candidates = [c for c in range(n_cols) if c != time_idx]
+        data_idx = [
+            c for c in candidates if (np.count_nonzero(~np.isnan(data[:, c])) / n_rows) > 0.1
+        ]
+    else:
+        data_idx = [resolve_column(s, col_headers) for s in data_columns]
+    if not data_idx:
+        raise ValueError(f"no valid data columns in {path.name}")
+
+    labels: list[str] = []
+    units: list[str] = []
+    for c in data_idx:
+        unit, label = _extract_units(col_headers[c])
+        labels.append(label)
+        units.append(unit)
+
+    if time_idx >= 0:
+        x_unit, x_name = _extract_units(col_headers[time_idx])
+        if not x_name:
+            x_name = col_headers[time_idx]
+    else:
+        x_name, x_unit = "Sample Index", ""
+
+    metadata: dict[str, Any] = {
+        "source": str(path),
+        "parser_name": "import_excel",
+        "x_column_name": x_name,
+        "x_column_unit": x_unit,
+        "sheet_name": sheet_name,
+        "all_column_names": col_headers,
+    }
+    return DataStruct.create(
+        time_vec, data[:, data_idx], labels=labels, units=units, metadata=metadata
+    )

quantized/io/hdf5.py

@@ -0,0 +1,192 @@
+"""Self-describing HDF5 exporter. Port of MATLAB ``+utilities/exportHDF5.m``.
+
+Writes a unified ``DataStruct`` (plus optional corrected data, correction
+parameters, and peak fits) to a hierarchical HDF5 file with a schema that is
+consistent across data types (VSM, PPMS, XRD, generic CSV/Excel). All parsers
+produce the same ``DataStruct`` layout, so one schema covers everything.
+
+Schema overview (v1.0, identical to the MATLAB original)::
+
+    /raw/           raw data (always written)
+    /corrected/     corrected data (optional)
+    /corrections/   xOff, yOff, bgSlope, bgInt (optional)
+    /peaks/         peak-fit results (optional)
+    /metadata/      common metadata attributes
+    /metadata/parserSpecific/   instrument-specific attributes
+
+String datasets (labels, units, peak status/model) are written as space-padded
+ASCII ``uint8`` matrices with an ``encoding='ASCII_padded_space'`` attribute,
+matching MATLAB so files round-trip across both implementations.
+
+Pure layer: ``DataStruct`` in -> ``.h5`` file out. No fastapi/pydantic imports.
+``h5py`` (BSD-3-Clause) is the only extra dependency; it is imported lazily so
+the rest of ``io/`` works without it installed.
+
+The tree-shaping logic lives in :mod:`quantized.io._hdf5_layout` (keeps both
+modules under the 500-line ceiling and makes the layout independently testable).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from quantized.datastruct import DataStruct
+from quantized.io import _hdf5_layout as layout
+
+__all__ = ["write_hdf5"]
+
+_VALID_EXTENSIONS = (".h5", ".hdf5")
+_TOOLBOX_NAME = "quantized"
+_HDF5_SCHEMA = "1.0"
+
+
+def write_hdf5(
+    data: DataStruct,
+    output_path: str | Path,
+    *,
+    corr_data: DataStruct | None = None,
+    corrections: Mapping[str, float] | None = None,
+    include_peaks: bool = False,
+    peaks: Sequence[Mapping[str, Any]] | None = None,
+    overwrite: bool = True,
+) -> None:
+    """Write ``data`` to a self-describing HDF5 file (port of ``exportHDF5``).
+
+    Parameters
+    ----------
+    data:
+        Unified ``DataStruct`` (from any parser). Written to ``/raw/``.
+    output_path:
+        Destination file; must end in ``.h5`` or ``.hdf5``. Its parent
+        directory must already exist (mirrors the MATLAB ``isfolder`` check).
+    corr_data:
+        Corrected ``DataStruct`` (same layout as ``data``); writes
+        ``/corrected/`` when given. ``None`` (or ``struct()`` upstream) skips it.
+    corrections:
+        Mapping with any of ``xOff``/``yOff``/``bgSlope``/``bgInt`` (also
+        accepts the quantized snake_case ``x_off``/``y_off``/``bg_slope``/
+        ``bg_int``). Writes ``/corrections/`` only when at least one finite,
+        value is present (mirrors MATLAB ``hasCorrections``).
+    include_peaks:
+        Write ``/peaks/`` from ``peaks`` (default ``False``).
+    peaks:
+        Sequence of peak mappings (``center``/``fwhm``/``height``/``bg``/
+        ``xRange``/``status``/``model``). Required when ``include_peaks=True``.
+    overwrite:
+        Delete an existing file first (default ``True``). When ``False`` and the
+        file exists, raise ``FileExistsError`` (mirrors MATLAB ``Overwrite``).
+
+    Raises
+    ------
+    ValueError
+        Bad file extension.
+    FileNotFoundError
+        Output directory does not exist.
+    FileExistsError
+        File exists and ``overwrite=False``.
+    ImportError
+        ``h5py`` is not installed.
+    """
+    out = Path(output_path)
+    _validate_path(out, overwrite=overwrite)
+
+    h5py = _import_h5py()
+
+    norm_corr = _normalize_corrections(corrections)
+    has_corr_data = corr_data is not None
+    peak_list = list(peaks) if peaks else []
+    has_peaks = include_peaks and len(peak_list) > 0
+    has_corrections = _has_corrections(norm_corr)
+    corrections_applied = has_corrections and any(v != 0.0 for v in norm_corr.values())
+
+    if out.exists():
+        out.unlink()
+
+    with h5py.File(out, "w") as hf:
+        # Root sentinel dataset + root-level attributes.
+        hf.create_dataset("file_schema_version", data=np.uint8(1).reshape(1, 1))
+        hf.attrs["toolboxName"] = _TOOLBOX_NAME
+        hf.attrs["hdf5Schema"] = _HDF5_SCHEMA
+        hf.attrs["createdAt"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S")
+        hf.attrs["hasCorrected"] = np.uint8(has_corr_data)
+        hf.attrs["hasPeaks"] = np.uint8(has_peaks)
+        hf.attrs["correctionsApplied"] = np.uint8(corrections_applied)
+
+        layout.write_data_group(hf, "raw", data)
+
+        if has_corr_data:
+            assert corr_data is not None  # narrowed by has_corr_data
+            layout.write_data_group(hf, "corrected", corr_data)
+
+        if has_corrections:
+            layout.write_corrections_group(hf, norm_corr)
+
+        if has_peaks:
+            layout.write_peaks_group(hf, peak_list)
+
+        layout.write_metadata_group(hf, data.metadata)
+
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+
+def _validate_path(out: Path, *, overwrite: bool) -> None:
+    """Validate extension, parent directory, and overwrite policy."""
+    if out.suffix.lower() not in _VALID_EXTENSIONS:
+        raise ValueError(
+            f"output_path must end in .h5 or .hdf5 (got: {out.suffix!r})"
+        )
+    parent = out.parent
+    if str(parent) not in ("", ".") and not parent.is_dir():
+        raise FileNotFoundError(f"Output directory does not exist: {parent}")
+    if not overwrite and out.exists():
+        raise FileExistsError(
+            f"File already exists and overwrite=False: {out}"
+        )
+
+
+def _import_h5py() -> Any:
+    """Lazily import ``h5py`` with an actionable error message."""
+    try:
+        import h5py  # noqa: PLC0415
+    except ImportError as exc:  # pragma: no cover - exercised only without h5py
+        raise ImportError(
+            "h5py is required for HDF5 export. Install it with: uv add h5py"
+        ) from exc
+    return h5py
+
+
+# Map quantized snake_case correction names onto the MATLAB attribute names.
+_CORRECTION_ALIASES: dict[str, str] = {
+    "x_off": "xOff",
+    "y_off": "yOff",
+    "bg_slope": "bgSlope",
+    "bg_int": "bgInt",
+}
+
+
+def _normalize_corrections(
+    corrections: Mapping[str, float] | None,
+) -> dict[str, float]:
+    """Coerce a corrections mapping to the four MATLAB-named float keys."""
+    if not corrections:
+        return {}
+    out: dict[str, float] = {}
+    for key, val in corrections.items():
+        name = _CORRECTION_ALIASES.get(key, key)
+        if name in layout.CORRECTION_FIELDS:
+            try:
+                out[name] = float(val)
+            except (TypeError, ValueError):
+                continue
+    return out
+
+
+def _has_corrections(norm_corr: Mapping[str, float]) -> bool:
+    """True when any normalized correction value is finite (mirrors MATLAB)."""
+    return any(np.isfinite(v) for v in norm_corr.values())