quantized-lab 0.8.0__py3-none-any.whl

quantized/io/import_filters.py

@@ -0,0 +1,178 @@
+"""Saved import filters: persist an :class:`~quantized.io.import_preview.ImportSettings`
+under a name + glob pattern so a messy instrument file only needs the import
+wizard's manual setup once (ORIGIN_GAP_PLAN #40).
+
+Persistence lives in the SERVER user config directory (via ``platformdirs``,
+MIT-licensed), never in the browser — the registry (a headless, server-side
+chokepoint; see :mod:`quantized.io.registry`) must be able to see saved
+filters, which rules out frontend-only storage. The directory is overridable
+via the ``QZ_CONFIG_DIR`` environment variable so tests never touch the real
+user config (and so a packaged app can be pointed at a portable location).
+
+Pure ``io`` layer — no fastapi/pydantic imports; :mod:`quantized.routes.import_wizard`
+is the thin adapter that exposes this as CRUD + "import with filter" endpoints.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import json
+import os
+from dataclasses import dataclass, replace
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import platformdirs
+
+from quantized.io.import_preview import ImportSettings
+
+__all__ = [
+    "ImportFilter",
+    "config_dir",
+    "delete_filter",
+    "load_filters",
+    "match_filter",
+    "save_filter",
+]
+
+_ENV_OVERRIDE = "QZ_CONFIG_DIR"
+_APP_NAME = "quantized"
+_FILTERS_FILENAME = "import_filters.json"
+
+
+@dataclass(frozen=True)
+class ImportFilter:
+    """A saved, named :class:`ImportSettings` bound to a filename glob.
+
+    ``updated`` is an ISO-8601 UTC timestamp, stamped by :func:`save_filter`;
+    it is the tie-break key in :func:`match_filter` (see there).
+    """
+
+    name: str
+    glob: str
+    settings: ImportSettings
+    updated: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "glob": self.glob,
+            "settings": self.settings.to_dict(),
+            "updated": self.updated,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> ImportFilter:
+        return cls(
+            name=str(payload.get("name", "")),
+            glob=str(payload.get("glob", "*")),
+            settings=ImportSettings.from_dict(payload.get("settings") or {}),
+            updated=str(payload.get("updated", "")),
+        )
+
+
+def config_dir() -> Path:
+    """The user config directory quantized persists filters (and later,
+    plugin state) under. ``QZ_CONFIG_DIR`` overrides it (tests set this to a
+    ``tmp_path`` so they never touch the real user config directory).
+    """
+    override = os.environ.get(_ENV_OVERRIDE)
+    base = Path(override) if override else Path(
+        platformdirs.user_config_dir(_APP_NAME, appauthor=False)
+    )
+    base.mkdir(parents=True, exist_ok=True)
+    return base
+
+
+def _filters_path() -> Path:
+    return config_dir() / _FILTERS_FILENAME
+
+
+def load_filters() -> list[ImportFilter]:
+    """All saved filters. Missing or corrupt files -> ``[]`` (never raises);
+    a malformed individual entry is skipped, not fatal to the rest."""
+    path = _filters_path()
+    if not path.is_file():
+        return []
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError, UnicodeDecodeError):
+        return []
+    if not isinstance(raw, list):
+        return []
+    filters: list[ImportFilter] = []
+    for item in raw:
+        if not isinstance(item, dict):
+            continue
+        try:
+            filters.append(ImportFilter.from_dict(item))
+        except (TypeError, ValueError):
+            continue
+    return filters
+
+
+def _write_filters(filters: list[ImportFilter]) -> None:
+    payload = [f.to_dict() for f in filters]
+    _filters_path().write_text(json.dumps(payload, indent=2), encoding="utf-8")
+
+
+def save_filter(filt: ImportFilter) -> ImportFilter:
+    """Upsert ``filt`` by name (case-sensitive) and persist it.
+
+    Stamps ``updated`` to now (UTC), overriding whatever was passed in, so the
+    tie-break in :func:`match_filter` always reflects the true save order.
+    """
+    if not filt.name.strip():
+        raise ValueError("filter name must not be empty")
+    stamped = replace(filt, updated=datetime.now(UTC).isoformat())
+    remaining = [f for f in load_filters() if f.name != stamped.name]
+    remaining.append(stamped)
+    _write_filters(remaining)
+    return stamped
+
+
+def delete_filter(name: str) -> bool:
+    """Remove the filter named ``name``. Returns whether one was removed."""
+    filters = load_filters()
+    remaining = [f for f in filters if f.name != name]
+    if len(remaining) == len(filters):
+        return False
+    _write_filters(remaining)
+    return True
+
+
+def _specificity(pattern: str) -> int:
+    """Rough glob specificity: count of literal (non-wildcard) characters.
+
+    ``"XYZ9000_*.dat"`` (11 literal chars) outranks ``"*.dat"`` (4) so a
+    narrowly-targeted saved filter wins over a broad one.
+    """
+    return sum(1 for c in pattern if c not in "*?")
+
+
+def match_filter(
+    path: str | Path, filters: list[ImportFilter] | None = None
+) -> ImportFilter | None:
+    """The best saved filter for ``path``'s filename, or ``None``.
+
+    Matching is case-insensitive glob (:mod:`fnmatch`) against the filename
+    only (not the full path). When more than one saved filter matches:
+
+    - the most SPECIFIC glob wins (highest literal-character count, per
+      :func:`_specificity`) — a filter aimed at one instrument's naming
+      convention should beat a catch-all;
+    - ties in specificity are broken by the most RECENTLY saved filter
+      (``updated``, ISO-8601 so it sorts lexicographically) — the user's
+      latest edit reflects their current intent.
+
+    ``filters`` lets callers pass an already-loaded list (e.g. the registry,
+    to avoid re-reading the JSON file per candidate extension); defaults to
+    :func:`load_filters`.
+    """
+    name = Path(path).name.lower()
+    candidates = load_filters() if filters is None else filters
+    matches = [f for f in candidates if fnmatch.fnmatch(name, f.glob.lower())]
+    if not matches:
+        return None
+    return max(matches, key=lambda f: (_specificity(f.glob), f.updated))

quantized/io/import_preview.py

@@ -0,0 +1,262 @@
+"""Interactive import engine: guess -> preview -> parse under explicit settings.
+
+ORIGIN_GAP_PLAN #40 (the wizard's backend). ``import_csv`` auto-detects and
+imports in one shot; the wizard instead needs to *show* the user what a file
+looks like under adjustable settings and re-preview on every tweak, then parse
+with the confirmed settings. This module provides that:
+
+- :class:`ImportSettings` — a serializable description of how to read a file
+  (delimiter, which absolute lines are the header / units / first data row,
+  column-name overrides, and a per-column role: ``x`` / ``y`` / ``error`` /
+  ``label`` / ``ignore``). This is also the persistable "import filter" shape;
+  binding a saved filter to a glob and consulting it from the registry is the
+  remaining (design) half of #40.
+- :func:`guess_settings` — a starting guess from the raw text (reusing the
+  ``delimited`` detectors).
+- :func:`preview_import` — parse the first rows under given settings and return
+  a table + resolved columns for the wizard to render.
+- :func:`parse_import` — parse the full text under settings into a
+  ``DataStruct``.
+
+Absolute line indices (over ``text.splitlines()``, comments/blanks included)
+so the wizard can number every line and let the user point at the header.
+Pure ``io`` layer — no fastapi/pydantic imports.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import asdict, dataclass
+from typing import Any
+
+import numpy as np
+
+from quantized.datastruct import DataStruct
+from quantized.io.delimited import (
+    _detect_delimiter,
+    _extract_units,
+    _looks_like_units_row,
+    _numeric_score,
+    _to_float,
+)
+
+__all__ = [
+    "DATA_ROLES",
+    "ImportSettings",
+    "guess_settings",
+    "parse_import",
+    "preview_import",
+]
+
+DATA_ROLES = ("x", "y", "error", "label", "ignore")
+_CHANNEL_ROLES = ("y", "error")  # numeric roles that become DataStruct channels
+# friendly delimiter aliases -> how to split
+_NAMED_DELIMS = {"auto": "auto", "comma": ",", "tab": "\t", "\\t": "\t",
+                 "semicolon": ";", "pipe": "|", "space": " ", "whitespace": " "}
+
+
+@dataclass(frozen=True)
+class ImportSettings:
+    """How to read a delimited file (also the persistable import-filter shape)."""
+
+    delimiter: str = "auto"
+    header_line: int | None = None
+    units_line: int | None = None
+    data_start_line: int = 0
+    column_names: list[str] | None = None
+    roles: list[str] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> ImportSettings:
+        allowed = {f for f in cls.__dataclass_fields__}
+        return cls(**{k: v for k, v in payload.items() if k in allowed})
+
+
+@dataclass
+class _Parsed:
+    lines: list[str]
+    delim: str
+    names: list[str]
+    units: list[str]
+    roles: list[str]
+    matrix: np.ndarray  # (n_rows, n_cols) float
+    data_start: int
+
+
+def _split(line: str, delim: str) -> list[str]:
+    if delim in (" ", "whitespace"):
+        return re.split(r"\s+", line.strip())
+    return line.split(delim)
+
+
+def _effective_ncols(rows: list[list[str]]) -> int:
+    """Column count ignoring trailing empty tokens (a trailing-delimiter row
+    like ``"1,2,"`` is 2 columns, not 3), while preserving empty *interior*
+    cells (``"1,,3"`` stays 3). Mirrors ``import_csv``'s trailing-column guard.
+    """
+    best = 0
+    for row in rows:
+        last = 0
+        for k, cell in enumerate(row):
+            if cell.strip():
+                last = k + 1
+        best = max(best, last)
+    return best
+
+
+def _resolve_delim(lines: list[str], setting: str) -> str:
+    d = _NAMED_DELIMS.get(setting.lower(), setting)
+    if d != "auto":
+        return d
+    non_empty = [ln for ln in lines if ln.strip()]
+    return _detect_delimiter(non_empty) if non_empty else ","
+
+
+def guess_settings(text: str) -> ImportSettings:
+    """Best-effort starting settings for ``text`` (the wizard's initial state)."""
+    lines = text.splitlines()
+    delim = _resolve_delim(lines, "auto")
+    tokens = [_split(ln, delim) for ln in lines]
+    scores = [_numeric_score(t) if ln.strip() else 0.0 for t, ln in zip(tokens, lines, strict=True)]
+    data_start = next((i for i, s in enumerate(scores) if s > 0.5), 0)
+
+    header_line: int | None = None
+    units_line: int | None = None
+    data_rows = [t for t in tokens[data_start:] if any(c.strip() for c in t)]
+    n_cols = _effective_ncols(data_rows)
+    # a units row just above the data, and a header above that
+    if data_start >= 2 and scores[data_start - 1] < 0.5 and _looks_like_units_row(
+        tokens[data_start - 1], n_cols
+    ):
+        units_line = data_start - 1
+        header_line = data_start - 2 if scores[data_start - 2] < 0.5 else None
+    elif data_start >= 1 and scores[data_start - 1] < 0.5:
+        header_line = data_start - 1
+
+    names = _resolve_names(tokens, header_line, n_cols)
+    roles = ["x"] + ["y"] * (n_cols - 1) if n_cols else []
+    return ImportSettings(
+        delimiter="auto", header_line=header_line, units_line=units_line,
+        data_start_line=data_start, column_names=names, roles=roles,
+    )
+
+
+def _resolve_names(tokens: list[list[str]], header_line: int | None, n_cols: int) -> list[str]:
+    if header_line is not None and 0 <= header_line < len(tokens):
+        raw = [c.strip() for c in tokens[header_line]]
+    else:
+        raw = []
+    names = [raw[k] if k < len(raw) and raw[k] else f"Col{k + 1}" for k in range(n_cols)]
+    return names
+
+
+def _parse_core(text: str, settings: ImportSettings) -> _Parsed:
+    lines = text.splitlines()
+    delim = _resolve_delim(lines, settings.delimiter)
+    tokens = [_split(ln, delim) for ln in lines]
+    ds = max(0, settings.data_start_line)
+    data_tokens = [t for t in tokens[ds:] if any(c.strip() for c in t)]
+    n_cols = _effective_ncols(data_tokens)
+    if settings.column_names:
+        names = [settings.column_names[k] if k < len(settings.column_names) else f"Col{k + 1}"
+                 for k in range(n_cols)]
+    else:
+        names = _resolve_names(tokens, settings.header_line, n_cols)
+    # split any "Name (unit)" embedded units out of the header names
+    units = [""] * n_cols
+    for k in range(n_cols):
+        u, lbl = _extract_units(names[k])
+        names[k], units[k] = lbl, u
+    # an explicit units row overrides
+    if settings.units_line is not None and 0 <= settings.units_line < len(tokens):
+        urow = [c.strip().strip("()[]{}") for c in tokens[settings.units_line]]
+        for k in range(min(n_cols, len(urow))):
+            if urow[k]:
+                units[k] = urow[k]
+    roles = _resolve_roles(settings.roles, n_cols)
+
+    matrix = np.full((len(data_tokens), n_cols), np.nan, dtype=float)
+    for i, row in enumerate(data_tokens):
+        for k in range(min(len(row), n_cols)):
+            matrix[i, k] = _to_float(row[k])
+    return _Parsed(lines, delim, names, units, roles, matrix, ds)
+
+
+def _resolve_roles(roles: list[str] | None, n_cols: int) -> list[str]:
+    if not roles:
+        return (["x"] + ["y"] * (n_cols - 1)) if n_cols else []
+    out = [roles[k] if k < len(roles) and roles[k] in DATA_ROLES else "y" for k in range(n_cols)]
+    return out
+
+
+def preview_import(text: str, settings: ImportSettings, *, max_rows: int = 20,
+                   max_lines: int = 60) -> dict[str, Any]:
+    """Parse the first ``max_rows`` under ``settings`` for the wizard to render.
+
+    Returns the raw lines (numbered, up to ``max_lines``), the resolved
+    delimiter, the header/units/data-start indices, one column descriptor per
+    column (name/unit/role/sample values), a preview row grid, and the total
+    data-row count.
+    """
+    p = _parse_core(text, settings)
+    n_rows, n_cols = p.matrix.shape
+    preview_rows = [
+        [None if np.isnan(v) else float(v) for v in p.matrix[i, :]]
+        for i in range(min(n_rows, max_rows))
+    ]
+    columns = [
+        {"index": k, "name": p.names[k], "unit": p.units[k], "role": p.roles[k]}
+        for k in range(n_cols)
+    ]
+    return {
+        "raw_lines": p.lines[:max_lines],
+        "n_lines": len(p.lines),
+        "delimiter": p.delim,
+        "header_line": settings.header_line,
+        "units_line": settings.units_line,
+        "data_start_line": p.data_start,
+        "columns": columns,
+        "rows": preview_rows,
+        "n_data_rows": int(n_rows),
+        "n_preview_rows": len(preview_rows),
+    }
+
+
+def parse_import(text: str, settings: ImportSettings) -> DataStruct:
+    """Parse the full ``text`` under ``settings`` into a ``DataStruct``.
+
+    The ``x`` role column becomes the axis (a 1..N sample index if none is
+    marked); ``y`` / ``error`` columns become channels; ``label`` / ``ignore``
+    columns are dropped (``DataStruct`` is numeric-only).
+    """
+    p = _parse_core(text, settings)
+    n_rows, n_cols = p.matrix.shape
+    if n_cols == 0 or n_rows == 0:
+        raise ValueError("no data rows found under these settings")
+
+    x_cols = [k for k in range(n_cols) if p.roles[k] == "x"]
+    chan_cols = [k for k in range(n_cols) if p.roles[k] in _CHANNEL_ROLES]
+    if not chan_cols:
+        raise ValueError("no y/error columns selected to import")
+    if x_cols:
+        x = p.matrix[:, x_cols[0]]
+        x_name, x_unit = p.names[x_cols[0]], p.units[x_cols[0]]
+    else:
+        x = np.arange(1, n_rows + 1, dtype=float)
+        x_name, x_unit = "Sample Index", ""
+
+    labels = [p.names[k] for k in chan_cols]
+    units = [p.units[k] for k in chan_cols]
+    metadata: dict[str, Any] = {
+        "parser_name": "import_preview",
+        "x_column_name": x_name,
+        "x_column_unit": x_unit,
+        "delimiter": p.delim,
+        "all_column_names": p.names,
+        "import_settings": settings.to_dict(),
+    }
+    return DataStruct.create(x, p.matrix[:, chan_cols], labels=labels, units=units,
+                             metadata=metadata)

quantized/io/jcamp.py

@@ -0,0 +1,179 @@
+"""JCAMP-DX spectroscopy parser (``.jdx`` / ``.dx``).
+
+JCAMP-DX (Joint Committee on Atomic and Molecular Physical Data — Data
+eXchange) is the manufacturer-independent text format for IR, Raman, UV-Vis,
+NMR and mass spectra. A file is a list of ``##LABEL= value`` records; the
+ordinate data follows an ``##XYDATA= (X++(Y..Y))`` (equally-spaced, ASDF-
+compressed) or ``##XYPOINTS=``/``##PEAK TABLE= (XY..XY)`` (explicit pairs)
+record.
+
+This parser reads the first spectral block of a file (compound ``LINK`` files
+note the extra blocks in metadata) and returns a single-channel DataStruct.
+The ASDF ordinate decoding (SQZ/DIF/DUP) lives in :mod:`_jcamp_asdf`.
+
+Correctness is self-checking: the decoded ordinate count must equal
+``##NPOINTS`` and the first ordinate must equal ``##FIRSTY`` (JCAMP's built-in
+integrity checks), plus DIF lines carry per-line Y-value checks.
+
+Sample corpus: ``nzhagen/jcamp`` (MIT) IR spectra + the R. J. Lancashire
+compression-form test suite.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from quantized.datastruct import DataStruct
+from quantized.io._jcamp_asdf import decode_xydata
+
+__all__ = ["import_jcamp", "is_jcamp"]
+
+_DATA_LABELS = {"XYDATA", "XYPOINTS", "PEAKTABLE"}
+
+
+def _norm_label(label: str) -> str:
+    """JCAMP label equivalence: case- and separator-insensitive."""
+    return label.upper().replace(" ", "").replace("-", "").replace("_", "").replace("/", "")
+
+
+def _strip_comment(line: str) -> str:
+    i = line.find("$$")
+    return line[:i] if i >= 0 else line
+
+
+def is_jcamp(path: Path) -> bool:
+    """Sniff a file as JCAMP-DX: the first data record must be ``##TITLE=``."""
+    try:
+        with Path(path).open("r", encoding="latin-1") as fh:
+            for _ in range(20):
+                line = fh.readline()
+                if not line:
+                    break
+                s = line.strip()
+                if s.startswith("##"):
+                    return _norm_label(s[2:].split("=", 1)[0]) == "TITLE"
+    except OSError:
+        return False
+    return False
+
+
+def _parse_records(text: str) -> tuple[dict[str, str], list[str], str, int]:
+    """Split into (header LDRs, data lines, data-kind, extra block count).
+
+    Reads the *first* data block; counts how many additional ``##TITLE`` blocks
+    follow (compound/LINK files).
+    """
+    header: dict[str, str] = {}
+    data_lines: list[str] = []
+    data_kind = ""
+    in_data = False
+    title_count = 0
+
+    for raw_line in text.splitlines():
+        line = _strip_comment(raw_line)
+        stripped = line.strip()
+        if stripped.startswith("##"):
+            in_data = False
+            label, _, value = stripped[2:].partition("=")
+            nlabel = _norm_label(label)
+            if nlabel == "TITLE":
+                title_count += 1
+                if title_count > 1:
+                    continue  # a later block; header already captured
+            if nlabel in _DATA_LABELS and not data_kind:
+                data_kind = nlabel
+                in_data = True
+            elif nlabel == "END":
+                in_data = False
+            else:
+                header.setdefault(nlabel, value.strip())
+        elif in_data and stripped:
+            data_lines.append(stripped)
+
+    return header, data_lines, data_kind, max(0, title_count - 1)
+
+
+def _num(header: dict[str, str], key: str, default: float) -> float:
+    try:
+        return float(header[key])
+    except (KeyError, ValueError):
+        return default
+
+
+def _decode_pairs(data_lines: list[str], xfactor: float, yfactor: float) -> tuple[
+    np.ndarray, np.ndarray
+]:
+    """Decode ``(XY..XY)`` explicit pairs (XYPOINTS / PEAK TABLE)."""
+    nums: list[float] = []
+    for line in data_lines:
+        for tok in line.replace(";", " ").replace(",", " ").split():
+            try:
+                nums.append(float(tok))
+            except ValueError:
+                continue
+    if len(nums) % 2:
+        nums = nums[:-1]
+    arr = np.asarray(nums, dtype=float).reshape(-1, 2)
+    return arr[:, 0] * xfactor, arr[:, 1] * yfactor
+
+
+def import_jcamp(filepath: str | Path) -> DataStruct:
+    """Import a JCAMP-DX ``.jdx``/``.dx`` spectrum (one channel).
+
+    Returns
+    -------
+    DataStruct
+        ``time`` = abscissa (wavenumber / wavelength / m/z, per ``##XUNITS``),
+        one ordinate channel (``##YUNITS``).
+    """
+    path = Path(filepath)
+    text = path.read_text(encoding="latin-1")
+    header, data_lines, data_kind, extra_blocks = _parse_records(text)
+    if not data_kind or not data_lines:
+        raise ValueError(f"no XYDATA/XYPOINTS/PEAK TABLE block found: {path.name}")
+
+    xfactor = _num(header, "XFACTOR", 1.0)
+    yfactor = _num(header, "YFACTOR", 1.0)
+
+    if data_kind == "XYDATA":
+        raw_y = np.asarray(decode_xydata(data_lines), dtype=float)
+        y = raw_y * yfactor
+        npoints = int(_num(header, "NPOINTS", len(y)))
+        if len(y) != npoints:
+            raise ValueError(
+                f"decoded {len(y)} ordinates but ##NPOINTS={npoints}: {path.name}"
+            )
+        firstx = _num(header, "FIRSTX", 0.0)
+        lastx = _num(header, "LASTX", float(len(y) - 1))
+        x = np.linspace(firstx, lastx, len(y)) if len(y) > 1 else np.asarray([firstx])
+        firsty = _num(header, "FIRSTY", y[0] if len(y) else 0.0)
+        # FIRSTY is an optional integrity check; 0 is a common "not set"
+        # placeholder, so only enforce it when a real value is given.
+        if len(y) and firsty != 0.0 and abs(y[0] - firsty) > 1e-3 * (abs(firsty) + 1.0):
+            raise ValueError(
+                f"first ordinate {y[0]:.6g} != ##FIRSTY={firsty:.6g}: {path.name}"
+            )
+    else:  # XYPOINTS / PEAK TABLE
+        x, y = _decode_pairs(data_lines, xfactor, yfactor)
+        if not len(x):
+            raise ValueError(f"no data points in {data_kind} block: {path.name}")
+
+    xunits = header.get("XUNITS", "")
+    yunits = header.get("YUNITS", "") or "Intensity"
+    metadata: dict[str, Any] = {
+        "source": str(path),
+        "parser_name": "import_jcamp",
+        "title": header.get("TITLE", path.stem),
+        "data_type": header.get("DATATYPE", ""),
+        "jcamp_version": header.get("JCAMPDX", ""),
+        "data_form": data_kind,
+        "x_column_name": header.get("DATATYPE", "") or "X",
+        "x_column_unit": xunits,
+        "num_points": int(len(y)),
+    }
+    if extra_blocks:
+        metadata["extra_blocks"] = extra_blocks  # compound/LINK file
+    return DataStruct.create(x, y, labels=[yunits.title()], units=[""], metadata=metadata)