quantized-lab 0.8.0__py3-none-any.whl

quantized/io/origin_com.py

@@ -0,0 +1,194 @@
+"""COM "Send to Origin" — push DataStructs into a RUNNING OriginPro instance.
+
+Plan item 25 (Windows-only, optional, feature-flagged; architecture guard
+#10): COM ("Send to Origin") is never a hard dependency and never a CI
+requirement. Everywhere else (macOS/Linux, Origin absent, the flag off, or a
+failed dispatch) callers fall back to the existing Origin-ASCII / ``.ogs``
+export (``io/origin.py``, routes ``/api/export/origin`` and
+``/api/export/origin-project``).
+
+Gated behind the ``QZ_ORIGIN_COM=1`` environment variable so a stray pywin32
+install on a Windows dev/CI box never silently starts dispatching COM.
+``win32com`` is imported LAZILY inside the two public functions, never at
+module scope, so this module imports cleanly on macOS/Linux and keeps the
+``io/`` pure-layer + no-hard-dependency guarantees (enforced by
+``tests/test_repo_integrity.py``).
+
+Reuses the LabTalk/COM surface proven in ``tools/origin_trial/
+generate_specimens.py`` (the only prior code in this repo to drive Origin's
+COM API): a single ``Origin.ApplicationSI`` instance, ``newbook`` + LabTalk
+``option:=lsname``, ``PutWorksheet(rows)`` for bulk numeric data (row-major,
+time column first, then one column per value channel), and a single
+``range ...; ....lname$=...; ....unit$=...;`` LabTalk ``Execute`` call per
+column for long names/units — direct COM property assignment on worksheet
+ranges is flaky in practice; LabTalk string commands are the reliable path
+(see that script's docstring for the hard-won gotchas: only one live
+``ApplicationSI`` instance at a time — a dead/killed server faults every
+subsequent call).
+
+Scope note: this pushes data + long names/units only (matches the plan-item
+ask); it does not set column designations (X/Y/Y-error types) the way the
+``.ogs`` exporter does — a deliberate smaller surface for the first cut,
+worth revisiting once item 25 is live-verified against real Origin.
+
+This repo's own tests never dispatch real COM — see ``tests/test_origin_com.py``
+(mock-based only, a fake ``win32com`` module injected via ``sys.modules``).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+from collections.abc import Sequence
+from typing import Any
+
+from quantized.datastruct import DataStruct
+
+__all__ = ["com_available", "send_to_origin"]
+
+_FLAG_ENV = "QZ_ORIGIN_COM"
+_UNAVAILABLE_HINT = (
+    "Use the Origin-ASCII / .ogs export instead "
+    "(POST /api/export/origin or /api/export/origin-project)."
+)
+
+
+def com_available() -> bool:
+    """True only on Windows, with ``QZ_ORIGIN_COM=1`` set, AND pywin32
+    importable. Never raises — every check is a soft probe, so callers
+    (routes, capability checks) can call this unconditionally on any platform.
+    """
+    if sys.platform != "win32":
+        return False
+    if os.environ.get(_FLAG_ENV) != "1":
+        return False
+    try:
+        import win32com.client  # noqa: F401  (lazy: optional Windows-only dep)
+    except ImportError:
+        return False
+    return True
+
+
+def _escape_lt(text: str) -> str:
+    """Make ``text`` safe inside a LabTalk string literal.
+
+    LabTalk has NO backslash escape — a ``\\"`` lands literally in Origin
+    (live-verified 2026-07-04, see docs/origin_re/validation_log.md), so
+    embedded double-quotes are downgraded to single quotes instead.
+    """
+    return text.replace('"', "'")
+
+
+def _sanitize_book_name(name: str, index: int, used: set[str]) -> str:
+    """A unique, LabTalk-legal workbook short name: non-word chars ->
+    underscore, never digit-leading, falls back to ``Book<N>`` when empty,
+    and de-duplicated against every name already handed out in this call."""
+    cleaned = re.sub(r"\W", "_", name).strip("_") or f"Book{index + 1}"
+    if cleaned[0].isdigit():
+        cleaned = f"B{cleaned}"
+    book, n = cleaned, 1
+    while book in used:
+        n += 1
+        book = f"{cleaned}{n}"
+    used.add(book)
+    return book
+
+
+def _label_column(app: Any, book: str, col: int, lname: str, unit: str) -> None:
+    """One LabTalk ``Execute`` assigning a column's long name + unit via a
+    ``range`` reference — mirrors ``tools/origin_trial/generate_specimens.py``
+    (``range ra = [Book]1!col(1); ra.lname$="..."; ra.unit$="...";``)."""
+    var = f"__c{col}"
+    parts = [f"range {var} = [{book}]1!col({col});"]
+    if lname:
+        parts.append(f'{var}.lname$ = "{_escape_lt(lname)}";')
+    if unit:
+        parts.append(f'{var}.unit$ = "{_escape_lt(unit)}";')
+    if len(parts) == 1:
+        return  # nothing to label
+    cmd = " ".join(parts)
+    if not app.Execute(cmd):
+        raise RuntimeError(f"Origin LabTalk command failed: {cmd}")
+
+
+def send_to_origin(
+    datasets: Sequence[DataStruct],
+    *,
+    book_names: Sequence[str] | None = None,
+) -> dict[str, Any]:
+    """Push each ``DataStruct`` into a new workbook in a RUNNING Origin
+    instance via COM, with long names/units set from ``.labels``/``.units``.
+
+    ``book_names[i]`` (if given and non-empty) names workbook ``i``;
+    otherwise falls back to ``metadata['origin_book']`` then ``Book<N>``.
+    Returns ``{"books": [<created workbook names>], "rows": [<row count per
+    book>]}``.
+
+    Raises ``RuntimeError`` (never a raw COM exception) when COM is
+    unavailable, Origin rejects a command, or dispatch fails — the caller
+    (the export route) maps that to an HTTP 409 pointing at the ``.ogs``
+    fallback.
+    """
+    if not datasets:
+        raise ValueError("send_to_origin needs at least one dataset")
+    if not com_available():
+        raise RuntimeError(
+            "Origin COM is unavailable on this machine (needs Windows, "
+            f"pywin32, {_FLAG_ENV}=1, and a running OriginPro instance). "
+            + _UNAVAILABLE_HINT
+        )
+
+    import win32com.client as win32  # lazy: optional Windows-only dep
+
+    try:
+        app = win32.gencache.EnsureDispatch("Origin.ApplicationSI")
+    except Exception as exc:  # pragma: no cover - real COM only, never in CI
+        raise RuntimeError(
+            "Could not connect to a running Origin instance via COM. Start "
+            "OriginPro first, then retry. " + _UNAVAILABLE_HINT
+        ) from exc
+
+    names = list(book_names) if book_names is not None else []
+    used: set[str] = set()
+    books: list[str] = []
+    rows_created: list[int] = []
+    try:
+        for i, ds in enumerate(datasets):
+            raw = names[i] if i < len(names) and names[i] else ""
+            if not raw:
+                raw = str(ds.metadata.get("origin_book", "")) or f"Book{i + 1}"
+            book = _sanitize_book_name(raw, i, used)
+
+            if not app.Execute(f"newbook name:={book} option:=lsname;"):
+                raise RuntimeError(f"Origin rejected 'newbook' for workbook {book!r}.")
+
+            cols = [ds.time.tolist()]
+            cols.extend(ds.values[:, c].tolist() for c in range(ds.n_channels))
+            rows = [list(r) for r in zip(*cols, strict=True)]
+            app.PutWorksheet(f"[{book}]1", rows, 0, 0)
+
+            x_name = str(
+                ds.metadata.get("x_column_name")
+                or ds.metadata.get("xColumnName")
+                or ds.metadata.get("x_column_long")  # writer.py's key family
+                or "X"
+            )
+            x_unit = str(
+                ds.metadata.get("x_column_unit")
+                or ds.metadata.get("xColumnUnit")
+                or ds.metadata.get("x_unit")
+                or ""
+            )
+            _label_column(app, book, 1, x_name, x_unit)
+            for k, (label, unit) in enumerate(zip(ds.labels, ds.units, strict=True)):
+                _label_column(app, book, k + 2, label, unit)
+
+            books.append(book)
+            rows_created.append(ds.n_points)
+    except RuntimeError:
+        raise
+    except Exception as exc:  # pragma: no cover - real COM only, never in CI
+        raise RuntimeError(f"Origin COM call failed: {exc}") from exc
+
+    return {"books": books, "rows": rows_created}

quantized/io/origin_project/__init__.py

@@ -0,0 +1,221 @@
+"""Origin project readers (``.opj`` / ``.opju``) — clean-room, no GPL liborigin.
+
+Package layout (split per plan item 15 to stay under the module ceiling as
+decoders grow): ``container`` (CPY block primitives), ``opj`` (worksheet data +
+names/units), ``opju`` (FPC column codec, see ``opju_codec``), ``windows``
+(windows-section metadata). Format notes: ``docs/origin_project_format.md`` +
+``docs/origin_re/``.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from pathlib import Path
+
+import numpy as np
+
+from quantized.datastruct import DataStruct
+from quantized.io.origin_project.container import OriginProjectError
+from quantized.io.origin_project.notes import notes_windows, parse_results_log, results_log
+from quantized.io.origin_project.opj import (
+    build_opj_books,
+    build_opj_primary,
+    parse_opj,
+    read_opj,
+    read_opj_books,
+)
+from quantized.io.origin_project.opju import (
+    build_opju_books,
+    build_opju_primary,
+    parse_opju,
+    read_opju,
+    read_opju_books,
+)
+from quantized.io.origin_project.tree import opj_folder_paths, opj_project_dates
+from quantized.io.origin_project.tree_opju import opju_folder_paths
+
+__all__ = [
+    "OriginProjectError",
+    "drop_empty_library_books",
+    "drop_nonactionable_figures",
+    "read_origin_books",
+    "read_origin_project",
+    "read_origin_project_all",
+]
+
+
+def _with_provenance(ds: DataStruct, path: Path, *, raw: bytes | None = None) -> DataStruct:
+    """Attach project-global provenance (results log + notes) to one DataStruct.
+
+    Both are project-global, so they ride only the primary dataset (and the
+    first book of a multi-book read) rather than being duplicated per book.
+    The file is read once and both scans run over the same bytes (``raw``
+    lets a caller that already has the bytes, e.g. :func:`read_origin_books`,
+    skip a second read of the file). ``path``'s suffix lets both scans
+    restrict themselves to the structurally-derived tail on ``.opj`` (see
+    ``notes.py``'s ``_tail_scan_start``) instead of the whole file.
+    """
+    raw = path.read_bytes() if raw is None else raw
+    suffix = path.suffix.lower()
+    extra: dict[str, object] = {}
+    log = results_log(raw, suffix=suffix)
+    if log:
+        extra["origin_results_log"] = log
+        records = parse_results_log(log)
+        if records:
+            extra["origin_results_log_records"] = records
+    notes = notes_windows(raw, suffix=suffix)
+    if notes:
+        extra["origin_notes"] = notes
+    # .opj project created/modified timestamps (tree.py::opj_project_dates,
+    # §13.2 #5); None on the 4.3227 variant / CPYUA bytes — key then absent.
+    dates = opj_project_dates(raw)
+    if dates:
+        extra["origin_project_dates"] = dates
+    if not extra:
+        return ds
+    return dataclasses.replace(ds, metadata={**ds.metadata, **extra})
+
+
+def read_origin_project(path: Path) -> DataStruct:
+    """Dispatch by extension to the clean-room ``.opj`` / ``.opju`` decoder.
+
+    Origin projects are proprietary binary files; quantized decodes them itself
+    (it will not bundle the GPL liborigin). Both containers recover worksheet
+    data with real column names/units; the project's results log (analysis
+    provenance) lands in ``metadata['origin_results_log']`` (raw text) and
+    ``metadata['origin_results_log_records']`` (parsed per-operation records,
+    when at least one parses) and any notes-window text in
+    ``metadata['origin_notes']`` when present.
+    """
+    reader = read_opju if path.suffix.lower() == ".opju" else read_opj
+    return _with_provenance(reader(path), path)
+
+
+def _with_folder_path(ds: DataStruct, folder_paths: dict[str, list[str]]) -> DataStruct:
+    """Attach ``metadata['origin_folder_path']`` (the Project Explorer folder
+    a book lives in, root-exclusive; ``[]`` when unknown or at the root) --
+    see ``tree.py``. A ``Book4@2``-style sheet pseudo-book (plan item 5)
+    resolves through its base book's name, since sheets aren't separate
+    Project Explorer windows."""
+    book = str(ds.metadata.get("origin_book", ""))
+    base_book, _, _sheet = book.partition("@")
+    folder_path = folder_paths.get(base_book, [])
+    return dataclasses.replace(ds, metadata={**ds.metadata, "origin_folder_path": folder_path})
+
+
+def read_origin_books(path: Path) -> list[DataStruct]:
+    """Every workbook in an Origin project as its own DataStruct (plan item 3).
+
+    ``read_origin_project`` keeps the registry's single-DataStruct contract
+    (largest book); this is the pure API a multi-dataset import flow (plan
+    item 16) builds on. The results log rides the first book only; every
+    book gets an ``origin_folder_path`` (plan item: Project Explorer folder
+    tree -- see ``tree.py``; decoded for ``.opj`` and both CPYUA ``.opju``
+    sub-versions (4.3811 + 4.3380), degrading to ``[]`` only on an unknown
+    container or a framing/consistency mismatch).
+    """
+    is_opju = path.suffix.lower() == ".opju"
+    books = read_opju_books(path) if is_opju else read_opj_books(path)
+    if not books:
+        return books
+    raw = path.read_bytes()
+    books[0] = _with_provenance(books[0], path, raw=raw)
+    folder_paths = opju_folder_paths(raw) if is_opju else opj_folder_paths(raw)
+    return [_with_folder_path(b, folder_paths) for b in books]
+
+
+def read_origin_project_all(
+    path: Path, *, raw: bytes | None = None
+) -> tuple[DataStruct, list[DataStruct]]:
+    """Parse an Origin project ONCE and return both ``read_origin_project``'s
+    primary DataStruct (with provenance) and ``read_origin_books``'s full
+    per-book list (with folder paths + book[0] provenance) — the combined
+    result of calling both entry points, without the redundant second
+    ``parse_opj``/``parse_opju`` (and repeated file reads) each independently
+    performs today.
+
+    Used by the ``/api/parsers`` import route: profiling a 122-book /
+    8.5M-cell project (``PNR.opj``, 121.56 MB) showed the duplicated
+    project-wide parse dominating the ~4s round-trip (2026-07-09) — calling
+    ``import_auto`` (-> ``read_origin_project`` -> a full parse) and then
+    ``read_origin_books`` (-> another full parse) back to back reads the file
+    from disk repeatedly and decodes every column twice.
+
+    Primary-book construction (``build_opj*_primary``) and all-books
+    construction (``build_opj*_books``) stay independent calls sharing only
+    the parsed intermediate — mirroring ``read_opj``/``read_opj_books``'s
+    separation — so a decode issue confined to one non-primary book still
+    degrades the book list to ``[]`` without taking down the primary import,
+    exactly as ``routes.parsers._import_with_books`` behaved before (its
+    ``except OriginProjectError`` around ``read_origin_books`` only ever
+    guarded against that isolated-book-build failure, since a shared-parse
+    failure would already have surfaced on the first, unguarded call).
+    """
+    is_opju = path.suffix.lower() == ".opju"
+    raw = path.read_bytes() if raw is None else raw
+
+    books: list[DataStruct]
+    if is_opju:
+        opju_parsed = parse_opju(path, raw=raw)
+        primary = _with_provenance(build_opju_primary(opju_parsed), path, raw=raw)
+        try:
+            books = build_opju_books(opju_parsed)
+        except OriginProjectError:
+            books = []
+    else:
+        opj_parsed = parse_opj(path, raw=raw)
+        primary = _with_provenance(build_opj_primary(opj_parsed), path, raw=raw)
+        try:
+            books = build_opj_books(opj_parsed)
+        except OriginProjectError:
+            books = []
+
+    if books:
+        books[0] = _with_provenance(books[0], path, raw=raw)
+        folder_paths = opju_folder_paths(raw) if is_opju else opj_folder_paths(raw)
+        books = [_with_folder_path(b, folder_paths) for b in books]
+    return primary, books
+
+
+def drop_empty_library_books(books: list[DataStruct]) -> list[DataStruct]:
+    """Presentation gate for the multi-book Library listing: hide books with no
+    plottable numeric data and no text content.
+
+    Origin fit/report sheets (plan item 4) surface as pseudo-books whose numeric
+    ``.values`` are empty -- their content is unresolved ``cell://`` reference
+    stubs in ``metadata['origin_report_sheets']``, not real data -- so listing
+    them as top-level Library entries floods it with empty rows (the Hc2 project
+    alone produced 48 ``Book2@N`` fit-report shells). Mirrors
+    ``drop_nonactionable_figures``: a gate applied at the import boundary, not in
+    the decoder, so ``read_origin_books`` stays a complete reader (the report
+    metadata still rides the returned DataStruct for a future fit-report surface).
+    Text-only books (real content in ``origin_text_columns``) are kept. Never
+    returns empty: a degenerate all-empty project passes through unchanged so the
+    Library still shows something.
+    """
+    kept = [
+        b
+        for b in books
+        if (b.values.size and int(np.count_nonzero(np.isfinite(b.values))) > 0)
+        or b.metadata.get("origin_text_columns")
+    ]
+    return kept or books
+
+
+def drop_nonactionable_figures(figs: list[dict[str, object]]) -> list[dict[str, object]]:
+    """Keep only figures a user can act on: those with bound curves OR a resolvable
+    source hint.
+
+    The layer-anchor scan also locates internal storage/thumbnail blocks that carry
+    a decodable axis record but no bound curves and no source reference -- the
+    decoder can find them, but they are not user graphs and cannot be restored onto
+    any imported dataset. Surfacing them just floods the Library's Figures section
+    with dead "SYSTEM" rows (the Hc2 project produced 32 of them). This is a
+    presentation gate, applied at the import boundary, not in the decoder -- so the
+    decoder stays a complete axis-record reader for the synthetic tests. Verified
+    against the corpus: keeps all 14 real figures (each has curves), drops all 32
+    non-actionable Hc2 anchors. (Moved here from ``figures_opju`` 2026-07-06 --
+    it pairs with ``drop_empty_library_books``, the sibling gate above.)
+    """
+    return [f for f in figs if f.get("curves") or str(f.get("source_hint") or "").strip()]