scistack-db 0.1.9.dev0__py3-none-any.whl

scidb/__init__.py

@@ -0,0 +1,144 @@
+"""
+SciStack: Scientific Data Versioning Framework
+
+A lightweight database framework for scientific computing that provides:
+- Type-safe serialization of numpy arrays, DataFrames, and custom types
+- Automatic content-based versioning
+- Flexible metadata-based addressing
+- DuckDB storage for data and lineage (via SciDuck)
+
+Example:
+    from scidb import configure_database, BaseVariable
+    import numpy as np
+
+    # Setup (single DuckDB file for data + lineage, auto-registers types)
+    db = configure_database("experiment.duckdb", ["subject", "session"])
+
+    class RawSignal(BaseVariable):
+        schema_version = 1
+
+    # Save/load
+    RawSignal.save(np.array([1, 2, 3]), subject=1, session="A")
+    raw = RawSignal.load(subject=1, session="A")
+"""
+
+# From scifor (Layer 1)
+from scifor import Col, PathInput, PathOutput, get_schema, set_schema
+
+from .across_variants import AcrossVariants
+from .artifact_stamp import read_artifact_stamp, stamp_artifact
+from .colname import ColName
+from .column_selection import ColumnSelection
+from .constant import Constant, constant
+from .database import configure_database, get_database, get_user_id
+from .discover import (
+    DiscoveryResult,
+    ModuleError,
+    ModuleExports,
+    PackageResult,
+    discover_module,
+    scan_package,
+    scan_project,
+)
+from .each_of import EachOf
+from .exceptions import (
+    AmbiguousParamError,
+    AmbiguousVersionError,
+    DatabaseLockedError,
+    DatabaseNotConfiguredError,
+    NotFoundError,
+    NotRegisteredError,
+    PipelineCycleError,
+    ReservedMetadataKeyError,
+    SciStackError,
+)
+from .exclusions import exclude_schema, include_schema, list_exclusions
+from .filters import raw_sql, schema_key
+from .fixed import Fixed
+
+# Batch execution (Layer 2 — DB-backed, no lineage)
+from .foreach import for_each
+from .foreach_config import ForEachConfig
+from .lineage_save import save
+from .log import Log
+from .merge import Merge
+from .pipeline import Pipeline, PipelineBinding, Step, active_pipeline, scistack
+from .state import (
+    check_combo_state,
+    check_multiple_nodes_state,
+    check_node_state,
+    check_pathinput_node_state,
+)
+from .variable import BaseVariable
+from .variant import Variant, branch_param
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Core classes
+    "BaseVariable",
+    "Constant",
+    "constant",
+    # Discovery
+    "scan_project",
+    "scan_package",
+    "discover_module",
+    "DiscoveryResult",
+    "PackageResult",
+    "ModuleExports",
+    "ModuleError",
+    # Configuration
+    "configure_database",
+    "get_database",
+    "get_user_id",
+    # Save
+    "save",
+    # Pipeline node staleness
+    "check_combo_state",
+    "check_node_state",
+    "check_pathinput_node_state",
+    "check_multiple_nodes_state",
+    # Logging
+    "Log",
+    # Batch execution
+    "for_each",
+    "scistack",
+    "Pipeline",
+    "PipelineBinding",
+    "Step",
+    "active_pipeline",
+    "Fixed",
+    "Variant",
+    "AcrossVariants",
+    "stamp_artifact",
+    "read_artifact_stamp",
+    "branch_param",
+    "Merge",
+    "ColumnSelection",
+    "ColName",
+    "EachOf",
+    "ForEachConfig",
+    "PathInput",
+    "PathOutput",
+    # Standalone / DataFrame support
+    "Col",
+    "set_schema",
+    "get_schema",
+    # Filter utilities
+    "raw_sql",
+    "schema_key",
+    # Schema exclusions
+    "exclude_schema",
+    "include_schema",
+    "list_exclusions",
+    # Exceptions
+    "SciStackError",
+    "NotRegisteredError",
+    "NotFoundError",
+    "DatabaseNotConfiguredError",
+    "ReservedMetadataKeyError",
+    "AmbiguousVersionError",
+    "AmbiguousParamError",
+    "DatabaseLockedError",
+    "PipelineCycleError",
+]

scidb/across_variants.py

@@ -0,0 +1,97 @@
+"""AcrossVariants pooling wrapper: opt IN to cross-variant aggregation."""
+
+from typing import Any
+
+
+class AcrossVariants:
+    """
+    Wrapper to deliberately pool ALL branch_param variants of an input into one
+    aggregated table, with each row's upstream branch_params attached as columns.
+
+    Aggregation-mode ``for_each`` (iterating a strict subset of the schema keys)
+    **auto-splits** by upstream branch_param signature by default: one call per
+    variant group, as if the user had written
+    ``EachOf(Variant(X, ...), Variant(X, ...))``. That default is right for
+    ordinary aggregation and for endpoint (``plot_``/``stat_``) functions — a
+    pooled t-test double-counts, and a figure mixing two filter settings is
+    wrong.
+
+    ``AcrossVariants`` is the explicit opt-out for **multiverse /
+    specification-curve analysis**: the one legitimate cross-variant use case,
+    where the analysis deliberately spans pipeline decisions to quantify
+    robustness. The pooled rows keep their variant identity — each namespaced
+    branch_param key (e.g. ``bandpass.low_hz``) becomes an ordinary DataFrame
+    column so the function can group by specification::
+
+        def robustness(df):
+            return df.groupby("bandpass.low_hz")["value"].mean()
+
+        for_each(robustness, {"df": AcrossVariants(Filtered)}, [Spec],
+                 subject=["S01", "S02"])
+
+    Composition:
+
+    - May wrap a variable type, a ``Fixed``, or a ``Variant``
+      (``AcrossVariants(Variant(X, low_hz=20))`` pins some params and pools
+      whatever variants remain).
+    - ``AcrossVariants(ColumnSelection)`` is an ERROR: the column selection
+      would drop the attached branch_param columns, silently defeating the
+      pooling contract.
+    - ``AcrossVariants(Merge(...))`` is an ERROR (mirrors ``Variant`` /
+      ``Fixed``): pool per constituent instead.
+    - ``AcrossVariants(EachOf(...))`` is an ERROR: ``EachOf`` must stay the
+      outermost wrapper.
+
+    In full-iteration mode (all schema keys iterated) each combo already sees
+    exactly one variant, so pooling is meaningless: the input behaves as if
+    unwrapped and a warning is logged.
+    """
+
+    def __init__(self, var_type: Any):
+        from .column_selection import ColumnSelection
+        from .each_of import EachOf
+        from .merge import Merge
+
+        if isinstance(var_type, Merge):
+            raise TypeError(
+                "AcrossVariants cannot wrap a Merge. branch_params are namespaced "
+                "per producing function, so pooling must happen per constituent: "
+                "Merge(AcrossVariants(A), B)."
+            )
+        if isinstance(var_type, EachOf):
+            raise TypeError(
+                "AcrossVariants cannot wrap an EachOf. EachOf must stay the "
+                "outermost wrapper."
+            )
+        if isinstance(var_type, ColumnSelection):
+            raise TypeError(
+                "AcrossVariants cannot wrap a ColumnSelection: the column "
+                "selection would drop the attached branch_param columns. "
+                "Select columns inside your function instead."
+            )
+        if isinstance(var_type, AcrossVariants):
+            var_type = var_type.var_type  # idempotent
+
+        self.var_type = var_type
+
+    def to_key(self) -> str:
+        """Return a canonical string for use as a version key.
+
+        A pooled run and an auto-split run of the same function must not
+        collide: this key lands in the ``__inputs`` config key (same mechanism
+        as ``Variant.to_key()``).
+        """
+        if hasattr(self.var_type, "to_key"):
+            inner_key = self.var_type.to_key()
+        elif isinstance(self.var_type, type):
+            inner_key = self.var_type.__name__
+        else:
+            inner_key = repr(self.var_type)
+        return f"AcrossVariants({inner_key})"
+
+    @property
+    def __name__(self) -> str:
+        """Display name for format_inputs and error messages."""
+        from .foreach import _input_type_name
+
+        return f"AcrossVariants({_input_type_name(self.var_type)})"

scidb/artifact_stamp.py

@@ -0,0 +1,301 @@
+"""Artifact provenance stamping (D4, endpoints-viz-and-stats-design.md).
+
+Embeds a JSON provenance blob INSIDE endpoint artifacts — ``plot_`` figures
+and ``stat_`` PDF reports — so a file found in a paper draft years later
+traces back to its exact DB record. Embedded metadata travels with the file
+(unlike a sidecar, which a copy/move/email can leave behind); the sidecar
+``<artifact>.provenance.json`` is strictly the fallback for formats that
+cannot be embedded safely.
+
+All stamping is dependency-free (stdlib only) and operates on the FILE, not
+the renderer (``savefig(metadata=)`` cannot include the record_id — it only
+exists after the save — and file-level stamping works identically for
+MATLAB-rendered figures whose paths cross the bridge).
+
+Formats:
+
+- **PNG** — a ``tEXt`` chunk (keyword ``scidb:provenance``) inserted after
+  ``IHDR``; CRC via ``zlib.crc32``.
+- **SVG** — a ``<metadata id="scidb-provenance">`` element inserted right
+  after the opening ``<svg …>`` tag.
+- **PDF** — an *incremental update*: a new object carrying
+  ``/scidb_provenance <hex-encoded JSON>`` is appended together with a new
+  xref subsection and a trailer chaining to the previous one via ``/Prev``.
+  Every original byte stays untouched. Works for classic-xref producers
+  (reportlab — csv-stats' writer — and matplotlib both qualify); PDFs whose
+  trailer cannot be parsed (e.g. xref-stream producers) fall back to the
+  sidecar. The READER never depends on the update being well-formed: it is a
+  backwards byte-scan for the marker.
+
+Failures never raise out of ``stamp_artifact`` — an artifact that renders but
+cannot be stamped must not fail a pipeline. Every stamp, fallback, and
+failure is logged.
+"""
+
+import json
+import re
+import zlib
+from pathlib import Path
+
+from .log import Log
+
+STAMP_VERSION = 1
+_PNG_SIG = b"\x89PNG\r\n\x1a\n"
+_PNG_KEYWORD = b"scidb:provenance"
+_SVG_MARKER = '<metadata id="scidb-provenance">'
+_PDF_KEY = b"/scidb_provenance"
+_SIDECAR_SUFFIX = ".provenance.json"
+
+
+# ---------------------------------------------------------------------------
+# PNG
+# ---------------------------------------------------------------------------
+
+
+def _png_chunks(data: bytes):
+    """Yield (offset, length, type, data_bytes) for each chunk."""
+    pos = len(_PNG_SIG)
+    while pos + 8 <= len(data):
+        length = int.from_bytes(data[pos : pos + 4], "big")
+        ctype = data[pos + 4 : pos + 8]
+        yield pos, length, ctype, data[pos + 8 : pos + 8 + length]
+        pos += 12 + length
+
+
+def _strip_png_stamp(data: bytes) -> bytes:
+    """Remove any existing scidb tEXt chunk (idempotent re-stamping)."""
+    for pos, length, ctype, cdata in _png_chunks(data):
+        if ctype == b"tEXt" and cdata.startswith(_PNG_KEYWORD + b"\x00"):
+            return data[:pos] + data[pos + 12 + length :]
+    return data
+
+
+def _stamp_png(data: bytes, payload: bytes) -> "bytes | None":
+    if not data.startswith(_PNG_SIG) or data[12:16] != b"IHDR":
+        return None
+    data = _strip_png_stamp(data)
+    ihdr_len = int.from_bytes(data[8:12], "big")
+    insert_at = len(_PNG_SIG) + 12 + ihdr_len  # after IHDR incl. its CRC
+    chunk_data = _PNG_KEYWORD + b"\x00" + payload
+    chunk = (
+        len(chunk_data).to_bytes(4, "big")
+        + b"tEXt"
+        + chunk_data
+        + zlib.crc32(b"tEXt" + chunk_data).to_bytes(4, "big")
+    )
+    return data[:insert_at] + chunk + data[insert_at:]
+
+
+def _read_png(data: bytes) -> "str | None":
+    if not data.startswith(_PNG_SIG):
+        return None
+    for _pos, _length, ctype, cdata in _png_chunks(data):
+        if ctype == b"tEXt" and cdata.startswith(_PNG_KEYWORD + b"\x00"):
+            return cdata[len(_PNG_KEYWORD) + 1 :].decode("latin-1")
+    return None
+
+
+# ---------------------------------------------------------------------------
+# SVG
+# ---------------------------------------------------------------------------
+
+
+def _xml_escape(s: str) -> str:
+    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+
+
+def _xml_unescape(s: str) -> str:
+    return s.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&")
+
+
+def _stamp_svg(data: bytes, payload: bytes) -> "bytes | None":
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        return None
+    # Strip an existing stamp element (idempotent re-stamping).
+    start = text.find(_SVG_MARKER)
+    if start != -1:
+        end = text.find("</metadata>", start)
+        if end != -1:
+            text = text[:start] + text[end + len("</metadata>") :]
+    svg_open = text.find("<svg")
+    if svg_open == -1:
+        return None
+    tag_end = text.find(">", svg_open)
+    if tag_end == -1 or text[tag_end - 1] == "/":  # self-closing root: no content
+        return None
+    element = _SVG_MARKER + _xml_escape(payload.decode("ascii")) + "</metadata>"
+    return (text[: tag_end + 1] + element + text[tag_end + 1 :]).encode("utf-8")
+
+
+def _read_svg(data: bytes) -> "str | None":
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        return None
+    start = text.find(_SVG_MARKER)
+    if start == -1:
+        return None
+    start += len(_SVG_MARKER)
+    end = text.find("</metadata>", start)
+    if end == -1:
+        return None
+    return _xml_unescape(text[start:end])
+
+
+# ---------------------------------------------------------------------------
+# PDF (incremental update; classic-xref producers)
+# ---------------------------------------------------------------------------
+
+
+def _stamp_pdf(data: bytes, payload: bytes) -> "bytes | None":
+    if not data.startswith(b"%PDF"):
+        return None
+    sx = data.rfind(b"startxref")
+    if sx == -1:
+        return None
+    m = re.match(rb"\s*(\d+)", data[sx + len(b"startxref") :])
+    if m is None:
+        return None
+    prev_xref = int(m.group(1))
+    # Classic trailer dict (reportlab / matplotlib). No trailer keyword →
+    # xref-stream PDF → caller falls back to the sidecar.
+    tidx = data.rfind(b"trailer", 0, sx)
+    if tidx == -1:
+        return None
+    tdict = data[tidx:sx]
+    root_m = re.search(rb"/Root\s+(\d+)\s+(\d+)\s+R", tdict)
+    size_m = re.search(rb"/Size\s+(\d+)", tdict)
+    if root_m is None or size_m is None:
+        return None
+    new_obj = int(size_m.group(1))  # /Size == next free object number
+
+    out = data if data.endswith(b"\n") else data + b"\n"
+    obj_offset = len(out)
+    out += b"%d 0 obj\n<< %s <%s> >>\nendobj\n" % (
+        new_obj,
+        _PDF_KEY,
+        payload.hex().encode("ascii"),
+    )
+    xref_offset = len(out)
+    out += b"xref\n%d 1\n%010d 00000 n \n" % (new_obj, obj_offset)
+    trailer = b"trailer\n<< /Size %d /Root %s %s R /Prev %d /ScidbProv %d 0 R >>\n" % (
+        new_obj + 1,
+        root_m.group(1),
+        root_m.group(2),
+        prev_xref,
+        new_obj,
+    )
+    out += trailer
+    out += b"startxref\n%d\n%%%%EOF\n" % xref_offset
+    return out
+
+
+def _read_pdf(data: bytes) -> "str | None":
+    # Backwards byte-scan: independent of xref flavor and update validity.
+    idx = data.rfind(_PDF_KEY + b" <")
+    if idx == -1:
+        return None
+    start = idx + len(_PDF_KEY) + 2
+    end = data.find(b">", start)
+    if end == -1:
+        return None
+    try:
+        return bytes.fromhex(data[start:end].decode("ascii")).decode("ascii")
+    except ValueError:
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+_STAMPERS = {".png": _stamp_png, ".svg": _stamp_svg, ".pdf": _stamp_pdf}
+_READERS = {".png": _read_png, ".svg": _read_svg, ".pdf": _read_pdf}
+
+
+def _sidecar_path(path: Path) -> Path:
+    return path.with_name(path.name + _SIDECAR_SUFFIX)
+
+
+def stamp_artifact(path: "str | Path", blob: dict) -> bool:
+    """Embed ``blob`` (JSON) into the artifact at ``path``.
+
+    Returns True when embedded IN the file; False when the sidecar fallback
+    was used or the file is missing. Never raises: a figure that renders but
+    cannot be stamped must not fail the pipeline.
+    """
+    p = Path(path)
+    payload = json.dumps(blob, sort_keys=True, ensure_ascii=True).encode("ascii")
+    if not p.is_file():
+        Log.warn(f"[artifact-stamp] {p}: file not found — nothing stamped")
+        return False
+
+    suffix = p.suffix.lower()
+    stamper = _STAMPERS.get(suffix)
+    reason = f"unsupported format {suffix!r}"
+    if stamper is not None:
+        try:
+            data = p.read_bytes()
+            new = stamper(data, payload)
+            if new is not None:
+                p.write_bytes(new)
+                Log.info(
+                    f"[artifact-stamp] {p.name}: embedded provenance "
+                    f"({suffix}, {len(payload)} bytes)"
+                )
+                return True
+            reason = f"{suffix} structure not recognized (e.g. xref-stream PDF)"
+        except Exception as exc:  # never fail the pipeline over a stamp
+            reason = f"{type(exc).__name__}: {exc}"
+
+    try:
+        _sidecar_path(p).write_text(payload.decode("ascii"))
+        Log.warn(
+            f"[artifact-stamp] {p.name}: could not embed ({reason}) — "
+            f"wrote sidecar {_sidecar_path(p).name}"
+        )
+    except Exception as exc:
+        Log.warn(
+            f"[artifact-stamp] {p.name}: could not embed ({reason}) and "
+            f"sidecar write failed ({type(exc).__name__}: {exc})"
+        )
+    return False
+
+
+def read_artifact_stamp(path: "str | Path") -> "dict | None":
+    """Read the provenance blob from an artifact (embedded or sidecar).
+
+    Returns the parsed dict, or None when no stamp is found.
+    """
+    p = Path(path)
+    raw: str | None = None
+    if p.is_file():
+        reader = _READERS.get(p.suffix.lower())
+        try:
+            data = p.read_bytes()
+            if reader is not None:
+                raw = reader(data)
+            else:
+                # Unknown extension: probe all formats by content.
+                for probe in (_read_png, _read_svg, _read_pdf):
+                    raw = probe(data)
+                    if raw is not None:
+                        break
+        except Exception:
+            raw = None
+    if raw is None:
+        sidecar = _sidecar_path(p)
+        if sidecar.is_file():
+            try:
+                raw = sidecar.read_text()
+            except Exception:
+                return None
+    if raw is None:
+        return None
+    try:
+        parsed = json.loads(raw)
+    except (ValueError, TypeError):
+        return None
+    return parsed if isinstance(parsed, dict) else None

scidb/colname.py

@@ -0,0 +1,57 @@
+"""ColName wrapper — resolves to the single data column name of a Variable type."""
+
+from typing import Any
+
+
+class ColName:
+    """
+    Marker that resolves to the single data column name of a DB-backed variable.
+
+    Use this when a function needs to know the name of the data column
+    but the function itself should stay framework-agnostic.
+
+    Two forms:
+
+    1. ``ColName(MyVar)`` — *static*. At for_each time it is replaced by the
+       string name of the single data column for that variable type. Raises
+       ValueError if the variable has 0 or 2+ data columns.
+
+    2. ``ColName()`` — *deferred*. Resolves per-column inside a ``for_columns``
+       iteration to the name of the column currently being fed to the function.
+       Requires at least one iterate input (``MyVar.for_columns()``); using it
+       without one is an error.
+
+    Example (static):
+        for_each(
+            analyze,
+            inputs={"table": MyVar, "col_name": ColName(MyVar)},
+            outputs=[Result],
+            subject=[1, 2, 3],
+        )
+
+        # The function is pure — no framework imports:
+        def analyze(table, col_name):
+            return table[col_name].mean()
+
+    Example (deferred, current for_columns column):
+        for_each(
+            analyze,
+            inputs={"df": MyVar.for_columns(), "col_name": ColName()},
+            outputs=[Result],
+            subject=[],
+        )
+    """
+
+    def __init__(self, var_type: Any = None):
+        """
+        Args:
+            var_type: The variable type (class) whose data column name will be
+                resolved (static form). Omit it for the deferred form, which
+                resolves to the current ``for_columns`` column.
+        """
+        self.var_type = var_type
+
+    @property
+    def is_deferred(self) -> bool:
+        """True for the no-arg form (resolves to the current for_columns column)."""
+        return self.var_type is None