rosetta-eclab 0.1.1__py3-none-any.whl

spike2/corpus.py

@@ -0,0 +1,162 @@
+"""Reference decoding of every corpus .mpr into named, typed columns.
+
+The v2 benchmark isolates the semantic step: which physical quantity is this
+column? Byte structure is code's job (shown 6/6 by the v1 code baseline), so
+here every file is decoded with yadg 7.0.1's own column map and offsets, and
+the map's names and units are the answer key. Flags are dropped; they are bit
+fields, not quantities.
+
+Usage:  python -m spike2.corpus          # prints a summary per file
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import warnings
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+
+ROOT = Path(__file__).resolve().parents[1]
+EXT = ROOT / "data" / "external"
+
+# Real format drifts (galvani 0.5.0 cannot read them): kept out of the dev split.
+DRIFT_FILES = {"gcpl.issue_230.mpr", "peis.issue_225.with_ece.mpr"}
+
+
+@dataclass
+class Column:
+    position: int  # index in the file's record layout (flags excluded)
+    raw_id: int  # full column ID (little-endian u2; for 11.50+ files at module offset 6, see docs/history.md)
+    ref_id: int  # ID yadg uses for its map lookup (its own reading % 256); names come from this
+    name: str
+    unit: str | None
+    dtype: str
+    values: np.ndarray = field(repr=False)
+
+
+@dataclass
+class FileData:
+    file: str
+    technique: str | None
+    version: int
+    n_points: int
+    columns: list[Column]
+    galvani_unknown_ids: list[int]
+
+
+def _modules(contents: bytes):
+    """Yield (name, version, module_data) like yadg's process_modules, without converting values."""
+    from yadg import dgutils
+    from yadg.extractors.eclab.mpr_columns import module_header_dtypes
+
+    for module in contents.split(b"MODULE")[1:]:
+        for mhd in module_header_dtypes:
+            try:
+                header = dgutils.read_value(module, 0x0000, mhd)
+                if len(module) == mhd.itemsize + header["length"]:
+                    version = header.get("newver", 0) + header["oldver"]
+                    break
+            except UnicodeDecodeError:
+                continue
+        else:
+            raise RuntimeError("unknown module header")
+        yield header["short_name"].strip(), version, module[mhd.itemsize:]
+
+
+def decode(path: Path) -> FileData:
+    from yadg.extractors.eclab.mpr import parse_columns, process_settings
+
+    logging.getLogger("yadg").setLevel(logging.CRITICAL)
+    raw = path.read_bytes()
+    technique, data, data_version = None, None, None
+    for name, version, mdata in _modules(raw):
+        if name == "VMP Set":
+            try:
+                technique, _, _ = process_settings(mdata, "11.50" if version >= 10 else "10.40")
+            except Exception:  # old galvani regression files: settings layout yadg does not read
+                technique = None
+        elif name == "VMP data":
+            data, data_version = mdata, version
+    if data is None:
+        raise ValueError("no data module")
+    n_points = int(np.frombuffer(data, offset=0, dtype="<u4", count=1)[0])
+    n_columns = int(np.frombuffer(data, offset=4, dtype="|u1", count=1)[0])
+    if data_version in (10, 11):
+        ids = np.frombuffer(data, offset=5, dtype=">u2", count=n_columns).tolist()
+        full_ids = np.frombuffer(data, offset=6, dtype="<u2", count=n_columns).tolist()
+        offset = 0x3EF
+    elif data_version in (2, 3):
+        ids = np.frombuffer(data, offset=5, dtype="<u2", count=n_columns).tolist()
+        full_ids = list(ids)
+        offset = 0x195 if data_version == 2 else 0x196
+    else:
+        raise NotImplementedError(f"data module version {data_version}")
+    names, dtypes, units, _flags = parse_columns(ids, technique)
+    arr = np.frombuffer(data, offset=offset, dtype=np.dtype(list(zip(names, dtypes))), count=n_points)
+    from galvani import BioLogic
+
+    km = {int(k): v for k, v in BioLogic.VMPdata_colID_dtype_map.items()}  # galvani 0.5.0: full IDs it knows
+    cols, pos = [], 0
+    # yadg's own flag test (on its reading of the IDs) decides which entries are flags
+    data_ids = [(i, f) for i, f in zip(ids, full_ids) if i not in (1, 2, 3, 21, 31, 65)]
+    galvani_unknown = []
+    k = 0
+    for name, dt, unit in zip(names, dtypes, units):
+        if name == "flags":
+            continue
+        yid, cid = data_ids[k]
+        k += 1
+        if name.startswith("unknown_"):
+            continue
+        ref = yid % 256
+        if cid not in km:  # galvani 0.5.0 looks up the full ID
+            galvani_unknown.append(cid)
+        cols.append(Column(position=pos, raw_id=cid, ref_id=ref, name=name, unit=unit, dtype=dt,
+                           values=np.asarray(arr[name])))
+        pos += 1
+    return FileData(file=path.name, technique=technique, version=data_version, n_points=n_points,
+                    columns=cols, galvani_unknown_ids=galvani_unknown)
+
+
+# Files that come from the same experiment or contributor (same GitHub issue/PR, the VSP set,
+# or sharing an identical non-constant column; checked 2026-10-14). Used for leave-one-group-out.
+_GROUP_TAGS = ("issue_149", "issue_185", "issue_225", "issue_226", "pr_182", "vsp_", "bio_logic")
+
+
+def source_group(name: str) -> str:
+    for tag in _GROUP_TAGS:
+        if tag in name:
+            return tag
+    return name
+
+
+def corpus_files() -> list[Path]:
+    return sorted(EXT.glob("*.mpr"))
+
+
+def load_all() -> tuple[list[FileData], dict[str, str]]:
+    out, failed = [], {}
+    warnings.filterwarnings("ignore")
+    for p in corpus_files():
+        try:
+            out.append(decode(p))
+        except Exception as exc:
+            failed[p.name] = f"{type(exc).__name__}: {exc}"
+    return out, failed
+
+
+def main() -> int:
+    files, failed = load_all()
+    for f in files:
+        print(f"{f.file:30s} {str(f.technique):8s} v{f.version:<3d} n={f.n_points:<6d} cols={len(f.columns)} "
+              f"galvani_unknown={f.galvani_unknown_ids}")
+        print("    " + "; ".join(f"{c.name} [{c.unit}] {c.dtype}" for c in f.columns))
+    print("\nfailed:", json.dumps(failed, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

spike2/dsl.py

@@ -0,0 +1,167 @@
+"""Safe expression language for hypotheses about a column.
+
+The model writes an expression over the file's known columns (aliases c0, c1,
+...) and code evaluates it with numpy: nothing is exec'd, only a whitelisted
+AST (numbers, column aliases, + - * / **, unary minus and the functions
+below). Code then fits target ~ a * expr + b and reports how much of the
+column the hypothesis explains, so the model never has to compute a number.
+"""
+
+from __future__ import annotations
+
+import ast
+
+import numpy as np
+
+MAX_LEN = 300
+
+
+def _diff0(x):
+    """First difference with a leading 0, same length as x."""
+    return np.concatenate([[0.0], np.diff(x)])
+
+
+def _cumsum_by(x, key):
+    """Cumulative sum that restarts whenever key changes value."""
+    x = np.nan_to_num(np.asarray(x, dtype="f8"))
+    key = np.asarray(key)
+    out = np.empty_like(x)
+    acc = 0.0
+    for i in range(len(x)):
+        if i and key[i] != key[i - 1]:
+            acc = 0.0
+        acc += x[i]
+        out[i] = acc
+    return out
+
+
+def _since_change(t, key):
+    """t minus the value of t at the most recent change of key (e.g. time since step start)."""
+    t = np.asarray(t, dtype="f8")
+    key = np.asarray(key)
+    out = np.empty_like(t)
+    start = t[0] if len(t) else 0.0
+    for i in range(len(t)):
+        if i and key[i] != key[i - 1]:
+            start = t[i]
+        out[i] = t[i] - start
+    return out
+
+
+def _shift(x, k=1):
+    k = int(k)
+    x = np.asarray(x, dtype="f8")
+    if k == 0:
+        return x.copy()
+    out = np.full_like(x, np.nan)
+    if k > 0:
+        out[k:] = x[:-k]
+    else:
+        out[:k] = x[-k:]
+    return out
+
+
+FUNCS = {
+    "abs": np.abs, "sqrt": np.sqrt, "log": np.log, "log10": np.log10, "exp": np.exp,
+    "sin": np.sin, "cos": np.cos, "tan": np.tan, "arctan": np.arctan, "arctan2": np.arctan2,
+    "degrees": np.degrees, "radians": np.radians, "sign": np.sign, "square": np.square,
+    "cumsum": lambda x: np.cumsum(np.nan_to_num(np.asarray(x, dtype="f8"))),
+    "diff0": _diff0, "cumsum_by": _cumsum_by, "since_change": _since_change, "shift": _shift,
+    "where_pos": lambda x, a, b: np.where(np.asarray(x) > 0, a, b),
+    "maximum": np.maximum, "minimum": np.minimum,
+}
+CONSTS = {"pi": np.pi}
+
+DOC = (
+    "Expression language: column aliases (c0, c1, ...), numbers, + - * / **, unary minus, constant pi, "
+    "and functions abs sqrt log log10 exp sin cos tan arctan arctan2 degrees radians sign square maximum minimum, "
+    "cumsum(x), diff0(x) (first difference, leading 0), cumsum_by(x, key) (cumulative sum restarting when key "
+    "changes), since_change(t, key) (t minus t at the last change of key), shift(x, k), where_pos(x, a, b). "
+    "Code fits TARGET = a*expr + b, so constant factors and offsets need not be exact."
+)
+
+
+class DSLError(ValueError):
+    pass
+
+
+def evaluate(expr: str, cols: dict[str, np.ndarray]) -> np.ndarray:
+    if not isinstance(expr, str) or not expr.strip():
+        raise DSLError("empty expression")
+    if len(expr) > MAX_LEN:
+        raise DSLError(f"expression longer than {MAX_LEN} characters")
+    try:
+        tree = ast.parse(expr.strip(), mode="eval")
+    except SyntaxError as exc:
+        raise DSLError(f"syntax error: {exc.msg}") from None
+
+    def ev(node):
+        if isinstance(node, ast.Expression):
+            return ev(node.body)
+        if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)) and not isinstance(node.value, bool):
+            return float(node.value)
+        if isinstance(node, ast.Name):
+            if node.id in cols:
+                return np.asarray(cols[node.id], dtype="f8")
+            if node.id in CONSTS:
+                return CONSTS[node.id]
+            raise DSLError(f"unknown name {node.id!r}; use column aliases {sorted(cols)[:8]}... or pi")
+        if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
+            v = ev(node.operand)
+            return -v if isinstance(node.op, ast.USub) else v
+        if isinstance(node, ast.BinOp):
+            a, b = ev(node.left), ev(node.right)
+            ops = {ast.Add: np.add, ast.Sub: np.subtract, ast.Mult: np.multiply, ast.Div: np.divide,
+                   ast.Pow: np.power}
+            for t, f in ops.items():
+                if isinstance(node.op, t):
+                    return f(a, b)
+            raise DSLError(f"operator {type(node.op).__name__} not allowed")
+        if isinstance(node, ast.Call):
+            if not isinstance(node.func, ast.Name) or node.func.id not in FUNCS:
+                name = getattr(node.func, "id", "?")
+                raise DSLError(f"function {name!r} not allowed; allowed: {sorted(FUNCS)}")
+            if node.keywords:
+                raise DSLError("keyword arguments are not allowed")
+            return FUNCS[node.func.id](*[ev(a) for a in node.args])
+        raise DSLError(f"{type(node).__name__} not allowed")
+
+    with np.errstate(all="ignore"):
+        try:
+            out = ev(tree)
+        except DSLError:
+            raise
+        except Exception as exc:
+            raise DSLError(f"{type(exc).__name__}: {exc}") from None
+    n = len(next(iter(cols.values())))
+    out = np.broadcast_to(np.asarray(out, dtype="f8"), (n,)).copy()
+    return out
+
+
+def fit(target: np.ndarray, pred: np.ndarray) -> dict:
+    """Least-squares target ~ a*pred + b over rows where both are finite."""
+    y = np.asarray(target, dtype="f8")
+    x = np.asarray(pred, dtype="f8")
+    m = np.isfinite(y) & np.isfinite(x)
+    n = int(m.sum())
+    if n < 3:
+        return {"rows_used": n, "error": "fewer than 3 finite rows"}
+    y, x = y[m], x[m]
+    if np.ptp(y) == 0:
+        return {"rows_used": n, "error": "TARGET is constant, so no hypothesis can be tested against it"}
+    if np.ptp(x) == 0:
+        return {"rows_used": n, "r2": 0.0, "error": "the expression is constant over these rows, so it explains nothing"}
+    rng = float(np.ptp(y))
+    a, b = np.polyfit(x, y, 1)
+    res = y - (a * x + b)
+    ss_tot = float(((y - y.mean()) ** 2).sum())
+    r2 = 1.0 - float((res ** 2).sum()) / ss_tot
+    return {
+        "rows_used": n,
+        "rows_total": int(len(target)),
+        "a": float(f"{a:.6g}"),
+        "b": float(f"{b:.6g}"),
+        "r2": round(r2, 5),
+        "max_abs_residual_over_range": round(float(np.abs(res).max()) / rng, 5),
+        "fraction_rows_within_1pct_of_range": round(float((np.abs(res) <= 0.01 * rng).mean()), 4),
+    }

spike2/elim.py

@@ -0,0 +1,52 @@
+"""Name elimination: the code-only trick that solves a column from its siblings' names.
+
+If the known columns include |Ewe h2|, |Ewe h3|, |Ewe h5|, ... then the
+vocabulary name one token away from at least two of them (|Ewe h4|) is the
+gap. That needs no data and no model, so tasks it settles are not evidence
+for the model (found by the harness review, 2026-10-14). The rule:
+
+  candidates = vocabulary names not already present among the known columns
+  score(v)   = number of known column names exactly one token away from v
+  answer     = the unique candidate with the highest score, if that score >= 2
+"""
+
+from __future__ import annotations
+
+import re
+
+from . import vocab
+
+_TOKEN = re.compile(r"[A-Za-z]+|\d+|[^A-Za-z\d\s]")
+
+
+def tokens(name: str) -> list[str]:
+    return _TOKEN.findall(name)
+
+
+def one_token_apart(a: str, b: str) -> bool:
+    """Siblings: same token count, exactly one token differs, and they share at least one word
+    (so "Ns"/"dQ" or "|C|"/"|I|", which share nothing or only punctuation, are not siblings)."""
+    ta, tb = tokens(a), tokens(b)
+    if len(ta) != len(tb) or len(ta) < 2:
+        return False
+    diff = sum(x != y for x, y in zip(ta, tb))
+    shared_word = any(x == y and x.isalpha() for x, y in zip(ta, tb))
+    return diff == 1 and shared_word
+
+
+def eliminate(known_names: list[str], vocabulary: list[str]) -> str | None:
+    present = {vocab.canonical(n) for n in known_names}
+    spellings = set(known_names) | present
+    best, best_score, tie = None, 0, False
+    for v in vocabulary:
+        if v in present:
+            continue
+        forms = [v] + [a for a, c in vocab.ALIAS.items() if c == v and a != v]
+        score = max(sum(one_token_apart(f, k) for k in spellings) for f in forms)
+        if score > best_score:
+            best, best_score, tie = v, score, False
+        elif score == best_score and score > 0:
+            tie = True
+    if best_score >= 2 and not tie:
+        return best
+    return None

spike2/features.py

@@ -0,0 +1,137 @@
+"""What code knows about one column: statistics, generic relations, kNN features.
+
+Everything the model is shown is computed here. The relation search is
+deliberately generic (each known column alone, linearly and as a time
+integral); products, ratios, resets and domain formulas are left to the
+model's hypotheses (dsl.py).
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from . import dsl
+
+
+def _f8(v) -> np.ndarray:
+    return np.asarray(v, dtype="f8")
+
+
+def is_constant(v) -> bool:
+    x = _f8(v)
+    x = x[np.isfinite(x)]
+    return x.size == 0 or float(np.max(x)) == float(np.min(x))
+
+
+def identical(a, b) -> bool:
+    a, b = _f8(a), _f8(b)
+    return a.shape == b.shape and np.allclose(a, b, rtol=1e-5, atol=1e-9, equal_nan=True)
+
+
+def stats(v, dtype: str) -> dict:
+    x = _f8(v)
+    fin = np.isfinite(x)
+    y = x[fin]
+    n = len(x)
+    if y.size == 0:
+        return {"rows": n, "finite_fraction": 0.0}
+    d = np.diff(y)
+    rng = float(np.ptp(y))
+    drops = int((d < -0.05 * rng).sum()) if rng > 0 else 0
+    rises = int((d > 0.05 * rng).sum()) if rng > 0 else 0
+    s = np.sign(y)
+    out = {
+        "rows": n,
+        "dtype": dtype,
+        "finite_fraction": round(float(fin.mean()), 4),
+        "min": float(f"{y.min():.6g}"),
+        "max": float(f"{y.max():.6g}"),
+        "mean": float(f"{y.mean():.6g}"),
+        "std": float(f"{y.std():.6g}"),
+        "distinct_values": int(np.unique(y).size),
+        "integer_valued_fraction": round(float(np.mean(np.isclose(y, np.round(y)))), 4),
+        "fraction_positive": round(float((y > 0).mean()), 4),
+        "fraction_negative": round(float((y < 0).mean()), 4),
+        "fraction_zero": round(float((y == 0).mean()), 4),
+        "non_decreasing_step_fraction": round(float((d >= 0).mean()), 4) if d.size else None,
+        "non_increasing_step_fraction": round(float((d <= 0).mean()), 4) if d.size else None,
+        "large_drops (>5% of range)": drops,
+        "large_rises (>5% of range)": rises,
+        "sign_changes": int((s[1:] * s[:-1] < 0).sum()),
+    }
+    pos = y[y > 0]
+    if pos.size > 3 and (pos.size == y.size):
+        r = pos[1:] / pos[:-1]
+        med = float(np.median(r))
+        if med != 1.0:
+            out["geometric_spacing_fraction"] = round(float(np.mean(np.abs(r / med - 1) < 0.05)), 4)
+    return out
+
+
+def relations(x, known: dict[str, np.ndarray], time_alias: str | None, top: int = 6) -> list[dict]:
+    """Generic search: x ~ a*c + b and x ~ a*cumsum(c*dt) + b for every known column c."""
+    out = []
+    dt = dsl._diff0(_f8(known[time_alias])) if time_alias else None
+    for alias, c in known.items():
+        c = _f8(c)
+        if is_constant(c):
+            continue
+        f = dsl.fit(x, c)
+        if "r2" in f:
+            out.append({"hypothesis": f"{alias}", **f})
+        if dt is not None and alias != time_alias:
+            g = dsl.fit(x, np.cumsum(np.nan_to_num(c * dt)))
+            if "r2" in g:
+                out.append({"hypothesis": f"cumsum({alias}*diff0({time_alias}))", **g})
+    out.sort(key=lambda r: -r["r2"])
+    return [r for r in out[:top] if r["r2"] >= 0.5]
+
+
+# --- kNN features (scale-free) -------------------------------------------------
+
+DTYPES = ["<f8", "<f4", "<u2", "<u4"]
+
+
+def knn_features(v, dtype: str, time_values=None, known_values: list | None = None) -> np.ndarray:
+    x = _f8(v)
+    y = x[np.isfinite(x)]
+    if y.size < 2:
+        y = np.zeros(2)
+    d = np.diff(y)
+    rng = float(np.ptp(y))
+    amax = float(np.abs(y).max())
+    feats = [1.0 if dtype == t else 0.0 for t in DTYPES]
+    feats += [
+        float(np.mean(np.isclose(y, np.round(y)))),
+        float((y > 0).mean()), float((y < 0).mean()), float((y == 0).mean()),
+        float((d >= 0).mean()), float((d <= 0).mean()),
+        float(np.unique(y).size / y.size),
+        float(np.log10(amax + 1e-12)),
+        float(np.log10(rng + 1e-12)),
+        float((d < -0.05 * rng).sum() / y.size) if rng > 0 else 0.0,
+    ]
+    corr_t = 0.0
+    if time_values is not None and not is_constant(time_values) and rng > 0:
+        t = _f8(time_values)
+        m = np.isfinite(t) & np.isfinite(x)
+        if m.sum() > 2:
+            corr_t = float(np.corrcoef(t[m], x[m])[0, 1])
+    feats.append(corr_t if np.isfinite(corr_t) else 0.0)
+    best = 0.0
+    for k in known_values or []:
+        k = _f8(k)
+        if is_constant(k) or rng == 0:
+            continue
+        m = np.isfinite(k) & np.isfinite(x)
+        if m.sum() > 2:
+            r = np.corrcoef(k[m], x[m])[0, 1]
+            if np.isfinite(r):
+                best = max(best, abs(float(r)))
+    feats.append(best)
+    return np.asarray(feats, dtype="f8")
+
+
+def fmt(v: float) -> str:
+    if v is None or not np.isfinite(v):
+        return "nan"
+    return f"{v:.6g}"

spike2/lock.py

@@ -0,0 +1,59 @@
+"""Freeze the pre-registration: hash everything that decides the v2 spike.
+
+Writes prereg2/LOCK.json with SHA-256 of every file in prereg2/ and spike2/,
+the corpus manifest, the task file
+(data/v2/tasks.jsonl, git-ignored, so its hash is the record), the dossier set,
+the vocabulary, package versions and the response mode per arm. Commit
+prereg2/ and spike2/ right after; run.py refuses the test split until the
+hashes match and both folders are committed and clean.
+
+Usage:  python -m spike2.lock --by "<name>" --response-mode json_schema
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import time
+from importlib.metadata import version
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def sha(p: Path) -> str:
+    """SHA-256 with CRLF normalised to LF, so a fresh clone gives the same hash (.gitattributes eol=lf)."""
+    return hashlib.sha256(p.read_bytes().replace(b"\r\n", b"\n")).hexdigest()
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--by", required=True)
+    ap.add_argument("--response-mode", required=True, choices=["json_schema", "json_object", "none"])
+    a = ap.parse_args()
+    files = sorted([*(ROOT / "prereg2").glob("*"), *(ROOT / "spike2").glob("*.py"),
+                    ROOT / "data" / "external" / "manifest.json"])
+    files = [f for f in files if f.is_file() and f.name != "LOCK.json"]
+    manifest = json.loads((ROOT / "prereg2" / "tasks_manifest.json").read_text(encoding="utf-8"))
+    lock = {
+        "locked_at": time.strftime("%Y-%m-%d %H:%M:%S"),
+        "locked_by": a.by,
+        "response_mode": a.response_mode,
+        "all_dossiers_sha256": manifest["all_dossiers_sha256"],
+        "vocabulary_sha256": manifest["vocabulary_sha256"],
+        "tasks_jsonl_sha256": sha(ROOT / "data" / "v2" / "tasks.jsonl"),
+        "packages": {p: version(p) for p in ("yadg", "galvani", "numpy", "openai")},
+        "files": {str(f.relative_to(ROOT)).replace("\\", "/"): sha(f) for f in files},
+    }
+    if lock["tasks_jsonl_sha256"] != manifest.get("tasks_jsonl_sha256"):
+        raise SystemExit("data/v2/tasks.jsonl does not match prereg2/tasks_manifest.json; rebuild with python -m spike2.tasks")
+    lock["combined_sha256"] = hashlib.sha256(json.dumps(lock["files"], sort_keys=True).encode()).hexdigest()
+    (ROOT / "prereg2" / "LOCK.json").write_text(json.dumps(lock, indent=1), encoding="utf-8", newline="\n")
+    print(f"prereg2/LOCK.json written; combined sha256 {lock['combined_sha256']}")
+    print('Now commit: git add prereg2 spike2 && git commit -m "prereg v2: freeze before the first test-split call"')
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())