rosetta-eclab 0.1.1__py3-none-any.whl

rosetta_eclab/misses.py

@@ -0,0 +1,39 @@
+"""List the namer's misses on a v2 split with the engine's ranking (development aid; never on the hold-out)."""
+
+from __future__ import annotations
+
+import sys
+import warnings
+
+from spike2 import corpus, score, tasks
+
+from .evaluate import answer
+
+
+def main() -> int:
+    warnings.filterwarnings("ignore")
+    sys.stdout.reconfigure(encoding="utf-8")
+    split = sys.argv[1] if len(sys.argv) > 1 else "test"
+    assert split in ("dev", "test", "holdout", "holdout2", "holdout3", "holdout4"), "development (spent) splits only"
+    for t in tasks.load(split):
+        if t["bucket"] not in ("main", "decoy"):
+            continue
+        x, dtype, kv, kc = tasks.hidden_and_known(t)
+        a = answer(x, dtype, list(kc.values()), t["technique"], tasks.task_vocabulary(t),
+                   exclude_group=corpus.source_group(t["file"]))
+        r = {"status": "ok", **a}
+        if t["bucket"] == "main" and score.correct(t, r):
+            continue
+        if t["bucket"] == "decoy" and not score.hallucinated(t, r):
+            continue
+        truth = t["truth"]["canonical"] if t["truth"] else f"DECOY {t['decoy']['kind']} {t['decoy'].get('source_name', '')}"
+        print(f"\n{t['task_id']} {t['file']} ({t['technique']}) truth={truth} -> {a['verdict']} {a['name']}")
+        print(f"   known: {', '.join(c.name for c in kc.values())}")
+        print(f"   evidence: {a.get('evidence', '')[:200]}")
+        for row in a.get("ranking", [])[:4]:
+            print(f"   {row}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rosetta_eclab/mprfile.py

@@ -0,0 +1,178 @@
+"""Read a BioLogic EC-Lab .mpr file, including columns the reference parser does not know.
+
+Known columns are typed and named with the reference parser's column map. For
+unknown column IDs the byte width is solved from the record size, the storage
+type is chosen by plausibility (finite, sensible magnitudes, integers for
+integer types), and the values are decoded so the namer can identify them.
+"""
+
+from __future__ import annotations
+
+import itertools
+import logging
+import warnings
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+
+FLAG_IDS = (1, 2, 3, 21, 31, 65)
+CANDIDATES = {1: ["|u1"], 2: ["<u2", "<i2"], 4: ["<f4", "<u4", "<i4"], 8: ["<f8", "<u8", "<i8"]}
+
+
+@dataclass
+class Col:
+    position: int
+    raw_id: int
+    name: str
+    unit: str | None
+    dtype: str
+    known: bool
+    values: np.ndarray = field(repr=False)
+
+
+@dataclass
+class MprData:
+    path: str
+    technique: str | None
+    n_points: int
+    record_size: int
+    columns: list[Col]
+    notes: list[str]
+
+    @property
+    def unknown(self) -> list[Col]:
+        return [c for c in self.columns if not c.known]
+
+
+def _modules(raw: bytes):
+    from yadg import dgutils
+    from yadg.extractors.eclab.mpr_columns import module_header_dtypes
+
+    for module in raw.split(b"MODULE")[1:]:
+        for mhd in module_header_dtypes:
+            try:
+                header = dgutils.read_value(module, 0x0000, mhd)
+                if len(module) == mhd.itemsize + header["length"]:
+                    yield header["short_name"].strip(), header.get("newver", 0) + header["oldver"], module[mhd.itemsize:]
+                    break
+            except UnicodeDecodeError:
+                continue
+
+
+def _plausibility(v: np.ndarray, dtype: str) -> float:
+    x = v.astype("f8")
+    fin = np.isfinite(x)
+    if fin.mean() < 0.999:
+        return 0.0
+    y = x[fin]
+    if y.size == 0:
+        return 0.0
+    if dtype.endswith(("f4", "f8")):
+        a = np.abs(y[y != 0])
+        if a.size and (np.mean((a < 1e-30) | (a > 1e15)) > 0.01):
+            return 0.05  # denormal-looking or astronomically large values: wrong type
+        return 1.0 if np.mean(np.isclose(y, np.round(y))) < 0.95 else 0.7
+    # integers: plausible when they are small relative to the type's range
+    span = float(np.max(np.abs(y))) if y.size else 0.0
+    width = np.dtype(dtype).itemsize
+    return 0.9 if span < 2 ** (8 * width - 4) else 0.2
+
+
+def read(path: str | Path, reference: str = "galvani") -> MprData:
+    from galvani import BioLogic
+    from yadg.extractors.eclab.mpr import parse_columns, process_settings
+
+    logging.getLogger("yadg").setLevel(logging.CRITICAL)
+    warnings.filterwarnings("ignore")
+    raw = Path(path).read_bytes()
+    if not raw.startswith(b"BIO-LOGIC MODULAR FILE"):
+        raise ValueError("not a BioLogic EC-Lab .mpr file")
+    technique, data, version = None, None, None
+    for name, ver, mdata in _modules(raw):
+        if name == "VMP Set":
+            try:
+                technique, _, _ = process_settings(mdata, "11.50" if ver >= 10 else "10.40")
+            except Exception:
+                technique = None
+        elif name == "VMP data":
+            data, version = mdata, ver
+    if data is None:
+        raise ValueError("no 'VMP data' module")
+    n_points = int(np.frombuffer(data, offset=0, dtype="<u4", count=1)[0])
+    n_columns = int(np.frombuffer(data, offset=4, dtype="|u1", count=1)[0])
+    if version in (10, 11):
+        yids = np.frombuffer(data, offset=5, dtype=">u2", count=n_columns).tolist()
+        full = np.frombuffer(data, offset=6, dtype="<u2", count=n_columns).tolist()
+        offset = 0x3EF
+    elif version in (2, 3):
+        yids = full = np.frombuffer(data, offset=5, dtype="<u2", count=n_columns).tolist()
+        offset = 0x195 if version == 2 else 0x196
+    else:
+        raise NotImplementedError(f"data module version {version} is not supported yet")
+    region = len(data) - offset
+    if n_points == 0 or region % n_points:
+        raise ValueError(f"record size is not a whole number of bytes ({region} / {n_points})")
+    rec = region // n_points
+
+    names, dtypes, units, _ = parse_columns(yids, technique)  # yadg's map: names for known columns
+    gmap = {int(k): v for k, v in BioLogic.VMPdata_colID_dtype_map.items()}
+    layout = []  # record layout in file order: (raw_id or None for the flags byte, ...)
+    for yid, fid in zip(yids, full):
+        if yid in FLAG_IDS:  # all flag IDs share one byte, at the position of the first one
+            if not any(entry[0] is None for entry in layout):
+                layout.append((None, "flags", None, "|u1", True))
+            continue
+        layout.append((fid, None, None, None, None))
+    # names/dtypes for data columns, in order, from yadg's map (it skips flags the same way)
+    data_names = [(n, d, u) for n, d, u in zip(names, dtypes, units) if n != "flags"]
+    filled, j = [], 0
+    for entry in layout:
+        if entry[0] is None:
+            filled.append(entry)
+            continue
+        n, d, u = data_names[j]
+        j += 1
+        fid = entry[0]
+        known_yadg = not n.startswith("unknown_")
+        known_ref = (fid in gmap) if reference == "galvani" else known_yadg
+        if known_ref and known_yadg:
+            filled.append((fid, n, u, d, True))
+        elif known_ref:  # galvani knows it, yadg does not
+            gname, gd = gmap[fid]
+            filled.append((fid, gname.split("/")[0], None, gd, True))
+        else:
+            filled.append((fid, None, None, None, False))
+    known_width = sum(np.dtype(d).itemsize for _, _, _, d, kn in filled if kn)
+    unknown_idx = [i for i, e in enumerate(filled) if not e[4]]
+    rest = rec - known_width
+    notes = []
+    best = None
+    if unknown_idx:
+        combos = [c for c in itertools.product((1, 2, 4, 8), repeat=len(unknown_idx)) if sum(c) == rest]
+        for widths in combos[:200]:
+            for types in itertools.product(*[CANDIDATES[w] for w in widths]):
+                trial = list(filled)
+                for i, t in zip(unknown_idx, types):
+                    trial[i] = (trial[i][0], None, None, t, False)
+                arr = np.frombuffer(data, offset=offset, count=n_points,
+                                    dtype=np.dtype([(f"f{i}", e[3]) for i, e in enumerate(trial)]))
+                score = sum(_plausibility(arr[f"f{i}"], t) for i, t in zip(unknown_idx, types))
+                if best is None or score > best[0]:
+                    best = (score, trial)
+        if best is None:
+            raise ValueError(f"cannot fit {len(unknown_idx)} unknown column(s) into {rest} unexplained bytes per record")
+        filled = best[1]
+        notes.append(f"{len(unknown_idx)} column ID(s) unknown to {reference}: "
+                     + ", ".join(str(filled[i][0]) for i in unknown_idx) + f"; widths solved from the record size ({rec} bytes)")
+    elif rest != 0:
+        raise ValueError(f"record size {rec} does not match the known columns ({known_width} bytes)")
+    arr = np.frombuffer(data, offset=offset, count=n_points,
+                        dtype=np.dtype([(f"f{i}", e[3]) for i, e in enumerate(filled)]))
+    cols, pos = [], 0
+    for i, (fid, n, u, d, kn) in enumerate(filled):
+        if fid is None:
+            continue  # flags byte
+        cols.append(Col(pos, fid, n or f"unknown {fid}", u, d, kn, np.asarray(arr[f"f{i}"])))
+        pos += 1
+    return MprData(str(path), technique, n_points, rec, cols, notes)

rosetta_eclab-0.1.1.dist-info/METADATA

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: rosetta-eclab
+Version: 0.1.1
+Summary: Name the columns of BioLogic EC-Lab .mpr files that your parser does not know - no language model, evidence for every answer
+Author: Nataniel Ataseven
+License-Expression: GPL-3.0-or-later
+Project-URL: Repository, https://github.com/Nataniel0212/rosetta-eclab
+Project-URL: Issues, https://github.com/Nataniel0212/rosetta-eclab/issues
+Keywords: electrochemistry,biologic,ec-lab,mpr,battery,potentiostat,file-format
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Chemistry
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=2.0
+Requires-Dist: yadg==7.0.1
+Requires-Dist: galvani==0.5.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
+
+# Rosetta
+
+**Rosetta names the columns of BioLogic EC-Lab `.mpr` files that your parser does not know. It shows the evidence for each answer, and it says so when the data cannot support a name.**
+
+When the instrument vendor adds a column ID, open-source parsers such as [galvani](https://github.com/echemdata/galvani) stop reading the file until someone works out what the new column is. Rosetta reads the file anyway. It works out the byte width and storage type of each unknown column from the record size, and then identifies the physical quantity from the data. It uses no language model and no network.
+
+```
+$ rosetta drift.mpr --self-check
+drift.mpr
+  technique GCPL, 12057 rows, 79 bytes per record
+  1 column ID(s) unknown to galvani: 950; widths solved from the record size (79 bytes)
+
+  column ID 950 (<f8): Q charge or discharge [C or mA·h]  (confidence 0.533)
+    evidence: relation: I*dt summed within half cycle: r2 1.0000, scale 0.0002778
+    also considered: dQ, step time
+
+  self-check: hid each of 11 known columns in turn; named 11, 11 correctly, 0 wrong, the rest abstained
+```
+
+*`drift.mpr` is `gcpl.mpr` from the yadg test suite, with one column ID changed to 950, an ID no parser knows (see `tests/test_cli.py`).*
+
+## Install
+
+Python 3.10 or later:
+
+```sh
+pip install rosetta-eclab
+```
+
+To run the tests or reproduce the benchmark, clone the repository instead:
+
+```sh
+git clone https://github.com/Nataniel0212/rosetta-eclab.git
+cd rosetta-eclab
+pip install ".[test]"
+```
+
+## Use
+
+```sh
+rosetta FILE.mpr                     # name every column unknown to galvani (the default reference parser)
+rosetta FILE.mpr --reference yadg    # ... unknown to yadg instead
+rosetta FILE.mpr --self-check        # also hide each known column in turn and report how many come back right
+rosetta FILE.mpr --json              # machine-readable output
+```
+
+Each unknown column gets one of three answers:
+- **NAME:** an EC-Lab quantity, with its unit, a confidence and the evidence, for example the relation that holds and how well it fits.
+- **DUPLICATE:** identical in every row to a known column.
+- **ABSTAIN:** the data does not support a name. Rosetta prefers this to a guess.
+
+`--self-check` shows how well Rosetta knows *your* kind of file. It hides each known column in turn and tries to name it.
+
+## How it works
+
+1. **Read the file** (`namer/mprfile.py`). Columns the reference parser knows are typed from its column map. For unknown IDs the byte width is solved from the record size, and the storage type is chosen by plausibility.
+2. **Code checks first.** Constant columns and exact copies of a known column are reported as such. Names that a family of siblings makes obvious are filled in, for example the missing harmonic `|Ewe h4|`.
+3. **Physics library** (`namer/library.py`). Every candidate quantity is tested against the file's other columns:
+   - **charges:** current × time, reset per half cycle, cycle or sequence
+   - **energies and power:** E × I and its integrals
+   - **impedance:** |Z| = √(Re² + Im²), the phase angle, Cs and Cp
+   - **clocks and step times**, and **counters** stepping by one
+   - **set points** that the measured potential or current holds to
+
+   Each test reports r² and the fitted scale, which must match the expected unit conversion. A potential can therefore not pass as a current. Fits are robust to the single glitch rows that real records have at step transitions.
+4. **Statistics fallback** (`namer/fallback.py`). A nearest-neighbour vote over column statistics from public files. It is used only for columns that are not noise, and it is vetoed when a defining relation of the guessed quantity fails in the file.
+5. **Abstain** when nothing above supports a name.
+
+## How accurate it is
+
+The numbers come from five hold-out sets of public EC-Lab files from Zenodo, Battery Data Format and navani: 373 real columns and 95 decoys. None of these files were used during development. Each set was frozen before the namer version that ran on it, and each was run once (`results/open_runs.md`).
+
+In every task one real column is hidden and must be named from the data. There are also decoys, where the right answer is not to name the column:
+- noise and random integers
+- *out-of-vocabulary* columns: real columns whose name the tool is not allowed to use, the way a brand-new quantity would look
+
+| | Rosetta (no model) | Claude Opus 5.5 | best code-only baseline |
+|---|---|---|---|
+| Real columns named correctly | 350/373 (93.8 %) | 359/373 (96.2 %) | 77–100 % per hold-out¹ |
+| Wrong names on real columns | **5** | 10 | – |
+| Decoys named (should abstain) | **11/95** | 16/95 | 95/95 |
+| Last hold-out (70 columns) | **70/70, 0 wrong, 1 decoy named** | 69/70, 1 wrong, 2 decoys named | 78.6 % |
+
+¹ There are two code-only baselines. A technique prior names the column that is typical for this technique and missing here. A nearest-neighbour vote compares the column with labelled columns from other files. Both are strong when files have standard column sets, but they name every decoy and cannot name a genuinely new column.
+
+The namer improved between hold-outs, so the total includes earlier versions. The history, including the language-model experiments that came first, is in [`docs/history.md`](https://github.com/Nataniel0212/rosetta-eclab/blob/main/docs/history.md).
+
+### Limitations
+
+- **Values EC-Lab computes after the fact** cannot be reconstructed reliably. For example, a capacitance is held per segment.
+- **Noise-like DC currents** in impedance records, and **external analog inputs** with no sibling column, usually get ABSTAIN.
+- **Two quantities with the same definition in a file** cannot be told apart from the data. In galvanostatic cycling, for example, `dQ` and `Q charge or discharge` reset at the same points. A prior over how common each name is for the technique decides between them, and the alternative is shown.
+- **Supported formats:** EC-Lab data-module versions 2, 3, 10 and 11. Very old files (version 0) are not supported yet.
+
+## Reproduce the benchmark
+
+```sh
+python scripts/fetch_testdata.py                 # development corpus (yadg + galvani test files)
+python scripts/fetch_holdout.py holdout5         # any hold-out; see DATA_SOURCES.md
+python -m spike2.tasks                           # v2 benchmark tasks (hashes in prereg2/tasks_manifest.json)
+python -m holdout.build holdout5                 # hold-out tasks (hash in holdout/manifest_holdout5.json)
+python -m namer.evaluate --split holdout5        # run the namer
+python -m spike2.baselines --split holdout5      # code-only baselines
+python -m bench.claude_cli --split holdout5 --score rules   # score (Claude runs need the Claude Code CLI)
+python -m namer.golden check                     # regression check: every answer must match tests/golden
+pytest tests
+```
+
+## Data and licence
+
+Rosetta is free software under the **GNU General Public License v3.0 or later** (see `LICENSE`). It uses [yadg](https://github.com/dgbowl/yadg) and [galvani](https://github.com/echemdata/galvani), both GPL. Measurement files are downloaded at run time and never committed. Sources, licences and attributions are listed in [`DATA_SOURCES.md`](https://github.com/Nataniel0212/rosetta-eclab/blob/main/DATA_SOURCES.md). Rosetta is not affiliated with BioLogic.

rosetta_eclab-0.1.1.dist-info/RECORD

@@ -0,0 +1,31 @@
+rosetta_eclab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rosetta_eclab/cli.py,sha256=oe_hdjIQFOtHIkXqYqeVf2ijG1HJVHE-F5OJnqy5E6k,5087
+rosetta_eclab/debug.py,sha256=ddsJm2aV-nUktFlIAOym7yXO0P-olMCv2T1aRObSzpk,3461
+rosetta_eclab/engine.py,sha256=YVQQ4zAoxJn193212Qe5FGYFUvh8IjP8EjE9u9Md5hU,9006
+rosetta_eclab/evaluate.py,sha256=ZoTh1LIrz-RJ5G1wiZ3flxU-ODPJXnV-BfCCKHkHaIM,9031
+rosetta_eclab/fallback.py,sha256=enxXyy-IJGm7zwgWrn-jvZsRPvC6rvcuUQg7rw1E9LQ,5935
+rosetta_eclab/golden.py,sha256=_oONIABpdWFHoHXW--0uEh-UwSi17OzEe5lawSJQZDU,2947
+rosetta_eclab/library.py,sha256=3LG7fOJYXRNgEtsvxQv3WuzgKVJCe3YkzngPMqSpQkM,35677
+rosetta_eclab/misses.py,sha256=XBw6yfCDFIkXV5GMtJNnyKqqqV7gESjjm1xIYd_qUKE,1550
+rosetta_eclab/mprfile.py,sha256=dDEVZe3aPzzqEKH10q44Re2y3wYnEkUwzNkRoF-jifU,7471
+rosetta_eclab/data/stats.json,sha256=MNp2xOiOZHCbmGfdACBDV5zWOoatIH7pKp3Be8rqtK8,28652
+rosetta_eclab/data/stats_features.npy,sha256=-bCgLTO5DNiJLXy-mXFhuRpX_X9jY1eTYk8C-h1hfkE,73728
+rosetta_eclab-0.1.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+spike2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spike2/agent.py,sha256=FtjyJbzynSidjfOsKit9fH-zWsbJ-Q2LwCX1SAVGPkE,17325
+spike2/api_check.py,sha256=EkyVF6UJwK-_GIPRI6Zi113McMKhMGbtHBTobAwnCt4,5725
+spike2/baselines.py,sha256=P6gpXGFdZVAQMQDBlgi01luTXUd63_bvlmI1d4n3Rgs,6530
+spike2/corpus.py,sha256=iH-WIZf6mnjO1EHKDc2esurJTeIAu-J9L1wV3m_gUJo,6081
+spike2/dsl.py,sha256=jlogDcHMFEuA3o-5ex-DhFXi55huIZ0hOOviNIeZ2Lw,6401
+spike2/elim.py,sha256=hkPjy0xHEwELjpU4UpICuhEL2fOa5c6BRNnl5tPljXI,1971
+spike2/features.py,sha256=BR_5zLGx85rMkWSoyRgnXjK4pC7VkM0fvhWkyHz8arY,4752
+spike2/lock.py,sha256=tyg354VnCjPzw6zvluWDOq--MZa1YUS2X42z76HVe9s,2708
+spike2/run.py,sha256=PtJszxAo2QiZfidjM6AtgQg5zG5bFPEPUiB4Pvup4cs,10407
+spike2/score.py,sha256=ypoeiUXOSW_t9FZdQQtPa1t4zgMWDnh1FEXEJe02Va0,10869
+spike2/tasks.py,sha256=bWm6QcEiUq5K8F4lDG4i4OKhw0H7RyLh6BikY6Ns_vQ,15805
+spike2/vocab.py,sha256=MtcNoLo5aLS-uUXiUJoih74Dvy6CpOlKX0cqCckSuJ8,4537
+rosetta_eclab-0.1.1.dist-info/METADATA,sha256=5Ti6e2N7nppDfmpFyRwM77zj4vMvZr3SqBfy9VWw3qw,8329
+rosetta_eclab-0.1.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+rosetta_eclab-0.1.1.dist-info/entry_points.txt,sha256=3XKxa49VWnGYZBrFuzpqS0qMW5hKAkBgyIsk2wfgSxk,51
+rosetta_eclab-0.1.1.dist-info/top_level.txt,sha256=8IvVzyW3bwuHVOAFu1n8uGLIn_M2F3jNORuvQH6p-1I,21
+rosetta_eclab-0.1.1.dist-info/RECORD,,

rosetta_eclab-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

rosetta_eclab-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+rosetta = rosetta_eclab.cli:main