cellproof 0.2.0__py3-none-any.whl

cellproof/report.py

@@ -0,0 +1,51 @@
+"""REPORT.md for people, report.json for tools."""
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+
+from cellproof.model import Key, Verdict, a1
+
+
+def _md(value) -> str:
+    return "(no value)" if value is None else str(value).replace("|", "\\|")
+
+
+def write_report(verdicts: list[Verdict], findings: list[str], out: Path, source: str,
+                 computed: frozenset[Key] | set[Key] = frozenset(), notes=()) -> Counter:
+    counts = Counter({"PASS": 0, "FAIL": 0, "UNRESOLVED": 0})
+    counts.update(v.status for v in verdicts)
+    findings = list(dict.fromkeys(findings))
+
+    data = {
+        "source": source,
+        "counts": dict(counts),
+        "findings": findings,
+        "regions": list(notes),
+        "cells": [{"cell": a1(v.key), "status": v.status, "expected": v.expected, "actual": v.actual,
+                   "reason": v.reason, "answer": "computed" if v.key in computed else "excel-saved"}
+                  for v in verdicts],
+    }
+    (out / "report.json").write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    lines = [f"# cellproof report: {source}", "", "| Status | Cells |", "|---|---|"]
+    lines += [f"| {status} | {counts[status]} |" for status in ("PASS", "FAIL", "UNRESOLVED")]
+    if computed:
+        lines += ["", f"{len(computed)} formula cells had no saved value. Their expected values were computed "
+                      "by the `formulas` library, not by Excel, and are marked (computed)."]
+    if notes:
+        lines += ["", "## Regions", ""] + [f"- {n}" for n in notes]
+    if findings:
+        lines += ["", "## Findings in the workbook", ""] + [f"- {f}" for f in findings]
+    failed = [v for v in verdicts if v.status == "FAIL"]
+    if failed:
+        lines += ["", "## Failed cells", "", "| Cell | Excel | SQL |", "|---|---|---|"]
+        lines += [f"| {a1(v.key)} | {_md(v.expected)}{' (computed)' if v.key in computed else ''} | {_md(v.actual)} |"
+                  for v in failed]
+    unresolved = [v for v in verdicts if v.status == "UNRESOLVED"]
+    if unresolved:
+        lines += ["", "## Unresolved cells", "", "| Cell | Reason |", "|---|---|"]
+        lines += [f"| {a1(v.key)} | {_md(v.reason)} |" for v in unresolved]
+    (out / "REPORT.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return counts

cellproof/verifier.py

@@ -0,0 +1,62 @@
+"""Run the compiled batches on DuckDB and compare every cell with Excel's saved value."""
+from __future__ import annotations
+
+import math
+import tempfile
+from pathlib import Path
+
+import duckdb
+
+from cellproof.compiler import READ_SEED, Compiled, batch_rows, write_seed
+from cellproof.model import Book, Cell, Key, Verdict, a1
+
+
+class SqlFailed(Exception):
+    """DuckDB rejected SQL that cellproof generated (a bug: every supported formula must run)."""
+
+
+def execute(book: Book, compiled: Compiled) -> dict[Key, tuple]:
+    """The same row SQL as the dbt project: the seed, then each batch appended in place. Returns formula cells only."""
+    con = duckdb.connect()
+    with tempfile.TemporaryDirectory() as tmp:
+        seed = Path(tmp) / "input_cells.csv"
+        write_seed(book, seed)
+        con.execute(f"CREATE TABLE cells AS SELECT * FROM {READ_SEED}", [str(seed)])
+    for batch in compiled.batches:
+        try:
+            con.execute(f"INSERT INTO cells {batch_rows(batch, 'cells')}")
+        except duckdb.Error as err:
+            # ponytail: the run stops; marking just this batch's cells (and their dependents) UNRESOLVED needs a recompile
+            raise SqlFailed(f"DuckDB could not run the SQL for {a1(batch[0].key)} and {len(batch) - 1} cells after it "
+                            f"({str(err).splitlines()[0]}). This is a bug in cellproof; please report it.") from err
+    wanted = {s.key for s in compiled.steps}
+    rows = con.execute("SELECT * FROM cells").fetchall()
+    return {(s, r, c): (k, n, t) for s, r, c, k, n, t in rows if (s, r, c) in wanted}
+
+
+def matches(cell: Cell, kind: str, num, txt) -> bool:
+    """Excel's saved value vs the SQL result. An Excel error matches only the same error code."""
+    if cell.kind == "e":
+        return kind == "e" and txt == cell.value
+    if kind == "e" or (num is None and txt is None):
+        return False
+    if cell.kind == "n":
+        return kind == "n" and math.isclose(cell.value, num, rel_tol=1e-9, abs_tol=1e-12)
+    if cell.kind == "b":
+        return kind == "b" and num == (1.0 if cell.value else 0.0)
+    if cell.kind == "s":
+        return kind == "s" and txt == cell.value
+    return False
+
+
+def verify(book: Book, compiled: Compiled) -> list[Verdict]:
+    got = execute(book, compiled)
+    out = []
+    for step in compiled.steps:
+        cell = book.cells[step.key]
+        kind, n, t = got[step.key]
+        actual = t if kind in ("s", "e") else (bool(n) if kind == "b" and n is not None else n)
+        out.append(Verdict(step.key, "PASS" if matches(cell, kind, n, t) else "FAIL", cell.value, actual))
+    for key, reason in compiled.unresolved.items():
+        out.append(Verdict(key, "UNRESOLVED", book.cells[key].value, None, reason))
+    return sorted(out, key=lambda v: v.key)

cellproof-0.2.0.dist-info/METADATA

@@ -0,0 +1,177 @@
+Metadata-Version: 2.5
+Name: cellproof
+Version: 0.2.0
+Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
+Project-URL: Homepage, https://github.com/Nikhil414/cellproof
+Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
+Author: Nikhil414
+License-Expression: MIT
+License-File: LICENSE
+Keywords: dbt,duckdb,excel,migration,spreadsheet,sql,xlsx
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
+Requires-Python: >=3.10
+Requires-Dist: duckdb>=1.1
+Requires-Dist: openpyxl>=3.1
+Provides-Extra: compute
+Requires-Dist: formulas<1.4,>=1.3.4; extra == 'compute'
+Provides-Extra: dbt
+Requires-Dist: dbt-duckdb>=1.8; extra == 'dbt'
+Provides-Extra: dev
+Requires-Dist: jinja2>=3.1; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# cellproof
+
+**Turn an Excel workbook into SQL, and prove every number matches.**
+
+(Released as `unexcel` up to 0.1.0; renamed to cellproof in 0.2.0.)
+
+Business logic often lives in a spreadsheet that one person understands. Moving it into a database is risky because
+nobody can show the new numbers equal the old ones. cellproof translates every formula of an `.xlsx` into SQL, runs that
+SQL on [DuckDB](https://duckdb.org), and compares each result with the value Excel itself saved in the file. Excel's
+saved values are a free answer key.
+
+```
+$ cellproof convert examples/functions.xlsx
+72 pass, 0 fail, 4 unresolved -> examples/functions_dbt/REPORT.md
+```
+
+Every formula cell gets one verdict:
+
+| Verdict | Meaning |
+|---|---|
+| **PASS** | the SQL gives the value Excel saved |
+| **FAIL** | the SQL gives a different value; the report shows both |
+| **UNRESOLVED** | cellproof could not translate the formula exactly; the report says why. It never guesses. |
+
+Exit code: `0` no FAIL, `1` at least one FAIL (or `dbt build` failed), `2` the workbook could not be read, `3`
+DuckDB rejected SQL that cellproof generated (a bug in cellproof; the message names the cell, please report it).
+
+## Install
+
+Python 3.10 or newer.
+
+```bash
+pip install cellproof
+pip install "cellproof[dbt]"      # also run dbt build (--dbt)
+pip install "cellproof[compute]"  # answer key for files never saved in Excel
+```
+
+Runtime dependencies are only `openpyxl` and `duckdb`.
+
+## Use
+
+```bash
+cellproof convert book.xlsx                 # writes book_dbt/ next to the file
+cellproof convert book.xlsx --out out_dir   # choose the folder
+cellproof convert book.xlsx --dbt           # also run `dbt build` on the generated project
+```
+
+The output folder holds:
+
+- **`REPORT.md` / `report.json`**: every verdict, failed cells with both values, unresolved cells with the reason,
+  findings (for example a number stored as text, which Excel skips in SUM), and which blocks of rows became readable
+  table models.
+- **A dbt project** (`dbt_project.yml`, `models/`, `seeds/input_cells.csv`) that recomputes the workbook on DuckDB.
+  It runs exactly the SQL the verifier checked.
+
+The workbook must have been saved by Excel (so the file holds Excel's values). For a file generated by code and never
+opened in Excel, install the `compute` extra: the [`formulas`](https://pypi.org/project/formulas/) library then
+computes the expected values, and the report labels them `computed` (a weaker answer key than Excel).
+
+## How it works
+
+```
+reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter -> report
+```
+
+1. **Read** the file twice with openpyxl: the formulas, and the values Excel saved. Dates become Excel serial numbers.
+2. **Find regions.** A block of rows where columns repeat one formula down (`=A2*1.1`, `=IF(B2>100,B2,0)`) is a
+   *region*. Running totals in a region (`=C1+B2`, `=SUM($B$2:B2)`) become SQL window functions. A region compiles to
+   one readable statement (`amount * 1.1 AS tax`) instead of one statement per cell.
+3. **Order** every formula after the cells it reads. Circular references are UNRESOLVED.
+4. **Compile** each formula to SQL with Excel's rules: Excel's type conversions (`"3"+1 = 4`), its error codes
+   (`#DIV/0!`, `#N/A`, `#VALUE!` ...) which must match exactly, the left-to-right order in which Excel reports errors,
+   and DuckDB guards (division by zero, `ln(0)`, overflow of `POWER`) so DuckDB never fails where Excel returns an
+   error. Cells outside regions run in batches ("cell mode").
+5. **Verify** on DuckDB in memory: numbers must match within a relative 1e-9, text and booleans exactly, errors by
+   code.
+6. **Emit** the dbt project and the report.
+
+93 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
+COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR,
+HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LEFT, LEN, LN, LOG,
+LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
+PROPER, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SQRT, SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
+SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VLOOKUP, WEEKDAY, XLOOKUP, XOR, YEAR.
+
+**Speed** (Windows 11 laptop, measured): a 200,000-formula workbook (four formula columns over 50,000 rows, one of them
+a running total) converts and verifies in 26 s, all PASS. 50,000 rows of VLOOKUP into a 1,000-row table: 4.6 s.
+COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
+
+## Limitations
+
+cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
+
+- **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
+- **Not supported:** array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`,
+  `INDIRECT`), functions outside the list above, external workbooks, structured table references (`Table1[@Col]`),
+  INDEX used as a reference, the 1904 date system in TEXT date formats.
+- **Approximate lookups on unsorted data** (MATCH/VLOOKUP/HLOOKUP with match type 1/-1): UNRESOLVED, because Excel's
+  answer then depends on its binary search.
+- **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Excel also reads `"1,000"`, `"$5"`, `"50%"`;
+  cellproof gives `#VALUE!`, which shows as a FAIL.
+- **Some function options:** SUBTOTAL 101-111 (hidden rows), DATEDIF units MD/YM/YD, WEEKDAY types 11-17, TEXT formats
+  outside the supported table, COUNT or MEDIAN of values typed into the formula, criteria taken from an empty cell.
+- **Floating point:** a running total becomes a fast window function only when Excel's saved values prove the different
+  addition order cannot matter; balances that cancel back to zero (ledgers) are added in Excel's order instead, which
+  is slower. Elsewhere SQL can still differ from Excel in the last digits, which would show as an honest FAIL. A
+  number that overflows becomes `inf` instead of `#NUM!`.
+- **Speed:** regions cover columns that repeat one formula over the same rows, lookups and *IF(S) criteria included.
+  A formula that is not part of such a column runs cell by cell (a lookup alone: about 7 s per 1,000). Running totals
+  other than plain sums (`=C1*1.05+B2`, `=MAX(C1,B2)`) also run cell by cell.
+- **The answer key:** a workbook must have been saved by Excel; `formulas` (the `compute` extra) is a weaker fallback
+  and is known to disagree with Excel in places.
+- **dbt:** the generated project targets DuckDB (`dbt-duckdb`). Other warehouses are not generated yet.
+
+## Security and privacy
+
+- **No network.** cellproof never sends workbook data anywhere; everything runs locally in DuckDB.
+- **Untrusted files:** workbook text only ever reaches SQL as escaped string literals; headers become SQL names only
+  after being reduced to `[a-z0-9_]`; braces are escaped so dbt's Jinja cannot run workbook text. A file that unzips to
+  more than 2 GB is refused. For extra XML hardening on untrusted files, `pip install defusedxml` (openpyxl then uses it
+  automatically).
+- **Output folder:** a rerun deletes the old `*.sql` files in `<out>/models/cellproof_cells/` so stale models cannot
+  linger; point `--out` at a folder of its own.
+- **`--dbt`** runs your installed `dbt` on the generated project (no shell).
+
+See [SECURITY.md](SECURITY.md) to report a vulnerability.
+
+## Development
+
+```bash
+python -m venv .venv && . .venv/bin/activate      # Windows: .venv\Scripts\activate
+pip install -e ".[dev,dbt,compute]"
+python -m pytest tests -q                          # ~8 min with dbt on PATH
+CELLPROOF_PERF=1 python -m pytest tests/test_perf.py -s   # the 200k-formula benchmark
+```
+
+Rules the code follows: never guess a value (anything not translated exactly is UNRESOLVED with a reason a user
+understands); a false PASS is the worst bug; no DuckDB exception may escape from a supported function; tests first; real
+Excel is the ground truth. Design notes: [DESIGN.md](DESIGN.md) and `docs/superpowers/specs/`.
+
+## License
+
+[MIT](LICENSE)

cellproof-0.2.0.dist-info/RECORD

@@ -0,0 +1,33 @@
+cellproof/__init__.py,sha256=qbhMh4mg2JMoCFflKwIRvApU63RR8AxvYq1OT__8sgI,27
+cellproof/__main__.py,sha256=VwT4B2Uhc4sjZoXMJicnl8mM0ND-lRucvdeeZKcpaWs,57
+cellproof/cli.py,sha256=Atc4jxDcP4ESybIaBttxjcFh3hqSL39qRuQV9mUS0Bg,2444
+cellproof/compiler.py,sha256=wGinNTP5dorROaEJMWzRZgz43a-z4K5j2j3WS1gkAxs,19519
+cellproof/compute.py,sha256=ONf-3wV9ZsxRFoTAEHrgsCdp7m4hRDssfWaYCOs47XU,2685
+cellproof/emitter.py,sha256=MQJea7LEwqEzFKrIG2qcHXXEZngm1kQuQIDnETyuRes,6344
+cellproof/expr.py,sha256=iexLxvwRnB4oT1HP2XiANQv8MYlZ6Lw-10AUM6DPeqk,8993
+cellproof/fenwick.py,sha256=N9-uMNlykMMhzJfkDEtNxsYeXFscR2BHYl6Vw-lHz4Q,1348
+cellproof/functions.py,sha256=XCejVodGJp290w_w2rW9oEpyjBBHMY2eXMJYlxb1MX8,4206
+cellproof/graph.py,sha256=rjFVe00e8iyK5Rd5ySoHD2SAqHCw4hVlPEoE_oL2SMY,8271
+cellproof/model.py,sha256=lox07H1v9bCKovagRbs7GhDMf5PtmOwpRh785ajJDiI,2423
+cellproof/parser.py,sha256=PL4qb-pNQwtNEw9rOj9g1bDTrvzOIb9aGzds5mHpbTA,4946
+cellproof/r1c1.py,sha256=xY0INUlfzjucM7A6LCqKo65fPQt0Z_kBezmVWT1yeWk,8436
+cellproof/rangeindex.py,sha256=qxL5D3AnULfohTHeLjqJTaxhgl4aoMUBPwRM4sCXpeU,4496
+cellproof/reader.py,sha256=mazmjCwq3QqRm9Wmwbr7u_o5A4vwC_0N35yGJ-z1PJE,3542
+cellproof/refs.py,sha256=q4j46GxhEDg97GzmhghMqnrDKOYwhgLTwrPIPsRk0V0,1807
+cellproof/region.py,sha256=OhLCxD4YEZpWeAXG9Ha4dOhGLZ2-fleMROYvYMk7a5U,19151
+cellproof/regions.py,sha256=6QYZsJaS1eyFX7FN_RgrUd_D4YYvD0695UWOgDnablE,11888
+cellproof/report.py,sha256=5uUuBENrKJI5GULMDQ4uN1Mx5NmCzr8GQBfv2NAgQHc,2365
+cellproof/verifier.py,sha256=LGzNsJF2BUhifjZ8bfQazBd7-nL8K3AJipEkFAYJZEY,2780
+cellproof/fn/__init__.py,sha256=FCehhK5QagM3aJJhMo92lTSj0uS_Z3FSbMZ57Ht_l9w,206
+cellproof/fn/aggregate.py,sha256=aKDPArE9rJhMnL9Cpvic6vZLK1AykR3uhCtWT7g9Ars,8040
+cellproof/fn/criteria.py,sha256=_A9ZbGnu8loKwYwZscwzffn5pQ3xGSnABLij_jBrIMk,9648
+cellproof/fn/dates.py,sha256=9SN-v143i1uFcR9V8Xa55KLQ2oyOGM1sqpS6MDY_IFU,8066
+cellproof/fn/logic.py,sha256=gQ4qAQsFxNAHA1yZRInaZw8Mjpv9kPeRdzLY50uG_n8,4742
+cellproof/fn/lookup.py,sha256=AFFJeKS6PahhW-UObvrZYVGgW_H1DyycBXciVpbRI24,14782
+cellproof/fn/maths.py,sha256=dZxxdo4EaTF6Tn7Wb651sWjxYtdOUBBWaGUDJ7QQPWw,4992
+cellproof/fn/text.py,sha256=xiw6O7HkEe-Ov3umtUijaEPG-0ogpyAhD31N16B3u5U,10872
+cellproof-0.2.0.dist-info/METADATA,sha256=olWEUvIiBD3MUsjCx9AvSt3JVThtFo9hpy752wWl-XU,9579
+cellproof-0.2.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+cellproof-0.2.0.dist-info/entry_points.txt,sha256=Fmzx5Byo9wZQWQrzM557_61ppO4wRmXrYfwAhwmsD3Y,49
+cellproof-0.2.0.dist-info/licenses/LICENSE,sha256=DwkdN_2ue-4IM8pDfuB5nJ387buNEFyKDadbaB-gu8E,1093
+cellproof-0.2.0.dist-info/RECORD,,

cellproof-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

cellproof-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cellproof = cellproof.cli:main

cellproof-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nikhil414 and cellproof contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.