cellproof 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cellproof-0.2.1 → cellproof-0.3.0}/AGENTS.md +2 -1
- cellproof-0.3.0/CHANGELOG.md +88 -0
- cellproof-0.3.0/INFO.md +153 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/PKG-INFO +39 -15
- {cellproof-0.2.1 → cellproof-0.3.0}/PROGRESS.md +22 -7
- {cellproof-0.2.1 → cellproof-0.3.0}/README.md +37 -13
- {cellproof-0.2.1 → cellproof-0.3.0}/pyproject.toml +2 -2
- cellproof-0.3.0/src/cellproof/__init__.py +1 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/cli.py +2 -1
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/compiler.py +42 -6
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/expr.py +11 -1
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fn/__init__.py +1 -1
- cellproof-0.3.0/src/cellproof/fn/aggregate.py +473 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fn/dates.py +117 -4
- cellproof-0.3.0/src/cellproof/fn/finance.py +103 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fn/logic.py +47 -4
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fn/lookup.py +167 -17
- cellproof-0.3.0/src/cellproof/fn/stats.py +116 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fn/text.py +2 -2
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/functions.py +2 -13
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/graph.py +2 -0
- cellproof-0.3.0/src/cellproof/impact.py +57 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/model.py +11 -1
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/parser.py +71 -7
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/reader.py +31 -5
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/refs.py +1 -1
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/region.py +1 -1
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/report.py +18 -2
- cellproof-0.3.0/src/cellproof/tables.py +162 -0
- cellproof-0.3.0/tests/test_arrays.py +166 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_corpus_findings.py +35 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_dates.py +26 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_emitter.py +11 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_excel_findings.py +8 -6
- cellproof-0.3.0/tests/test_finance_stats.py +127 -0
- cellproof-0.3.0/tests/test_impact.py +115 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_implicit.py +6 -5
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_logic.py +36 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_lookup.py +29 -0
- cellproof-0.3.0/tests/test_lookups.py +136 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_parser.py +1 -1
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_region.py +5 -0
- cellproof-0.3.0/tests/test_tables.py +75 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/xlsxkit.py +40 -8
- cellproof-0.2.1/CHANGELOG.md +0 -42
- cellproof-0.2.1/src/cellproof/__init__.py +0 -1
- cellproof-0.2.1/src/cellproof/fn/aggregate.py +0 -197
- cellproof-0.2.1/tests/test_lookups.py +0 -49
- {cellproof-0.2.1 → cellproof-0.3.0}/.github/workflows/ci.yml +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/.github/workflows/publish.yml +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/.gitignore +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/DESIGN.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/LICENSE +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/PLAN-2a.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/PLAN-2b.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/PLAN.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/SECURITY.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/docs/JOURNEY.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/examples/commission.xlsx +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/examples/errors.xlsx +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/examples/functions.xlsx +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/examples/functions_unsaved.xlsx +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/examples/make_commission.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/examples/make_errors.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/examples/make_functions.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/__main__.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/compute.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/emitter.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fenwick.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fn/criteria.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/fn/maths.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/r1c1.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/rangeindex.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/regions.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/src/cellproof/verifier.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/bench_shapes.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/corpus.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_aggregate.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_batches.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_cli.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_compiler.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_compute.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_criteria.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_errors.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_excel_saved.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_fenwick.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_functions.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_graph.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_index.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_lexed.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_limits.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_maths.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_perf.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_r1c1.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_rangeindex.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_reader.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_refs.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_regions.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_regions_differential.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_regions_split.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_report.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_review_regressions.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_running.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_shapes.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_text.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_types.py +0 -0
- {cellproof-0.2.1 → cellproof-0.3.0}/tests/test_verifier.py +0 -0
|
@@ -28,7 +28,8 @@ CELLPROOF_PERF=1 python -m pytest tests/test_perf.py -s # 200k-formula benchmar
|
|
|
28
28
|
| `expr.py` | `Expr` (SQL value + SQL error code), Excel coercions |
|
|
29
29
|
| `compiler.py` | cell mode: formula -> SQL in batches |
|
|
30
30
|
| `region.py` | a region -> one set-based statement (window functions for running totals) |
|
|
31
|
-
| `functions.py`, `fn/` | the
|
|
31
|
+
| `functions.py`, `fn/` | the 121 Excel functions (`fn/finance.py`, `fn/stats.py` new in 0.3.0; LET in `fn/logic.py`) |
|
|
32
|
+
| `tables.py` | structured references to Excel tables -> A1 text, applied by the reader |
|
|
32
33
|
| `verifier.py` | runs everything on DuckDB and compares with Excel's saved values |
|
|
33
34
|
| `emitter.py` | writes the dbt project; `--dbt` runs `dbt build` |
|
|
34
35
|
| `report.py`, `cli.py`, `compute.py` | REPORT.md/report.json, the CLI, the optional `formulas` answer key |
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.3.0 (2026-10-31)
|
|
4
|
+
|
|
5
|
+
- **Finance:** PMT, PV, FV, NPER, NPV. Microsoft's closed forms; NPV skips text, TRUE/FALSE and empty cells in a
|
|
6
|
+
reference without spending a period on them (checked against pycel's Excel-saved npv.xlsx).
|
|
7
|
+
- **Statistics:** LARGE, SMALL, RANK(.EQ), STDEV(.S/.P/P), VAR(.S/.P/P), PERCENTILE(.INC), QUARTILE(.INC). Real Excel
|
|
8
|
+
(formulas' test.xlsx) taught two rules: k's error comes before an error in the range, and QUARTILE checks quart
|
|
9
|
+
before truncating it (QUARTILE(x, -0.1) is #NUM!).
|
|
10
|
+
- **Working days:** NETWORKDAYS and WORKDAY, with a holidays range.
|
|
11
|
+
- **Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `#All`,
|
|
12
|
+
`#Data`, `#Headers`, `#Totals`, `Sales[[Rep]:[Amount]]`) are rewritten to A1 references by the reader; a
|
|
13
|
+
calculated table column compiles as a region.
|
|
14
|
+
- **LET**, nested LETs and a name bound to a range (`LET(r, A1:A9, SUM(r)/COUNT(r))`) included.
|
|
15
|
+
- **Not guessed:** edge cases of the new functions that were never checked in real Excel (zero periods, rate <= -100%,
|
|
16
|
+
payment type other than 0/1, a k that is not whole, ...) are UNRESOLVED with that reason.
|
|
17
|
+
- **Fixes found by the public corpus** (FAIL 92 -> 16, all 16 explained: 10 in LibreOffice files, 6 last-digit
|
|
18
|
+
cancellations around NPV): INDEX with a negative row or column is #VALUE!, not #REF!; SWITCH reads an empty cell as
|
|
19
|
+
0 (it matches 0, not FALSE or ""); a range where one value is expected inside an array formula is UNRESOLVED
|
|
20
|
+
instead of implicitly intersected (Excel evaluates it per element there).
|
|
21
|
+
- **Array arithmetic in aggregates**, the first slices of array evaluation: `SUM(A1:A10*B1:B10)`,
|
|
22
|
+
`SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`, `SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`,
|
|
23
|
+
`SUM(A1:A9^2)` and friends evaluate element by element like modern Excel - an empty element is 0, text converts
|
|
24
|
+
when it is a number and is #VALUE! otherwise, comparisons rank numbers < text < TRUE/FALSE case-insensitively,
|
|
25
|
+
concatenation writes numbers the Excel way, and the first error wins. SUM, SUMPRODUCT, MAX, MIN, AVERAGE, COUNT,
|
|
26
|
+
COUNTA and PRODUCT take arrays; IF, IFERROR and IFNA take them as their arguments. A single-cell array formula
|
|
27
|
+
(Ctrl+Shift+Enter) now compiles; one over several cells is UNRESOLVED. Plain-entered range arithmetic or IF as an
|
|
28
|
+
aggregate argument follows modern Excel (the element-wise answer), not the legacy first-row intersection - a
|
|
29
|
+
workbook an old Excel last saved can show FAILs there, with both values shown.
|
|
30
|
+
- **Approximate lookups on unsorted data** (MATCH type 1, VLOOKUP/HLOOKUP approximate, LOOKUP) over a plain numeric
|
|
31
|
+
input table: cellproof now compiles Excel's binary-search path itself - the answer is the last probe the search
|
|
32
|
+
moves right from, not the sorted bracket, and it matches Excel's own answers on files we checked (the 9.99E+307
|
|
33
|
+
last-number trick included). The SQL refuses table values that changed after translation (#STALE, UNRESOLVED)
|
|
34
|
+
instead of answering from a stale search. MATCH type -1, text, and tables with empty cells or formulas stay
|
|
35
|
+
UNRESOLVED: Excel's answer there is not derivable from the file.
|
|
36
|
+
- **Where to start:** the report groups the unresolved cells by the reason to fix first and ranks the reasons by
|
|
37
|
+
reach: a cell that reads an unresolved cell is attributed to the reason above it, so the top row of
|
|
38
|
+
"Where to start" names the fix that lets the most cells try again. report.json gains `unresolved_groups` and a
|
|
39
|
+
`blocked_by` root per unresolved cell.
|
|
40
|
+
|
|
41
|
+
## 0.2.2 (2026-10-27)
|
|
42
|
+
|
|
43
|
+
- **LOOKUP** (vector and array form). Like the other approximate lookups, UNRESOLVED on data out of order.
|
|
44
|
+
- **YEARFRAC**, all five bases, the basis written in the formula or computed. The 1,992 YEARFRAC cells of pycel's
|
|
45
|
+
test workbook all PASS; a date before 1 March 1900 (Excel's 29 February 1900) is UNRESOLVED.
|
|
46
|
+
- **Array constants** (`{0,50,100}`, `{1,"a";2,"b"}`) wherever a range is taken: `LOOKUP(x,{0,50,100},{"F","C","A"})`,
|
|
47
|
+
`MATCH(x,{...},0)`, `SUM({...})`. Used as one value (`A1*{1,2}`) they stay UNRESOLVED.
|
|
48
|
+
|
|
49
|
+
## 0.2.1 (2026-10-20)
|
|
50
|
+
|
|
51
|
+
Found by running all 85 public test workbooks of pycel, formulas and xlcalculator (`python tests/corpus.py DIR`):
|
|
52
|
+
FAIL 229 -> 10 (all 10 in files LibreOffice saved, which the report now says), no crashes.
|
|
53
|
+
|
|
54
|
+
- **Reading:** a defined name that is a formula of names (`=h+i`) no longer crashes the load; a date-formatted number
|
|
55
|
+
keeps its serial (Excel's 29 February 1900 and date-formatted negatives were lost); text escaped by the file format
|
|
56
|
+
(`_x000D_`, written for a carriage return) is decoded as Excel does.
|
|
57
|
+
- **AND/OR/XOR** skip text and empty cells in references, and typed text other than TRUE/FALSE, like Excel.
|
|
58
|
+
- **An empty or omitted result is 0:** `IF(c,,"x")`, `IFERROR(A1,"x")` with A1 empty.
|
|
59
|
+
- **EDATE/EOMONTH/MROUND/VALUE** refuse TRUE/FALSE (`#VALUE!`); `EOMONTH(0,0)` is 31.
|
|
60
|
+
- **SUMPRODUCT** of ranges of different sizes returns an error inside them before `#VALUE!`.
|
|
61
|
+
- **PROPER** accepts another cell's value (DuckDB refused the SQL) and capitalizes accented letters.
|
|
62
|
+
- **Locale-dependent text** (`"1,000"`, `"1/2/2020"`) is UNRESOLVED instead of a wrong `#VALUE!`.
|
|
63
|
+
- **Report:** warns when the file was not last saved by Excel; CHAR above 127 in a Mac-calculated file is UNRESOLVED.
|
|
64
|
+
|
|
65
|
+
From a user's practice workbook (3 PASS / 47 UNRESOLVED -> 48 PASS / 2 UNRESOLVED):
|
|
66
|
+
|
|
67
|
+
- **Implicit intersection:** a one-column or one-row range where one value is expected takes the formula's own row or
|
|
68
|
+
column (`=IF(E$3:E$41="F",B$3:B$41)`); outside it is `#VALUE!`. A range IF passes into another function is
|
|
69
|
+
UNRESOLVED (Excel gives that function the whole range).
|
|
70
|
+
- **`INDIRECT("Sales")`** of fixed text is the reference it names (dependencies included).
|
|
71
|
+
- **MODE / MODE.SNGL.**
|
|
72
|
+
- **A bare word is a name, never a column:** `=i` without a name `i` was read as column I.
|
|
73
|
+
|
|
74
|
+
## 0.2.0 (2026-10-19)
|
|
75
|
+
|
|
76
|
+
- **Renamed** from `unexcel` to `cellproof` (the old name clashed with an R package): `pip install cellproof`, command
|
|
77
|
+
`cellproof convert`, import `cellproof`, `CELLPROOF_PERF` for the benchmark, dbt project and database `cellproof`.
|
|
78
|
+
- **Lookups row-wise:** VLOOKUP, HLOOKUP, MATCH, INDEX and XLOOKUP pasted down a column compile into one statement.
|
|
79
|
+
1,000 VLOOKUPs: verify 5.7 s -> 0.1 s.
|
|
80
|
+
- **Criteria row-wise:** COUNTIFS (and SUMIF/AVERAGEIF(S) over a range holding an error) with a criterion from the row
|
|
81
|
+
no longer fall back to cell mode. 1,000 COUNTIFS: verify 29 s -> 0.1 s.
|
|
82
|
+
- **Faster compile:** a pasted-down formula is tokenized once per run. 200k-formula benchmark 31 s -> 26 s; 50k
|
|
83
|
+
VLOOKUPs 8.9 s -> 4.6 s.
|
|
84
|
+
- A lookup outside a region is about 20% slower than in 0.1.0 (one SQL form for both modes).
|
|
85
|
+
|
|
86
|
+
## 0.1.0 (2026-10-18)
|
|
87
|
+
|
|
88
|
+
First release, as `unexcel`.
|
cellproof-0.3.0/INFO.md
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# INFO: handoff for the next coding agent
|
|
2
|
+
|
|
3
|
+
You are taking over development of **cellproof**. Read this file first, then `AGENTS.md` (rules), `README.md` (what the
|
|
4
|
+
tool does and its limits) and `PROGRESS.md` (state and next steps). This file is the short version of all of them plus
|
|
5
|
+
the things that are not written down elsewhere.
|
|
6
|
+
|
|
7
|
+
Last updated: 2026-10-27. Current release: 0.2.2 (PyPI: `cellproof`). Repo: github.com/Nikhil414/cellproof (public).
|
|
8
|
+
|
|
9
|
+
## What we are building
|
|
10
|
+
|
|
11
|
+
cellproof turns an Excel workbook (`.xlsx`/`.xlsm`) into SQL and **proves every number matches**.
|
|
12
|
+
|
|
13
|
+
1. Read the formulas and the values Excel saved in the file (Excel's saved values are a free answer key).
|
|
14
|
+
2. Translate every formula to SQL (DuckDB dialect), with Excel's exact rules: type coercions, error codes, error order.
|
|
15
|
+
3. Run the SQL on DuckDB and compare each result with Excel's saved value.
|
|
16
|
+
4. Emit a dbt project that recomputes the workbook, plus `REPORT.md` / `report.json`.
|
|
17
|
+
|
|
18
|
+
Every formula cell gets one verdict:
|
|
19
|
+
|
|
20
|
+
- **PASS**: SQL result equals Excel's saved value.
|
|
21
|
+
- **FAIL**: SQL result differs; the report shows both values.
|
|
22
|
+
- **UNRESOLVED**: cellproof cannot translate it exactly; the report says why (the text of an `Unsupported` exception).
|
|
23
|
+
|
|
24
|
+
## End goal
|
|
25
|
+
|
|
26
|
+
A finance or analytics person who has business logic trapped in a spreadsheet can move it to a database and **show
|
|
27
|
+
evidence** that nothing changed. The product is trust, not coverage: an honest UNRESOLVED beats a guessed PASS.
|
|
28
|
+
|
|
29
|
+
Longer term:
|
|
30
|
+
|
|
31
|
+
- Cover the Excel features real business workbooks use (array formulas are the biggest missing piece).
|
|
32
|
+
- Stay fast on big workbooks (200k formulas verifies in about 26 s today).
|
|
33
|
+
- Plan 3 (not started): an AI retry loop for cells that come out UNRESOLVED or FAIL. It may see **formula structure
|
|
34
|
+
only, never cell values**.
|
|
35
|
+
- Other warehouses besides DuckDB for the dbt output (not started).
|
|
36
|
+
|
|
37
|
+
## Non-negotiable rules (from AGENTS.md)
|
|
38
|
+
|
|
39
|
+
- **Never guess a value.** Anything not translated exactly raises `Unsupported("<reason a user understands>")`.
|
|
40
|
+
- **A false PASS is the worst bug.** When unsure, produce an honest FAIL or UNRESOLVED.
|
|
41
|
+
- **No DuckDB exception may escape from a supported function.** Guard every domain with `CASE`.
|
|
42
|
+
- **Regions must equal cell mode.** `compile_book(book, regions=False)` is the reference; region tests compare to it.
|
|
43
|
+
- **Runtime dependencies stay `openpyxl` and `duckdb`.** Anything else goes in an optional extra.
|
|
44
|
+
- **Cell values never leave the machine.** No network calls with workbook data.
|
|
45
|
+
- **Tests first.** Console output is ASCII-only. Measure before claiming speed.
|
|
46
|
+
- **Real Excel is the ground truth.** The `formulas` library is only a labelled fallback answer key.
|
|
47
|
+
- Commits made by an agent end with a `Co-Authored-By:` trailer naming the agent.
|
|
48
|
+
|
|
49
|
+
## Where we are (2026-10-27)
|
|
50
|
+
|
|
51
|
+
Shipped and on `main` / PyPI 0.2.2:
|
|
52
|
+
|
|
53
|
+
- Full pipeline: reader, parser, dependency graph, cell-mode compiler, region (table) mode, verifier, dbt emitter, report, CLI.
|
|
54
|
+
- 96 Excel functions (list in README). Latest additions: LOOKUP, YEARFRAC (all 5 bases, dynamic basis), array constants
|
|
55
|
+
such as `{0,50,100}` wherever a range is taken.
|
|
56
|
+
- Region mode: a column repeating one formula compiles to one set-based statement; running totals become window
|
|
57
|
+
functions; VLOOKUP/HLOOKUP/MATCH/INDEX/XLOOKUP and the *IF(S) family pasted down compile to joins.
|
|
58
|
+
- Speed: 200k formulas in 26 s; 50k-row VLOOKUP in 4.6 s.
|
|
59
|
+
- Tests: about 430 pass, 8 skip (dbt tests skip when `dbt` is not on PATH). Whole suite takes 4 to 8 minutes.
|
|
60
|
+
- Validation: 3 workbooks saved by real Excel all PASS. 85 public test workbooks (pycel, formulas, xlcalculator):
|
|
61
|
+
6,874 PASS, 10 FAIL (all in LibreOffice-saved files, flagged in their reports), 16,764 UNRESOLVED, no crashes.
|
|
62
|
+
Run it with `python tests/corpus.py DIR`.
|
|
63
|
+
- CI: Windows, macOS and Linux x Python 3.10 to 3.13. Release = GitHub release, which triggers the PyPI publish workflow.
|
|
64
|
+
|
|
65
|
+
## What to do next (in order)
|
|
66
|
+
|
|
67
|
+
1. **Array formulas and array evaluation.** `SUM(A1:A10*B1:B10)`, `MAX(IF(c, A1:A9, 0))`, `A1*{1,2}`. Today these are
|
|
68
|
+
UNRESOLVED. This is the largest remaining source of UNRESOLVED cells in business workbooks. It needs element-wise
|
|
69
|
+
evaluation of ranges, which is a new design piece: write a short spec in `docs/superpowers/specs/` first, then a plan
|
|
70
|
+
in `docs/superpowers/plans/`, then build test-first. Existing building blocks you can reuse:
|
|
71
|
+
- Array constants are parsed into hidden-sheet `RefNode`s; the values live in an `ARRAYS` dict keyed by content hash.
|
|
72
|
+
The hidden sheet name is `#array-<hash>` (see gotchas for why).
|
|
73
|
+
- Regions already compile column-wise statements; element-wise range math may fit there.
|
|
74
|
+
2. **Test on more real analyst workbooks.** One user workbook found 4 gaps the 85 public files did not. Every real
|
|
75
|
+
workbook is worth more than another synthetic test. Turn each finding into a test in
|
|
76
|
+
`tests/test_corpus_findings.py`.
|
|
77
|
+
3. **Read time.** openpyxl loads the workbook twice (8.3 s of the 26 s). Also the per-cell dependency loop (11.3 s compile).
|
|
78
|
+
4. **Plan 3**: AI retry loop (structure only, never values).
|
|
79
|
+
|
|
80
|
+
Known smaller gaps are listed in `PROGRESS.md` under "Known gaps".
|
|
81
|
+
|
|
82
|
+
## Code map (`src/cellproof/`)
|
|
83
|
+
|
|
84
|
+
| Module | Job |
|
|
85
|
+
|---|---|
|
|
86
|
+
| `model.py` | `Cell`, `Book`, `Verdict`, `Unsupported` (its message becomes the UNRESOLVED reason) |
|
|
87
|
+
| `reader.py` | loads the workbook twice with openpyxl (formulas, saved values); refuses zip bombs |
|
|
88
|
+
| `parser.py`, `refs.py` | formula text to AST with Excel precedence; reference text to `Area` |
|
|
89
|
+
| `r1c1.py` | how a reference behaves when a formula is pasted down |
|
|
90
|
+
| `regions.py` | finds regions, running totals, splits on cycles |
|
|
91
|
+
| `graph.py`, `fenwick.py`, `rangeindex.py` | dependency order without expanding ranges; cycle detection |
|
|
92
|
+
| `expr.py` | `Expr` (SQL value + SQL error code), Excel coercions, sentinels such as `#1900` |
|
|
93
|
+
| `compiler.py` | cell mode: formula to SQL in batches |
|
|
94
|
+
| `region.py` | a region to one set-based statement |
|
|
95
|
+
| `functions.py`, `fn/` | the Excel functions (`fn/lookup.py`, `fn/dates.py`, ...) |
|
|
96
|
+
| `verifier.py` | runs everything on DuckDB and compares with Excel's saved values |
|
|
97
|
+
| `emitter.py` | writes the dbt project; `--dbt` runs `dbt build` |
|
|
98
|
+
| `report.py`, `cli.py`, `compute.py` | REPORT.md/report.json, the CLI, the optional `formulas` answer key |
|
|
99
|
+
|
|
100
|
+
## How to work here
|
|
101
|
+
|
|
102
|
+
Setup (Mac/Linux):
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
python -m venv .venv && . .venv/bin/activate
|
|
106
|
+
pip install -e ".[dev,dbt,compute]"
|
|
107
|
+
python -m pytest tests -q # dbt must be on PATH (venv active) or the dbt tests skip
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Workflow that has worked for every piece so far:
|
|
111
|
+
|
|
112
|
+
1. Write the failing tests first (TDD). Add a region-mode test next to every new function; it must equal cell mode.
|
|
113
|
+
2. Implement the smallest thing that is exactly right. If it cannot be exact, raise `Unsupported` with a plain reason.
|
|
114
|
+
3. Run the whole suite. After a change that can affect results, re-run the corpus and compare PASS/FAIL/UNRESOLVED
|
|
115
|
+
counts. FAIL must not go up.
|
|
116
|
+
4. Update `README.md` (function list and limitations), `CHANGELOG.md` (under "Unreleased") and `PROGRESS.md`.
|
|
117
|
+
5. When Excel's behaviour is uncertain, **do not guess from memory**. Remove the unverified test case, or find an
|
|
118
|
+
Excel-saved workbook that proves it. (Example: a YEARFRAC test with an unverified expected value was dropped.)
|
|
119
|
+
|
|
120
|
+
## Gotchas learned the hard way
|
|
121
|
+
|
|
122
|
+
- **Braces in sheet names break things.** Python `.format()` and dbt's Jinja both treat `{...}` as placeholders. The
|
|
123
|
+
array sheet used to be `{cellproof-array-hash}` and failed in region mode and in dbt; it is now `#array-<hash>`. Never
|
|
124
|
+
put `{` or `}` in generated names. Detect array sheets by membership in the `ARRAYS` dict, not by a string prefix.
|
|
125
|
+
- **Error order matters.** Excel reports errors left to right, and errors typed into a formula come before errors inside
|
|
126
|
+
referenced cells. YEARFRAC needed the basis error checked before date errors (52 corpus FAILs until fixed).
|
|
127
|
+
- **Parser tests that expect "unparseable" input go stale** when the parser learns new syntax (`{1,2}` was one). Pick a
|
|
128
|
+
form that is still invalid (`{1,A1}`).
|
|
129
|
+
- **Excel facts verified against real Excel** are in `PROGRESS.md` ("What Excel taught us") with tests in
|
|
130
|
+
`tests/test_excel_findings.py`. Read them before touching COUNTIF, DAYS360, TEXT, MOD, POWER or date code.
|
|
131
|
+
- **Approximate lookups on unsorted data** (LOOKUP, MATCH/VLOOKUP with type 1/-1) stay UNRESOLVED. Excel's answer there
|
|
132
|
+
depends on its binary search.
|
|
133
|
+
- **Text to number** only handles plain numbers. Locale-dependent text (`"1,000"`, `"$5"`, `"50%"`) is UNRESOLVED.
|
|
134
|
+
- Console output must stay ASCII (the tool is also used on Windows consoles).
|
|
135
|
+
- No linter is installed in the dev environment (ruff and pyflakes are missing). Keep the existing style by hand.
|
|
136
|
+
|
|
137
|
+
## Repo and release facts
|
|
138
|
+
|
|
139
|
+
- Public repo `Nikhil414/cellproof` (remote `origin`), single-commit-history start. **Never push the old history
|
|
140
|
+
publicly.** A private backup `Nikhil414/unexcel-dev` exists (remote `dev`, branch `public-main` mirrors `main`).
|
|
141
|
+
The project was called `unexcel` until 0.2.0; older plans and specs still use that name.
|
|
142
|
+
- Release: bump `version` in `pyproject.toml` and `src/cellproof/__init__.py`, date the CHANGELOG, run the tests,
|
|
143
|
+
commit, push `main`, then `gh release create vX.Y.Z`. The release triggers the PyPI publish workflow. Confirm CI is
|
|
144
|
+
green first.
|
|
145
|
+
- Only push, tag or release when the owner asks.
|
|
146
|
+
- The owner prefers decisions to questions: decide, build, test, security-check, and explain what you did.
|
|
147
|
+
|
|
148
|
+
## Safety checklist before every commit
|
|
149
|
+
|
|
150
|
+
- No workbook data, secrets or tokens in the repo, tests or logs.
|
|
151
|
+
- No new runtime dependency.
|
|
152
|
+
- No network call that carries workbook data.
|
|
153
|
+
- A false PASS is possible? Make it an UNRESOLVED instead.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cellproof
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Nikhil414/cellproof
|
|
6
6
|
Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
|
|
@@ -21,7 +21,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
21
21
|
Classifier: Topic :: Database
|
|
22
22
|
Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
|
|
23
23
|
Requires-Python: >=3.10
|
|
24
|
-
Requires-Dist: duckdb>=1.
|
|
24
|
+
Requires-Dist: duckdb>=1.3
|
|
25
25
|
Requires-Dist: openpyxl>=3.1
|
|
26
26
|
Provides-Extra: compute
|
|
27
27
|
Requires-Dist: formulas<1.4,>=1.3.4; extra == 'compute'
|
|
@@ -83,7 +83,8 @@ The output folder holds:
|
|
|
83
83
|
|
|
84
84
|
- **`REPORT.md` / `report.json`**: every verdict, failed cells with both values, unresolved cells with the reason,
|
|
85
85
|
findings (for example a number stored as text, which Excel skips in SUM), and which blocks of rows became readable
|
|
86
|
-
table models.
|
|
86
|
+
table models. The unresolved reasons are ranked by how many cells depend on them ("Where to start"), so a long list
|
|
87
|
+
of unresolved cells says what to fix first.
|
|
87
88
|
- **A dbt project** (`dbt_project.yml`, `models/`, `seeds/input_cells.csv`) that recomputes the workbook on DuckDB.
|
|
88
89
|
It runs exactly the SQL the verifier checked.
|
|
89
90
|
|
|
@@ -110,12 +111,18 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
110
111
|
code.
|
|
111
112
|
6. **Emit** the dbt project and the report.
|
|
112
113
|
|
|
113
|
-
|
|
114
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR,
|
|
115
|
-
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LEFT, LEN,
|
|
116
|
-
LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA,
|
|
117
|
-
|
|
118
|
-
|
|
114
|
+
121 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
115
|
+
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR, FV,
|
|
116
|
+
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
|
|
117
|
+
LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
|
|
118
|
+
NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
|
|
119
|
+
RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
120
|
+
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
121
|
+
YEAR, YEARFRAC.
|
|
122
|
+
|
|
123
|
+
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
124
|
+
`Sales[[Rep]:[Amount]]`) are rewritten to plain references when the file is read, so a calculated table column
|
|
125
|
+
compiles like any column pasted down.
|
|
119
126
|
|
|
120
127
|
**Speed** (Windows 11 laptop, measured): a 200,000-formula workbook (four formula columns over 50,000 rows, one of them
|
|
121
128
|
a running total) converts and verifies in 26 s, all PASS. 50,000 rows of VLOOKUP into a 1,000-row table: 4.6 s.
|
|
@@ -126,21 +133,38 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
|
|
|
126
133
|
cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
|
|
127
134
|
|
|
128
135
|
- **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
|
|
129
|
-
- **
|
|
136
|
+
- **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...)
|
|
137
|
+
and inside array arithmetic; where one value is expected (`A1*{1,2}`) they need array evaluation and are UNRESOLVED.
|
|
138
|
+
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
139
|
+
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
140
|
+
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
141
|
+
numbers < text < TRUE/FALSE case-insensitively, concatenation writes numbers the Excel way, and the first error
|
|
142
|
+
wins. A single-cell array formula (Ctrl+Shift+Enter) compiles; an array formula over several cells is UNRESOLVED
|
|
143
|
+
(spilling is not read yet). A plain-entered range arithmetic or IF as an aggregate argument follows modern Excel,
|
|
144
|
+
not the legacy first-row intersection - a workbook an old Excel last saved can show FAILs there, with both values
|
|
145
|
+
shown.
|
|
146
|
+
- **Not supported:** multi-cell array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
|
|
130
147
|
`INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
|
|
131
|
-
external workbooks, structured
|
|
148
|
+
external workbooks, a structured reference naming several separate columns (`T[[A],[C]]`), INDEX used as a reference, the 1904 date system in
|
|
132
149
|
TEXT date formats.
|
|
133
150
|
- **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
|
|
134
|
-
Excel does (implicit intersection).
|
|
135
|
-
|
|
136
|
-
- **Approximate lookups on unsorted data** (MATCH/VLOOKUP/HLOOKUP with match type 1
|
|
137
|
-
|
|
151
|
+
Excel does (implicit intersection). As an aggregate's argument the element-wise answer is compiled instead (see
|
|
152
|
+
array arithmetic above). IFERROR, IFS and CHOOSE over arrays are still UNRESOLVED.
|
|
153
|
+
- **Approximate lookups on unsorted data** (LOOKUP, MATCH/VLOOKUP/HLOOKUP with match type 1): compiled for plain
|
|
154
|
+
numeric tables written as inputs (up to about 1,000 values): the SQL carries Excel's binary-search path itself, and
|
|
155
|
+
refuses changed table values instead of answering from a stale search. MATCH with type -1, text tables, tables with
|
|
156
|
+
empty cells or formulas, and bigger tables stay UNRESOLVED, because Excel's answer there is not derivable from the
|
|
157
|
+
file (LibreOffice, the only open source that emulates the search, provably differs from Excel in places).
|
|
138
158
|
- **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Text Excel reads with the computer's language
|
|
139
159
|
settings (`"1,000"`, `"$5"`, `"50%"`, `"1/2/2020"`, `"2:00 PM"`, `"Jan 5"`) makes the cell UNRESOLVED, because the
|
|
140
160
|
file does not record those settings.
|
|
141
161
|
- **Who saved the file:** the report warns when it was last saved by another program (LibreOffice, a script): the
|
|
142
162
|
saved values then follow that program's rules. In a file Excel for Mac calculated, `CHAR` above 127 is UNRESOLVED
|
|
143
163
|
(the Mac character set differs).
|
|
164
|
+
- **Edge cases not checked in real Excel yet** are UNRESOLVED with that reason, never guessed: PMT/PV/FV/NPER with
|
|
165
|
+
zero periods, a rate of -100% or less or a payment type other than 0 or 1; NPV at a rate of -100%; LARGE/SMALL with
|
|
166
|
+
a k that is not whole; RANK over a range holding an error; NETWORKDAYS/WORKDAY with text or errors among the
|
|
167
|
+
holidays. NETWORKDAYS.INTL, WORKDAY.INTL, IRR, RATE and the other finance and statistics functions are not in yet.
|
|
144
168
|
- **Some function options:** SUBTOTAL 101-111 (hidden rows), DATEDIF units MD/YM/YD, WEEKDAY types 11-17, TEXT formats
|
|
145
169
|
outside the supported table, COUNT or MEDIAN of values typed into the formula, criteria taken from an empty cell.
|
|
146
170
|
- **Floating point:** a running total becomes a fast window function only when Excel's saved values prove the different
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# cellproof: progress
|
|
2
2
|
|
|
3
|
-
Last updated: 2026-10-
|
|
3
|
+
Last updated: 2026-10-31.
|
|
4
4
|
|
|
5
5
|
## State
|
|
6
6
|
|
|
@@ -18,17 +18,31 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
|
|
|
18
18
|
| Tokenize once | a formula pasted down a column is tokenized once; the next rows' text is predicted and only re-tokenized when it differs |
|
|
19
19
|
| Implicit intersection | a one-column range where one value is expected takes the formula's row; `INDIRECT` of fixed text; MODE (found in a user's workbook) |
|
|
20
20
|
| Criteria row-wise | the *IF(S) family with a criterion from the row (`=COUNTIFS($E:$E,">"&A2,...)`) compiles into the region's statement too |
|
|
21
|
+
| Impact-ranked unresolveds | the report groups UNRESOLVED cells by the reason to fix first ("Where to start"): the compiler records which unresolved cell stopped a compile (`Unsupported.dep`), each chained cell is attributed to the first blocker its chain reaches, reasons are ranked by reach; report.json gains `unresolved_groups` and per-cell `blocked_by` |
|
|
22
|
+
| Approximate lookups unsorted | MATCH type 1 / VLOOKUP / HLOOKUP / LOOKUP over a plain numeric input table out of order compiles Excel's binary-search path (a decision tree of probes, depths and x-intervals, one aggregate in SQL; verified differentially against the bisection and against real Excel answers). A changed table is #STALE -> UNRESOLVED, never a stale answer. Type -1 and text stay UNRESOLVED (Excel's rule there is not derivable; LibreOffice differs). examples/functions.xlsx: 4 UNRESOLVED -> 2 |
|
|
23
|
+
| Arrays slices 1+2 | element-wise arrays in the aggregates: `SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`. An array argument is one relation with the cells table's columns (a row per element position, blanks and all), which SUM/PRODUCT/MAX/MIN/AVERAGE/COUNT/COUNTA/SUMPRODUCT read like a range; IF picks one element per position (the untaken branch's error never shows). Text converts in arithmetic ("3"->3, else #VALUE!); comparisons rank numbers < text < TRUE/FALSE case-insensitively. SUM moved from functions.py onto the aggregate machinery. A single-cell CSE formula compiles (`Cell.ref` kept from the file); multi-cell stays UNRESOLVED. Plain-entered range arithmetic/IF as an aggregate argument now follows modern Excel, not the legacy intersection - a legacy-saved workbook can FAIL there (a real user workbook's MAX(IF(...))=99 artifact surfaces as a FAIL with both values shown). Next: IFERROR/IFS/CHOOSE over arrays, & and ^ on arrays, then legacy multi-cell array formulas and dynamic arrays |
|
|
24
|
+
| 0.3.0 functions | PMT, PV, FV, NPER, NPV; LARGE, SMALL, RANK(.EQ), STDEV/VAR (S/P), PERCENTILE(.INC), QUARTILE(.INC); NETWORKDAYS, WORKDAY. Edge cases not checked in real Excel are an UNCHECKED marker (UNRESOLVED), like LOCALE |
|
|
25
|
+
| Excel tables | structured references rewritten to A1 text by the reader (`tables.py`); [#This Row] becomes a same-row reference, so a calculated column is a region |
|
|
26
|
+
| LET | names (stored `_xlpm.x`) substituted by their value's tree, shadowing respected; the graph skips them as dependencies |
|
|
21
27
|
| Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
|
|
22
28
|
|
|
23
|
-
Tests:
|
|
24
|
-
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.0
|
|
29
|
+
Tests: 492 (+10 skipped without dbt/Excel files), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
|
|
30
|
+
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.0, 0.2.1, 0.2.2 and 0.3.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
|
|
25
31
|
A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
|
|
26
32
|
|
|
33
|
+
## Public test workbooks (2026-10-31)
|
|
34
|
+
|
|
35
|
+
After 0.3.0's functions, tables and LET: 7,528 PASS, 16 FAIL, 16,104 UNRESOLVED (was 7,108 / 92 / 16,448 on 0.2.2 +
|
|
36
|
+
arrays). The 16 FAIL: 10 in LibreOffice files, 6 in pycel's npv.xlsx where `=C7-A7` subtracts two numbers equal to
|
|
37
|
+
1e-13 relative (NPV's last digits differ from Excel's own arithmetic; every NPV cell itself PASSes). The 82 new FAILs
|
|
38
|
+
the arrays work exposed were INDEX with a negative index, SWITCH with an empty cell and ranges inside single-cell
|
|
39
|
+
array formulas - all fixed.
|
|
40
|
+
|
|
27
41
|
## Public test workbooks (2026-10-19)
|
|
28
42
|
|
|
29
43
|
`python tests/corpus.py DIR` runs the 85 .xlsx/.xlsm test files of pycel, formulas and xlcalculator (Excel-saved,
|
|
30
|
-
except 8 LibreOffice ones):
|
|
31
|
-
UNRESOLVED, no crashes. The UNRESOLVED are mostly functions those files test on purpose: CONVERT, YEARFRAC,
|
|
44
|
+
except 8 LibreOffice ones): 6,874 PASS, 10 FAIL (all in LibreOffice files, flagged in their reports), 16,764
|
|
45
|
+
UNRESOLVED, no crashes (was 4,693 PASS before LOOKUP, YEARFRAC and array constants). The UNRESOLVED are mostly functions those files test on purpose: CONVERT, YEARFRAC,
|
|
32
46
|
DEC2HEX/BIN/OCT, array formulas and `{1,2}` constants, CODE/UNICODE, LOOKUP. For business workbooks the ones that
|
|
33
47
|
matter are array formulas, array constants, LOOKUP and YEARFRAC. Findings are tests in `tests/test_corpus_findings.py`.
|
|
34
48
|
|
|
@@ -80,8 +94,9 @@ What Excel taught us (each has a test in `tests/test_excel_findings.py`):
|
|
|
80
94
|
|
|
81
95
|
## Next
|
|
82
96
|
|
|
83
|
-
1.
|
|
84
|
-
|
|
97
|
+
1. Array evaluation, next slices: IFS/CHOOSE over arrays, then legacy multi-cell array formulas and dynamic
|
|
98
|
+
arrays. (Done: element-wise arithmetic, comparisons, IF, IFERROR/IFNA, & and ^ inside the aggregates - see
|
|
99
|
+
Arrays slices 1-3 above.)
|
|
85
100
|
2. Test on more real analyst workbooks (one user workbook found 4 gaps the 85 public files did not).
|
|
86
101
|
3. Read time (openpyxl loads the workbook twice) and the per-cell dependency loop.
|
|
87
102
|
4. Plan 3: AI retry loop for failed cells (it sees formula structure only, never values).
|
|
@@ -49,7 +49,8 @@ The output folder holds:
|
|
|
49
49
|
|
|
50
50
|
- **`REPORT.md` / `report.json`**: every verdict, failed cells with both values, unresolved cells with the reason,
|
|
51
51
|
findings (for example a number stored as text, which Excel skips in SUM), and which blocks of rows became readable
|
|
52
|
-
table models.
|
|
52
|
+
table models. The unresolved reasons are ranked by how many cells depend on them ("Where to start"), so a long list
|
|
53
|
+
of unresolved cells says what to fix first.
|
|
53
54
|
- **A dbt project** (`dbt_project.yml`, `models/`, `seeds/input_cells.csv`) that recomputes the workbook on DuckDB.
|
|
54
55
|
It runs exactly the SQL the verifier checked.
|
|
55
56
|
|
|
@@ -76,12 +77,18 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
76
77
|
code.
|
|
77
78
|
6. **Emit** the dbt project and the report.
|
|
78
79
|
|
|
79
|
-
|
|
80
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR,
|
|
81
|
-
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LEFT, LEN,
|
|
82
|
-
LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA,
|
|
83
|
-
|
|
84
|
-
|
|
80
|
+
121 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
81
|
+
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR, FV,
|
|
82
|
+
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
|
|
83
|
+
LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
|
|
84
|
+
NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
|
|
85
|
+
RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
86
|
+
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
87
|
+
YEAR, YEARFRAC.
|
|
88
|
+
|
|
89
|
+
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
90
|
+
`Sales[[Rep]:[Amount]]`) are rewritten to plain references when the file is read, so a calculated table column
|
|
91
|
+
compiles like any column pasted down.
|
|
85
92
|
|
|
86
93
|
**Speed** (Windows 11 laptop, measured): a 200,000-formula workbook (four formula columns over 50,000 rows, one of them
|
|
87
94
|
a running total) converts and verifies in 26 s, all PASS. 50,000 rows of VLOOKUP into a 1,000-row table: 4.6 s.
|
|
@@ -92,21 +99,38 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
|
|
|
92
99
|
cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
|
|
93
100
|
|
|
94
101
|
- **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
|
|
95
|
-
- **
|
|
102
|
+
- **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...)
|
|
103
|
+
and inside array arithmetic; where one value is expected (`A1*{1,2}`) they need array evaluation and are UNRESOLVED.
|
|
104
|
+
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
105
|
+
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
106
|
+
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
107
|
+
numbers < text < TRUE/FALSE case-insensitively, concatenation writes numbers the Excel way, and the first error
|
|
108
|
+
wins. A single-cell array formula (Ctrl+Shift+Enter) compiles; an array formula over several cells is UNRESOLVED
|
|
109
|
+
(spilling is not read yet). A plain-entered range arithmetic or IF as an aggregate argument follows modern Excel,
|
|
110
|
+
not the legacy first-row intersection - a workbook an old Excel last saved can show FAILs there, with both values
|
|
111
|
+
shown.
|
|
112
|
+
- **Not supported:** multi-cell array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
|
|
96
113
|
`INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
|
|
97
|
-
external workbooks, structured
|
|
114
|
+
external workbooks, a structured reference naming several separate columns (`T[[A],[C]]`), INDEX used as a reference, the 1904 date system in
|
|
98
115
|
TEXT date formats.
|
|
99
116
|
- **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
|
|
100
|
-
Excel does (implicit intersection).
|
|
101
|
-
|
|
102
|
-
- **Approximate lookups on unsorted data** (MATCH/VLOOKUP/HLOOKUP with match type 1
|
|
103
|
-
|
|
117
|
+
Excel does (implicit intersection). As an aggregate's argument the element-wise answer is compiled instead (see
|
|
118
|
+
array arithmetic above). IFERROR, IFS and CHOOSE over arrays are still UNRESOLVED.
|
|
119
|
+
- **Approximate lookups on unsorted data** (LOOKUP, MATCH/VLOOKUP/HLOOKUP with match type 1): compiled for plain
|
|
120
|
+
numeric tables written as inputs (up to about 1,000 values): the SQL carries Excel's binary-search path itself, and
|
|
121
|
+
refuses changed table values instead of answering from a stale search. MATCH with type -1, text tables, tables with
|
|
122
|
+
empty cells or formulas, and bigger tables stay UNRESOLVED, because Excel's answer there is not derivable from the
|
|
123
|
+
file (LibreOffice, the only open source that emulates the search, provably differs from Excel in places).
|
|
104
124
|
- **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Text Excel reads with the computer's language
|
|
105
125
|
settings (`"1,000"`, `"$5"`, `"50%"`, `"1/2/2020"`, `"2:00 PM"`, `"Jan 5"`) makes the cell UNRESOLVED, because the
|
|
106
126
|
file does not record those settings.
|
|
107
127
|
- **Who saved the file:** the report warns when it was last saved by another program (LibreOffice, a script): the
|
|
108
128
|
saved values then follow that program's rules. In a file Excel for Mac calculated, `CHAR` above 127 is UNRESOLVED
|
|
109
129
|
(the Mac character set differs).
|
|
130
|
+
- **Edge cases not checked in real Excel yet** are UNRESOLVED with that reason, never guessed: PMT/PV/FV/NPER with
|
|
131
|
+
zero periods, a rate of -100% or less or a payment type other than 0 or 1; NPV at a rate of -100%; LARGE/SMALL with
|
|
132
|
+
a k that is not whole; RANK over a range holding an error; NETWORKDAYS/WORKDAY with text or errors among the
|
|
133
|
+
holidays. NETWORKDAYS.INTL, WORKDAY.INTL, IRR, RATE and the other finance and statistics functions are not in yet.
|
|
110
134
|
- **Some function options:** SUBTOTAL 101-111 (hidden rows), DATEDIF units MD/YM/YD, WEEKDAY types 11-17, TEXT formats
|
|
111
135
|
outside the supported table, COUNT or MEDIAN of values typed into the formula, criteria taken from an empty cell.
|
|
112
136
|
- **Floating point:** a running total becomes a fast window function only when Excel's saved values prove the different
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cellproof"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -26,7 +26,7 @@ classifiers = [
|
|
|
26
26
|
"Topic :: Database",
|
|
27
27
|
"Topic :: Office/Business :: Financial :: Spreadsheet",
|
|
28
28
|
]
|
|
29
|
-
dependencies = ["openpyxl>=3.1", "duckdb>=1.
|
|
29
|
+
dependencies = ["openpyxl>=3.1", "duckdb>=1.3"]
|
|
30
30
|
|
|
31
31
|
[project.optional-dependencies]
|
|
32
32
|
dev = ["pytest>=8", "jinja2>=3.1"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|
|
@@ -41,7 +41,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
41
41
|
return 3
|
|
42
42
|
out = args.out or args.workbook.with_name(args.workbook.stem + "_dbt")
|
|
43
43
|
write_project(book, compiled, out, args.workbook.stem)
|
|
44
|
-
counts = write_report(verdicts, compiled.findings, out, args.workbook.name, computed, compiled.notes
|
|
44
|
+
counts = write_report(verdicts, compiled.findings, out, args.workbook.name, computed, compiled.notes,
|
|
45
|
+
compiled.chains)
|
|
45
46
|
if computed:
|
|
46
47
|
print(f"{len(computed)} expected values computed by the formulas library (the file had none saved)")
|
|
47
48
|
print(f"{counts['PASS']} pass, {counts['FAIL']} fail, {counts['UNRESOLVED']} unresolved -> {out / 'REPORT.md'}")
|