cellproof 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cellproof-0.2.0 → cellproof-0.2.1}/AGENTS.md +1 -1
- cellproof-0.2.1/CHANGELOG.md +42 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/PKG-INFO +16 -8
- {cellproof-0.2.0 → cellproof-0.2.1}/PROGRESS.md +18 -6
- {cellproof-0.2.0 → cellproof-0.2.1}/README.md +15 -7
- cellproof-0.2.1/docs/JOURNEY.md +266 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/pyproject.toml +1 -1
- cellproof-0.2.1/src/cellproof/__init__.py +1 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/compiler.py +43 -2
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/expr.py +38 -2
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/aggregate.py +17 -1
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/dates.py +7 -5
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/logic.py +7 -8
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/maths.py +4 -1
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/text.py +9 -5
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/functions.py +38 -9
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/model.py +1 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/parser.py +8 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/r1c1.py +33 -3
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/reader.py +64 -6
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/refs.py +2 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/verifier.py +4 -0
- cellproof-0.2.1/tests/corpus.py +50 -0
- cellproof-0.2.1/tests/test_corpus_findings.py +142 -0
- cellproof-0.2.1/tests/test_implicit.py +77 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_lexed.py +1 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_reader.py +17 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_text.py +2 -1
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_types.py +1 -1
- cellproof-0.2.0/CHANGELOG.md +0 -17
- cellproof-0.2.0/src/cellproof/__init__.py +0 -1
- {cellproof-0.2.0 → cellproof-0.2.1}/.github/workflows/ci.yml +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/.github/workflows/publish.yml +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/.gitignore +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/DESIGN.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/LICENSE +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/PLAN-2a.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/PLAN-2b.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/PLAN.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/SECURITY.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/examples/commission.xlsx +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/examples/errors.xlsx +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/examples/functions.xlsx +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/examples/functions_unsaved.xlsx +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/examples/make_commission.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/examples/make_errors.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/examples/make_functions.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/__main__.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/cli.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/compute.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/emitter.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fenwick.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/__init__.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/criteria.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/fn/lookup.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/graph.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/rangeindex.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/region.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/regions.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/src/cellproof/report.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/bench_shapes.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_aggregate.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_batches.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_cli.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_compiler.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_compute.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_criteria.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_dates.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_emitter.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_errors.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_excel_findings.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_excel_saved.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_fenwick.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_functions.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_graph.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_index.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_limits.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_logic.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_lookup.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_lookups.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_maths.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_parser.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_perf.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_r1c1.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_rangeindex.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_refs.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_region.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_regions.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_regions_differential.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_regions_split.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_report.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_review_regressions.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_running.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_shapes.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/test_verifier.py +0 -0
- {cellproof-0.2.0 → cellproof-0.2.1}/tests/xlsxkit.py +0 -0
|
@@ -28,7 +28,7 @@ CELLPROOF_PERF=1 python -m pytest tests/test_perf.py -s # 200k-formula benchmar
|
|
|
28
28
|
| `expr.py` | `Expr` (SQL value + SQL error code), Excel coercions |
|
|
29
29
|
| `compiler.py` | cell mode: formula -> SQL in batches |
|
|
30
30
|
| `region.py` | a region -> one set-based statement (window functions for running totals) |
|
|
31
|
-
| `functions.py`, `fn/` | the
|
|
31
|
+
| `functions.py`, `fn/` | the 94 Excel functions |
|
|
32
32
|
| `verifier.py` | runs everything on DuckDB and compares with Excel's saved values |
|
|
33
33
|
| `emitter.py` | writes the dbt project; `--dbt` runs `dbt build` |
|
|
34
34
|
| `report.py`, `cli.py`, `compute.py` | REPORT.md/report.json, the CLI, the optional `formulas` answer key |
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.2.1 (2026-10-20)
|
|
4
|
+
|
|
5
|
+
Found by running all 85 public test workbooks of pycel, formulas and xlcalculator (`python tests/corpus.py DIR`):
|
|
6
|
+
FAIL 229 -> 10 (all 10 in files LibreOffice saved, which the report now says), no crashes.
|
|
7
|
+
|
|
8
|
+
- **Reading:** a defined name that is a formula of names (`=h+i`) no longer crashes the load; a date-formatted number
|
|
9
|
+
keeps its serial (Excel's 29 February 1900 and date-formatted negatives were lost); text escaped by the file format
|
|
10
|
+
(`_x000D_`, written for a carriage return) is decoded as Excel does.
|
|
11
|
+
- **AND/OR/XOR** skip text and empty cells in references, and typed text other than TRUE/FALSE, like Excel.
|
|
12
|
+
- **An empty or omitted result is 0:** `IF(c,,"x")`, `IFERROR(A1,"x")` with A1 empty.
|
|
13
|
+
- **EDATE/EOMONTH/MROUND/VALUE** refuse TRUE/FALSE (`#VALUE!`); `EOMONTH(0,0)` is 31.
|
|
14
|
+
- **SUMPRODUCT** of ranges of different sizes returns an error inside them before `#VALUE!`.
|
|
15
|
+
- **PROPER** accepts another cell's value (DuckDB refused the SQL) and capitalizes accented letters.
|
|
16
|
+
- **Locale-dependent text** (`"1,000"`, `"1/2/2020"`) is UNRESOLVED instead of a wrong `#VALUE!`.
|
|
17
|
+
- **Report:** warns when the file was not last saved by Excel; CHAR above 127 in a Mac-calculated file is UNRESOLVED.
|
|
18
|
+
|
|
19
|
+
From a user's practice workbook (3 PASS / 47 UNRESOLVED -> 48 PASS / 2 UNRESOLVED):
|
|
20
|
+
|
|
21
|
+
- **Implicit intersection:** a one-column or one-row range where one value is expected takes the formula's own row or
|
|
22
|
+
column (`=IF(E$3:E$41="F",B$3:B$41)`); outside it is `#VALUE!`. A range IF passes into another function is
|
|
23
|
+
UNRESOLVED (Excel gives that function the whole range).
|
|
24
|
+
- **`INDIRECT("Sales")`** of fixed text is the reference it names (dependencies included).
|
|
25
|
+
- **MODE / MODE.SNGL.**
|
|
26
|
+
- **A bare word is a name, never a column:** `=i` without a name `i` was read as column I.
|
|
27
|
+
|
|
28
|
+
## 0.2.0 (2026-10-19)
|
|
29
|
+
|
|
30
|
+
- **Renamed** from `unexcel` to `cellproof` (the old name clashed with an R package): `pip install cellproof`, command
|
|
31
|
+
`cellproof convert`, import `cellproof`, `CELLPROOF_PERF` for the benchmark, dbt project and database `cellproof`.
|
|
32
|
+
- **Lookups row-wise:** VLOOKUP, HLOOKUP, MATCH, INDEX and XLOOKUP pasted down a column compile into one statement.
|
|
33
|
+
1,000 VLOOKUPs: verify 5.7 s -> 0.1 s.
|
|
34
|
+
- **Criteria row-wise:** COUNTIFS (and SUMIF/AVERAGEIF(S) over a range holding an error) with a criterion from the row
|
|
35
|
+
no longer fall back to cell mode. 1,000 COUNTIFS: verify 29 s -> 0.1 s.
|
|
36
|
+
- **Faster compile:** a pasted-down formula is tokenized once per run. 200k-formula benchmark 31 s -> 26 s; 50k
|
|
37
|
+
VLOOKUPs 8.9 s -> 4.6 s.
|
|
38
|
+
- A lookup outside a region is about 20% slower than in 0.1.0 (one SQL form for both modes).
|
|
39
|
+
|
|
40
|
+
## 0.1.0 (2026-10-18)
|
|
41
|
+
|
|
42
|
+
First release, as `unexcel`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cellproof
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Nikhil414/cellproof
|
|
6
6
|
Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
|
|
@@ -110,10 +110,10 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
110
110
|
code.
|
|
111
111
|
6. **Emit** the dbt project and the report.
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
94 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
114
114
|
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR,
|
|
115
115
|
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LEFT, LEN, LN, LOG,
|
|
116
|
-
LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
|
|
116
|
+
LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
|
|
117
117
|
PROPER, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SQRT, SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
118
118
|
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VLOOKUP, WEEKDAY, XLOOKUP, XOR, YEAR.
|
|
119
119
|
|
|
@@ -126,13 +126,21 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
|
|
|
126
126
|
cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
|
|
127
127
|
|
|
128
128
|
- **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
|
|
129
|
-
- **Not supported:** array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`,
|
|
130
|
-
`INDIRECT`
|
|
131
|
-
INDEX used as a reference, the 1904 date system in
|
|
129
|
+
- **Not supported:** array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
|
|
130
|
+
`INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
|
|
131
|
+
external workbooks, structured table references (`Table1[@Col]`), INDEX used as a reference, the 1904 date system in
|
|
132
|
+
TEXT date formats.
|
|
133
|
+
- **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
|
|
134
|
+
Excel does (implicit intersection). A range that IF/IFERROR/CHOOSE hands to another function
|
|
135
|
+
(`MAX(IF(c,A1:A9,0))`, where Excel uses the whole range) is UNRESOLVED.
|
|
132
136
|
- **Approximate lookups on unsorted data** (MATCH/VLOOKUP/HLOOKUP with match type 1/-1): UNRESOLVED, because Excel's
|
|
133
137
|
answer then depends on its binary search.
|
|
134
|
-
- **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Excel
|
|
135
|
-
|
|
138
|
+
- **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Text Excel reads with the computer's language
|
|
139
|
+
settings (`"1,000"`, `"$5"`, `"50%"`, `"1/2/2020"`, `"2:00 PM"`, `"Jan 5"`) makes the cell UNRESOLVED, because the
|
|
140
|
+
file does not record those settings.
|
|
141
|
+
- **Who saved the file:** the report warns when it was last saved by another program (LibreOffice, a script): the
|
|
142
|
+
saved values then follow that program's rules. In a file Excel for Mac calculated, `CHAR` above 127 is UNRESOLVED
|
|
143
|
+
(the Mac character set differs).
|
|
136
144
|
- **Some function options:** SUBTOTAL 101-111 (hidden rows), DATEDIF units MD/YM/YD, WEEKDAY types 11-17, TEXT formats
|
|
137
145
|
outside the supported table, COUNT or MEDIAN of values typed into the formula, criteria taken from an empty cell.
|
|
138
146
|
- **Floating point:** a running total becomes a fast window function only when Excel's saved values prove the different
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# cellproof: progress
|
|
2
2
|
|
|
3
|
-
Last updated: 2026-10-
|
|
3
|
+
Last updated: 2026-10-20.
|
|
4
4
|
|
|
5
5
|
## State
|
|
6
6
|
|
|
@@ -16,11 +16,21 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
|
|
|
16
16
|
| Slice 2 | a region with a cell between its columns splits instead of dissolving; running totals (`=C1+B2`, `=SUM($B$2:B2)`) become window functions |
|
|
17
17
|
| Lookups row-wise | VLOOKUP, HLOOKUP, MATCH, INDEX, XLOOKUP pasted down compile into the region's statement (one join each instead of one query per cell) |
|
|
18
18
|
| Tokenize once | a formula pasted down a column is tokenized once; the next rows' text is predicted and only re-tokenized when it differs |
|
|
19
|
+
| Implicit intersection | a one-column range where one value is expected takes the formula's row; `INDIRECT` of fixed text; MODE (found in a user's workbook) |
|
|
19
20
|
| Criteria row-wise | the *IF(S) family with a criterion from the row (`=COUNTIFS($E:$E,">"&A2,...)`) compiles into the region's statement too |
|
|
20
21
|
| Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
|
|
21
22
|
|
|
22
|
-
Tests: about
|
|
23
|
-
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.0
|
|
23
|
+
Tests: about 425 (dbt build included), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
|
|
24
|
+
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.0 (2026-10-19) and 0.2.1 (2026-10-20) are on PyPI as `cellproof` (see CHANGELOG.md).
|
|
25
|
+
A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
|
|
26
|
+
|
|
27
|
+
## Public test workbooks (2026-10-19)
|
|
28
|
+
|
|
29
|
+
`python tests/corpus.py DIR` runs the 85 .xlsx/.xlsm test files of pycel, formulas and xlcalculator (Excel-saved,
|
|
30
|
+
except 8 LibreOffice ones): 4,693 PASS, 10 FAIL (all in LibreOffice files, flagged in their reports), 18,945
|
|
31
|
+
UNRESOLVED, no crashes. The UNRESOLVED are mostly functions those files test on purpose: CONVERT, YEARFRAC,
|
|
32
|
+
DEC2HEX/BIN/OCT, array formulas and `{1,2}` constants, CODE/UNICODE, LOOKUP. For business workbooks the ones that
|
|
33
|
+
matter are array formulas, array constants, LOOKUP and YEARFRAC. Findings are tests in `tests/test_corpus_findings.py`.
|
|
24
34
|
|
|
25
35
|
## Measured speed (Windows 11 laptop)
|
|
26
36
|
|
|
@@ -70,6 +80,8 @@ What Excel taught us (each has a test in `tests/test_excel_findings.py`):
|
|
|
70
80
|
|
|
71
81
|
## Next
|
|
72
82
|
|
|
73
|
-
1.
|
|
74
|
-
|
|
75
|
-
|
|
83
|
+
1. Close the UNRESOLVED gaps business workbooks hit: LOOKUP, YEARFRAC, array constants (`{1,2,3}`), simple array
|
|
84
|
+
formulas (`SUM(A1:A10*B1:B10)`, `MAX(IF(...))`).
|
|
85
|
+
2. Test on more real analyst workbooks (one user workbook found 4 gaps the 85 public files did not).
|
|
86
|
+
3. Read time (openpyxl loads the workbook twice) and the per-cell dependency loop.
|
|
87
|
+
4. Plan 3: AI retry loop for failed cells (it sees formula structure only, never values).
|
|
@@ -76,10 +76,10 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
76
76
|
code.
|
|
77
77
|
6. **Emit** the dbt project and the report.
|
|
78
78
|
|
|
79
|
-
|
|
79
|
+
94 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
80
80
|
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR,
|
|
81
81
|
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LEFT, LEN, LN, LOG,
|
|
82
|
-
LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
|
|
82
|
+
LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
|
|
83
83
|
PROPER, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SQRT, SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
84
84
|
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VLOOKUP, WEEKDAY, XLOOKUP, XOR, YEAR.
|
|
85
85
|
|
|
@@ -92,13 +92,21 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
|
|
|
92
92
|
cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
|
|
93
93
|
|
|
94
94
|
- **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
|
|
95
|
-
- **Not supported:** array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`,
|
|
96
|
-
`INDIRECT`
|
|
97
|
-
INDEX used as a reference, the 1904 date system in
|
|
95
|
+
- **Not supported:** array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
|
|
96
|
+
`INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
|
|
97
|
+
external workbooks, structured table references (`Table1[@Col]`), INDEX used as a reference, the 1904 date system in
|
|
98
|
+
TEXT date formats.
|
|
99
|
+
- **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
|
|
100
|
+
Excel does (implicit intersection). A range that IF/IFERROR/CHOOSE hands to another function
|
|
101
|
+
(`MAX(IF(c,A1:A9,0))`, where Excel uses the whole range) is UNRESOLVED.
|
|
98
102
|
- **Approximate lookups on unsorted data** (MATCH/VLOOKUP/HLOOKUP with match type 1/-1): UNRESOLVED, because Excel's
|
|
99
103
|
answer then depends on its binary search.
|
|
100
|
-
- **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Excel
|
|
101
|
-
|
|
104
|
+
- **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Text Excel reads with the computer's language
|
|
105
|
+
settings (`"1,000"`, `"$5"`, `"50%"`, `"1/2/2020"`, `"2:00 PM"`, `"Jan 5"`) makes the cell UNRESOLVED, because the
|
|
106
|
+
file does not record those settings.
|
|
107
|
+
- **Who saved the file:** the report warns when it was last saved by another program (LibreOffice, a script): the
|
|
108
|
+
saved values then follow that program's rules. In a file Excel for Mac calculated, `CHAR` above 127 is UNRESOLVED
|
|
109
|
+
(the Mac character set differs).
|
|
102
110
|
- **Some function options:** SUBTOTAL 101-111 (hidden rows), DATEDIF units MD/YM/YD, WEEKDAY types 11-17, TEXT formats
|
|
103
111
|
outside the supported table, COUNT or MEDIAN of values typed into the formula, criteria taken from an empty cell.
|
|
104
112
|
- **Floating point:** a running total becomes a fast window function only when Excel's saved values prove the different
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
# cellproof: the whole journey, from idea to today
|
|
2
|
+
|
|
3
|
+
This file tells the story of the project from the first design (2026-10-11) to now (2026-10-20): what was built, in
|
|
4
|
+
what order, why, what broke, and what was learned. The technical details of each step live in `DESIGN.md`,
|
|
5
|
+
`docs/superpowers/specs/`, `docs/superpowers/plans/`, `PROGRESS.md` and `CHANGELOG.md`; this is the readable summary.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1. The idea (2026-10-11)
|
|
10
|
+
|
|
11
|
+
**Problem.** A lot of business logic lives in Excel workbooks that one person understands. Moving that logic into a
|
|
12
|
+
database is slow and risky, because nobody can prove that the new numbers equal the old ones. Existing tools copy the
|
|
13
|
+
*data* out of Excel; none translated the *formulas*.
|
|
14
|
+
|
|
15
|
+
**Idea.** A command-line tool that:
|
|
16
|
+
|
|
17
|
+
1. reads every formula of an `.xlsx`,
|
|
18
|
+
2. translates each one into SQL,
|
|
19
|
+
3. runs that SQL on DuckDB (a fast local database),
|
|
20
|
+
4. compares every result with the value Excel itself saved in the file.
|
|
21
|
+
|
|
22
|
+
Excel's saved values are a free answer key. Every formula cell gets one verdict:
|
|
23
|
+
|
|
24
|
+
| Verdict | Meaning |
|
|
25
|
+
|---|---|
|
|
26
|
+
| PASS | the SQL gives the value Excel saved |
|
|
27
|
+
| FAIL | the SQL gives a different value; the report shows both |
|
|
28
|
+
| UNRESOLVED | the tool could not translate it exactly, and says why. It never guesses. |
|
|
29
|
+
|
|
30
|
+
The output is a **dbt project** (the standard way data teams ship SQL), so the result is something a team can use.
|
|
31
|
+
|
|
32
|
+
**Rules decided on day one** (they still hold):
|
|
33
|
+
|
|
34
|
+
- Never guess a value. Anything not translated exactly is UNRESOLVED with a reason a user understands.
|
|
35
|
+
- A false PASS is the worst possible bug. When unsure, give an honest FAIL or UNRESOLVED.
|
|
36
|
+
- Real Excel is the ground truth, not our idea of what Excel does.
|
|
37
|
+
- Cell values never leave the machine. No network calls with workbook data.
|
|
38
|
+
- Only two runtime dependencies: `openpyxl` (reads the file) and `duckdb` (runs the SQL).
|
|
39
|
+
- Tests first, and measure before claiming speed.
|
|
40
|
+
|
|
41
|
+
The working name was **unexcel**, already known to clash with an R package.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 2. Timeline at a glance
|
|
46
|
+
|
|
47
|
+
| Date | Step | Result |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| 2026-10-11 | Design + Plan 1: the core | reader, parser, references, dependency order, SQL compiler, verifier, dbt output, report, CLI |
|
|
50
|
+
| 2026-10-11 | Plan 2a: scale and Excel errors | batches (no crash at ~1k formulas), Excel error codes, safety limits |
|
|
51
|
+
| 2026-10-12 | Plan 2b: function library | 93 Excel functions, run-time types, Excel's conversions |
|
|
52
|
+
| 2026-10-14 | First check against real Excel | 3 workbooks saved by real Excel: every cell PASS (4 UNRESOLVED on purpose) |
|
|
53
|
+
| 2026-10-14 | Plan 2c: speed in cell mode | faster dependency order, range index, smarter batches |
|
|
54
|
+
| 2026-10-16 | Plan 2d: table mode | a column of one pasted-down formula becomes one readable SQL statement |
|
|
55
|
+
| 2026-10-18 | Slice 2 | regions split instead of dissolving; running totals as window functions |
|
|
56
|
+
| 2026-10-18 | Open-source release 0.1.0 | public GitHub repo (fresh history), MIT license, CI, PyPI as `unexcel` |
|
|
57
|
+
| 2026-10-19 | Lookups and criteria row-wise | VLOOKUP/INDEX/MATCH/XLOOKUP and COUNTIFS/SUMIFS in table mode |
|
|
58
|
+
| 2026-10-19 | Faster compile | a pasted-down formula is tokenized once |
|
|
59
|
+
| 2026-10-19 | Renamed to **cellproof**, 0.2.0 | new name, new PyPI package, repo renamed |
|
|
60
|
+
| 2026-10-19 | 85 public test workbooks | FAIL 229 -> 10 (all 10 in LibreOffice-saved files), 0 crashes |
|
|
61
|
+
| 2026-10-19 | First real user workbook | 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED |
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## 3. Step by step
|
|
66
|
+
|
|
67
|
+
### 3.1 Plan 1: the core (2026-10-11)
|
|
68
|
+
|
|
69
|
+
Built test-first, one module at a time:
|
|
70
|
+
|
|
71
|
+
- **Reader**: loads the workbook twice with openpyxl, once for formulas and once for the values Excel saved.
|
|
72
|
+
- **Parser**: formula text to a tree, with Excel's operator precedence (`-2^2` is 4 in Excel).
|
|
73
|
+
- **References**: `A1`, `Sheet!B2:C9`, whole columns, defined names.
|
|
74
|
+
- **Dependency order**: every cell after the cells it reads; circular references detected.
|
|
75
|
+
- **Compiler (cell mode)**: each formula becomes a SQL expression.
|
|
76
|
+
- **Verifier**: runs everything on DuckDB and compares with Excel's saved values.
|
|
77
|
+
- **Emitter**: writes the dbt project; `--dbt` also runs `dbt build`.
|
|
78
|
+
- **Report**: `REPORT.md` and `report.json`.
|
|
79
|
+
- **CLI**: `unexcel convert workbook.xlsx`.
|
|
80
|
+
|
|
81
|
+
The first end-to-end test was a small commission workbook.
|
|
82
|
+
|
|
83
|
+
### 3.2 Plan 2a: scale and Excel errors (2026-10-11)
|
|
84
|
+
|
|
85
|
+
- The first version crashed around 1,000 formulas (one giant SQL statement). Fix: run formulas in **batches** over one
|
|
86
|
+
shared cells table.
|
|
87
|
+
- Excel **error codes** (`#DIV/0!`, `#N/A`, `#VALUE!`...) now flow through operators and functions, and must match
|
|
88
|
+
exactly.
|
|
89
|
+
- Safety: a deeply nested formula or one whose SQL would grow exponentially becomes UNRESOLVED instead of crashing or
|
|
90
|
+
eating memory; a styled cell on the last row no longer costs seconds or gigabytes.
|
|
91
|
+
- Optional fallback answer key (the `formulas` library) for files with no saved values, always labelled as such.
|
|
92
|
+
- A 200,000-formula benchmark was added: 390 s at that point.
|
|
93
|
+
|
|
94
|
+
### 3.3 Plan 2b: the function library (2026-10-12)
|
|
95
|
+
|
|
96
|
+
- **93 Excel functions** (SUM, IF, VLOOKUP, XLOOKUP, INDEX/MATCH, SUMIFS, TEXT, dates, ...).
|
|
97
|
+
- Run-time types: a cell's type can be known only when the SQL runs ("any").
|
|
98
|
+
- One criteria engine for the whole COUNTIF/SUMIF/AVERAGEIF(S) family (`">"&B1`, wildcards, `"<>"`).
|
|
99
|
+
|
|
100
|
+
### 3.4 First check against real Excel (2026-10-14)
|
|
101
|
+
|
|
102
|
+
Three workbooks were saved by real Microsoft Excel and checked cell by cell. Excel taught us things no document says,
|
|
103
|
+
each now a test (`tests/test_excel_findings.py`):
|
|
104
|
+
|
|
105
|
+
- COUNTIF `"<>5"` skips only the number 5 (the text "5" is counted).
|
|
106
|
+
- DAYS360, `TEXT(-0.001,"0.00")`, `MOD(1,0.1)`, dates inside 1900, `(-8)^(1/3)`, error order in `SUM(H3,1/0)`.
|
|
107
|
+
- Approximate MATCH/VLOOKUP on unsorted data depends on Excel's binary search, so it is UNRESOLVED.
|
|
108
|
+
|
|
109
|
+
Result: every cell of the three workbooks PASS, 4 UNRESOLVED on purpose.
|
|
110
|
+
|
|
111
|
+
### 3.5 Plan 2c: speed in cell mode (2026-10-14)
|
|
112
|
+
|
|
113
|
+
Measured first, then four changes: a Fenwick tree for dependency order without expanding ranges, a range index,
|
|
114
|
+
batches closed by SQL size instead of cell count, smaller dbt models.
|
|
115
|
+
|
|
116
|
+
### 3.6 Plan 2d: table mode (2026-10-16)
|
|
117
|
+
|
|
118
|
+
Most real sheets repeat one formula down a column (`=B2*C2`, `=B3*C3`, ...). Such a column is now a **region**:
|
|
119
|
+
compiled once into one readable SQL statement (a "wide model" with real column names), instead of one expression per
|
|
120
|
+
cell. Hard parts solved along the way: safe SQL names from headers, regions that would create false cycles, proving
|
|
121
|
+
that table mode gives exactly the same answers as cell mode (a test rule since then).
|
|
122
|
+
|
|
123
|
+
### 3.7 Slice 2 (2026-10-18)
|
|
124
|
+
|
|
125
|
+
- A region with another cell between its columns is split instead of thrown away.
|
|
126
|
+
- Running totals (`=C1+B2`, `=SUM($B$2:B2)`) become SQL window functions, but only when Excel's saved values prove
|
|
127
|
+
the different addition order cannot change the answer. A ledger whose balance cancels back to zero stays in
|
|
128
|
+
Excel's exact order.
|
|
129
|
+
- 200k-formula benchmark: **390 s -> 31 s**, all PASS.
|
|
130
|
+
|
|
131
|
+
### 3.8 Open-source release 0.1.0 (2026-10-18)
|
|
132
|
+
|
|
133
|
+
- MIT license, README with an honest limits section, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13.
|
|
134
|
+
- Security review: zip bombs refused, crafted numbers beyond Excel's range UNRESOLVED, workbook text only reaches SQL
|
|
135
|
+
as escaped literals.
|
|
136
|
+
- **Privacy**: the public repository was created with a fresh one-commit history and a no-reply author address.
|
|
137
|
+
- Published to PyPI as `unexcel` 0.1.0 with trusted publishing (no token stored anywhere).
|
|
138
|
+
|
|
139
|
+
### 3.9 Lookups and criteria in table mode (2026-10-19)
|
|
140
|
+
|
|
141
|
+
- **Problem**: a VLOOKUP pasted down a column was still one query per cell. DuckDB could not plan the SQL form the
|
|
142
|
+
lookups used inside a region.
|
|
143
|
+
- **Fix**: every lookup became an aggregate (`arg_min` by position) over a table that does not depend on the row,
|
|
144
|
+
which DuckDB turns into one join for the whole column.
|
|
145
|
+
- 1,000 VLOOKUPs: verify **5.7 s -> 0.1 s**. 50,000 VLOOKUPs: all PASS.
|
|
146
|
+
- Same idea for COUNTIFS/SUMIFS with a criterion from the row:
|
|
147
|
+
- 1,000 COUNTIFS: verify **29 s -> 0.1 s**.
|
|
148
|
+
|
|
149
|
+
### 3.10 Faster compile (2026-10-19)
|
|
150
|
+
|
|
151
|
+
Profiling showed the formula tokenizer ran twice per cell and was most of the compile time. Now the first cell of a
|
|
152
|
+
pasted-down column is tokenized and the next rows are predicted by shifting row numbers; a cell is tokenized again only
|
|
153
|
+
when the prediction does not match its real text.
|
|
154
|
+
|
|
155
|
+
- 50k VLOOKUPs: compile **6.1 s -> 1.8 s**.
|
|
156
|
+
- 200k benchmark: **31 s -> 26 s**.
|
|
157
|
+
|
|
158
|
+
### 3.11 Renamed to cellproof, 0.2.0 (2026-10-19)
|
|
159
|
+
|
|
160
|
+
- `unexcel` clashed with an R package. Candidates were checked on PyPI, CRAN, npm and GitHub.
|
|
161
|
+
- **cellproof** was free everywhere and says what is unique: every cell is proven against Excel.
|
|
162
|
+
- Package, command, import, dbt project and GitHub repository were renamed (old links redirect).
|
|
163
|
+
- 0.2.0 was published to PyPI as `cellproof`. The old `unexcel` 0.1.0 stays on PyPI under the old name.
|
|
164
|
+
|
|
165
|
+
### 3.12 Testing on 85 public workbooks (2026-10-19)
|
|
166
|
+
|
|
167
|
+
The tool was run on every test workbook of three other Excel formula engines (pycel, formulas, xlcalculator), most
|
|
168
|
+
of them saved by real Excel. First run: 1 crash, 229 FAIL. Each FAIL was traced to its cause:
|
|
169
|
+
|
|
170
|
+
- **Bugs in cellproof, fixed**:
|
|
171
|
+
- a name defined as a formula crashed the reader;
|
|
172
|
+
- Excel's 29 February 1900 and date-formatted negative numbers were read wrong;
|
|
173
|
+
- escaped control characters in text were not decoded;
|
|
174
|
+
- AND/OR must skip text and empty cells;
|
|
175
|
+
- an empty result of IF/IFERROR is 0, not "";
|
|
176
|
+
- EDATE/EOMONTH/MROUND/VALUE refuse TRUE/FALSE;
|
|
177
|
+
- error order in SUMPRODUCT;
|
|
178
|
+
- PROPER of another cell built SQL DuckDB rejected.
|
|
179
|
+
- **Not bugs, now handled honestly**:
|
|
180
|
+
- 8 files were saved by LibreOffice, not Excel, so their saved values follow LibreOffice; the report now warns.
|
|
181
|
+
- One file was calculated by Excel for Mac, where CHAR above 127 gives other characters; now UNRESOLVED.
|
|
182
|
+
- Text like `"1/2/2020"` or `"1,000"` is read by Excel using the computer's language settings, which the file does
|
|
183
|
+
not record; now UNRESOLVED instead of a wrong answer.
|
|
184
|
+
- One of my own earlier assumptions (`AND(TRUE,"0")` is `#VALUE!`) was wrong: real Excel says TRUE. The test was
|
|
185
|
+
corrected to match Excel, never the other way round.
|
|
186
|
+
|
|
187
|
+
Result: **FAIL 229 -> 10** (all 10 in the LibreOffice files), **0 crashes**. `python tests/corpus.py DIR` repeats it.
|
|
188
|
+
|
|
189
|
+
### 3.13 The first real user workbook (2026-10-19)
|
|
190
|
+
|
|
191
|
+
A practice workbook full of array-style formulas: **3 PASS, 0 FAIL, 47 UNRESOLVED**. Added:
|
|
192
|
+
|
|
193
|
+
- **Implicit intersection**: `=IF(E$3:E$41="F",B$3:B$41)` uses the formula's own row, as Excel does (40 cells).
|
|
194
|
+
- **`INDIRECT("English")`** of fixed text is just the named range.
|
|
195
|
+
- **MODE**.
|
|
196
|
+
|
|
197
|
+
The first version of implicit intersection gave 89 for `=MAX(IF(E3:E41="F",math,0))` where Excel saved 99: the
|
|
198
|
+
verifier caught it as a FAIL. Investigation showed Excel passes the whole `math` range through IF into MAX, so that
|
|
199
|
+
case is now honestly UNRESOLVED. The same investigation found an older bug: a bare word like `=i` with no matching
|
|
200
|
+
name was read as the whole column I.
|
|
201
|
+
|
|
202
|
+
Result: **48 PASS, 0 FAIL, 2 UNRESOLVED** (one real array formula, one range passed through IF).
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## 4. How it works now (short version)
|
|
207
|
+
|
|
208
|
+
```
|
|
209
|
+
workbook.xlsx
|
|
210
|
+
| reader: formulas + Excel's saved values (and which program saved the file)
|
|
211
|
+
v
|
|
212
|
+
parse every formula -> find columns of one pasted-down formula (regions)
|
|
213
|
+
|
|
|
214
|
+
v
|
|
215
|
+
dependency order (regions as one block) -> SQL: one statement per region, small batches for the rest
|
|
216
|
+
|
|
|
217
|
+
v
|
|
218
|
+
run on DuckDB -> compare every cell with Excel's saved value -> PASS / FAIL / UNRESOLVED
|
|
219
|
+
|
|
|
220
|
+
v
|
|
221
|
+
REPORT.md + report.json + a dbt project (wide readable models per region)
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
---
|
|
225
|
+
|
|
226
|
+
## 5. Where things stand (2026-10-20)
|
|
227
|
+
|
|
228
|
+
| | |
|
|
229
|
+
|---|---|
|
|
230
|
+
| Public repository | github.com/Nikhil414/cellproof (MIT, CI green) |
|
|
231
|
+
| PyPI | `pip install cellproof` (0.2.1) |
|
|
232
|
+
| Tests | 425 passing (dbt build included) |
|
|
233
|
+
| Functions | 94 Excel functions |
|
|
234
|
+
| Real Excel checks | 3 own workbooks all PASS; 85 public workbooks: only FAILs in LibreOffice files; 1 user workbook 48/0/2 |
|
|
235
|
+
|
|
236
|
+
**Speed** (Windows 11 laptop):
|
|
237
|
+
|
|
238
|
+
| Workbook | Time |
|
|
239
|
+
|---|---|
|
|
240
|
+
| 200k formulas (4 columns x 50k rows, a running total) | 26 s, all PASS (was 390 s) |
|
|
241
|
+
| 50k VLOOKUPs into a 1,000-row table | 4.6 s, all PASS |
|
|
242
|
+
| 1,000 COUNTIFS with a criterion from the row | verify 0.1 s (was 29 s) |
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## 6. What we learned
|
|
247
|
+
|
|
248
|
+
1. **Real Excel is the only reliable reference.** Several "obvious" Excel behaviours turned out wrong once checked
|
|
249
|
+
against files Excel saved.
|
|
250
|
+
2. **The answer key catches our own mistakes.** Every wrong translation so far showed up as a FAIL, including two of
|
|
251
|
+
my own wrong fixes; none slipped through as a false PASS.
|
|
252
|
+
3. **Real files find what tests do not.** One user workbook needed three new features; 85 public workbooks found
|
|
253
|
+
eight bugs.
|
|
254
|
+
4. **Not every FAIL is our bug.** A file can be saved by LibreOffice, calculated on a Mac, or depend on language
|
|
255
|
+
settings. The tool now says so instead of blaming itself or guessing.
|
|
256
|
+
5. **Measure, then optimize.** Every speed-up came from a profile or a benchmark, not from a hunch.
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
## 7. What is next
|
|
261
|
+
|
|
262
|
+
1. Release 0.2.1 with the fixes since 0.2.0.
|
|
263
|
+
2. Launch prep: a short demo and a sample report on the README front page; then posts on LinkedIn and Twitter/X.
|
|
264
|
+
3. Array formulas and `{1,2}` array constants (the biggest remaining gap for real workbooks), LOOKUP, YEARFRAC.
|
|
265
|
+
4. More real workbooks from users.
|
|
266
|
+
5. Plan 3: an AI retry for cells that fail, which sees only the formula structure, never the values.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.1"
|
|
@@ -29,6 +29,7 @@ SEED_HEADER = ["sheet", "row_num", "col_num", "kind", "num", "txt"]
|
|
|
29
29
|
READ_SEED = ("read_csv(?, header = true, quote = '\"', escape = '\"', columns = {'sheet': 'VARCHAR', "
|
|
30
30
|
"'row_num': 'INTEGER', 'col_num': 'INTEGER', 'kind': 'VARCHAR', 'num': 'DOUBLE', 'txt': 'VARCHAR'})")
|
|
31
31
|
VOLATILE = {"NOW", "TODAY", "RAND", "RANDBETWEEN", "OFFSET", "INDIRECT"}
|
|
32
|
+
PASS_THROUGH = {"IF", "IFS", "IFERROR", "IFNA", "CHOOSE", "SWITCH"} # can return a range given as an argument
|
|
32
33
|
ARITH = {"+", "-", "*"}
|
|
33
34
|
KIND = {"num": "n", "txt": "s", "bool": "b", "err": "e", "any": "a"} # a: decided when the SQL runs
|
|
34
35
|
TYPE = {"n": "num", "s": "txt", "b": "bool"}
|
|
@@ -108,11 +109,39 @@ class Compiler:
|
|
|
108
109
|
self.findings: list[str] = []
|
|
109
110
|
self.index = RangeIndex(book)
|
|
110
111
|
self.rowrefs: dict[str, Expr] | None = None # same-row placeholders of a region (see regions)
|
|
112
|
+
self.at: Key | None = None # the formula cell being compiled in cell mode (implicit intersection)
|
|
113
|
+
self.ctx: list[str] = [] # what consumes the node being compiled: "op", "fn" or "pass" (PASS_THROUGH)
|
|
111
114
|
|
|
112
115
|
def compile_cell(self, cell: Cell) -> Expr:
|
|
113
116
|
if cell.array:
|
|
114
117
|
raise Unsupported("array formula")
|
|
115
|
-
|
|
118
|
+
self.at = cell.key
|
|
119
|
+
try:
|
|
120
|
+
return finalize(self.expr(parse(cell.formula), cell.sheet))
|
|
121
|
+
finally:
|
|
122
|
+
self.at = None
|
|
123
|
+
|
|
124
|
+
def _intersect(self, area, text: str) -> Expr:
|
|
125
|
+
"""A range where one value is expected: Excel's implicit intersection takes the cell in the formula's own row
|
|
126
|
+
(a one-column range) or column (a one-row range), and is #VALUE! when there is none."""
|
|
127
|
+
if self.at is None or area.sheet != self.at[0] or (area.r1 != area.r2 and area.c1 != area.c2):
|
|
128
|
+
raise Unsupported(f"range {text} used as a single value")
|
|
129
|
+
# real Excel: in MAX(IF(c, A1:A9, 0)) IF hands MAX the whole range; only an operator, a function's own
|
|
130
|
+
# argument or the cell itself takes one value of it
|
|
131
|
+
for consumer in reversed(self.ctx):
|
|
132
|
+
if consumer == "op":
|
|
133
|
+
break
|
|
134
|
+
if consumer == "fn":
|
|
135
|
+
if self.ctx[-1] == "pass":
|
|
136
|
+
raise Unsupported(f"range {text} passed through IF/CHOOSE into another function (Excel may use "
|
|
137
|
+
"the whole range there) not supported yet")
|
|
138
|
+
break
|
|
139
|
+
_, row, col = self.at
|
|
140
|
+
if area.c1 == area.c2 and area.r1 <= row <= area.r2:
|
|
141
|
+
return self.cell_value((area.sheet, row, area.c1))
|
|
142
|
+
if area.r1 == area.r2 and area.c1 <= col <= area.c2:
|
|
143
|
+
return self.cell_value((area.sheet, area.r1, col))
|
|
144
|
+
return error("#VALUE!")
|
|
116
145
|
|
|
117
146
|
def expr(self, node, sheet: str) -> Expr:
|
|
118
147
|
e = self._expr(node, sheet)
|
|
@@ -136,8 +165,17 @@ class Compiler:
|
|
|
136
165
|
return self.rowrefs[node.text]
|
|
137
166
|
area = resolve(node.text, sheet, self.book)
|
|
138
167
|
if not area.is_single:
|
|
139
|
-
|
|
168
|
+
return self._intersect(area, node.text)
|
|
140
169
|
return self.cell_value(area.key)
|
|
170
|
+
kind = "op" if isinstance(node, (Unary, Postfix, Binary)) else "pass" if (
|
|
171
|
+
isinstance(node, Call) and node.name in PASS_THROUGH) else "fn"
|
|
172
|
+
self.ctx.append(kind)
|
|
173
|
+
try:
|
|
174
|
+
return self._compound(node, sheet)
|
|
175
|
+
finally:
|
|
176
|
+
self.ctx.pop()
|
|
177
|
+
|
|
178
|
+
def _compound(self, node, sheet: str) -> Expr:
|
|
141
179
|
if isinstance(node, Unary):
|
|
142
180
|
x = num(self.expr(node.operand, sheet))
|
|
143
181
|
return Expr(f"(-{x.sql})", "num", x.err)
|
|
@@ -264,6 +302,9 @@ def compile_book(book: Book, regions: bool = True) -> Compiled:
|
|
|
264
302
|
scan = find_regions(book) if regions else RegionScan([], [])
|
|
265
303
|
order, unresolved, kept = order_with_regions(book, scan.regions)
|
|
266
304
|
comp = Compiler(book)
|
|
305
|
+
if book.app and not (book.app.startswith("Microsoft") and "Compatible" not in book.app): # openpyxl: "Compatible"
|
|
306
|
+
comp.findings.append(f"the file was last saved by {book.app}, not Microsoft Excel: its saved values follow that "
|
|
307
|
+
"program's rules, so a FAIL can be a difference between that program and Excel")
|
|
267
308
|
batches: list = []
|
|
268
309
|
current: list[Step] = []
|
|
269
310
|
weight = 0
|
|
@@ -9,6 +9,14 @@ from cellproof.model import Key, Unsupported
|
|
|
9
9
|
|
|
10
10
|
CELLS = "__cellproof_cells__" # every cell known before the current batch; replaced when SQL is rendered
|
|
11
11
|
NO_ERR = "NULL" # the err of an expression that cannot fail
|
|
12
|
+
# Not an Excel error: text Excel would read as a number using the computer's language settings ("1/2/2020", "1,000",
|
|
13
|
+
# "2:00 PM"), which the file does not record. The verifier reports a cell holding it as UNRESOLVED.
|
|
14
|
+
LOCALE = "#LOCALE"
|
|
15
|
+
LOCALE_REASON = ("reads text as a number, date or time (like \"1/2/2020\" or \"1,000\"): Excel does that with the "
|
|
16
|
+
"computer's language settings, which the file does not record")
|
|
17
|
+
PLATFORM = "#PLATFORM" # not an Excel error either: Excel for Mac answers differently (CHAR above 127)
|
|
18
|
+
PLATFORM_REASON = "CHAR above 127 gives a different character in Excel for Mac, which last calculated this file"
|
|
19
|
+
SETTINGS = {LOCALE: LOCALE_REASON, PLATFORM: PLATFORM_REASON} # markers the verifier reports as UNRESOLVED
|
|
12
20
|
Type = Literal["num", "txt", "bool", "blank", "err", "any"]
|
|
13
21
|
|
|
14
22
|
|
|
@@ -80,6 +88,16 @@ def _value_error(bad: str) -> str:
|
|
|
80
88
|
return f"(CASE WHEN {bad} THEN '#VALUE!' END)"
|
|
81
89
|
|
|
82
90
|
|
|
91
|
+
MONTHS = "jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _text_error(t: str, n: str) -> str:
|
|
95
|
+
"""#VALUE! when text t does not read as a number (n is NULL), LOCALE when it might in some language settings."""
|
|
96
|
+
local = (f"(regexp_matches({t}, '[0-9]\\s*[-/:.,]\\s*[0-9]|[0-9]\\s*%|[$\u20ac\u00a3\u00a5]') "
|
|
97
|
+
f"OR regexp_matches(lower({t}), '[0-9]\\s*[ap][.]?m|[0-9][^a-z]*({MONTHS})|({MONTHS})[a-z]*[^a-z0-9]*[0-9]'))")
|
|
98
|
+
return f"(CASE WHEN {n} IS NULL THEN (CASE WHEN {local} THEN '{LOCALE}' ELSE '#VALUE!' END) END)"
|
|
99
|
+
|
|
100
|
+
|
|
83
101
|
def num(e: Expr) -> Expr:
|
|
84
102
|
if e.type == "num":
|
|
85
103
|
return e
|
|
@@ -91,10 +109,26 @@ def num(e: Expr) -> Expr:
|
|
|
91
109
|
return Expr("CAST(NULL AS DOUBLE)", "num", e.err)
|
|
92
110
|
if e.type == "txt":
|
|
93
111
|
n = parse_num(e.sql)
|
|
94
|
-
return Expr(n, "num", errs(e.err,
|
|
112
|
+
return Expr(n, "num", errs(e.err, _text_error(e.sql, n)))
|
|
95
113
|
n = parse_num(e.txt)
|
|
96
114
|
return Expr(f"(CASE WHEN {e.kind} = 's' THEN {n} ELSE {e.sql} END)", "num",
|
|
97
|
-
errs(e.err,
|
|
115
|
+
errs(e.err, f"(CASE WHEN {e.kind} = 's' THEN {_text_error(e.txt, n)} END)"))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def num_strict(e: Expr) -> Expr:
|
|
119
|
+
"""A number argument that real Excel refuses as TRUE/FALSE (EDATE, EOMONTH, MROUND, VALUE): #VALUE!."""
|
|
120
|
+
if e.type == "bool":
|
|
121
|
+
return Expr("CAST(NULL AS DOUBLE)", "num", errs(e.err, "'#VALUE!'"))
|
|
122
|
+
n = num(e)
|
|
123
|
+
if e.type == "any":
|
|
124
|
+
return Expr(n.sql, "num", errs(n.err, f"(CASE WHEN {e.kind} = 'b' THEN '#VALUE!' END)"))
|
|
125
|
+
return n
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def keep_locale(err: str) -> str:
|
|
129
|
+
"""err if it is a SETTINGS marker, else NULL: functions that look at errors (IFERROR, ISERROR) must not hide it."""
|
|
130
|
+
codes = ", ".join(f"'{k}'" for k in SETTINGS)
|
|
131
|
+
return NO_ERR if err == NO_ERR else f"(CASE WHEN ({err}) IN ({codes}) THEN ({err}) END)"
|
|
98
132
|
|
|
99
133
|
|
|
100
134
|
def _bool_text(b: str) -> str:
|
|
@@ -161,6 +195,8 @@ def unify(a: Expr, b: Expr) -> tuple[Expr, Expr]:
|
|
|
161
195
|
|
|
162
196
|
|
|
163
197
|
def unify_all(es: list[Expr]) -> list[Expr]:
|
|
198
|
+
# an empty cell or an omitted argument returned as a result is 0 (IF(TRUE,,"x"), IFERROR(A1,"x") with A1 empty)
|
|
199
|
+
es = [ZERO if e.type == "blank" else e for e in es]
|
|
164
200
|
fixed = {e.type for e in es} - set(FLEX)
|
|
165
201
|
if not fixed:
|
|
166
202
|
return [num(e) for e in es]
|