cellproof 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cellproof-0.3.0 → cellproof-0.4.0}/CHANGELOG.md +30 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/PKG-INFO +19 -10
- {cellproof-0.3.0 → cellproof-0.4.0}/PROGRESS.md +5 -4
- {cellproof-0.3.0 → cellproof-0.4.0}/README.md +18 -9
- {cellproof-0.3.0 → cellproof-0.4.0}/pyproject.toml +1 -1
- cellproof-0.4.0/src/cellproof/__init__.py +1 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/compiler.py +56 -7
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/expr.py +2 -2
- cellproof-0.4.0/src/cellproof/fn/__init__.py +2 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/dates.py +3 -2
- cellproof-0.4.0/src/cellproof/fn/dynamic.py +207 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/logic.py +36 -2
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/lookup.py +10 -2
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/text.py +11 -6
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/model.py +1 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/parser.py +1 -1
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/reader.py +108 -2
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_arrays.py +4 -3
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_corpus_findings.py +6 -5
- cellproof-0.4.0/tests/test_dynamic.py +92 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_finance_stats.py +8 -0
- cellproof-0.4.0/tests/test_spill.py +100 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_text.py +12 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/xlsxkit.py +4 -2
- cellproof-0.3.0/src/cellproof/__init__.py +0 -1
- cellproof-0.3.0/src/cellproof/fn/__init__.py +0 -2
- {cellproof-0.3.0 → cellproof-0.4.0}/.github/workflows/ci.yml +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/.github/workflows/publish.yml +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/.gitignore +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/AGENTS.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/DESIGN.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/INFO.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/LICENSE +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/PLAN-2a.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/PLAN-2b.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/PLAN.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/SECURITY.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/docs/JOURNEY.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/examples/commission.xlsx +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/examples/errors.xlsx +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/examples/functions.xlsx +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/examples/functions_unsaved.xlsx +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/examples/make_commission.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/examples/make_errors.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/examples/make_functions.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/__main__.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/cli.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/compute.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/emitter.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fenwick.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/aggregate.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/criteria.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/finance.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/maths.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/fn/stats.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/functions.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/graph.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/impact.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/r1c1.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/rangeindex.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/refs.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/region.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/regions.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/report.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/tables.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/src/cellproof/verifier.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/bench_shapes.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/corpus.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_aggregate.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_batches.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_cli.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_compiler.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_compute.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_criteria.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_dates.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_emitter.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_errors.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_excel_findings.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_excel_saved.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_fenwick.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_functions.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_graph.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_impact.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_implicit.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_index.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_lexed.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_limits.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_logic.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_lookup.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_lookups.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_maths.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_parser.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_perf.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_r1c1.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_rangeindex.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_reader.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_refs.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_region.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_regions.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_regions_differential.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_regions_split.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_report.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_review_regressions.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_running.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_shapes.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_tables.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_types.py +0 -0
- {cellproof-0.3.0 → cellproof-0.4.0}/tests/test_verifier.py +0 -0
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.4.0 (2026-10-31)
|
|
4
|
+
|
|
5
|
+
- **Array formulas over a block** (legacy Ctrl+Shift+Enter blocks and dynamic-array spills) compile cell by cell:
|
|
6
|
+
a range where one value is expected takes this cell's element, a range one row (column) high repeats down
|
|
7
|
+
(across). Rules learned from formulas' Excel-saved test.xlsx: a block cell past the array is #N/A whatever the
|
|
8
|
+
formula does with errors (`{=IFERROR(G9:Q9,R9)}`), but an IS function's own range is padded with #N/A elements
|
|
9
|
+
(`{=ISERROR(C8:Q8)}` past Q8 is TRUE); EDATE, EOMONTH, MROUND and YEARFRAC do not take a range there (#VALUE!).
|
|
10
|
+
A single-cell array formula is its first element (`{=MONTH(I6:J6)}`).
|
|
11
|
+
- **FILTER, SORT, UNIQUE** spill their results and nest in each other (`SORT(UNIQUE(FILTER(...)))`): FILTER by rows
|
|
12
|
+
or columns with if_empty, SORT by one key (ascending or descending, equal keys keep their order), UNIQUE with
|
|
13
|
+
exactly_once, text compared ignoring case. A dynamic spill (Excel's cm mark on the cell, now read) whose result
|
|
14
|
+
is not exactly as big as the block Excel saved shows #SPILL! in its first cell, a FAIL. Not guessed (UNRESOLVED):
|
|
15
|
+
an empty FILTER without if_empty (#CALC! in Microsoft's docs, #VALUE! in an Excel-saved file), SORT keys that
|
|
16
|
+
are empty, errors or text with punctuation, UNIQUE over empty cells or errors.
|
|
17
|
+
- **Excel 365 error codes** (#CALC!, #SPILL!, #FIELD!, ...) are read as errors; functions stored `_xlfn._xlws.` are
|
|
18
|
+
recognised.
|
|
19
|
+
- **Security:** an array block in a crafted file can no longer overwrite another cell's formula or create cells the
|
|
20
|
+
file does not hold.
|
|
21
|
+
- **`A1#`** (ANCHORARRAY) is the block Excel spilled A1 into; **`@`** (SINGLE) is implicit intersection.
|
|
22
|
+
- **INDEX with a row or column of 0** is the whole column or row: element by element in an array formula,
|
|
23
|
+
UNRESOLVED elsewhere (was a wrong #REF!).
|
|
24
|
+
- **Fixes:** SUBSTITUTE with an instance number and cell arguments stopped the run on DuckDB 2.0 (exit 3), and
|
|
25
|
+
refuses TRUE as the instance like Excel; ISBLANK of a range read its first cell only.
|
|
26
|
+
|
|
27
|
+
## 0.3.1 (2026-10-31)
|
|
28
|
+
|
|
29
|
+
- **WORKDAY with a huge holidays range** (`WORKDAY(A1,5,I1:XFD1048576)`): the days it searched were sized by the
|
|
30
|
+
range's area, so one workbook could make DuckDB fail or exhaust memory. The search now covers 2 days per holiday
|
|
31
|
+
actually in the range, never past Excel's last date.
|
|
32
|
+
|
|
3
33
|
## 0.3.0 (2026-10-31)
|
|
4
34
|
|
|
5
35
|
- **Finance:** PMT, PV, FV, NPER, NPV. Microsoft's closed forms; NPV skips text, TRUE/FALSE and empty cells in a
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cellproof
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Nikhil414/cellproof
|
|
6
6
|
Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
|
|
@@ -111,13 +111,13 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
111
111
|
code.
|
|
112
112
|
6. **Emit** the dbt project and the report.
|
|
113
113
|
|
|
114
|
-
|
|
115
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR, FV,
|
|
114
|
+
125 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
115
|
+
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
|
|
116
116
|
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
|
|
117
117
|
LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
|
|
118
118
|
NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
|
|
119
119
|
RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
120
|
-
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
120
|
+
SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
121
121
|
YEAR, YEARFRAC.
|
|
122
122
|
|
|
123
123
|
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
@@ -133,23 +133,32 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
|
|
|
133
133
|
cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
|
|
134
134
|
|
|
135
135
|
- **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
|
|
136
|
-
- **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...)
|
|
137
|
-
and
|
|
136
|
+
- **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...),
|
|
137
|
+
inside array arithmetic and in array formulas; where one value is expected in a plain formula (`A1*{1,2}`) they are
|
|
138
|
+
UNRESOLVED.
|
|
139
|
+
- **Array formulas over a block and dynamic arrays** (legacy Ctrl+Shift+Enter blocks, Excel 365 spills): every cell of
|
|
140
|
+
the block is verified. A range where one value is expected takes the cell's element (one row or column repeats);
|
|
141
|
+
past the array a cell is #N/A, while an IS function's own range is padded per element, as real Excel does. `A1#`
|
|
142
|
+
and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
|
|
143
|
+
result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
|
|
144
|
+
computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
|
|
145
|
+
Not yet: SORTBY, SEQUENCE, XMATCH, TAKE/DROP/CHOOSECOLS and the other array functions, FILTER/SORT/UNIQUE inside
|
|
146
|
+
another calculation (`ROWS(UNIQUE(...))`), SORT by several keys or by columns, SORT keys that are empty, errors or
|
|
147
|
+
text with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
|
|
138
148
|
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
139
149
|
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
140
150
|
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
141
151
|
numbers < text < TRUE/FALSE case-insensitively, concatenation writes numbers the Excel way, and the first error
|
|
142
|
-
wins. A
|
|
143
|
-
(spilling is not read yet). A plain-entered range arithmetic or IF as an aggregate argument follows modern Excel,
|
|
152
|
+
wins. A plain-entered range arithmetic or IF as an aggregate argument follows modern Excel,
|
|
144
153
|
not the legacy first-row intersection - a workbook an old Excel last saved can show FAILs there, with both values
|
|
145
154
|
shown.
|
|
146
|
-
- **Not supported:**
|
|
155
|
+
- **Not supported:** volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
|
|
147
156
|
`INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
|
|
148
157
|
external workbooks, a structured reference naming several separate columns (`T[[A],[C]]`), INDEX used as a reference, the 1904 date system in
|
|
149
158
|
TEXT date formats.
|
|
150
159
|
- **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
|
|
151
160
|
Excel does (implicit intersection). As an aggregate's argument the element-wise answer is compiled instead (see
|
|
152
|
-
array arithmetic above)
|
|
161
|
+
array arithmetic above); in an array formula, element by element (see array formulas above).
|
|
153
162
|
- **Approximate lookups on unsorted data** (LOOKUP, MATCH/VLOOKUP/HLOOKUP with match type 1): compiled for plain
|
|
154
163
|
numeric tables written as inputs (up to about 1,000 values): the SQL carries Excel's binary-search path itself, and
|
|
155
164
|
refuses changed table values instead of answering from a stale search. MATCH with type -1, text tables, tables with
|
|
@@ -24,10 +24,12 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
|
|
|
24
24
|
| 0.3.0 functions | PMT, PV, FV, NPER, NPV; LARGE, SMALL, RANK(.EQ), STDEV/VAR (S/P), PERCENTILE(.INC), QUARTILE(.INC); NETWORKDAYS, WORKDAY. Edge cases not checked in real Excel are an UNCHECKED marker (UNRESOLVED), like LOCALE |
|
|
25
25
|
| Excel tables | structured references rewritten to A1 text by the reader (`tables.py`); [#This Row] becomes a same-row reference, so a calculated column is a region |
|
|
26
26
|
| LET | names (stored `_xlpm.x`) substituted by their value's tree, shadowing respected; the graph skips them as dependencies |
|
|
27
|
+
| Array blocks (0.4.0) | each cell of a CSE block or dynamic spill is a formula cell computing its element (`Compiler.spill`, `_element`); per-cell #N/A past the array, IS functions padded per element; ANCHORARRAY, SINGLE; blocks never overwrite other formulas |
|
|
28
|
+
| FILTER/SORT/UNIQUE (0.4.0) | `fn/dynamic.py`: arrays as relations (i, j, value), nested via MATERIALIZED CTEs; dynamic anchors check the result size against the saved block (`Cell.dynamic` from the file's cm mark) |
|
|
27
29
|
| Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
|
|
28
30
|
|
|
29
31
|
Tests: 492 (+10 skipped without dbt/Excel files), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
|
|
30
|
-
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.
|
|
32
|
+
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1 and 0.4.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
|
|
31
33
|
A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
|
|
32
34
|
|
|
33
35
|
## Public test workbooks (2026-10-31)
|
|
@@ -94,9 +96,8 @@ What Excel taught us (each has a test in `tests/test_excel_findings.py`):
|
|
|
94
96
|
|
|
95
97
|
## Next
|
|
96
98
|
|
|
97
|
-
1.
|
|
98
|
-
|
|
99
|
-
Arrays slices 1-3 above.)
|
|
99
|
+
1. Arrays, next: FILTER/SORT/UNIQUE inside other calculations (`ROWS(UNIQUE())`, `SUM(FILTER())`), SORTBY,
|
|
100
|
+
SEQUENCE, XMATCH (384 corpus cells), TAKE/DROP/CHOOSECOLS/TOCOL; one evaluation per spill instead of per cell.
|
|
100
101
|
2. Test on more real analyst workbooks (one user workbook found 4 gaps the 85 public files did not).
|
|
101
102
|
3. Read time (openpyxl loads the workbook twice) and the per-cell dependency loop.
|
|
102
103
|
4. Plan 3: AI retry loop for failed cells (it sees formula structure only, never values).
|
|
@@ -77,13 +77,13 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
77
77
|
code.
|
|
78
78
|
6. **Emit** the dbt project and the report.
|
|
79
79
|
|
|
80
|
-
|
|
81
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR, FV,
|
|
80
|
+
125 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
81
|
+
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
|
|
82
82
|
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
|
|
83
83
|
LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
|
|
84
84
|
NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
|
|
85
85
|
RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
86
|
-
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
86
|
+
SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
87
87
|
YEAR, YEARFRAC.
|
|
88
88
|
|
|
89
89
|
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
@@ -99,23 +99,32 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
|
|
|
99
99
|
cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
|
|
100
100
|
|
|
101
101
|
- **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
|
|
102
|
-
- **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...)
|
|
103
|
-
and
|
|
102
|
+
- **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...),
|
|
103
|
+
inside array arithmetic and in array formulas; where one value is expected in a plain formula (`A1*{1,2}`) they are
|
|
104
|
+
UNRESOLVED.
|
|
105
|
+
- **Array formulas over a block and dynamic arrays** (legacy Ctrl+Shift+Enter blocks, Excel 365 spills): every cell of
|
|
106
|
+
the block is verified. A range where one value is expected takes the cell's element (one row or column repeats);
|
|
107
|
+
past the array a cell is #N/A, while an IS function's own range is padded per element, as real Excel does. `A1#`
|
|
108
|
+
and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
|
|
109
|
+
result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
|
|
110
|
+
computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
|
|
111
|
+
Not yet: SORTBY, SEQUENCE, XMATCH, TAKE/DROP/CHOOSECOLS and the other array functions, FILTER/SORT/UNIQUE inside
|
|
112
|
+
another calculation (`ROWS(UNIQUE(...))`), SORT by several keys or by columns, SORT keys that are empty, errors or
|
|
113
|
+
text with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
|
|
104
114
|
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
105
115
|
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
106
116
|
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
107
117
|
numbers < text < TRUE/FALSE case-insensitively, concatenation writes numbers the Excel way, and the first error
|
|
108
|
-
wins. A
|
|
109
|
-
(spilling is not read yet). A plain-entered range arithmetic or IF as an aggregate argument follows modern Excel,
|
|
118
|
+
wins. A plain-entered range arithmetic or IF as an aggregate argument follows modern Excel,
|
|
110
119
|
not the legacy first-row intersection - a workbook an old Excel last saved can show FAILs there, with both values
|
|
111
120
|
shown.
|
|
112
|
-
- **Not supported:**
|
|
121
|
+
- **Not supported:** volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
|
|
113
122
|
`INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
|
|
114
123
|
external workbooks, a structured reference naming several separate columns (`T[[A],[C]]`), INDEX used as a reference, the 1904 date system in
|
|
115
124
|
TEXT date formats.
|
|
116
125
|
- **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
|
|
117
126
|
Excel does (implicit intersection). As an aggregate's argument the element-wise answer is compiled instead (see
|
|
118
|
-
array arithmetic above)
|
|
127
|
+
array arithmetic above); in an array formula, element by element (see array formulas above).
|
|
119
128
|
- **Approximate lookups on unsorted data** (LOOKUP, MATCH/VLOOKUP/HLOOKUP with match type 1): compiled for plain
|
|
120
129
|
numeric tables written as inputs (up to about 1,000 values): the SQL carries Excel's binary-search path itself, and
|
|
121
130
|
refuses changed table values instead of answering from a stale search. MATCH with type -1, text tables, tables with
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.4.0"
|
|
@@ -15,8 +15,11 @@ from dataclasses import field as dc_field
|
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
|
|
17
17
|
import cellproof.fn # noqa: F401 (registers the function library)
|
|
18
|
+
from openpyxl.utils import range_boundaries
|
|
19
|
+
|
|
18
20
|
from cellproof.expr import (BLANK, CELLS, NO_ERR, Expr, any_of, compare, error, errs, finalize, lit_num, lit_str, num,
|
|
19
21
|
txt, where_cell)
|
|
22
|
+
from cellproof.fn.dynamic import DYNAMIC, element
|
|
20
23
|
from cellproof.functions import FUNCS
|
|
21
24
|
from cellproof.graph import formula_order
|
|
22
25
|
from cellproof.model import Book, Cell, Key, Unsupported, a1
|
|
@@ -102,6 +105,11 @@ class Keys:
|
|
|
102
105
|
return self._comp.index.text_cells(a.sheet, a.r1, a.c1, a.r2, a.c2)
|
|
103
106
|
|
|
104
107
|
|
|
108
|
+
# Analysis ToolPak functions do not take an array where one value is expected: {arguments that are one value}
|
|
109
|
+
NO_LIFT = {"EDATE": 2, "EOMONTH": 2, "MROUND": 2, "YEARFRAC": 3, "NETWORKDAYS": 2, "WORKDAY": 2}
|
|
110
|
+
NO_LIFT_CHECKED = {"EDATE", "EOMONTH", "MROUND", "YEARFRAC"} # #VALUE! there, in formulas' Excel-saved test.xlsx
|
|
111
|
+
|
|
112
|
+
|
|
105
113
|
class Compiler:
|
|
106
114
|
def __init__(self, book: Book):
|
|
107
115
|
add_arrays(book)
|
|
@@ -114,6 +122,8 @@ class Compiler:
|
|
|
114
122
|
self.at: Key | None = None # the formula cell being compiled in cell mode (implicit intersection)
|
|
115
123
|
self.ctx: list[str] = [] # what consumes the node being compiled: "op", "fn" or "pass" (PASS_THROUGH)
|
|
116
124
|
self.wid = 0 # unique aliases w1, w2... for the element sources of one array argument
|
|
125
|
+
self.spill: tuple[int, int] | None = None # array formula: this cell's (row, col) offset in its block
|
|
126
|
+
self.spill_dims: list[tuple[int, int]] = [] # sizes of the ranges taken element by element (see _element)
|
|
117
127
|
|
|
118
128
|
def walias(self) -> str:
|
|
119
129
|
self.wid += 1
|
|
@@ -122,24 +132,39 @@ class Compiler:
|
|
|
122
132
|
def compile_cell(self, cell: Cell) -> Expr:
|
|
123
133
|
if cell.key in self.book.unread:
|
|
124
134
|
raise Unsupported(self.book.unread[cell.key])
|
|
125
|
-
if cell.array and cell.ref and ":" in cell.ref:
|
|
126
|
-
raise Unsupported("array formula over several cells") # spilling into the neighbours is not read yet
|
|
127
135
|
self.at = cell.key
|
|
136
|
+
self.spill, self.spill_dims = None, []
|
|
137
|
+
if cell.array:
|
|
138
|
+
c1, r1, _, _ = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
139
|
+
self.spill = (cell.row - r1, cell.col - c1)
|
|
128
140
|
try:
|
|
129
|
-
|
|
141
|
+
tree = parse(cell.formula)
|
|
142
|
+
if self.spill is not None and isinstance(tree, Call) and tree.name in DYNAMIC:
|
|
143
|
+
c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
144
|
+
return element(self, tree, cell.sheet, *self.spill, (r2 - r1 + 1, c2 - c1 + 1), cell.dynamic)
|
|
145
|
+
e = finalize(self.expr(tree, cell.sheet))
|
|
146
|
+
if self.spill is not None and self.spill_dims:
|
|
147
|
+
# real Excel (formulas' test.xlsx): the array is as big as its biggest range, one row (column)
|
|
148
|
+
# repeating; a block cell beyond it is #N/A whatever the formula does with errors -
|
|
149
|
+
# {=IFERROR(G9:Q9,R9)} one cell past Q9 is #N/A, not R9
|
|
150
|
+
rows = max(d[0] for d in self.spill_dims)
|
|
151
|
+
cols = max(d[1] for d in self.spill_dims)
|
|
152
|
+
i, j = self.spill
|
|
153
|
+
if (rows > 1 and i >= rows) or (cols > 1 and j >= cols):
|
|
154
|
+
return error("#N/A")
|
|
155
|
+
return e
|
|
130
156
|
finally:
|
|
131
|
-
self.at = None
|
|
157
|
+
self.at, self.spill = None, None
|
|
132
158
|
|
|
133
159
|
def _intersect(self, area, text: str) -> Expr:
|
|
134
160
|
"""A range where one value is expected: Excel's implicit intersection takes the cell in the formula's own row
|
|
135
161
|
(a one-column range) or column (a one-row range), and is #VALUE! when there is none."""
|
|
162
|
+
if self.spill is not None:
|
|
163
|
+
return self._element(area)
|
|
136
164
|
if area.sheet in self.book.arrays:
|
|
137
165
|
raise Unsupported("array constant used where one value is expected (needs array evaluation) not supported yet")
|
|
138
166
|
if self.at is None or area.sheet != self.at[0] or (area.r1 != area.r2 and area.c1 != area.c2):
|
|
139
167
|
raise Unsupported(f"range {text} used as a single value")
|
|
140
|
-
if self.book.cells[self.at].array: # real Excel: {=MONTH(A1:B1)} evaluates per element, shows the first
|
|
141
|
-
raise Unsupported(f"range {text} where one value is expected in an array formula (needs array "
|
|
142
|
-
"evaluation) not supported yet")
|
|
143
168
|
# real Excel: in MAX(IF(c, A1:A9, 0)) IF hands MAX the whole range; only an operator, a function's own
|
|
144
169
|
# argument or the cell itself takes one value of it
|
|
145
170
|
for consumer in reversed(self.ctx):
|
|
@@ -157,6 +182,23 @@ class Compiler:
|
|
|
157
182
|
return self.cell_value((area.sheet, area.r1, col))
|
|
158
183
|
return error("#VALUE!")
|
|
159
184
|
|
|
185
|
+
def element_key(self, area) -> Key | None:
|
|
186
|
+
"""In an array formula a range where one value is expected is taken element by element: this cell's element,
|
|
187
|
+
the only row (column) of a range one row (column) high for every row (column); None beyond its size, where
|
|
188
|
+
Excel pads the range with #N/A (so ISERROR there is TRUE: real Excel, formulas' test.xlsx)."""
|
|
189
|
+
rows, cols = area.r2 - area.r1 + 1, area.c2 - area.c1 + 1
|
|
190
|
+
i, j = self.spill
|
|
191
|
+
i, j = (0 if rows == 1 else i), (0 if cols == 1 else j)
|
|
192
|
+
return None if i >= rows or j >= cols else (area.sheet, area.r1 + i, area.c1 + j)
|
|
193
|
+
|
|
194
|
+
def _element(self, area, sized: bool = True) -> Expr:
|
|
195
|
+
"""sized=False: an IS function's own argument, which real Excel pads with #N/A without sizing the array
|
|
196
|
+
({=ISERROR(C8:Q8)} past Q8 is TRUE)."""
|
|
197
|
+
if sized:
|
|
198
|
+
self.spill_dims.append((area.r2 - area.r1 + 1, area.c2 - area.c1 + 1))
|
|
199
|
+
key = self.element_key(area)
|
|
200
|
+
return error("#N/A") if key is None else self.cell_value(key)
|
|
201
|
+
|
|
160
202
|
def expr(self, node, sheet: str) -> Expr:
|
|
161
203
|
e = self._expr(node, sheet)
|
|
162
204
|
if sum(len(part) for part in (e.sql, e.err, e.txt, e.kind)) > MAX_SQL:
|
|
@@ -204,6 +246,13 @@ class Compiler:
|
|
|
204
246
|
fn = FUNCS.get(node.name)
|
|
205
247
|
if fn is None:
|
|
206
248
|
raise Unsupported(f"function {node.name} not supported yet")
|
|
249
|
+
if self.spill is not None and node.name in NO_LIFT:
|
|
250
|
+
ranged = [i for i, a in enumerate(node.args) if i < NO_LIFT[node.name] and isinstance(a, RefNode)
|
|
251
|
+
and not resolve(a.text, sheet, self.book).is_single]
|
|
252
|
+
if ranged and node.name in NO_LIFT_CHECKED:
|
|
253
|
+
return error("#VALUE!") # real Excel: {=EDATE(E9:I9,C9:G9)} is #VALUE! in every cell
|
|
254
|
+
if ranged:
|
|
255
|
+
raise Unsupported(f"{node.name} of a range in an array formula not supported yet")
|
|
207
256
|
return fn(self, node.args, sheet)
|
|
208
257
|
raise Unsupported(f"cannot translate {node!r}")
|
|
209
258
|
|
|
@@ -23,8 +23,8 @@ STALE_REASON = ("the lookup table's values changed after the workbook was transl
|
|
|
23
23
|
"out of date: re-run cellproof")
|
|
24
24
|
UNCHECKED = "#UNCHECKED" # nor this: an edge case of a function whose Excel answer was never checked in real Excel
|
|
25
25
|
UNCHECKED_REASON = ("an edge case of this function (like zero periods, a rate of -100% or less, a payment type other "
|
|
26
|
-
"than 0 or 1, a k that is not a whole number,
|
|
27
|
-
"been checked against real Excel yet")
|
|
26
|
+
"than 0 or 1, a k that is not a whole number, an error in the range, or INDEX of a whole row or column "
|
|
27
|
+
"outside an array formula) whose Excel answer has not been checked against real Excel yet")
|
|
28
28
|
SETTINGS = {LOCALE: LOCALE_REASON, PLATFORM: PLATFORM_REASON, EARLY1900: EARLY1900_REASON,
|
|
29
29
|
STALE: STALE_REASON, UNCHECKED: UNCHECKED_REASON} # reported as UNRESOLVED
|
|
30
30
|
Type = Literal["num", "txt", "bool", "blank", "err", "any"]
|
|
@@ -270,10 +270,11 @@ def workday(c, args, sheet):
|
|
|
270
270
|
s = _serial(c, args[0], sheet)
|
|
271
271
|
d = num(c.expr(args[1], sheet))
|
|
272
272
|
rng, odd = _holidays(c, args, sheet, "WORKDAY", 2)
|
|
273
|
-
|
|
273
|
+
# the holidays actually there, not the range's area (H:XFD would be 3E10 days); never past Excel's last date
|
|
274
|
+
extra = "0" if rng is None else f"2 * (SELECT COUNT(*) FROM ({_holiday_serials(rng)}) hol)"
|
|
274
275
|
n = f"CAST(abs(trunc({d.sql})) AS BIGINT)"
|
|
275
276
|
step = f"(CASE WHEN {d.sql} < 0 THEN -1 ELSE 1 END)"
|
|
276
|
-
span = f"({n} * 7 // 5 + 7 + {extra})" #
|
|
277
|
+
span = f"LEAST({n} * 7 // 5 + 7 + {extra}, {MAX_SERIAL + 7})" # room for n working days and each holiday
|
|
277
278
|
start = f"CAST({s.sql} AS BIGINT)"
|
|
278
279
|
skip = "" if rng is None else f" AND x NOT IN ({_holiday_serials(rng)})"
|
|
279
280
|
days = (f"SELECT x, row_number() OVER (ORDER BY x * {step}) AS rn FROM "
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""FILTER, SORT, UNIQUE: functions whose result is an array as big as the data says.
|
|
2
|
+
|
|
3
|
+
An array is a relation: one row per element with its 1-based position (i, j) and the cells table's value columns
|
|
4
|
+
(kind, num, txt, err; kind NULL for an empty cell). Ranges, array constants and these functions nest in any order
|
|
5
|
+
(SORT(UNIQUE(FILTER(...)))). Each cell of the block Excel spilled the result into picks its own element
|
|
6
|
+
(compiler.compile_cell); the anchor of a dynamic spill also checks the result is exactly as big as that block.
|
|
7
|
+
|
|
8
|
+
What Excel's answer was not checked for is UNCHECKED (UNRESOLVED), never guessed: SORT keys that are empty, errors
|
|
9
|
+
or text with characters other than letters, digits and spaces (Excel's text order is its own collation), and
|
|
10
|
+
UNIQUE over empty cells or errors.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str
|
|
17
|
+
from cellproof.fn.aggregate import _array, _base, _truth
|
|
18
|
+
from cellproof.fn.lookup import int_literal
|
|
19
|
+
from cellproof.functions import register
|
|
20
|
+
from cellproof.model import Unsupported
|
|
21
|
+
from cellproof.parser import Bool, Call, RefNode
|
|
22
|
+
from cellproof.refs import MAX_ROW, resolve
|
|
23
|
+
|
|
24
|
+
DYNAMIC = {"FILTER", "SORT", "UNIQUE"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class Rel:
|
|
29
|
+
sql: str # a subquery with columns i, j, kind, num, txt, err
|
|
30
|
+
rows: int | None # its size when known before it runs
|
|
31
|
+
cols: int | None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _uid(c) -> str:
|
|
35
|
+
return "d" + c.walias()[1:]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _range(c, node, sheet: str) -> Rel:
|
|
39
|
+
area = resolve(node.text, sheet, c.book)
|
|
40
|
+
r2 = area.r2
|
|
41
|
+
if r2 == MAX_ROW: # ponytail: a whole column stops at the last row any sheet holds (its cells are all empty)
|
|
42
|
+
r2 = max(area.r1, max(c.book.max_row.values(), default=1))
|
|
43
|
+
c.ready(area)
|
|
44
|
+
sql = (f"(SELECT CAST(gr.r - {area.r1} + 1 AS BIGINT) AS i, CAST(gc.c - {area.c1} + 1 AS BIGINT) AS j, "
|
|
45
|
+
f"x.kind AS kind, x.num AS num, x.txt AS txt, CASE WHEN x.kind = 'e' THEN x.txt END AS err "
|
|
46
|
+
f"FROM range({area.r1}, {r2 + 1}) gr(r) CROSS JOIN range({area.c1}, {area.c2 + 1}) gc(c) "
|
|
47
|
+
f"LEFT JOIN {CELLS} x ON x.sheet = {lit_str(area.sheet).sql} AND x.row_num = gr.r AND x.col_num = gc.c)")
|
|
48
|
+
return Rel(sql, r2 - area.r1 + 1, area.c2 - area.c1 + 1)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def relation(c, node, sheet: str) -> Rel:
|
|
52
|
+
"""An array argument as a relation: a range, an array constant, or FILTER/SORT/UNIQUE of one."""
|
|
53
|
+
if isinstance(node, RefNode):
|
|
54
|
+
return _range(c, node, sheet)
|
|
55
|
+
if isinstance(node, Call) and node.name in BUILDERS:
|
|
56
|
+
return BUILDERS[node.name](c, node.args, sheet)
|
|
57
|
+
raise Unsupported("an array argument other than a range, an array constant, FILTER, SORT or UNIQUE "
|
|
58
|
+
"not supported yet")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _whole(u: str, ctes: list[str], body: str, bad: str, empty: str | None = None) -> str:
|
|
62
|
+
"""WITH ctes, {u}_w AS body: that relation, or one element holding the error `bad` when it is set, or `empty`
|
|
63
|
+
(a one-row SELECT) when it has no elements. Every input is a MATERIALIZED CTE: written and run once, however
|
|
64
|
+
deep FILTER/SORT/UNIQUE nest."""
|
|
65
|
+
ctes = ctes + [f"{u}_w AS MATERIALIZED {body}", f"{u}_b AS MATERIALIZED (SELECT ({bad}) AS e)"]
|
|
66
|
+
parts = [f"SELECT * FROM {u}_w WHERE (SELECT e FROM {u}_b) IS NULL",
|
|
67
|
+
f"SELECT 1, 1, 'e', NULL, NULL, e FROM {u}_b WHERE e IS NOT NULL"]
|
|
68
|
+
if empty is not None:
|
|
69
|
+
parts.append(f"{empty} WHERE (SELECT e FROM {u}_b) IS NULL AND NOT EXISTS (SELECT 1 FROM {u}_w)")
|
|
70
|
+
return f"(WITH {', '.join(ctes)} " + " UNION ALL ".join(parts) + ")"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _flag(node, name: str, default: bool) -> bool:
|
|
74
|
+
if node is None:
|
|
75
|
+
return default
|
|
76
|
+
if isinstance(node, Bool):
|
|
77
|
+
return node.value
|
|
78
|
+
n = int_literal(node)
|
|
79
|
+
if n is None:
|
|
80
|
+
raise Unsupported(f"{name} with a by_col/exactly_once argument computed at run time not supported yet")
|
|
81
|
+
return n != 0
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def filter_(c, args, sheet: str) -> Rel:
|
|
85
|
+
"""FILTER(array, include, [if_empty]): the rows (include one column high as the array) or the columns (include
|
|
86
|
+
one row as wide) whose include element is true; none -> if_empty (without it, UNCHECKED: see below). A text include element is #VALUE!,
|
|
87
|
+
an error one is that error, for the whole result."""
|
|
88
|
+
if not 2 <= len(args) <= 3:
|
|
89
|
+
raise Unsupported("FILTER needs two or three arguments")
|
|
90
|
+
a = relation(c, args[0], sheet)
|
|
91
|
+
inc = _base(c, args[1], sheet, "FILTER") if isinstance(args[1], RefNode) else _array(c, args[1], sheet, "FILTER", "raw")
|
|
92
|
+
if not inc.srcs:
|
|
93
|
+
raise Unsupported("FILTER with an include that is one value not supported yet")
|
|
94
|
+
if a.rows is None or a.cols is None:
|
|
95
|
+
raise Unsupported("FILTER of an array whose size is only known at run time not supported yet")
|
|
96
|
+
ih, iw = inc.shape[0] + 1, inc.shape[1] + 1
|
|
97
|
+
first = inc.srcs[0].area
|
|
98
|
+
if iw == 1 and ih == a.rows:
|
|
99
|
+
pos, other = f"(row_num - {first.r1} + 1)", "i"
|
|
100
|
+
elif ih == 1 and iw == a.cols:
|
|
101
|
+
pos, other = f"(col_num - {first.c1} + 1)", "j"
|
|
102
|
+
else:
|
|
103
|
+
return Rel("(SELECT 1 AS i, 1 AS j, 'e' AS kind, NULL AS num, NULL AS txt, '#VALUE!' AS err)", 1, 1)
|
|
104
|
+
keep, kerr = _truth("kind", "num", "txt", "err")
|
|
105
|
+
u = _uid(c)
|
|
106
|
+
ctes = [f"{u}_a AS MATERIALIZED {a.sql}",
|
|
107
|
+
f"{u}_f AS MATERIALIZED (SELECT {pos} AS k, {keep} AS keep, {kerr} AS kerr FROM {inc.rel()} {u}_src)"]
|
|
108
|
+
bad = f"(SELECT kerr FROM {u}_f WHERE kerr IS NOT NULL ORDER BY k LIMIT 1)"
|
|
109
|
+
kept = (f"(SELECT {'dense_rank() OVER (ORDER BY a.i)' if other == 'i' else 'a.i'} AS i, "
|
|
110
|
+
f"{'dense_rank() OVER (ORDER BY a.j)' if other == 'j' else 'a.j'} AS j, a.kind, a.num, a.txt, a.err "
|
|
111
|
+
f"FROM {u}_a a JOIN {u}_f f ON f.k = a.{other} WHERE f.keep)")
|
|
112
|
+
empty = None
|
|
113
|
+
if len(args) == 3 and args[2] is not None:
|
|
114
|
+
e = as_any(c.expr(args[2], sheet))
|
|
115
|
+
empty = f"SELECT 1, 1, {e.kind}, {e.sql}, {e.txt}, {e.err}"
|
|
116
|
+
else: # real Excel disagrees with itself: #CALC! in Microsoft's docs, #VALUE! in formulas' test.xlsx
|
|
117
|
+
empty = f"SELECT 1, 1, 'e', NULL, NULL, '{UNCHECKED}'"
|
|
118
|
+
return Rel(_whole(u, ctes, kept, bad, empty), None, None)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
SAFE_TEXT = "'^[A-Za-z0-9 ]*$'"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def sort_(c, args, sheet: str) -> Rel:
|
|
125
|
+
"""SORT(array, [sort_index], [sort_order], [by_col]): rows ordered by one column, numbers before text before
|
|
126
|
+
TRUE/FALSE, text ignoring case; rows with equal keys keep their order."""
|
|
127
|
+
if not 1 <= len(args) <= 4:
|
|
128
|
+
raise Unsupported("SORT needs one to four arguments")
|
|
129
|
+
a = relation(c, args[0], sheet)
|
|
130
|
+
idx = 1 if len(args) < 2 or args[1] is None else int_literal(args[1])
|
|
131
|
+
order = 1 if len(args) < 3 or args[2] is None else int_literal(args[2])
|
|
132
|
+
if idx is None or order is None:
|
|
133
|
+
raise Unsupported("SORT with a sort_index or sort_order that is not one whole number not supported yet")
|
|
134
|
+
if _flag(args[3] if len(args) == 4 else None, "SORT", False):
|
|
135
|
+
raise Unsupported("SORT by columns not supported yet")
|
|
136
|
+
if order not in (1, -1) or idx < 1 or (a.cols is not None and idx > a.cols):
|
|
137
|
+
return Rel("(SELECT 1 AS i, 1 AS j, 'e' AS kind, NULL AS num, NULL AS txt, '#VALUE!' AS err)", 1, 1)
|
|
138
|
+
d = "ASC" if order == 1 else "DESC"
|
|
139
|
+
u = _uid(c)
|
|
140
|
+
ctes = [f"{u}_a AS MATERIALIZED {a.sql}", f"{u}_k AS MATERIALIZED (SELECT * FROM {u}_a WHERE j = {idx})"]
|
|
141
|
+
odd = (f"(SELECT CASE WHEN EXISTS (SELECT 1 FROM {u}_k WHERE kind IS NULL OR kind = 'e' OR "
|
|
142
|
+
f"(kind = 's' AND NOT regexp_matches(COALESCE(txt, ''), {SAFE_TEXT}))) THEN '{UNCHECKED}' END)")
|
|
143
|
+
rank = "CASE kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END"
|
|
144
|
+
new = (f"(SELECT i AS oi, row_number() OVER (ORDER BY {rank} {d}, num {d}, lower(COALESCE(txt, '')) {d}, i) "
|
|
145
|
+
f"AS ni FROM {u}_k)")
|
|
146
|
+
sql = f"(SELECT n.ni AS i, a.j, a.kind, a.num, a.txt, a.err FROM {u}_a a JOIN {new} n ON n.oi = a.i)"
|
|
147
|
+
return Rel(_whole(u, ctes, sql, odd), a.rows, a.cols)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def unique_(c, args, sheet: str) -> Rel:
|
|
151
|
+
"""UNIQUE(array, [by_col], [exactly_once]): the distinct rows in the order first seen (text ignoring case);
|
|
152
|
+
exactly_once keeps only the rows that occur once."""
|
|
153
|
+
if not 1 <= len(args) <= 3:
|
|
154
|
+
raise Unsupported("UNIQUE needs one to three arguments")
|
|
155
|
+
a = relation(c, args[0], sheet)
|
|
156
|
+
if _flag(args[1] if len(args) > 1 else None, "UNIQUE", False):
|
|
157
|
+
raise Unsupported("UNIQUE by columns not supported yet")
|
|
158
|
+
once = _flag(args[2] if len(args) > 2 else None, "UNIQUE", False)
|
|
159
|
+
u = _uid(c)
|
|
160
|
+
key = ("CASE kind WHEN 's' THEN 's' || lower(COALESCE(txt, '')) WHEN 'b' THEN 'b' || CAST(num AS VARCHAR) "
|
|
161
|
+
"ELSE 'n' || CAST(num AS VARCHAR) END")
|
|
162
|
+
ctes = [f"{u}_a AS MATERIALIZED {a.sql}"]
|
|
163
|
+
odd = f"(SELECT CASE WHEN EXISTS (SELECT 1 FROM {u}_a WHERE kind IS NULL OR kind = 'e') THEN '{UNCHECKED}' END)"
|
|
164
|
+
rows = f"(SELECT i, string_agg({key}, chr(31) ORDER BY j) AS rk FROM {u}_a GROUP BY i)"
|
|
165
|
+
groups = f"(SELECT rk, min(i) AS first, COUNT(*) AS n FROM {rows} g GROUP BY rk)"
|
|
166
|
+
keep = f"(SELECT first, row_number() OVER (ORDER BY first) AS ni FROM {groups} h" + (
|
|
167
|
+
" WHERE n = 1)" if once else ")")
|
|
168
|
+
sql = f"(SELECT k.ni AS i, a.j, a.kind, a.num, a.txt, a.err FROM {u}_a a JOIN {keep} k ON k.first = a.i)"
|
|
169
|
+
empty = "SELECT 1, 1, 'e', NULL, NULL, '#CALC!'"
|
|
170
|
+
return Rel(_whole(u, ctes, sql, odd, empty), None, a.cols)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
BUILDERS = {"FILTER": filter_, "SORT": sort_, "UNIQUE": unique_}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def element(c, node, sheet: str, i: int, j: int, block: tuple[int, int], dynamic: bool) -> Expr:
|
|
177
|
+
"""This cell's element (i, j) of the array, 0-based. A result one row (column) high repeats down (across), as in
|
|
178
|
+
a legacy block; past the result a cell is #N/A. The anchor of a dynamic spill is #SPILL! when the result is not
|
|
179
|
+
exactly as big as the block Excel spilled it into (the translation disagrees with Excel's own result)."""
|
|
180
|
+
rel = relation(c, node, sheet).sql
|
|
181
|
+
u = _uid(c)
|
|
182
|
+
# ponytail: the relation runs once per value column (4x); one struct per cell if big spills get slow
|
|
183
|
+
head = (f"WITH {u}_r AS MATERIALIZED {rel}, {u}_d AS (SELECT max(i) AS h, max(j) AS w FROM {u}_r), "
|
|
184
|
+
f"{u}_e AS (SELECT r.* FROM {u}_r r, {u}_d d WHERE r.i = (CASE WHEN d.h = 1 THEN 1 ELSE {i + 1} END) "
|
|
185
|
+
f"AND r.j = (CASE WHEN d.w = 1 THEN 1 ELSE {j + 1} END))")
|
|
186
|
+
|
|
187
|
+
def get(col: str) -> str:
|
|
188
|
+
return f"({head} SELECT {col} FROM {u}_e)"
|
|
189
|
+
kind = f"COALESCE(NULLIF({get('kind')}, 'e'), 'n')" # an empty element spills as 0
|
|
190
|
+
num = f"COALESCE({get('num')}, CAST(0 AS DOUBLE))"
|
|
191
|
+
txt = f"COALESCE({get('txt')}, '')"
|
|
192
|
+
size = f"(SELECT h <> {block[0]} OR w <> {block[1]} FROM {u}_d)" if dynamic and (i, j) == (0, 0) else "FALSE"
|
|
193
|
+
markers = ", ".join(f"'{m}'" for m in SETTINGS)
|
|
194
|
+
first = f"(SELECT err FROM {u}_r WHERE i = 1 AND j = 1)" # an UNRESOLVED marker wins over the size check
|
|
195
|
+
err = (f"({head} SELECT CASE WHEN {first} IN ({markers}) THEN {first} WHEN {size} THEN '#SPILL!' WHEN NOT EXISTS (SELECT 1 FROM {u}_e) THEN '#N/A' "
|
|
196
|
+
f"ELSE (SELECT err FROM {u}_e) END)") # no FROM: a missing element still gives one row
|
|
197
|
+
return Expr(num, "any", err, txt, kind)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _inside(name):
|
|
201
|
+
def fn(c, args, sheet):
|
|
202
|
+
raise Unsupported(f"{name} inside another calculation not supported yet")
|
|
203
|
+
register(name)(fn)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
for _n in DYNAMIC:
|
|
207
|
+
_inside(_n)
|
|
@@ -77,10 +77,20 @@ def ifna(c, args, sheet):
|
|
|
77
77
|
return pick(f"(NOT {na})", value, fallback, f"(CASE WHEN {na} THEN {fallback.err} ELSE {value.err} END)")
|
|
78
78
|
|
|
79
79
|
|
|
80
|
+
def is_arg(c, node, sheet: str) -> Expr:
|
|
81
|
+
"""An IS function's argument: in an array formula a range is padded per element, not sized (see _element)."""
|
|
82
|
+
if c.spill is not None and isinstance(node, RefNode):
|
|
83
|
+
from cellproof.refs import resolve
|
|
84
|
+
area = resolve(node.text, sheet, c.book)
|
|
85
|
+
if not area.is_single:
|
|
86
|
+
return c._element(area, sized=False)
|
|
87
|
+
return c.expr(node, sheet)
|
|
88
|
+
|
|
89
|
+
|
|
80
90
|
def _is(name: str, test):
|
|
81
91
|
def fn(c, args, sheet):
|
|
82
92
|
arity(name, args, 1, 1)
|
|
83
|
-
x = c
|
|
93
|
+
x = is_arg(c, args[0], sheet)
|
|
84
94
|
return Expr(test(x), "bool", keep_locale(x.err))
|
|
85
95
|
register(name)(fn)
|
|
86
96
|
|
|
@@ -109,7 +119,14 @@ def isblank(c, args, sheet):
|
|
|
109
119
|
if not isinstance(args[0], RefNode):
|
|
110
120
|
return Expr("FALSE", "bool") # ISBLANK("") is FALSE: only an empty cell is blank
|
|
111
121
|
area = resolve(args[0].text, sheet, c.book)
|
|
112
|
-
|
|
122
|
+
key = area.key if area.is_single else None
|
|
123
|
+
if not area.is_single:
|
|
124
|
+
if c.spill is None:
|
|
125
|
+
raise Unsupported("ISBLANK of a range where one value is expected not supported yet")
|
|
126
|
+
key = c.element_key(area)
|
|
127
|
+
if key is None: # beyond the range: Excel's #N/A padding, which is not blank
|
|
128
|
+
return Expr("FALSE", "bool")
|
|
129
|
+
cell = c.book.cells.get(key)
|
|
113
130
|
empty = cell is None or (cell.formula is None and cell.kind == "blank")
|
|
114
131
|
return Expr("TRUE" if empty else "FALSE", "bool")
|
|
115
132
|
|
|
@@ -171,3 +188,20 @@ def let(c, args, sheet):
|
|
|
171
188
|
raise Unsupported(f"LET name {getattr(name, 'text', '?')!r} is not a name")
|
|
172
189
|
env[name.text.lower()] = _subst(args[i + 1], env)
|
|
173
190
|
return c.expr(_subst(args[-1], env), sheet)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
@register("SINGLE")
|
|
194
|
+
def single(c, args, sheet):
|
|
195
|
+
"""@x (stored _xlfn.SINGLE(x)): implicit intersection, even inside an array formula."""
|
|
196
|
+
arity("SINGLE", args, 1, 1)
|
|
197
|
+
a = args[0]
|
|
198
|
+
if isinstance(a, RefNode):
|
|
199
|
+
from cellproof.refs import resolve
|
|
200
|
+
area = resolve(a.text, sheet, c.book)
|
|
201
|
+
if not area.is_single:
|
|
202
|
+
spill, c.spill = c.spill, None
|
|
203
|
+
try:
|
|
204
|
+
return c._intersect(area, a.text)
|
|
205
|
+
finally:
|
|
206
|
+
c.spill = spill
|
|
207
|
+
return c.expr(a, sheet)
|