cellproof 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cellproof-0.4.0 → cellproof-0.5.0}/CHANGELOG.md +14 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/PKG-INFO +9 -8
- {cellproof-0.4.0 → cellproof-0.5.0}/PROGRESS.md +1 -1
- {cellproof-0.4.0 → cellproof-0.5.0}/README.md +8 -7
- {cellproof-0.4.0 → cellproof-0.5.0}/pyproject.toml +1 -1
- cellproof-0.5.0/src/cellproof/__init__.py +1 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/compiler.py +38 -6
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/emitter.py +4 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/aggregate.py +7 -1
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/dynamic.py +168 -24
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/lookup.py +40 -1
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_dynamic.py +44 -3
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_emitter.py +27 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_lookups.py +19 -0
- cellproof-0.4.0/src/cellproof/__init__.py +0 -1
- {cellproof-0.4.0 → cellproof-0.5.0}/.github/workflows/ci.yml +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/.github/workflows/publish.yml +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/.gitignore +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/AGENTS.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/DESIGN.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/INFO.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/LICENSE +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/PLAN-2a.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/PLAN-2b.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/PLAN.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/SECURITY.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/docs/JOURNEY.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/examples/commission.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/examples/errors.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/examples/functions.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/examples/functions_unsaved.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/examples/make_commission.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/examples/make_errors.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/examples/make_functions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/__main__.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/cli.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/compute.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/expr.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fenwick.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/__init__.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/criteria.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/dates.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/finance.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/logic.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/maths.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/stats.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/text.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/functions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/graph.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/impact.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/model.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/parser.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/r1c1.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/rangeindex.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/reader.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/refs.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/region.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/regions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/report.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/tables.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/verifier.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/bench_shapes.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/corpus.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_aggregate.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_arrays.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_batches.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_cli.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_compiler.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_compute.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_corpus_findings.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_criteria.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_dates.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_errors.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_excel_findings.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_excel_saved.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_fenwick.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_finance_stats.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_functions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_graph.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_impact.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_implicit.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_index.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_lexed.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_limits.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_logic.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_lookup.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_maths.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_parser.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_perf.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_r1c1.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_rangeindex.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_reader.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_refs.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_region.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_regions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_regions_differential.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_regions_split.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_report.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_review_regressions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_running.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_shapes.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_spill.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_tables.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_text.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_types.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_verifier.py +0 -0
- {cellproof-0.4.0 → cellproof-0.5.0}/tests/xlsxkit.py +0 -0
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.5.0 (2026-10-31)
|
|
4
|
+
|
|
5
|
+
- **FILTER, SORT, UNIQUE inside other calculations:** `SUM(FILTER(...))`, `COUNTA(UNIQUE(...))`, AVERAGE, MIN, MAX,
|
|
6
|
+
PRODUCT, STDEV, VAR of them; **ROWS** and **COLUMNS** (of a range or of these). An empty element reaching an
|
|
7
|
+
aggregate is UNRESOLVED (whether Excel skips it or takes 0 was not checked), even through COUNT.
|
|
8
|
+
- **XMATCH** (exact, next smaller/larger, wildcards; first-to-last or last-to-first), **SORTBY** (several keys; by
|
|
9
|
+
rows, or by columns when the keys are one row: real Excel), **SEQUENCE** (more than a million elements is
|
|
10
|
+
UNRESOLVED and builds nothing). FILTER's if_empty can be a range, which spills whole.
|
|
11
|
+
- **Spills are one statement:** a spilled block is one batch computing its relation once (verify 1,334 cells
|
|
12
|
+
31.0 s -> 0.03 s; 33,334 cells in 0.7 s), and one dbt model.
|
|
13
|
+
- **Not guessed:** XLOOKUP/XMATCH next smaller/larger across types or over text with punctuation (Excel ranks numbers
|
|
14
|
+
< text < TRUE/FALSE and orders text by its own collation), and exact text matches with a German sharp s (Excel
|
|
15
|
+
matched "ss" to it). Some of these used to PASS by luck; they are UNRESOLVED now.
|
|
16
|
+
|
|
3
17
|
## 0.4.0 (2026-10-31)
|
|
4
18
|
|
|
5
19
|
- **Array formulas over a block** (legacy Ctrl+Shift+Enter blocks and dynamic-array spills) compile cell by cell:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cellproof
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Nikhil414/cellproof
|
|
6
6
|
Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
|
|
@@ -111,13 +111,13 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
111
111
|
code.
|
|
112
112
|
6. **Emit** the dbt project and the report.
|
|
113
113
|
|
|
114
|
-
|
|
115
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
|
|
114
|
+
130 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
115
|
+
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, COLUMNS, EXACT, EXP, FILTER, FIND, FLOOR, FV,
|
|
116
116
|
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
|
|
117
117
|
LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
|
|
118
118
|
NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
|
|
119
|
-
RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
120
|
-
SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
119
|
+
RIGHT, ROUND(UP/DOWN), ROWS, SEQUENCE, SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
120
|
+
SORT, SORTBY, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR,
|
|
121
121
|
YEAR, YEARFRAC.
|
|
122
122
|
|
|
123
123
|
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
@@ -142,9 +142,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
|
|
|
142
142
|
and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
|
|
143
143
|
result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
|
|
144
144
|
computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
145
|
+
SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
|
|
146
|
+
Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
|
|
147
|
+
calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
|
|
148
|
+
with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
|
|
148
149
|
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
149
150
|
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
150
151
|
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
@@ -29,7 +29,7 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
|
|
|
29
29
|
| Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
|
|
30
30
|
|
|
31
31
|
Tests: 492 (+10 skipped without dbt/Excel files), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
|
|
32
|
-
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1 and 0.
|
|
32
|
+
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1, 0.4.0 and 0.5.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
|
|
33
33
|
A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
|
|
34
34
|
|
|
35
35
|
## Public test workbooks (2026-10-31)
|
|
@@ -77,13 +77,13 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
77
77
|
code.
|
|
78
78
|
6. **Emit** the dbt project and the report.
|
|
79
79
|
|
|
80
|
-
|
|
81
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
|
|
80
|
+
130 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
|
|
81
|
+
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, COLUMNS, EXACT, EXP, FILTER, FIND, FLOOR, FV,
|
|
82
82
|
HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
|
|
83
83
|
LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
|
|
84
84
|
NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
|
|
85
|
-
RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
86
|
-
SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
|
|
85
|
+
RIGHT, ROUND(UP/DOWN), ROWS, SEQUENCE, SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
86
|
+
SORT, SORTBY, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR,
|
|
87
87
|
YEAR, YEARFRAC.
|
|
88
88
|
|
|
89
89
|
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
@@ -108,9 +108,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
|
|
|
108
108
|
and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
|
|
109
109
|
result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
|
|
110
110
|
computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
111
|
+
SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
|
|
112
|
+
Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
|
|
113
|
+
calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
|
|
114
|
+
with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
|
|
114
115
|
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
115
116
|
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
116
117
|
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.5.0"
|
|
@@ -19,7 +19,7 @@ from openpyxl.utils import range_boundaries
|
|
|
19
19
|
|
|
20
20
|
from cellproof.expr import (BLANK, CELLS, NO_ERR, Expr, any_of, compare, error, errs, finalize, lit_num, lit_str, num,
|
|
21
21
|
txt, where_cell)
|
|
22
|
-
from cellproof.fn.dynamic import DYNAMIC,
|
|
22
|
+
from cellproof.fn.dynamic import DYNAMIC, spill_batch
|
|
23
23
|
from cellproof.functions import FUNCS
|
|
24
24
|
from cellproof.graph import formula_order
|
|
25
25
|
from cellproof.model import Book, Cell, Key, Unsupported, a1
|
|
@@ -138,11 +138,7 @@ class Compiler:
|
|
|
138
138
|
c1, r1, _, _ = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
139
139
|
self.spill = (cell.row - r1, cell.col - c1)
|
|
140
140
|
try:
|
|
141
|
-
|
|
142
|
-
if self.spill is not None and isinstance(tree, Call) and tree.name in DYNAMIC:
|
|
143
|
-
c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
144
|
-
return element(self, tree, cell.sheet, *self.spill, (r2 - r1 + 1, c2 - c1 + 1), cell.dynamic)
|
|
145
|
-
e = finalize(self.expr(tree, cell.sheet))
|
|
141
|
+
e = finalize(self.expr(parse(cell.formula), cell.sheet))
|
|
146
142
|
if self.spill is not None and self.spill_dims:
|
|
147
143
|
# real Excel (formulas' test.xlsx): the array is as big as its biggest range, one row (column)
|
|
148
144
|
# repeating; a block cell beyond it is #N/A whatever the formula does with errors -
|
|
@@ -411,10 +407,46 @@ def compile_book(book: Book, regions: bool = True) -> Compiled:
|
|
|
411
407
|
notes.append(f"{r.sheet} rows {r.r1}-{r.r2}, columns {letters(r.cols)} stay in cell mode: they could not "
|
|
412
408
|
f"be ordered as one block ({why})")
|
|
413
409
|
used_models: set[str] = set()
|
|
410
|
+
spilled: set[Key] = set()
|
|
411
|
+
|
|
412
|
+
def add_spill(cell) -> bool:
|
|
413
|
+
"""FILTER/SORT/UNIQUE/... spilled into a block: one batch for the whole block (fn/dynamic.spill_batch)."""
|
|
414
|
+
nonlocal weight
|
|
415
|
+
if not cell.array:
|
|
416
|
+
return False
|
|
417
|
+
try:
|
|
418
|
+
tree = parse(cell.formula)
|
|
419
|
+
except Unsupported:
|
|
420
|
+
return False
|
|
421
|
+
if not (isinstance(tree, Call) and tree.name in DYNAMIC):
|
|
422
|
+
return False
|
|
423
|
+
c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
424
|
+
block = [k for k in ((cell.sheet, r, c) for r in range(r1, r2 + 1) for c in range(c1, c2 + 1))
|
|
425
|
+
if k in book.cells and book.cells[k].array and book.cells[k].ref == cell.ref
|
|
426
|
+
and book.cells[k].formula == cell.formula]
|
|
427
|
+
spilled.update(block)
|
|
428
|
+
_close(comp, batches, current)
|
|
429
|
+
weight = 0
|
|
430
|
+
comp.at = (cell.sheet, r1, c1)
|
|
431
|
+
try:
|
|
432
|
+
_finish(comp, batches, spill_batch(comp, tree, cell.sheet, block, (r1, c1, r2, c2), cell.dynamic))
|
|
433
|
+
except Unsupported as err:
|
|
434
|
+
for k in block:
|
|
435
|
+
unresolved[k] = str(err)
|
|
436
|
+
if err.dep is not None:
|
|
437
|
+
chains[k] = err.dep
|
|
438
|
+
except RecursionError:
|
|
439
|
+
for k in block:
|
|
440
|
+
unresolved[k] = DEEP
|
|
441
|
+
finally:
|
|
442
|
+
comp.at = None
|
|
443
|
+
return True
|
|
414
444
|
|
|
415
445
|
def add(key) -> None:
|
|
416
446
|
nonlocal weight
|
|
417
447
|
cell = book.cells[key]
|
|
448
|
+
if key in spilled or add_spill(cell):
|
|
449
|
+
return
|
|
418
450
|
if cell.kind == "blank":
|
|
419
451
|
unresolved[key] = NO_VALUE
|
|
420
452
|
return
|
|
@@ -140,6 +140,10 @@ def write_project(book: Book, compiled: Compiled, out: Path, stem: str) -> None:
|
|
|
140
140
|
(batches_dir / f"{model_name(k)}.sql").write_text(dbt_region_cells(batch) + "\n", encoding="utf-8")
|
|
141
141
|
schema += (f' - name: {batch.model}\n description: "Region: rows of one sheet that repeat one formula down their rows, '
|
|
142
142
|
f'computed in one statement. Wide and readable; its {model_name(k)} twin holds the same values as long rows."\n')
|
|
143
|
+
elif hasattr(batch, "render_rows"): # a spill: its one statement, reading earlier cells as all_cells
|
|
144
|
+
body = _escape(batch.render_rows("all_cells"))
|
|
145
|
+
body = body.replace("WITH\n", f"WITH\nall_cells AS (\n{_union(sources_for(k))}\n),\n", 1)
|
|
146
|
+
(batches_dir / f"{model_name(k)}.sql").write_text(body + "\n", encoding="utf-8")
|
|
143
147
|
else:
|
|
144
148
|
(batches_dir / f"{model_name(k)}.sql").write_text(dbt_sql(batch, sources_for(k)) + "\n", encoding="utf-8")
|
|
145
149
|
if k % CHECKPOINT == 0 or k == n:
|
|
@@ -286,9 +286,12 @@ def _rng(c, node, sheet: str, skip_subtotals: bool = False) -> Rng:
|
|
|
286
286
|
def _split(c, args, sheet: str, fname: str, skip_subtotals: bool = False) -> tuple[list, list, list]:
|
|
287
287
|
"""The arguments as (ranges, arrays, values). A range arithmetic or an IF over ranges is an array; a
|
|
288
288
|
calculation over plain values stays a value, evaluated before the cells are read (MAX(H3, 1/0) is #DIV/0!)."""
|
|
289
|
+
from cellproof.fn.dynamic import DYNAMIC, as_array
|
|
289
290
|
ranges, arrays, values = [], [], []
|
|
290
291
|
for a in args:
|
|
291
|
-
if isinstance(a,
|
|
292
|
+
if isinstance(a, Call) and a.name in DYNAMIC: # SUM(FILTER(...)): the relation, read like an array
|
|
293
|
+
arrays.append(as_array(c, a, sheet))
|
|
294
|
+
elif isinstance(a, RefNode):
|
|
292
295
|
ranges.append(_rng(c, a, sheet, skip_subtotals))
|
|
293
296
|
elif isinstance(a, Binary) or (isinstance(a, Unary) and a.op == "-") or (
|
|
294
297
|
isinstance(a, Call) and a.name in ("IF", "IFERROR", "IFNA")):
|
|
@@ -363,6 +366,9 @@ def _plain(name, how):
|
|
|
363
366
|
result = how(c, ranges, arrays, values)
|
|
364
367
|
if name not in ("COUNT", "COUNTA", "SUM"): # SUM's error order is already the rule inside _sum
|
|
365
368
|
result = replace(result, err=errs(_errs(c, ranges, arrays, values), result.err))
|
|
369
|
+
markers = [a.unchecked() for a in arrays if hasattr(a, "unchecked")] # FILTER/SORT/UNIQUE: never hidden
|
|
370
|
+
if markers:
|
|
371
|
+
result = replace(result, err=errs(*markers, result.err))
|
|
366
372
|
return result
|
|
367
373
|
register(name)(fn)
|
|
368
374
|
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
from dataclasses import dataclass
|
|
15
15
|
|
|
16
|
-
from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str
|
|
16
|
+
from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str, num
|
|
17
17
|
from cellproof.fn.aggregate import _array, _base, _truth
|
|
18
18
|
from cellproof.fn.lookup import int_literal
|
|
19
19
|
from cellproof.functions import register
|
|
@@ -21,7 +21,8 @@ from cellproof.model import Unsupported
|
|
|
21
21
|
from cellproof.parser import Bool, Call, RefNode
|
|
22
22
|
from cellproof.refs import MAX_ROW, resolve
|
|
23
23
|
|
|
24
|
-
DYNAMIC = {"FILTER", "SORT", "UNIQUE"}
|
|
24
|
+
DYNAMIC = {"FILTER", "SORT", "SORTBY", "UNIQUE", "SEQUENCE"}
|
|
25
|
+
MAX_ELEMENTS = 1_000_000 # SEQUENCE bigger than this is UNCHECKED and builds nothing (a crafted file's 1E9 x 1E9)
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
@dataclass(frozen=True)
|
|
@@ -110,7 +111,9 @@ def filter_(c, args, sheet: str) -> Rel:
|
|
|
110
111
|
f"{'dense_rank() OVER (ORDER BY a.j)' if other == 'j' else 'a.j'} AS j, a.kind, a.num, a.txt, a.err "
|
|
111
112
|
f"FROM {u}_a a JOIN {u}_f f ON f.k = a.{other} WHERE f.keep)")
|
|
112
113
|
empty = None
|
|
113
|
-
if len(args) == 3 and args[2]
|
|
114
|
+
if len(args) == 3 and isinstance(args[2], RefNode) and not resolve(args[2].text, sheet, c.book).is_single:
|
|
115
|
+
empty = f"SELECT * FROM {_range(c, args[2], sheet).sql} ie" # a range spills whole (formulas' test.xlsx)
|
|
116
|
+
elif len(args) == 3 and args[2] is not None:
|
|
114
117
|
e = as_any(c.expr(args[2], sheet))
|
|
115
118
|
empty = f"SELECT 1, 1, {e.kind}, {e.sql}, {e.txt}, {e.err}"
|
|
116
119
|
else: # real Excel disagrees with itself: #CALC! in Microsoft's docs, #VALUE! in formulas' test.xlsx
|
|
@@ -170,31 +173,172 @@ def unique_(c, args, sheet: str) -> Rel:
|
|
|
170
173
|
return Rel(_whole(u, ctes, sql, odd, empty), None, a.cols)
|
|
171
174
|
|
|
172
175
|
|
|
173
|
-
|
|
176
|
+
def _keys_safe(table: str) -> str:
|
|
177
|
+
return (f"(SELECT CASE WHEN EXISTS (SELECT 1 FROM {table} WHERE kind IS NULL OR kind = 'e' OR "
|
|
178
|
+
f"(kind = 's' AND NOT regexp_matches(COALESCE(txt, ''), {SAFE_TEXT}))) THEN '{UNCHECKED}' END)")
|
|
174
179
|
|
|
175
180
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
+
RANK = "CASE kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def sortby(c, args, sheet: str) -> Rel:
|
|
185
|
+
"""SORTBY(array, by1, [order1], by2, [order2], ...): rows ordered by one-column ranges as high as the array, or
|
|
186
|
+
columns by one-row ranges as wide (real Excel, formulas' test.xlsx); the first key first, equal keys keep their
|
|
187
|
+
order. Keys follow SORT's rules (and what is not checked)."""
|
|
188
|
+
if len(args) < 2:
|
|
189
|
+
raise Unsupported("SORTBY needs an array and a by_array")
|
|
190
|
+
a = relation(c, args[0], sheet)
|
|
191
|
+
pairs, rest = [], list(args[1:])
|
|
192
|
+
while rest:
|
|
193
|
+
by = rest.pop(0)
|
|
194
|
+
order = 1 if not rest or rest[0] is None else int_literal(rest[0])
|
|
195
|
+
if rest:
|
|
196
|
+
rest.pop(0)
|
|
197
|
+
if order is None:
|
|
198
|
+
raise Unsupported("SORTBY with a sort order that is not one whole number not supported yet")
|
|
199
|
+
if not isinstance(by, RefNode):
|
|
200
|
+
raise Unsupported("SORTBY by something other than a range not supported yet")
|
|
201
|
+
pairs.append((_range(c, by, sheet), order))
|
|
202
|
+
if a.rows is None or a.cols is None:
|
|
203
|
+
raise Unsupported("SORTBY of an array whose size is only known at run time not supported yet")
|
|
204
|
+
if all(by.cols == 1 and by.rows == a.rows for by, _ in pairs):
|
|
205
|
+
axis, across = "i", "j"
|
|
206
|
+
elif all(by.rows == 1 and by.cols == a.cols for by, _ in pairs):
|
|
207
|
+
axis, across = "j", "i"
|
|
208
|
+
else:
|
|
209
|
+
return Rel("(SELECT 1 AS i, 1 AS j, 'e' AS kind, NULL AS num, NULL AS txt, '#VALUE!' AS err)", 1, 1)
|
|
181
210
|
u = _uid(c)
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
211
|
+
bad = [f"CASE WHEN {order} NOT IN (1, -1) THEN '#VALUE!' END" for _, order in pairs]
|
|
212
|
+
ctes = [f"{u}_a AS MATERIALIZED {a.sql}"]
|
|
213
|
+
order_by, joins = [], []
|
|
214
|
+
for n, (by, order) in enumerate(pairs):
|
|
215
|
+
ctes.append(f"{u}_k{n} AS MATERIALIZED (SELECT {axis} AS p, kind, num, txt FROM {by.sql} b WHERE {across} = 1)")
|
|
216
|
+
bad.append(_keys_safe(f"{u}_k{n}"))
|
|
217
|
+
d = "ASC" if order == 1 else "DESC"
|
|
218
|
+
k = f"k{n}"
|
|
219
|
+
joins.append(f"JOIN {u}_k{n} {k} ON {k}.p = a.p")
|
|
220
|
+
order_by += [f"(CASE {k}.kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END) {d}", f"{k}.num {d}",
|
|
221
|
+
f"lower(COALESCE({k}.txt, '')) {d}"]
|
|
222
|
+
new = (f"(SELECT a.p AS op, row_number() OVER (ORDER BY {', '.join(order_by)}, a.p) AS np "
|
|
223
|
+
f"FROM (SELECT DISTINCT {axis} AS p FROM {u}_a) a {' '.join(joins)})")
|
|
224
|
+
i_sql, j_sql = ("n.np", "a.j") if axis == "i" else ("a.i", "n.np")
|
|
225
|
+
sql = (f"(SELECT {i_sql} AS i, {j_sql} AS j, a.kind, a.num, a.txt, a.err FROM {u}_a a "
|
|
226
|
+
f"JOIN {new} n ON n.op = a.{axis})")
|
|
227
|
+
first_bad = "COALESCE(" + ", ".join(f"({b})" for b in bad) + ")"
|
|
228
|
+
return Rel(_whole(u, ctes, sql, first_bad), a.rows, a.cols)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def sequence(c, args, sheet: str) -> Rel:
|
|
232
|
+
"""SEQUENCE(rows, [columns], [start], [step]): start, start + step, ... row by row; rows and columns are
|
|
233
|
+
truncated. Fewer than one row or column, or more than MAX_ELEMENTS elements, is UNCHECKED."""
|
|
234
|
+
if not 1 <= len(args) <= 4:
|
|
235
|
+
raise Unsupported("SEQUENCE needs one to four arguments")
|
|
236
|
+
one = Expr("CAST(1 AS DOUBLE)", "num")
|
|
237
|
+
rows, cols, start, step = (num(c.expr(a, sheet)) if a is not None else one
|
|
238
|
+
for a in list(args) + [None] * (4 - len(args)))
|
|
239
|
+
h, w = f"trunc({rows.sql})", f"trunc({cols.sql})"
|
|
240
|
+
ok = f"({h} >= 1 AND {w} >= 1 AND {h} * {w} <= {MAX_ELEMENTS})"
|
|
241
|
+
n = lambda x: f"(CASE WHEN {ok} THEN CAST({x} AS BIGINT) ELSE 0 END)" # noqa: E731 - builds nothing when not ok
|
|
242
|
+
body = (f"(SELECT gi.i AS i, gj.j AS j, 'n' AS kind, {start.sql} + (({'gi.i'} - 1) * {w} + gj.j - 1) * "
|
|
243
|
+
f"{step.sql} AS num, CAST(NULL AS VARCHAR) AS txt, CAST(NULL AS VARCHAR) AS err "
|
|
244
|
+
f"FROM range(1, {n(h)} + 1) gi(i) CROSS JOIN range(1, {n(w)} + 1) gj(j))")
|
|
245
|
+
u = _uid(c)
|
|
246
|
+
bad = (f"COALESCE({rows.err}, {cols.err}, {start.err}, {step.err}, "
|
|
247
|
+
f"CASE WHEN NOT {ok} THEN '{UNCHECKED}' END)")
|
|
248
|
+
return Rel(_whole(u, [], body, bad), None, None)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
BUILDERS = {"FILTER": filter_, "SORT": sort_, "SORTBY": sortby, "UNIQUE": unique_, "SEQUENCE": sequence}
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class SpillBatch(list):
|
|
255
|
+
"""The Step of every cell of a spilled block, and one set-based statement computing them all: the relation once,
|
|
256
|
+
each cell joined to its element (like region.RegionBatch)."""
|
|
257
|
+
|
|
258
|
+
def __init__(self, steps, sql: str):
|
|
259
|
+
super().__init__(steps)
|
|
260
|
+
self.sql = sql
|
|
261
|
+
|
|
262
|
+
def render_rows(self, prev: str) -> str:
|
|
263
|
+
return self.sql.replace(CELLS, prev)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def spill_batch(c, node, sheet: str, block: list, box: tuple, dynamic: bool) -> SpillBatch:
|
|
267
|
+
"""A result one row (column) high repeats down (across), as in a legacy block; past the result a cell is #N/A.
|
|
268
|
+
The anchor of a dynamic spill is #SPILL! when the result is not exactly as big as the block Excel spilled it
|
|
269
|
+
into (the translation disagrees with Excel's own result); an UNRESOLVED marker wins over that check."""
|
|
270
|
+
from cellproof.compiler import Step
|
|
271
|
+
r1, c1, r2, c2 = box
|
|
272
|
+
rel = relation(c, node, sheet).sql
|
|
193
273
|
markers = ", ".join(f"'{m}'" for m in SETTINGS)
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
274
|
+
pos = ", ".join(f"({r}, {col}, {r - r1 + 1}, {col - c1 + 1})" for _, r, col in block)
|
|
275
|
+
size = (f"(p.pi = 1 AND p.pj = 1 AND (d.h <> {r2 - r1 + 1} OR d.w <> {c2 - c1 + 1}))" if dynamic else "FALSE")
|
|
276
|
+
err = (f"CASE WHEN f.e IN ({markers}) THEN f.e WHEN {size} THEN '#SPILL!' WHEN s.i IS NULL THEN '#N/A' "
|
|
277
|
+
f"ELSE s.err END")
|
|
278
|
+
sql = (f"WITH\nspill AS MATERIALIZED {rel},\n"
|
|
279
|
+
f"d AS (SELECT max(i) AS h, max(j) AS w FROM spill),\n"
|
|
280
|
+
f"f AS (SELECT (SELECT err FROM spill WHERE i = 1 AND j = 1) AS e),\n"
|
|
281
|
+
f"cellpos AS (SELECT * FROM (VALUES {pos}) v(row_num, col_num, pi, pj)),\n"
|
|
282
|
+
f"picked AS (SELECT p.row_num, p.col_num, s.kind, s.num, s.txt, {err} AS e FROM cellpos p CROSS JOIN d "
|
|
283
|
+
f"CROSS JOIN f LEFT JOIN spill s ON s.i = (CASE WHEN d.h = 1 THEN 1 ELSE p.pi END) "
|
|
284
|
+
f"AND s.j = (CASE WHEN d.w = 1 THEN 1 ELSE p.pj END))\n"
|
|
285
|
+
# an empty element spills as 0; an error is kind 'e' with its code as text, as in the cells table
|
|
286
|
+
f"SELECT {lit_str(sheet).sql} AS sheet, row_num, col_num, "
|
|
287
|
+
f"CASE WHEN e IS NOT NULL THEN 'e' WHEN kind IS NULL THEN 'n' ELSE kind END AS kind, "
|
|
288
|
+
f"CASE WHEN e IS NOT NULL THEN NULL WHEN kind IS NULL THEN CAST(0 AS DOUBLE) ELSE num END AS num, "
|
|
289
|
+
f"CASE WHEN e IS NOT NULL THEN e WHEN kind = 's' THEN COALESCE(txt, '') END AS txt FROM picked")
|
|
290
|
+
steps = [Step(k, Expr("NULL", "any", "'#N/A'"), f"s{n}") for n, k in enumerate(block, 1)]
|
|
291
|
+
return SpillBatch(steps, sql)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
@dataclass(frozen=True)
|
|
295
|
+
class RelArray:
|
|
296
|
+
"""A FILTER/SORT/UNIQUE result as the aggregates read an array (aggregate.Arr's agg/err_sql/rel)."""
|
|
297
|
+
sql: str
|
|
298
|
+
srcs: tuple = ("relation",)
|
|
299
|
+
shape: None = None
|
|
300
|
+
|
|
301
|
+
def rel(self) -> str:
|
|
302
|
+
# what an aggregate does with an empty element (skip it, or take it as 0) was not checked: UNCHECKED
|
|
303
|
+
return (f"(SELECT i AS row_num, j AS col_num, CASE WHEN kind IS NULL THEN 'e' ELSE kind END AS kind, num, "
|
|
304
|
+
f"txt, CASE WHEN kind IS NULL THEN '{UNCHECKED}' ELSE err END AS err FROM {self.sql} ra)")
|
|
305
|
+
|
|
306
|
+
def agg(self, fn: str, kinds: str = "('n')") -> str:
|
|
307
|
+
return f"(SELECT {fn} FROM {self.rel()} WHERE kind IN {kinds})"
|
|
308
|
+
|
|
309
|
+
def err_sql(self) -> str:
|
|
310
|
+
return f"(SELECT err FROM {self.rel()} WHERE err IS NOT NULL ORDER BY row_num, col_num LIMIT 1)"
|
|
311
|
+
|
|
312
|
+
def unchecked(self) -> str:
|
|
313
|
+
"""An UNRESOLVED marker in the relation, which even COUNT (that skips errors) must not hide."""
|
|
314
|
+
markers = ", ".join(f"'{m}'" for m in SETTINGS)
|
|
315
|
+
return f"(SELECT err FROM {self.rel()} WHERE err IN ({markers}) LIMIT 1)"
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def as_array(c, node, sheet: str) -> RelArray:
|
|
319
|
+
return RelArray(relation(c, node, sheet).sql)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _size(name: str, col: str):
|
|
323
|
+
def fn(c, args, sheet):
|
|
324
|
+
if len(args) != 1:
|
|
325
|
+
raise Unsupported(f"{name} needs one argument")
|
|
326
|
+
a = args[0]
|
|
327
|
+
if isinstance(a, RefNode):
|
|
328
|
+
area = resolve(a.text, sheet, c.book)
|
|
329
|
+
return Expr(f"CAST({area.r2 - area.r1 + 1 if col == 'i' else area.c2 - area.c1 + 1} AS DOUBLE)", "num")
|
|
330
|
+
if isinstance(a, Call) and a.name in DYNAMIC:
|
|
331
|
+
rel = relation(c, a, sheet).sql
|
|
332
|
+
# an error result (FILTER's #CALC!, an UNCHECKED marker) is the answer, not a size of one
|
|
333
|
+
first = f"(SELECT err FROM {rel} rs WHERE i = 1 AND j = 1 AND kind = 'e' AND "
|
|
334
|
+
whole = f"{first} (SELECT COUNT(*) FROM {rel} rc) = 1)"
|
|
335
|
+
return Expr(f"(SELECT CAST(max({col}) AS DOUBLE) FROM {rel} rm)", "num", whole)
|
|
336
|
+
raise Unsupported(f"{name} of something other than a range, FILTER, SORT or UNIQUE not supported yet")
|
|
337
|
+
register(name)(fn)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
_size("ROWS", "i")
|
|
341
|
+
_size("COLUMNS", "j")
|
|
198
342
|
|
|
199
343
|
|
|
200
344
|
def _inside(name):
|
|
@@ -241,6 +241,19 @@ class Search:
|
|
|
241
241
|
cond, key = _near("k", x, ">=")
|
|
242
242
|
self.agg, self.key = "arg_min", f"({key}, {'-' if last else ''}{self.pos})"
|
|
243
243
|
self.src = f"FROM {table} WHERE {where_area(line, 'k')} AND {cond}"
|
|
244
|
+
# Not guessed (UNCHECKED): next smaller/larger across types or over text with punctuation (real Excel
|
|
245
|
+
# ranks numbers < text < TRUE/FALSE and orders text by its own collation: formulas' test.xlsx), and an
|
|
246
|
+
# exact text match with a German sharp s (Excel matched "ss" to it there)
|
|
247
|
+
kind = {"num": "'n'", "txt": "'s'", "bool": "'b'"}.get(x.type, x.kind)
|
|
248
|
+
xt = x.txt if x.type == "any" else x.sql if x.type == "txt" else "NULL"
|
|
249
|
+
cells = f"FROM {CELLS} k WHERE {where_area(line, 'k')}"
|
|
250
|
+
if mode == "exact":
|
|
251
|
+
odd = f"(strpos(COALESCE({xt}, ''), 'ß') > 0 OR EXISTS (SELECT 1 {cells} AND strpos(COALESCE(k.txt, ''), 'ß') > 0))"
|
|
252
|
+
else:
|
|
253
|
+
safe = "regexp_matches(COALESCE({t}, ''), '^[A-Za-z0-9 ]*$')"
|
|
254
|
+
odd = (f"(NOT {safe.format(t=xt)} OR EXISTS (SELECT 1 {cells} AND (k.kind <> {kind} OR "
|
|
255
|
+
f"(k.kind = 's' AND NOT {safe.format(t='k.txt')}))))")
|
|
256
|
+
self.unchecked = f"(CASE WHEN {odd} THEN '{UNCHECKED}' END)"
|
|
244
257
|
|
|
245
258
|
def pick(self, value: str, default: str | None = None) -> str:
|
|
246
259
|
if default is not None: # arg_min skips a NULL value: the cell would look not found
|
|
@@ -363,13 +376,15 @@ def xlookup(c, args, sheet):
|
|
|
363
376
|
else f"r.col_num = k.col_num + {back.c1 - look.c1} AND r.row_num = {back.r1}")
|
|
364
377
|
search = Search(x, look, mode, wild=match_mode == 2, last=search_mode == -1, back=(lit_str(back.sheet).sql, hit))
|
|
365
378
|
keys = c.ready(back)
|
|
366
|
-
found = _read(search.pick, _type_of(c, back, keys),
|
|
379
|
+
found = _read(search.pick, _type_of(c, back, keys),
|
|
380
|
+
errs(x.err, search.unchecked, _found_error(search.pick, "#N/A")))
|
|
367
381
|
if len(args) < 4 or args[3] is None:
|
|
368
382
|
return found
|
|
369
383
|
missing = c.expr(args[3], sheet)
|
|
370
384
|
found, missing = unify_all([found, missing])
|
|
371
385
|
hit_sql = search.hit()
|
|
372
386
|
pick_err = f"(CASE WHEN {hit_sql} THEN {found.err} ELSE {missing.err} END)"
|
|
387
|
+
pick_err = errs(search.unchecked, pick_err)
|
|
373
388
|
|
|
374
389
|
def case(a: str, b: str) -> str:
|
|
375
390
|
return f"(CASE WHEN {hit_sql} THEN {a} ELSE {b} END)"
|
|
@@ -458,3 +473,27 @@ def choose(c, args, sheet):
|
|
|
458
473
|
err = errs(i.err, bad, case([v.err for v in values]) if any(v.err != NO_ERR for v in values) else NO_ERR)
|
|
459
474
|
return Expr(case([v.sql for v in values]), values[0].type, err, case([v.txt for v in values]),
|
|
460
475
|
case([v.kind for v in values]))
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
@register("XMATCH")
|
|
479
|
+
def xmatch(c, args, sheet):
|
|
480
|
+
"""XMATCH(lookup, array, [match_mode], [search_mode]): the position XLOOKUP's search finds (exact, next smaller
|
|
481
|
+
-1, next larger 1, wildcards 2; first to last 1 or last to first -1)."""
|
|
482
|
+
arity("XMATCH", args, 2, 4)
|
|
483
|
+
modes = []
|
|
484
|
+
for i, default in ((2, 0), (3, 1)):
|
|
485
|
+
node = args[i] if len(args) > i else None
|
|
486
|
+
mode = default if node is None else int_literal(node)
|
|
487
|
+
if mode is None:
|
|
488
|
+
raise Unsupported("XMATCH modes must be numbers written in the formula")
|
|
489
|
+
modes.append(mode)
|
|
490
|
+
match_mode, search_mode = modes
|
|
491
|
+
if match_mode not in (0, -1, 1, 2) or search_mode not in (1, -1):
|
|
492
|
+
raise Unsupported("XMATCH binary search modes not supported yet")
|
|
493
|
+
line = _line(c, args[1], sheet, "XMATCH")
|
|
494
|
+
x = _xlookup_x(c, args, sheet)
|
|
495
|
+
found = Search(x, line, {0: "exact", 2: "exact", -1: "le", 1: "ge"}[match_mode], wild=match_mode == 2,
|
|
496
|
+
last=search_mode == -1)
|
|
497
|
+
first = line.r1 if line.c1 == line.c2 else line.c1
|
|
498
|
+
return Expr(found.pick(f"CAST({found.pos} - {first} + 1 AS DOUBLE)"), "num",
|
|
499
|
+
errs(x.err, found.unchecked, f"(CASE WHEN NOT {found.hit()} THEN '#N/A' END)"))
|
|
@@ -84,9 +84,50 @@ def test_what_excel_was_not_asked_is_not_guessed(tmp_path):
|
|
|
84
84
|
**spill("K6", "_xlfn._xlws.FILTER(A2:A5,C2:C5>100)", [["#CALC!"]]),
|
|
85
85
|
# empty sort keys, in a block bigger than the one element: UNRESOLVED, not a #SPILL! FAIL
|
|
86
86
|
**spill("L6", "_xlfn._xlws.SORT(G1:G3)", [[0.0], [0.0], [1.0]]),
|
|
87
|
-
"J8": ("=ROWS(_xlfn.UNIQUE(B2:B5))", 3.0),
|
|
88
87
|
"J9": ("=_xlfn._xlws.FILTER(A2:A5,C2:C5>50)", "Asha"), # not an array formula
|
|
89
88
|
})
|
|
90
|
-
assert set(bad) == {("S", 6, 10), ("S", 6, 11), ("S", 6, 12), ("S", 7, 12), ("S", 8, 12), ("S",
|
|
91
|
-
("S", 9, 10)}
|
|
89
|
+
assert set(bad) == {("S", 6, 10), ("S", 6, 11), ("S", 6, 12), ("S", 7, 12), ("S", 8, 12), ("S", 9, 10)}
|
|
92
90
|
assert all(v[0] == "UNRESOLVED" for v in bad.values())
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_filter_sort_unique_inside_aggregates(tmp_path):
|
|
94
|
+
_, bad = check(tmp_path, {
|
|
95
|
+
"J1": ('=SUM(_xlfn._xlws.FILTER(C2:C5,B2:B5="West"))', 140.0),
|
|
96
|
+
"J2": ("=COUNTA(_xlfn.UNIQUE(B2:B5))", 3.0),
|
|
97
|
+
"J3": ("=ROWS(_xlfn.UNIQUE(C2:C5))", 3.0), "J4": ("=COLUMNS(A1:C5)", 3.0), "J5": ("=ROWS(A2:C5)", 4.0),
|
|
98
|
+
"J6": ("=MAX(_xlfn._xlws.SORT(C2:C5))", 80.0),
|
|
99
|
+
"J7": ('=AVERAGE(_xlfn._xlws.FILTER(C2:C5,C2:C5<70))', 140 / 3),
|
|
100
|
+
"J8": ('=SUM(_xlfn._xlws.FILTER(C2:C5,C2:C5>100))', "#CALC!"), # nothing kept, no if_empty
|
|
101
|
+
"J9": ('=COUNT(_xlfn._xlws.FILTER(G1:G3,G1:G3<5))', 1.0), # keeps two empty cells: not checked
|
|
102
|
+
})
|
|
103
|
+
assert set(bad) == {("S", 8, 10), ("S", 9, 10)} and all(v[0] == "UNRESOLVED" for v in bad.values())
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_sortby_and_sequence(tmp_path):
|
|
107
|
+
_, bad = check(tmp_path, {
|
|
108
|
+
**spill("J1", "_xlfn.SORTBY(A2:A5,C2:C5,-1)", [["Asha"], ["Mina"], ["Ravi"], ["Vikram"]]),
|
|
109
|
+
**spill("K1", "_xlfn.SORTBY(A2:A5,C2:C5,1,A2:A5,-1)", [["Vikram"], ["Ravi"], ["Mina"], ["Asha"]]),
|
|
110
|
+
**spill("L1", "_xlfn.SORTBY(A2:A5,C2:C3)", [["#VALUE!"]]), # by_array of another height
|
|
111
|
+
**spill("M1", "_xlfn.SEQUENCE(2,3)", [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]),
|
|
112
|
+
**spill("M4", "_xlfn.SEQUENCE(3,1,10,-5)", [[10.0], [5.0], [0.0]]),
|
|
113
|
+
**spill("M8", "_xlfn.SEQUENCE(2.9)", [[1.0], [2.0]]), # truncated
|
|
114
|
+
"M11": ("=SUM(_xlfn.SEQUENCE(100))", 5050.0),
|
|
115
|
+
**spill("Q1", "_xlfn.SEQUENCE(0)", [["#CALC!"]]), # not checked
|
|
116
|
+
**spill("Q2", "_xlfn.SEQUENCE(1E9,1E9)", [["#NUM!"]]), # never built
|
|
117
|
+
})
|
|
118
|
+
assert set(bad) == {("S", 1, 17), ("S", 2, 17)} and all(v[0] == "UNRESOLVED" for v in bad.values())
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_corpus_findings_for_sortby_filter_and_xmatch(tmp_path):
|
|
122
|
+
# formulas' test.xlsx (Excel-saved): SORTBY by one-row arrays sorts the columns; FILTER's if_empty range
|
|
123
|
+
# spills whole; XMATCH next-smaller across types or over punctuation, and a sharp s, are not guessed
|
|
124
|
+
_, bad = check(tmp_path, {
|
|
125
|
+
**spill("J1", "_xlfn.SORTBY(C2:C3,{2},1)", [[80.0], [40.0]]),
|
|
126
|
+
**spill("K1", "_xlfn.SORTBY(E1:G1,{3,1,2},1)", [[0.0, 1.0, 1.0]]),
|
|
127
|
+
**spill("K3", "_xlfn.SORTBY({1,2,3;4,5,6},{3,2,2},1,{5,1,4},-1)", [[3.0, 2.0, 1.0], [6.0, 5.0, 4.0]]),
|
|
128
|
+
**spill("K6", '_xlfn._xlws.FILTER(A2:A5,C2:C5>100,B2:B3)', [["West"], ["East"]]),
|
|
129
|
+
"N1": ('=_xlfn.XMATCH("*b?u*",{-1.1,2.1,3.1,4.1},-1)', 4.0),
|
|
130
|
+
"N2": ('=_xlfn.XMATCH("Mina",A2:A5,-1)', 3.0), # same type, plain text: compiled
|
|
131
|
+
"N3": ('=_xlfn.XMATCH("ss",{1,"ß"},0)', 2.0),
|
|
132
|
+
})
|
|
133
|
+
assert set(bad) == {("S", 1, 14), ("S", 3, 14)} and all(v[0] == "UNRESOLVED" for v in bad.values())
|
|
@@ -238,3 +238,30 @@ def test_array_constants_reach_the_dbt_models(tmp_path):
|
|
|
238
238
|
write_project(book, compiled, tmp_path, "arrays")
|
|
239
239
|
got = {(s, r, c): (k, n, t) for s, r, c, k, n, t in _run_models(tmp_path)}
|
|
240
240
|
assert got == execute(book, compiled)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _spill_book(tmp_path):
|
|
244
|
+
from test_dynamic import DATA, spill
|
|
245
|
+
from xlsxkit import write_xlsx
|
|
246
|
+
from cellproof.reader import load
|
|
247
|
+
wb = tmp_path / "spill.xlsx"
|
|
248
|
+
write_xlsx(wb, {"S": {**DATA, **spill("J1", "_xlfn._xlws.SORT(_xlfn.UNIQUE(B2:B5))", [["East"], ["North"], ["West"]]),
|
|
249
|
+
**spill("K1", "_xlfn._xlws.FILTER(A2:C5,C2:C5>50)", [["Asha", "West", 80.0],
|
|
250
|
+
["Mina", "west", 60.0]]),
|
|
251
|
+
"M1": ("=J2&K2", "NorthMina")}}) # a cell reading spilled cells
|
|
252
|
+
return wb, load(wb)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def test_spill_models_equal_the_verifier(tmp_path):
|
|
256
|
+
_, book = _spill_book(tmp_path)
|
|
257
|
+
compiled = compile_book(book)
|
|
258
|
+
assert compiled.unresolved == {}
|
|
259
|
+
write_project(book, compiled, tmp_path / "out", "spill")
|
|
260
|
+
got = {(s, r, c): (k, n, t) for s, r, c, k, n, t in _run_models(tmp_path / "out")}
|
|
261
|
+
assert got == execute(book, compiled)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@pytest.mark.skipif(shutil.which("dbt") is None, reason="needs: pip install -e .[dbt]")
|
|
265
|
+
def test_dbt_build_with_spills(tmp_path):
|
|
266
|
+
wb, _ = _spill_book(tmp_path)
|
|
267
|
+
assert main(["convert", str(wb), "--out", str(tmp_path / "out"), "--dbt"]) == 0
|
|
@@ -134,3 +134,22 @@ def test_the_walk_matches_excel_s_bisection_on_random_tables():
|
|
|
134
134
|
want = bisection(vec, x)
|
|
135
135
|
kind, num, txt = got[("F", i, 1)]
|
|
136
136
|
assert (kind == "e" and txt == "#N/A") if want is None else num == want + 1
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def test_xmatch_modes():
|
|
140
|
+
# Microsoft's XMATCH examples: exact, next smaller/larger, wildcard, last to first
|
|
141
|
+
from cellproof.compiler import compile_book
|
|
142
|
+
from cellproof.verifier import execute
|
|
143
|
+
from xlsxkit import book_of
|
|
144
|
+
s = {"A1": "Apple", "A2": "Grape", "A3": "Pear", "A4": "Banana", "A5": "Cherry", "A6": "Grape",
|
|
145
|
+
"B1": 10.0, "B2": 20.0, "B3": 30.0, "B4": 40.0,
|
|
146
|
+
"C1": ('=_xlfn.XMATCH("Grape",A1:A6)', 2.0), "C2": ('=_xlfn.XMATCH("Grape",A1:A6,0,-1)', 6.0),
|
|
147
|
+
"C3": ('=_xlfn.XMATCH("gra*",A1:A6,2)', 2.0), "C4": ("=_xlfn.XMATCH(25,B1:B4,-1)", 2.0),
|
|
148
|
+
"C5": ("=_xlfn.XMATCH(25,B1:B4,1)", 3.0), "C6": ('=_xlfn.XMATCH("Kiwi",A1:A6)', "#N/A"),
|
|
149
|
+
"C7": ("=_xlfn.XMATCH(25,B1:B4,0,2)", 0.0)}
|
|
150
|
+
book = book_of({"S": s})
|
|
151
|
+
compiled = compile_book(book)
|
|
152
|
+
got = execute(book, compiled)
|
|
153
|
+
assert [got[("S", r, 3)] for r in range(1, 7)] == [("n", 2.0, None), ("n", 6.0, None), ("n", 2.0, None),
|
|
154
|
+
("n", 2.0, None), ("n", 3.0, None), ("e", None, "#N/A")]
|
|
155
|
+
assert "binary search" in compiled.unresolved[("S", 7, 3)]
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.4.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md
RENAMED
|
File without changes
|
{cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md
RENAMED
|
File without changes
|
{cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-18-running-totals-design.md
RENAMED
|
File without changes
|
{cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|