cellproof 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cellproof-0.4.0 → cellproof-0.6.0}/CHANGELOG.md +26 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/PKG-INFO +14 -12
- {cellproof-0.4.0 → cellproof-0.6.0}/PROGRESS.md +1 -1
- {cellproof-0.4.0 → cellproof-0.6.0}/README.md +13 -11
- {cellproof-0.4.0 → cellproof-0.6.0}/pyproject.toml +1 -1
- cellproof-0.6.0/src/cellproof/__init__.py +1 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/cli.py +10 -6
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/compiler.py +41 -8
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/emitter.py +4 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/aggregate.py +7 -1
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/dynamic.py +168 -24
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/finance.py +63 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/logic.py +70 -2
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/lookup.py +40 -1
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/text.py +35 -1
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/regions.py +2 -1
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/verifier.py +25 -2
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_cli.py +42 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_dynamic.py +44 -3
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_emitter.py +27 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_finance_stats.py +28 -0
- cellproof-0.6.0/tests/test_functions.py +81 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_lookups.py +19 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_region.py +1 -1
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_spill.py +11 -0
- cellproof-0.4.0/src/cellproof/__init__.py +0 -1
- cellproof-0.4.0/tests/test_functions.py +0 -52
- {cellproof-0.4.0 → cellproof-0.6.0}/.github/workflows/ci.yml +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/.github/workflows/publish.yml +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/.gitignore +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/AGENTS.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/DESIGN.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/INFO.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/LICENSE +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/PLAN-2a.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/PLAN-2b.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/PLAN.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/SECURITY.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/docs/JOURNEY.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/examples/commission.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/examples/errors.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/examples/functions.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/examples/functions_unsaved.xlsx +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/examples/make_commission.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/examples/make_errors.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/examples/make_functions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/__main__.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/compute.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/expr.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fenwick.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/__init__.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/criteria.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/dates.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/maths.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/stats.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/functions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/graph.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/impact.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/model.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/parser.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/r1c1.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/rangeindex.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/reader.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/refs.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/region.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/report.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/tables.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/bench_shapes.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/corpus.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_aggregate.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_arrays.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_batches.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_compiler.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_compute.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_corpus_findings.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_criteria.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_dates.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_errors.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_excel_findings.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_excel_saved.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_fenwick.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_graph.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_impact.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_implicit.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_index.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_lexed.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_limits.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_logic.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_lookup.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_maths.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_parser.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_perf.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_r1c1.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_rangeindex.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_reader.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_refs.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_regions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_regions_differential.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_regions_split.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_report.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_review_regressions.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_running.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_shapes.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_tables.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_text.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_types.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_verifier.py +0 -0
- {cellproof-0.4.0 → cellproof-0.6.0}/tests/xlsxkit.py +0 -0
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.6.0 (2026-10-31)
|
|
4
|
+
|
|
5
|
+
- **A batch DuckDB rejects no longer stops the run:** its cells are UNRESOLVED with DuckDB's message, the book is
|
|
6
|
+
compiled again so every cell reading them is UNRESOLVED too (none can read a missing value), and the report is
|
|
7
|
+
written. The exit code is still 3: it is a bug in cellproof, please report it.
|
|
8
|
+
- **ROW, COLUMN** (of the formula's cell, of a cell, and in an array formula of a range: one column of row numbers,
|
|
9
|
+
one row of column numbers - real Excel), **CODE** (Windows-1252, CHAR's inverse; a character outside it, or above
|
|
10
|
+
127 in a file Excel for Mac calculated, is UNRESOLVED), **UNICODE, UNICHAR, N** (of a range in an array formula:
|
|
11
|
+
its first cell), **ISODD, ISEVEN** (no ranges in an array formula: #VALUE!, like EDATE).
|
|
12
|
+
- **IRR and RATE** by Newton's method in SQL (a recursive CTE). Not guessed: cash flows with several sign changes
|
|
13
|
+
(several roots), an error among IRR's cells, a RATE whose answer is 0, nper not a positive whole number.
|
|
14
|
+
|
|
15
|
+
## 0.5.0 (2026-10-31)
|
|
16
|
+
|
|
17
|
+
- **FILTER, SORT, UNIQUE inside other calculations:** `SUM(FILTER(...))`, `COUNTA(UNIQUE(...))`, AVERAGE, MIN, MAX,
|
|
18
|
+
PRODUCT, STDEV, VAR of them; **ROWS** and **COLUMNS** (of a range or of these). An empty element reaching an
|
|
19
|
+
aggregate is UNRESOLVED (whether Excel skips it or takes 0 was not checked), even through COUNT.
|
|
20
|
+
- **XMATCH** (exact, next smaller/larger, wildcards; first-to-last or last-to-first), **SORTBY** (several keys; by
|
|
21
|
+
rows, or by columns when the keys are one row: real Excel), **SEQUENCE** (more than a million elements is
|
|
22
|
+
UNRESOLVED and builds nothing). FILTER's if_empty can be a range, which spills whole.
|
|
23
|
+
- **Spills are one statement:** a spilled block is one batch computing its relation once (verify 1,334 cells
|
|
24
|
+
31.0 s -> 0.03 s; 33,334 cells in 0.7 s), and one dbt model.
|
|
25
|
+
- **Not guessed:** XLOOKUP/XMATCH next smaller/larger across types or over text with punctuation (Excel ranks numbers
|
|
26
|
+
< text < TRUE/FALSE and orders text by its own collation), and exact text matches with a German sharp s (Excel
|
|
27
|
+
matched "ss" to it). Some of these used to PASS by luck; they are UNRESOLVED now.
|
|
28
|
+
|
|
3
29
|
## 0.4.0 (2026-10-31)
|
|
4
30
|
|
|
5
31
|
- **Array formulas over a block** (legacy Ctrl+Shift+Enter blocks and dynamic-array spills) compile cell by cell:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cellproof
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Nikhil414/cellproof
|
|
6
6
|
Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
|
|
@@ -111,14 +111,15 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
111
111
|
code.
|
|
112
112
|
6. **Emit** the dbt project and the report.
|
|
113
113
|
|
|
114
|
-
|
|
115
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE,
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
114
|
+
140 Excel functions are supported: @ (SINGLE), ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CODE,
|
|
115
|
+
COLUMN, COLUMNS, CONCAT(ENATE), COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE,
|
|
116
|
+
EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV, HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IRR, IS* (BLANK, ERR,
|
|
117
|
+
ERROR, LOGICAL, NA, NUMBER, TEXT), ISEVEN, ISODD, LARGE, LEFT, LEN, LET, LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX,
|
|
118
|
+
MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, N, NA, NETWORKDAYS, NOT, NPER, NPV, OR, PERCENTILE(.INC),
|
|
119
|
+
PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), RATE, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), ROW, ROWS,
|
|
120
|
+
SEARCH, SEQUENCE, SIGN, SMALL, SORT, SORTBY, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
121
|
+
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNICHAR, UNICODE, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP,
|
|
122
|
+
WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR, YEAR, YEARFRAC.
|
|
122
123
|
|
|
123
124
|
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
124
125
|
`Sales[[Rep]:[Amount]]`) are rewritten to plain references when the file is read, so a calculated table column
|
|
@@ -142,9 +143,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
|
|
|
142
143
|
and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
|
|
143
144
|
result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
|
|
144
145
|
computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
|
|
147
|
+
Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
|
|
148
|
+
calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
|
|
149
|
+
with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
|
|
148
150
|
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
149
151
|
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
150
152
|
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
@@ -29,7 +29,7 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
|
|
|
29
29
|
| Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
|
|
30
30
|
|
|
31
31
|
Tests: 492 (+10 skipped without dbt/Excel files), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
|
|
32
|
-
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1 and 0.
|
|
32
|
+
purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1, 0.4.0, 0.5.0 and 0.6.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
|
|
33
33
|
A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
|
|
34
34
|
|
|
35
35
|
## Public test workbooks (2026-10-31)
|
|
@@ -77,14 +77,15 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
|
|
|
77
77
|
code.
|
|
78
78
|
6. **Emit** the dbt project and the report.
|
|
79
79
|
|
|
80
|
-
|
|
81
|
-
COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE,
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
80
|
+
140 Excel functions are supported: @ (SINGLE), ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CODE,
|
|
81
|
+
COLUMN, COLUMNS, CONCAT(ENATE), COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE,
|
|
82
|
+
EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV, HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IRR, IS* (BLANK, ERR,
|
|
83
|
+
ERROR, LOGICAL, NA, NUMBER, TEXT), ISEVEN, ISODD, LARGE, LEFT, LEN, LET, LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX,
|
|
84
|
+
MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, N, NA, NETWORKDAYS, NOT, NPER, NPV, OR, PERCENTILE(.INC),
|
|
85
|
+
PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), RATE, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), ROW, ROWS,
|
|
86
|
+
SEARCH, SEQUENCE, SIGN, SMALL, SORT, SORTBY, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
|
|
87
|
+
SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNICHAR, UNICODE, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP,
|
|
88
|
+
WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR, YEAR, YEARFRAC.
|
|
88
89
|
|
|
89
90
|
**Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
|
|
90
91
|
`Sales[[Rep]:[Amount]]`) are rewritten to plain references when the file is read, so a calculated table column
|
|
@@ -108,9 +109,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
|
|
|
108
109
|
and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
|
|
109
110
|
result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
|
|
110
111
|
computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
112
|
+
SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
|
|
113
|
+
Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
|
|
114
|
+
calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
|
|
115
|
+
with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
|
|
114
116
|
- **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
|
|
115
117
|
`SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
|
|
116
118
|
empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.6.0"
|
|
@@ -10,7 +10,7 @@ from cellproof.compute import fill_missing
|
|
|
10
10
|
from cellproof.emitter import run_dbt, write_project
|
|
11
11
|
from cellproof.reader import UnreadableWorkbook, load
|
|
12
12
|
from cellproof.report import write_report
|
|
13
|
-
from cellproof.verifier import SqlFailed,
|
|
13
|
+
from cellproof.verifier import SqlFailed, verify_book
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
def main(argv: list[str] | None = None) -> int:
|
|
@@ -31,14 +31,18 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
31
31
|
print(f"error: {err}", file=sys.stderr)
|
|
32
32
|
return 2
|
|
33
33
|
book, computed, why = fill_missing(book, args.workbook)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
def compile_fn(b):
|
|
35
|
+
compiled = compile_book(b)
|
|
36
|
+
if why:
|
|
37
|
+
compiled.unresolved.update({k: f"{r} ({why})" for k, r in compiled.unresolved.items() if r == NO_VALUE})
|
|
38
|
+
return compiled
|
|
37
39
|
try:
|
|
38
|
-
verdicts =
|
|
40
|
+
compiled, verdicts, failures = verify_book(book, compile_fn)
|
|
39
41
|
except SqlFailed as err:
|
|
40
42
|
print(f"error: {err}", file=sys.stderr)
|
|
41
43
|
return 3
|
|
44
|
+
for failure in failures: # those cells are UNRESOLVED in the report; the run goes on
|
|
45
|
+
print(f"error: {failure}", file=sys.stderr)
|
|
42
46
|
out = args.out or args.workbook.with_name(args.workbook.stem + "_dbt")
|
|
43
47
|
write_project(book, compiled, out, args.workbook.stem)
|
|
44
48
|
counts = write_report(verdicts, compiled.findings, out, args.workbook.name, computed, compiled.notes,
|
|
@@ -46,7 +50,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
46
50
|
if computed:
|
|
47
51
|
print(f"{len(computed)} expected values computed by the formulas library (the file had none saved)")
|
|
48
52
|
print(f"{counts['PASS']} pass, {counts['FAIL']} fail, {counts['UNRESOLVED']} unresolved -> {out / 'REPORT.md'}")
|
|
49
|
-
code = 1 if counts["FAIL"] else 0
|
|
53
|
+
code = 3 if failures else 1 if counts["FAIL"] else 0
|
|
50
54
|
if args.dbt:
|
|
51
55
|
ok, log = run_dbt(out)
|
|
52
56
|
print("dbt build: ok" if ok else f"dbt build failed:\n{log}")
|
|
@@ -19,7 +19,7 @@ from openpyxl.utils import range_boundaries
|
|
|
19
19
|
|
|
20
20
|
from cellproof.expr import (BLANK, CELLS, NO_ERR, Expr, any_of, compare, error, errs, finalize, lit_num, lit_str, num,
|
|
21
21
|
txt, where_cell)
|
|
22
|
-
from cellproof.fn.dynamic import DYNAMIC,
|
|
22
|
+
from cellproof.fn.dynamic import DYNAMIC, spill_batch
|
|
23
23
|
from cellproof.functions import FUNCS
|
|
24
24
|
from cellproof.graph import formula_order
|
|
25
25
|
from cellproof.model import Book, Cell, Key, Unsupported, a1
|
|
@@ -106,8 +106,9 @@ class Keys:
|
|
|
106
106
|
|
|
107
107
|
|
|
108
108
|
# Analysis ToolPak functions do not take an array where one value is expected: {arguments that are one value}
|
|
109
|
-
NO_LIFT = {"EDATE": 2, "EOMONTH": 2, "MROUND": 2, "YEARFRAC": 3, "NETWORKDAYS": 2, "WORKDAY": 2
|
|
110
|
-
|
|
109
|
+
NO_LIFT = {"EDATE": 2, "EOMONTH": 2, "MROUND": 2, "YEARFRAC": 3, "NETWORKDAYS": 2, "WORKDAY": 2, "ISEVEN": 1,
|
|
110
|
+
"ISODD": 1}
|
|
111
|
+
NO_LIFT_CHECKED = {"EDATE", "EOMONTH", "MROUND", "YEARFRAC", "ISEVEN", "ISODD"} # #VALUE!: formulas' test.xlsx
|
|
111
112
|
|
|
112
113
|
|
|
113
114
|
class Compiler:
|
|
@@ -138,11 +139,7 @@ class Compiler:
|
|
|
138
139
|
c1, r1, _, _ = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
139
140
|
self.spill = (cell.row - r1, cell.col - c1)
|
|
140
141
|
try:
|
|
141
|
-
|
|
142
|
-
if self.spill is not None and isinstance(tree, Call) and tree.name in DYNAMIC:
|
|
143
|
-
c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
144
|
-
return element(self, tree, cell.sheet, *self.spill, (r2 - r1 + 1, c2 - c1 + 1), cell.dynamic)
|
|
145
|
-
e = finalize(self.expr(tree, cell.sheet))
|
|
142
|
+
e = finalize(self.expr(parse(cell.formula), cell.sheet))
|
|
146
143
|
if self.spill is not None and self.spill_dims:
|
|
147
144
|
# real Excel (formulas' test.xlsx): the array is as big as its biggest range, one row (column)
|
|
148
145
|
# repeating; a block cell beyond it is #N/A whatever the formula does with errors -
|
|
@@ -411,10 +408,46 @@ def compile_book(book: Book, regions: bool = True) -> Compiled:
|
|
|
411
408
|
notes.append(f"{r.sheet} rows {r.r1}-{r.r2}, columns {letters(r.cols)} stay in cell mode: they could not "
|
|
412
409
|
f"be ordered as one block ({why})")
|
|
413
410
|
used_models: set[str] = set()
|
|
411
|
+
spilled: set[Key] = set()
|
|
412
|
+
|
|
413
|
+
def add_spill(cell) -> bool:
|
|
414
|
+
"""FILTER/SORT/UNIQUE/... spilled into a block: one batch for the whole block (fn/dynamic.spill_batch)."""
|
|
415
|
+
nonlocal weight
|
|
416
|
+
if not cell.array or cell.key in book.unread: # compile_cell gives the reason
|
|
417
|
+
return False
|
|
418
|
+
try:
|
|
419
|
+
tree = parse(cell.formula)
|
|
420
|
+
except Unsupported:
|
|
421
|
+
return False
|
|
422
|
+
if not (isinstance(tree, Call) and tree.name in DYNAMIC):
|
|
423
|
+
return False
|
|
424
|
+
c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
|
|
425
|
+
block = [k for k in ((cell.sheet, r, c) for r in range(r1, r2 + 1) for c in range(c1, c2 + 1))
|
|
426
|
+
if k in book.cells and book.cells[k].array and book.cells[k].ref == cell.ref
|
|
427
|
+
and book.cells[k].formula == cell.formula]
|
|
428
|
+
spilled.update(block)
|
|
429
|
+
_close(comp, batches, current)
|
|
430
|
+
weight = 0
|
|
431
|
+
comp.at = (cell.sheet, r1, c1)
|
|
432
|
+
try:
|
|
433
|
+
_finish(comp, batches, spill_batch(comp, tree, cell.sheet, block, (r1, c1, r2, c2), cell.dynamic))
|
|
434
|
+
except Unsupported as err:
|
|
435
|
+
for k in block:
|
|
436
|
+
unresolved[k] = str(err)
|
|
437
|
+
if err.dep is not None:
|
|
438
|
+
chains[k] = err.dep
|
|
439
|
+
except RecursionError:
|
|
440
|
+
for k in block:
|
|
441
|
+
unresolved[k] = DEEP
|
|
442
|
+
finally:
|
|
443
|
+
comp.at = None
|
|
444
|
+
return True
|
|
414
445
|
|
|
415
446
|
def add(key) -> None:
|
|
416
447
|
nonlocal weight
|
|
417
448
|
cell = book.cells[key]
|
|
449
|
+
if key in spilled or add_spill(cell):
|
|
450
|
+
return
|
|
418
451
|
if cell.kind == "blank":
|
|
419
452
|
unresolved[key] = NO_VALUE
|
|
420
453
|
return
|
|
@@ -140,6 +140,10 @@ def write_project(book: Book, compiled: Compiled, out: Path, stem: str) -> None:
|
|
|
140
140
|
(batches_dir / f"{model_name(k)}.sql").write_text(dbt_region_cells(batch) + "\n", encoding="utf-8")
|
|
141
141
|
schema += (f' - name: {batch.model}\n description: "Region: rows of one sheet that repeat one formula down their rows, '
|
|
142
142
|
f'computed in one statement. Wide and readable; its {model_name(k)} twin holds the same values as long rows."\n')
|
|
143
|
+
elif hasattr(batch, "render_rows"): # a spill: its one statement, reading earlier cells as all_cells
|
|
144
|
+
body = _escape(batch.render_rows("all_cells"))
|
|
145
|
+
body = body.replace("WITH\n", f"WITH\nall_cells AS (\n{_union(sources_for(k))}\n),\n", 1)
|
|
146
|
+
(batches_dir / f"{model_name(k)}.sql").write_text(body + "\n", encoding="utf-8")
|
|
143
147
|
else:
|
|
144
148
|
(batches_dir / f"{model_name(k)}.sql").write_text(dbt_sql(batch, sources_for(k)) + "\n", encoding="utf-8")
|
|
145
149
|
if k % CHECKPOINT == 0 or k == n:
|
|
@@ -286,9 +286,12 @@ def _rng(c, node, sheet: str, skip_subtotals: bool = False) -> Rng:
|
|
|
286
286
|
def _split(c, args, sheet: str, fname: str, skip_subtotals: bool = False) -> tuple[list, list, list]:
|
|
287
287
|
"""The arguments as (ranges, arrays, values). A range arithmetic or an IF over ranges is an array; a
|
|
288
288
|
calculation over plain values stays a value, evaluated before the cells are read (MAX(H3, 1/0) is #DIV/0!)."""
|
|
289
|
+
from cellproof.fn.dynamic import DYNAMIC, as_array
|
|
289
290
|
ranges, arrays, values = [], [], []
|
|
290
291
|
for a in args:
|
|
291
|
-
if isinstance(a,
|
|
292
|
+
if isinstance(a, Call) and a.name in DYNAMIC: # SUM(FILTER(...)): the relation, read like an array
|
|
293
|
+
arrays.append(as_array(c, a, sheet))
|
|
294
|
+
elif isinstance(a, RefNode):
|
|
292
295
|
ranges.append(_rng(c, a, sheet, skip_subtotals))
|
|
293
296
|
elif isinstance(a, Binary) or (isinstance(a, Unary) and a.op == "-") or (
|
|
294
297
|
isinstance(a, Call) and a.name in ("IF", "IFERROR", "IFNA")):
|
|
@@ -363,6 +366,9 @@ def _plain(name, how):
|
|
|
363
366
|
result = how(c, ranges, arrays, values)
|
|
364
367
|
if name not in ("COUNT", "COUNTA", "SUM"): # SUM's error order is already the rule inside _sum
|
|
365
368
|
result = replace(result, err=errs(_errs(c, ranges, arrays, values), result.err))
|
|
369
|
+
markers = [a.unchecked() for a in arrays if hasattr(a, "unchecked")] # FILTER/SORT/UNIQUE: never hidden
|
|
370
|
+
if markers:
|
|
371
|
+
result = replace(result, err=errs(*markers, result.err))
|
|
366
372
|
return result
|
|
367
373
|
register(name)(fn)
|
|
368
374
|
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
from dataclasses import dataclass
|
|
15
15
|
|
|
16
|
-
from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str
|
|
16
|
+
from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str, num
|
|
17
17
|
from cellproof.fn.aggregate import _array, _base, _truth
|
|
18
18
|
from cellproof.fn.lookup import int_literal
|
|
19
19
|
from cellproof.functions import register
|
|
@@ -21,7 +21,8 @@ from cellproof.model import Unsupported
|
|
|
21
21
|
from cellproof.parser import Bool, Call, RefNode
|
|
22
22
|
from cellproof.refs import MAX_ROW, resolve
|
|
23
23
|
|
|
24
|
-
DYNAMIC = {"FILTER", "SORT", "UNIQUE"}
|
|
24
|
+
DYNAMIC = {"FILTER", "SORT", "SORTBY", "UNIQUE", "SEQUENCE"}
|
|
25
|
+
MAX_ELEMENTS = 1_000_000 # SEQUENCE bigger than this is UNCHECKED and builds nothing (a crafted file's 1E9 x 1E9)
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
@dataclass(frozen=True)
|
|
@@ -110,7 +111,9 @@ def filter_(c, args, sheet: str) -> Rel:
|
|
|
110
111
|
f"{'dense_rank() OVER (ORDER BY a.j)' if other == 'j' else 'a.j'} AS j, a.kind, a.num, a.txt, a.err "
|
|
111
112
|
f"FROM {u}_a a JOIN {u}_f f ON f.k = a.{other} WHERE f.keep)")
|
|
112
113
|
empty = None
|
|
113
|
-
if len(args) == 3 and args[2]
|
|
114
|
+
if len(args) == 3 and isinstance(args[2], RefNode) and not resolve(args[2].text, sheet, c.book).is_single:
|
|
115
|
+
empty = f"SELECT * FROM {_range(c, args[2], sheet).sql} ie" # a range spills whole (formulas' test.xlsx)
|
|
116
|
+
elif len(args) == 3 and args[2] is not None:
|
|
114
117
|
e = as_any(c.expr(args[2], sheet))
|
|
115
118
|
empty = f"SELECT 1, 1, {e.kind}, {e.sql}, {e.txt}, {e.err}"
|
|
116
119
|
else: # real Excel disagrees with itself: #CALC! in Microsoft's docs, #VALUE! in formulas' test.xlsx
|
|
@@ -170,31 +173,172 @@ def unique_(c, args, sheet: str) -> Rel:
|
|
|
170
173
|
return Rel(_whole(u, ctes, sql, odd, empty), None, a.cols)
|
|
171
174
|
|
|
172
175
|
|
|
173
|
-
|
|
176
|
+
def _keys_safe(table: str) -> str:
|
|
177
|
+
return (f"(SELECT CASE WHEN EXISTS (SELECT 1 FROM {table} WHERE kind IS NULL OR kind = 'e' OR "
|
|
178
|
+
f"(kind = 's' AND NOT regexp_matches(COALESCE(txt, ''), {SAFE_TEXT}))) THEN '{UNCHECKED}' END)")
|
|
174
179
|
|
|
175
180
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
+
RANK = "CASE kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def sortby(c, args, sheet: str) -> Rel:
|
|
185
|
+
"""SORTBY(array, by1, [order1], by2, [order2], ...): rows ordered by one-column ranges as high as the array, or
|
|
186
|
+
columns by one-row ranges as wide (real Excel, formulas' test.xlsx); the first key first, equal keys keep their
|
|
187
|
+
order. Keys follow SORT's rules (and what is not checked)."""
|
|
188
|
+
if len(args) < 2:
|
|
189
|
+
raise Unsupported("SORTBY needs an array and a by_array")
|
|
190
|
+
a = relation(c, args[0], sheet)
|
|
191
|
+
pairs, rest = [], list(args[1:])
|
|
192
|
+
while rest:
|
|
193
|
+
by = rest.pop(0)
|
|
194
|
+
order = 1 if not rest or rest[0] is None else int_literal(rest[0])
|
|
195
|
+
if rest:
|
|
196
|
+
rest.pop(0)
|
|
197
|
+
if order is None:
|
|
198
|
+
raise Unsupported("SORTBY with a sort order that is not one whole number not supported yet")
|
|
199
|
+
if not isinstance(by, RefNode):
|
|
200
|
+
raise Unsupported("SORTBY by something other than a range not supported yet")
|
|
201
|
+
pairs.append((_range(c, by, sheet), order))
|
|
202
|
+
if a.rows is None or a.cols is None:
|
|
203
|
+
raise Unsupported("SORTBY of an array whose size is only known at run time not supported yet")
|
|
204
|
+
if all(by.cols == 1 and by.rows == a.rows for by, _ in pairs):
|
|
205
|
+
axis, across = "i", "j"
|
|
206
|
+
elif all(by.rows == 1 and by.cols == a.cols for by, _ in pairs):
|
|
207
|
+
axis, across = "j", "i"
|
|
208
|
+
else:
|
|
209
|
+
return Rel("(SELECT 1 AS i, 1 AS j, 'e' AS kind, NULL AS num, NULL AS txt, '#VALUE!' AS err)", 1, 1)
|
|
181
210
|
u = _uid(c)
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
211
|
+
bad = [f"CASE WHEN {order} NOT IN (1, -1) THEN '#VALUE!' END" for _, order in pairs]
|
|
212
|
+
ctes = [f"{u}_a AS MATERIALIZED {a.sql}"]
|
|
213
|
+
order_by, joins = [], []
|
|
214
|
+
for n, (by, order) in enumerate(pairs):
|
|
215
|
+
ctes.append(f"{u}_k{n} AS MATERIALIZED (SELECT {axis} AS p, kind, num, txt FROM {by.sql} b WHERE {across} = 1)")
|
|
216
|
+
bad.append(_keys_safe(f"{u}_k{n}"))
|
|
217
|
+
d = "ASC" if order == 1 else "DESC"
|
|
218
|
+
k = f"k{n}"
|
|
219
|
+
joins.append(f"JOIN {u}_k{n} {k} ON {k}.p = a.p")
|
|
220
|
+
order_by += [f"(CASE {k}.kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END) {d}", f"{k}.num {d}",
|
|
221
|
+
f"lower(COALESCE({k}.txt, '')) {d}"]
|
|
222
|
+
new = (f"(SELECT a.p AS op, row_number() OVER (ORDER BY {', '.join(order_by)}, a.p) AS np "
|
|
223
|
+
f"FROM (SELECT DISTINCT {axis} AS p FROM {u}_a) a {' '.join(joins)})")
|
|
224
|
+
i_sql, j_sql = ("n.np", "a.j") if axis == "i" else ("a.i", "n.np")
|
|
225
|
+
sql = (f"(SELECT {i_sql} AS i, {j_sql} AS j, a.kind, a.num, a.txt, a.err FROM {u}_a a "
|
|
226
|
+
f"JOIN {new} n ON n.op = a.{axis})")
|
|
227
|
+
first_bad = "COALESCE(" + ", ".join(f"({b})" for b in bad) + ")"
|
|
228
|
+
return Rel(_whole(u, ctes, sql, first_bad), a.rows, a.cols)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def sequence(c, args, sheet: str) -> Rel:
|
|
232
|
+
"""SEQUENCE(rows, [columns], [start], [step]): start, start + step, ... row by row; rows and columns are
|
|
233
|
+
truncated. Fewer than one row or column, or more than MAX_ELEMENTS elements, is UNCHECKED."""
|
|
234
|
+
if not 1 <= len(args) <= 4:
|
|
235
|
+
raise Unsupported("SEQUENCE needs one to four arguments")
|
|
236
|
+
one = Expr("CAST(1 AS DOUBLE)", "num")
|
|
237
|
+
rows, cols, start, step = (num(c.expr(a, sheet)) if a is not None else one
|
|
238
|
+
for a in list(args) + [None] * (4 - len(args)))
|
|
239
|
+
h, w = f"trunc({rows.sql})", f"trunc({cols.sql})"
|
|
240
|
+
ok = f"({h} >= 1 AND {w} >= 1 AND {h} * {w} <= {MAX_ELEMENTS})"
|
|
241
|
+
n = lambda x: f"(CASE WHEN {ok} THEN CAST({x} AS BIGINT) ELSE 0 END)" # noqa: E731 - builds nothing when not ok
|
|
242
|
+
body = (f"(SELECT gi.i AS i, gj.j AS j, 'n' AS kind, {start.sql} + (({'gi.i'} - 1) * {w} + gj.j - 1) * "
|
|
243
|
+
f"{step.sql} AS num, CAST(NULL AS VARCHAR) AS txt, CAST(NULL AS VARCHAR) AS err "
|
|
244
|
+
f"FROM range(1, {n(h)} + 1) gi(i) CROSS JOIN range(1, {n(w)} + 1) gj(j))")
|
|
245
|
+
u = _uid(c)
|
|
246
|
+
bad = (f"COALESCE({rows.err}, {cols.err}, {start.err}, {step.err}, "
|
|
247
|
+
f"CASE WHEN NOT {ok} THEN '{UNCHECKED}' END)")
|
|
248
|
+
return Rel(_whole(u, [], body, bad), None, None)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
BUILDERS = {"FILTER": filter_, "SORT": sort_, "SORTBY": sortby, "UNIQUE": unique_, "SEQUENCE": sequence}
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class SpillBatch(list):
|
|
255
|
+
"""The Step of every cell of a spilled block, and one set-based statement computing them all: the relation once,
|
|
256
|
+
each cell joined to its element (like region.RegionBatch)."""
|
|
257
|
+
|
|
258
|
+
def __init__(self, steps, sql: str):
|
|
259
|
+
super().__init__(steps)
|
|
260
|
+
self.sql = sql
|
|
261
|
+
|
|
262
|
+
def render_rows(self, prev: str) -> str:
|
|
263
|
+
return self.sql.replace(CELLS, prev)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def spill_batch(c, node, sheet: str, block: list, box: tuple, dynamic: bool) -> SpillBatch:
|
|
267
|
+
"""A result one row (column) high repeats down (across), as in a legacy block; past the result a cell is #N/A.
|
|
268
|
+
The anchor of a dynamic spill is #SPILL! when the result is not exactly as big as the block Excel spilled it
|
|
269
|
+
into (the translation disagrees with Excel's own result); an UNRESOLVED marker wins over that check."""
|
|
270
|
+
from cellproof.compiler import Step
|
|
271
|
+
r1, c1, r2, c2 = box
|
|
272
|
+
rel = relation(c, node, sheet).sql
|
|
193
273
|
markers = ", ".join(f"'{m}'" for m in SETTINGS)
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
274
|
+
pos = ", ".join(f"({r}, {col}, {r - r1 + 1}, {col - c1 + 1})" for _, r, col in block)
|
|
275
|
+
size = (f"(p.pi = 1 AND p.pj = 1 AND (d.h <> {r2 - r1 + 1} OR d.w <> {c2 - c1 + 1}))" if dynamic else "FALSE")
|
|
276
|
+
err = (f"CASE WHEN f.e IN ({markers}) THEN f.e WHEN {size} THEN '#SPILL!' WHEN s.i IS NULL THEN '#N/A' "
|
|
277
|
+
f"ELSE s.err END")
|
|
278
|
+
sql = (f"WITH\nspill AS MATERIALIZED {rel},\n"
|
|
279
|
+
f"d AS (SELECT max(i) AS h, max(j) AS w FROM spill),\n"
|
|
280
|
+
f"f AS (SELECT (SELECT err FROM spill WHERE i = 1 AND j = 1) AS e),\n"
|
|
281
|
+
f"cellpos AS (SELECT * FROM (VALUES {pos}) v(row_num, col_num, pi, pj)),\n"
|
|
282
|
+
f"picked AS (SELECT p.row_num, p.col_num, s.kind, s.num, s.txt, {err} AS e FROM cellpos p CROSS JOIN d "
|
|
283
|
+
f"CROSS JOIN f LEFT JOIN spill s ON s.i = (CASE WHEN d.h = 1 THEN 1 ELSE p.pi END) "
|
|
284
|
+
f"AND s.j = (CASE WHEN d.w = 1 THEN 1 ELSE p.pj END))\n"
|
|
285
|
+
# an empty element spills as 0; an error is kind 'e' with its code as text, as in the cells table
|
|
286
|
+
f"SELECT {lit_str(sheet).sql} AS sheet, row_num, col_num, "
|
|
287
|
+
f"CASE WHEN e IS NOT NULL THEN 'e' WHEN kind IS NULL THEN 'n' ELSE kind END AS kind, "
|
|
288
|
+
f"CASE WHEN e IS NOT NULL THEN NULL WHEN kind IS NULL THEN CAST(0 AS DOUBLE) ELSE num END AS num, "
|
|
289
|
+
f"CASE WHEN e IS NOT NULL THEN e WHEN kind = 's' THEN COALESCE(txt, '') END AS txt FROM picked")
|
|
290
|
+
steps = [Step(k, Expr("NULL", "any", "'#N/A'"), f"s{n}") for n, k in enumerate(block, 1)]
|
|
291
|
+
return SpillBatch(steps, sql)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
@dataclass(frozen=True)
|
|
295
|
+
class RelArray:
|
|
296
|
+
"""A FILTER/SORT/UNIQUE result as the aggregates read an array (aggregate.Arr's agg/err_sql/rel)."""
|
|
297
|
+
sql: str
|
|
298
|
+
srcs: tuple = ("relation",)
|
|
299
|
+
shape: None = None
|
|
300
|
+
|
|
301
|
+
def rel(self) -> str:
|
|
302
|
+
# what an aggregate does with an empty element (skip it, or take it as 0) was not checked: UNCHECKED
|
|
303
|
+
return (f"(SELECT i AS row_num, j AS col_num, CASE WHEN kind IS NULL THEN 'e' ELSE kind END AS kind, num, "
|
|
304
|
+
f"txt, CASE WHEN kind IS NULL THEN '{UNCHECKED}' ELSE err END AS err FROM {self.sql} ra)")
|
|
305
|
+
|
|
306
|
+
def agg(self, fn: str, kinds: str = "('n')") -> str:
|
|
307
|
+
return f"(SELECT {fn} FROM {self.rel()} WHERE kind IN {kinds})"
|
|
308
|
+
|
|
309
|
+
def err_sql(self) -> str:
|
|
310
|
+
return f"(SELECT err FROM {self.rel()} WHERE err IS NOT NULL ORDER BY row_num, col_num LIMIT 1)"
|
|
311
|
+
|
|
312
|
+
def unchecked(self) -> str:
|
|
313
|
+
"""An UNRESOLVED marker in the relation, which even COUNT (that skips errors) must not hide."""
|
|
314
|
+
markers = ", ".join(f"'{m}'" for m in SETTINGS)
|
|
315
|
+
return f"(SELECT err FROM {self.rel()} WHERE err IN ({markers}) LIMIT 1)"
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def as_array(c, node, sheet: str) -> RelArray:
|
|
319
|
+
return RelArray(relation(c, node, sheet).sql)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _size(name: str, col: str):
|
|
323
|
+
def fn(c, args, sheet):
|
|
324
|
+
if len(args) != 1:
|
|
325
|
+
raise Unsupported(f"{name} needs one argument")
|
|
326
|
+
a = args[0]
|
|
327
|
+
if isinstance(a, RefNode):
|
|
328
|
+
area = resolve(a.text, sheet, c.book)
|
|
329
|
+
return Expr(f"CAST({area.r2 - area.r1 + 1 if col == 'i' else area.c2 - area.c1 + 1} AS DOUBLE)", "num")
|
|
330
|
+
if isinstance(a, Call) and a.name in DYNAMIC:
|
|
331
|
+
rel = relation(c, a, sheet).sql
|
|
332
|
+
# an error result (FILTER's #CALC!, an UNCHECKED marker) is the answer, not a size of one
|
|
333
|
+
first = f"(SELECT err FROM {rel} rs WHERE i = 1 AND j = 1 AND kind = 'e' AND "
|
|
334
|
+
whole = f"{first} (SELECT COUNT(*) FROM {rel} rc) = 1)"
|
|
335
|
+
return Expr(f"(SELECT CAST(max({col}) AS DOUBLE) FROM {rel} rm)", "num", whole)
|
|
336
|
+
raise Unsupported(f"{name} of something other than a range, FILTER, SORT or UNIQUE not supported yet")
|
|
337
|
+
register(name)(fn)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
_size("ROWS", "i")
|
|
341
|
+
_size("COLUMNS", "j")
|
|
198
342
|
|
|
199
343
|
|
|
200
344
|
def _inside(name):
|
|
@@ -101,3 +101,66 @@ def npv(c, args, sheet):
|
|
|
101
101
|
sql = (f"(SELECT COALESCE(SUM(num / POWER(1 + {r.sql}, p)), CAST(0 AS DOUBLE)) FROM ({periods}) npv_periods)")
|
|
102
102
|
return Expr(sql, "num", errs(r.err, *(v.err for v in typed), *(g.err(c) for g in ranges),
|
|
103
103
|
_unchecked(f"{r.sql} = -1")))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
NEWTON_STEPS = 100
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _newton(ctes: str, f: str, fp: str, guess: str) -> tuple[str, str]:
|
|
110
|
+
"""(root SQL, converged SQL) of Newton's method from guess; f and fp are SQL of the column r (and the CTEs).
|
|
111
|
+
Excel stops at 0.00001 percent; this runs to the last digits, which differ from Excel's last iterate by far less
|
|
112
|
+
than the verifier's 1e-9. Not converging, or leaving r > -1, is #NUM!, as in Excel."""
|
|
113
|
+
it = (f"WITH RECURSIVE {ctes}, it(n, r, d) AS (SELECT 0, CAST({guess} AS DOUBLE), CAST(NULL AS DOUBLE) "
|
|
114
|
+
f"UNION ALL SELECT n + 1, r - fv / fpv, fv / fpv FROM (SELECT it.n, it.r, {f} AS fv, {fp} AS fpv FROM it, "
|
|
115
|
+
f"args WHERE it.n < {NEWTON_STEPS} AND it.r > -1 AND (it.d IS NULL OR abs(it.d) > 1e-13 * greatest(1, "
|
|
116
|
+
f"abs(it.r)))) s) SELECT {{}} FROM it ORDER BY n DESC LIMIT 1")
|
|
117
|
+
ok = "(d IS NOT NULL AND isfinite(r) AND r > -1 AND abs(d) <= 1e-10 * greatest(1, abs(r)))"
|
|
118
|
+
return f"({it.format('CASE WHEN ' + ok + ' THEN r END')})", f"({it.format(ok)})"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@register("IRR")
|
|
122
|
+
def irr(c, args, sheet):
|
|
123
|
+
"""IRR(values, [guess]): the rate where the NPV of the values (the first one now) is 0. Text, TRUE/FALSE and empty
|
|
124
|
+
cells are skipped. Several sign changes can have several roots, and which one Excel's iteration lands on was
|
|
125
|
+
not checked: UNCHECKED."""
|
|
126
|
+
arity("IRR", args, 1, 2)
|
|
127
|
+
if not isinstance(args[0], RefNode):
|
|
128
|
+
raise Unsupported("IRR of values typed into the formula not supported yet")
|
|
129
|
+
rng = _rng(c, args[0], sheet)
|
|
130
|
+
guess = num(c.expr(args[1], sheet)) if len(args) == 2 and args[1] is not None else Expr("CAST(0.1 AS DOUBLE)", "num")
|
|
131
|
+
where = f"FROM {CELLS} WHERE {rng.where()} AND kind = 'n'"
|
|
132
|
+
ctes = f"args AS (SELECT list(num ORDER BY row_num, col_num) AS v {where})"
|
|
133
|
+
f = "list_sum(list_transform(v, lambda x, i: x / pow(1 + it.r, i - 1)))"
|
|
134
|
+
fp = "list_sum(list_transform(v, lambda x, i: -(i - 1) * x / pow(1 + it.r, i)))"
|
|
135
|
+
root, ok = _newton(ctes, f, fp, guess.sql)
|
|
136
|
+
signs = f"(SELECT COUNT(*) FILTER (WHERE num > 0) > 0 AND COUNT(*) FILTER (WHERE num < 0) > 0 {where})"
|
|
137
|
+
flips = (f"(SELECT COUNT(*) FROM (SELECT sign(num) AS s, lag(sign(num)) OVER (ORDER BY row_num, col_num) AS p "
|
|
138
|
+
f"{where} AND num <> 0) irr_signs WHERE s <> p)")
|
|
139
|
+
# real Excel (formulas' test.xlsx): an error in an array constant is #VALUE!; in cells, not checked
|
|
140
|
+
error_in = rng.err(c)
|
|
141
|
+
if error_in != "NULL":
|
|
142
|
+
error_in = (f"(CASE WHEN {error_in} IS NOT NULL THEN "
|
|
143
|
+
f"'{'#VALUE!' if rng.area.sheet in c.book.arrays else UNCHECKED}' END)")
|
|
144
|
+
return Expr(root, "num", errs(error_in, guess.err, f"(CASE WHEN NOT {signs} THEN '#NUM!' END)",
|
|
145
|
+
_unchecked(f"{flips} > 1"), f"(CASE WHEN NOT {ok} THEN '#NUM!' END)"))
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@register("RATE")
|
|
149
|
+
def rate(c, args, sheet):
|
|
150
|
+
"""RATE(nper, pmt, pv, [fv], [type], [guess]): the rate solving PMT's equation, by Newton's method."""
|
|
151
|
+
arity("RATE", args, 3, 6)
|
|
152
|
+
a = _args(c, args[:5], sheet, "RATE", 3, 5)
|
|
153
|
+
n, pmt, pv, fv, t = (x.sql for x in a)
|
|
154
|
+
guess = num(c.expr(args[5], sheet)) if len(args) == 6 and args[5] is not None else Expr("CAST(0.1 AS DOUBLE)", "num")
|
|
155
|
+
ctes = f"args AS (SELECT {n} AS n_, {pmt} AS pmt, {pv} AS pv, {fv} AS fv, {t} AS t)"
|
|
156
|
+
g, gp = "pow(1 + it.r, n_)", "(n_ * pow(1 + it.r, n_ - 1))"
|
|
157
|
+
f = f"(pv * {g} + pmt * (1 + it.r * t) * ({g} - 1) / it.r + fv)"
|
|
158
|
+
fp = (f"(pv * {gp} + pmt * (t * ({g} - 1) / it.r + (1 + it.r * t) * ({gp} * it.r - ({g} - 1)) / "
|
|
159
|
+
f"(it.r * it.r)))")
|
|
160
|
+
root, ok = _newton(ctes, f, fp, guess.sql)
|
|
161
|
+
return Expr(root, "num", errs(*(x.err for x in a), guess.err, _type(a[4]),
|
|
162
|
+
# a rate of 0 solves it (the payments only repay): Excel's answer there unchecked
|
|
163
|
+
_unchecked(f"{n} <= 0 OR {n} <> floor({n}) OR abs({pv} + {pmt} * {n} + {fv}) "
|
|
164
|
+
f"<= 1e-9 * greatest(abs({pv}), abs({pmt} * {n}), abs({fv}), 1) "
|
|
165
|
+
f"OR abs({root}) < 1e-9"),
|
|
166
|
+
f"(CASE WHEN NOT {ok} THEN '#NUM!' END)"))
|