cellproof 0.4.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. {cellproof-0.4.0 → cellproof-0.6.0}/CHANGELOG.md +26 -0
  2. {cellproof-0.4.0 → cellproof-0.6.0}/PKG-INFO +14 -12
  3. {cellproof-0.4.0 → cellproof-0.6.0}/PROGRESS.md +1 -1
  4. {cellproof-0.4.0 → cellproof-0.6.0}/README.md +13 -11
  5. {cellproof-0.4.0 → cellproof-0.6.0}/pyproject.toml +1 -1
  6. cellproof-0.6.0/src/cellproof/__init__.py +1 -0
  7. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/cli.py +10 -6
  8. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/compiler.py +41 -8
  9. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/emitter.py +4 -0
  10. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/aggregate.py +7 -1
  11. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/dynamic.py +168 -24
  12. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/finance.py +63 -0
  13. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/logic.py +70 -2
  14. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/lookup.py +40 -1
  15. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/text.py +35 -1
  16. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/regions.py +2 -1
  17. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/verifier.py +25 -2
  18. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_cli.py +42 -0
  19. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_dynamic.py +44 -3
  20. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_emitter.py +27 -0
  21. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_finance_stats.py +28 -0
  22. cellproof-0.6.0/tests/test_functions.py +81 -0
  23. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_lookups.py +19 -0
  24. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_region.py +1 -1
  25. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_spill.py +11 -0
  26. cellproof-0.4.0/src/cellproof/__init__.py +0 -1
  27. cellproof-0.4.0/tests/test_functions.py +0 -52
  28. {cellproof-0.4.0 → cellproof-0.6.0}/.github/workflows/ci.yml +0 -0
  29. {cellproof-0.4.0 → cellproof-0.6.0}/.github/workflows/publish.yml +0 -0
  30. {cellproof-0.4.0 → cellproof-0.6.0}/.gitignore +0 -0
  31. {cellproof-0.4.0 → cellproof-0.6.0}/AGENTS.md +0 -0
  32. {cellproof-0.4.0 → cellproof-0.6.0}/DESIGN.md +0 -0
  33. {cellproof-0.4.0 → cellproof-0.6.0}/INFO.md +0 -0
  34. {cellproof-0.4.0 → cellproof-0.6.0}/LICENSE +0 -0
  35. {cellproof-0.4.0 → cellproof-0.6.0}/PLAN-2a.md +0 -0
  36. {cellproof-0.4.0 → cellproof-0.6.0}/PLAN-2b.md +0 -0
  37. {cellproof-0.4.0 → cellproof-0.6.0}/PLAN.md +0 -0
  38. {cellproof-0.4.0 → cellproof-0.6.0}/SECURITY.md +0 -0
  39. {cellproof-0.4.0 → cellproof-0.6.0}/docs/JOURNEY.md +0 -0
  40. {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
  41. {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
  42. {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
  43. {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
  44. {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
  45. {cellproof-0.4.0 → cellproof-0.6.0}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
  46. {cellproof-0.4.0 → cellproof-0.6.0}/examples/commission.xlsx +0 -0
  47. {cellproof-0.4.0 → cellproof-0.6.0}/examples/errors.xlsx +0 -0
  48. {cellproof-0.4.0 → cellproof-0.6.0}/examples/functions.xlsx +0 -0
  49. {cellproof-0.4.0 → cellproof-0.6.0}/examples/functions_unsaved.xlsx +0 -0
  50. {cellproof-0.4.0 → cellproof-0.6.0}/examples/make_commission.py +0 -0
  51. {cellproof-0.4.0 → cellproof-0.6.0}/examples/make_errors.py +0 -0
  52. {cellproof-0.4.0 → cellproof-0.6.0}/examples/make_functions.py +0 -0
  53. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/__main__.py +0 -0
  54. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/compute.py +0 -0
  55. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/expr.py +0 -0
  56. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fenwick.py +0 -0
  57. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/__init__.py +0 -0
  58. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/criteria.py +0 -0
  59. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/dates.py +0 -0
  60. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/maths.py +0 -0
  61. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/fn/stats.py +0 -0
  62. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/functions.py +0 -0
  63. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/graph.py +0 -0
  64. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/impact.py +0 -0
  65. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/model.py +0 -0
  66. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/parser.py +0 -0
  67. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/r1c1.py +0 -0
  68. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/rangeindex.py +0 -0
  69. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/reader.py +0 -0
  70. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/refs.py +0 -0
  71. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/region.py +0 -0
  72. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/report.py +0 -0
  73. {cellproof-0.4.0 → cellproof-0.6.0}/src/cellproof/tables.py +0 -0
  74. {cellproof-0.4.0 → cellproof-0.6.0}/tests/bench_shapes.py +0 -0
  75. {cellproof-0.4.0 → cellproof-0.6.0}/tests/corpus.py +0 -0
  76. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_aggregate.py +0 -0
  77. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_arrays.py +0 -0
  78. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_batches.py +0 -0
  79. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_compiler.py +0 -0
  80. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_compute.py +0 -0
  81. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_corpus_findings.py +0 -0
  82. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_criteria.py +0 -0
  83. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_dates.py +0 -0
  84. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_errors.py +0 -0
  85. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_excel_findings.py +0 -0
  86. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_excel_saved.py +0 -0
  87. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_fenwick.py +0 -0
  88. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_graph.py +0 -0
  89. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_impact.py +0 -0
  90. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_implicit.py +0 -0
  91. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_index.py +0 -0
  92. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_lexed.py +0 -0
  93. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_limits.py +0 -0
  94. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_logic.py +0 -0
  95. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_lookup.py +0 -0
  96. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_maths.py +0 -0
  97. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_parser.py +0 -0
  98. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_perf.py +0 -0
  99. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_r1c1.py +0 -0
  100. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_rangeindex.py +0 -0
  101. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_reader.py +0 -0
  102. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_refs.py +0 -0
  103. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_regions.py +0 -0
  104. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_regions_differential.py +0 -0
  105. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_regions_split.py +0 -0
  106. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_report.py +0 -0
  107. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_review_regressions.py +0 -0
  108. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_running.py +0 -0
  109. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_shapes.py +0 -0
  110. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_tables.py +0 -0
  111. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_text.py +0 -0
  112. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_types.py +0 -0
  113. {cellproof-0.4.0 → cellproof-0.6.0}/tests/test_verifier.py +0 -0
  114. {cellproof-0.4.0 → cellproof-0.6.0}/tests/xlsxkit.py +0 -0
@@ -1,5 +1,31 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.6.0 (2026-10-31)
4
+
5
+ - **A batch DuckDB rejects no longer stops the run:** its cells are UNRESOLVED with DuckDB's message, the book is
6
+ compiled again so every cell reading them is UNRESOLVED too (none can read a missing value), and the report is
7
+ written. The exit code is still 3: it is a bug in cellproof, please report it.
8
+ - **ROW, COLUMN** (of the formula's cell, of a cell, and in an array formula of a range: one column of row numbers,
9
+ one row of column numbers - real Excel), **CODE** (Windows-1252, CHAR's inverse; a character outside it, or above
10
+ 127 in a file Excel for Mac calculated, is UNRESOLVED), **UNICODE, UNICHAR, N** (of a range in an array formula:
11
+ its first cell), **ISODD, ISEVEN** (no ranges in an array formula: #VALUE!, like EDATE).
12
+ - **IRR and RATE** by Newton's method in SQL (a recursive CTE). Not guessed: cash flows with several sign changes
13
+ (several roots), an error among IRR's cells, a RATE whose answer is 0, nper not a positive whole number.
14
+
15
+ ## 0.5.0 (2026-10-31)
16
+
17
+ - **FILTER, SORT, UNIQUE inside other calculations:** `SUM(FILTER(...))`, `COUNTA(UNIQUE(...))`, AVERAGE, MIN, MAX,
18
+ PRODUCT, STDEV, VAR of them; **ROWS** and **COLUMNS** (of a range or of these). An empty element reaching an
19
+ aggregate is UNRESOLVED (whether Excel skips it or takes 0 was not checked), even through COUNT.
20
+ - **XMATCH** (exact, next smaller/larger, wildcards; first-to-last or last-to-first), **SORTBY** (several keys; by
21
+ rows, or by columns when the keys are one row: real Excel), **SEQUENCE** (more than a million elements is
22
+ UNRESOLVED and builds nothing). FILTER's if_empty can be a range, which spills whole.
23
+ - **Spills are one statement:** a spilled block is one batch computing its relation once (verify 1,334 cells
24
+ 31.0 s -> 0.03 s; 33,334 cells in 0.7 s), and one dbt model.
25
+ - **Not guessed:** XLOOKUP/XMATCH next smaller/larger across types or over text with punctuation (Excel ranks numbers
26
+ < text < TRUE/FALSE and orders text by its own collation), and exact text matches with a German sharp s (Excel
27
+ matched "ss" to it). Some of these used to PASS by luck; they are UNRESOLVED now.
28
+
3
29
  ## 0.4.0 (2026-10-31)
4
30
 
5
31
  - **Array formulas over a block** (legacy Ctrl+Shift+Enter blocks and dynamic-array spills) compile cell by cell:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cellproof
3
- Version: 0.4.0
3
+ Version: 0.6.0
4
4
  Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
5
5
  Project-URL: Homepage, https://github.com/Nikhil414/cellproof
6
6
  Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
@@ -111,14 +111,15 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
111
111
  code.
112
112
  6. **Emit** the dbt project and the report.
113
113
 
114
- 125 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
115
- COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
116
- HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
117
- LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
118
- NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
119
- RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
120
- SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
121
- YEAR, YEARFRAC.
114
+ 140 Excel functions are supported: @ (SINGLE), ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CODE,
115
+ COLUMN, COLUMNS, CONCAT(ENATE), COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE,
116
+ EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV, HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IRR, IS* (BLANK, ERR,
117
+ ERROR, LOGICAL, NA, NUMBER, TEXT), ISEVEN, ISODD, LARGE, LEFT, LEN, LET, LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX,
118
+ MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, N, NA, NETWORKDAYS, NOT, NPER, NPV, OR, PERCENTILE(.INC),
119
+ PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), RATE, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), ROW, ROWS,
120
+ SEARCH, SEQUENCE, SIGN, SMALL, SORT, SORTBY, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
121
+ SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNICHAR, UNICODE, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP,
122
+ WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR, YEAR, YEARFRAC.
122
123
 
123
124
  **Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
124
125
  `Sales[[Rep]:[Amount]]`) are rewritten to plain references when the file is read, so a calculated table column
@@ -142,9 +143,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
142
143
  and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
143
144
  result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
144
145
  computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
145
- Not yet: SORTBY, SEQUENCE, XMATCH, TAKE/DROP/CHOOSECOLS and the other array functions, FILTER/SORT/UNIQUE inside
146
- another calculation (`ROWS(UNIQUE(...))`), SORT by several keys or by columns, SORT keys that are empty, errors or
147
- text with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
146
+ SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
147
+ Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
148
+ calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
149
+ with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
148
150
  - **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
149
151
  `SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
150
152
  empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
@@ -29,7 +29,7 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
29
29
  | Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
30
30
 
31
31
  Tests: 492 (+10 skipped without dbt/Excel files), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
32
- purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1 and 0.4.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
32
+ purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1, 0.4.0, 0.5.0 and 0.6.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
33
33
  A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
34
34
 
35
35
  ## Public test workbooks (2026-10-31)
@@ -77,14 +77,15 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
77
77
  code.
78
78
  6. **Emit** the dbt project and the report.
79
79
 
80
- 125 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
81
- COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
82
- HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
83
- LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
84
- NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
85
- RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
86
- SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
87
- YEAR, YEARFRAC.
80
+ 140 Excel functions are supported: @ (SINGLE), ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CODE,
81
+ COLUMN, COLUMNS, CONCAT(ENATE), COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE,
82
+ EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV, HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IRR, IS* (BLANK, ERR,
83
+ ERROR, LOGICAL, NA, NUMBER, TEXT), ISEVEN, ISODD, LARGE, LEFT, LEN, LET, LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX,
84
+ MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, N, NA, NETWORKDAYS, NOT, NPER, NPV, OR, PERCENTILE(.INC),
85
+ PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), RATE, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), ROW, ROWS,
86
+ SEARCH, SEQUENCE, SIGN, SMALL, SORT, SORTBY, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
87
+ SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNICHAR, UNICODE, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP,
88
+ WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR, YEAR, YEARFRAC.
88
89
 
89
90
  **Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
90
91
  `Sales[[Rep]:[Amount]]`) are rewritten to plain references when the file is read, so a calculated table column
@@ -108,9 +109,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
108
109
  and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
109
110
  result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
110
111
  computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
111
- Not yet: SORTBY, SEQUENCE, XMATCH, TAKE/DROP/CHOOSECOLS and the other array functions, FILTER/SORT/UNIQUE inside
112
- another calculation (`ROWS(UNIQUE(...))`), SORT by several keys or by columns, SORT keys that are empty, errors or
113
- text with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
112
+ SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
113
+ Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
114
+ calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
115
+ with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
114
116
  - **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
115
117
  `SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
116
118
  empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cellproof"
7
- version = "0.4.0"
7
+ version = "0.6.0"
8
8
  description = "Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -0,0 +1 @@
1
+ __version__ = "0.6.0"
@@ -10,7 +10,7 @@ from cellproof.compute import fill_missing
10
10
  from cellproof.emitter import run_dbt, write_project
11
11
  from cellproof.reader import UnreadableWorkbook, load
12
12
  from cellproof.report import write_report
13
- from cellproof.verifier import SqlFailed, verify
13
+ from cellproof.verifier import SqlFailed, verify_book
14
14
 
15
15
 
16
16
  def main(argv: list[str] | None = None) -> int:
@@ -31,14 +31,18 @@ def main(argv: list[str] | None = None) -> int:
31
31
  print(f"error: {err}", file=sys.stderr)
32
32
  return 2
33
33
  book, computed, why = fill_missing(book, args.workbook)
34
- compiled = compile_book(book)
35
- if why:
36
- compiled.unresolved.update({k: f"{r} ({why})" for k, r in compiled.unresolved.items() if r == NO_VALUE})
34
+ def compile_fn(b):
35
+ compiled = compile_book(b)
36
+ if why:
37
+ compiled.unresolved.update({k: f"{r} ({why})" for k, r in compiled.unresolved.items() if r == NO_VALUE})
38
+ return compiled
37
39
  try:
38
- verdicts = verify(book, compiled)
40
+ compiled, verdicts, failures = verify_book(book, compile_fn)
39
41
  except SqlFailed as err:
40
42
  print(f"error: {err}", file=sys.stderr)
41
43
  return 3
44
+ for failure in failures: # those cells are UNRESOLVED in the report; the run goes on
45
+ print(f"error: {failure}", file=sys.stderr)
42
46
  out = args.out or args.workbook.with_name(args.workbook.stem + "_dbt")
43
47
  write_project(book, compiled, out, args.workbook.stem)
44
48
  counts = write_report(verdicts, compiled.findings, out, args.workbook.name, computed, compiled.notes,
@@ -46,7 +50,7 @@ def main(argv: list[str] | None = None) -> int:
46
50
  if computed:
47
51
  print(f"{len(computed)} expected values computed by the formulas library (the file had none saved)")
48
52
  print(f"{counts['PASS']} pass, {counts['FAIL']} fail, {counts['UNRESOLVED']} unresolved -> {out / 'REPORT.md'}")
49
- code = 1 if counts["FAIL"] else 0
53
+ code = 3 if failures else 1 if counts["FAIL"] else 0
50
54
  if args.dbt:
51
55
  ok, log = run_dbt(out)
52
56
  print("dbt build: ok" if ok else f"dbt build failed:\n{log}")
@@ -19,7 +19,7 @@ from openpyxl.utils import range_boundaries
19
19
 
20
20
  from cellproof.expr import (BLANK, CELLS, NO_ERR, Expr, any_of, compare, error, errs, finalize, lit_num, lit_str, num,
21
21
  txt, where_cell)
22
- from cellproof.fn.dynamic import DYNAMIC, element
22
+ from cellproof.fn.dynamic import DYNAMIC, spill_batch
23
23
  from cellproof.functions import FUNCS
24
24
  from cellproof.graph import formula_order
25
25
  from cellproof.model import Book, Cell, Key, Unsupported, a1
@@ -106,8 +106,9 @@ class Keys:
106
106
 
107
107
 
108
108
  # Analysis ToolPak functions do not take an array where one value is expected: {arguments that are one value}
109
- NO_LIFT = {"EDATE": 2, "EOMONTH": 2, "MROUND": 2, "YEARFRAC": 3, "NETWORKDAYS": 2, "WORKDAY": 2}
110
- NO_LIFT_CHECKED = {"EDATE", "EOMONTH", "MROUND", "YEARFRAC"} # #VALUE! there, in formulas' Excel-saved test.xlsx
109
+ NO_LIFT = {"EDATE": 2, "EOMONTH": 2, "MROUND": 2, "YEARFRAC": 3, "NETWORKDAYS": 2, "WORKDAY": 2, "ISEVEN": 1,
110
+ "ISODD": 1}
111
+ NO_LIFT_CHECKED = {"EDATE", "EOMONTH", "MROUND", "YEARFRAC", "ISEVEN", "ISODD"} # #VALUE!: formulas' test.xlsx
111
112
 
112
113
 
113
114
  class Compiler:
@@ -138,11 +139,7 @@ class Compiler:
138
139
  c1, r1, _, _ = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
139
140
  self.spill = (cell.row - r1, cell.col - c1)
140
141
  try:
141
- tree = parse(cell.formula)
142
- if self.spill is not None and isinstance(tree, Call) and tree.name in DYNAMIC:
143
- c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
144
- return element(self, tree, cell.sheet, *self.spill, (r2 - r1 + 1, c2 - c1 + 1), cell.dynamic)
145
- e = finalize(self.expr(tree, cell.sheet))
142
+ e = finalize(self.expr(parse(cell.formula), cell.sheet))
146
143
  if self.spill is not None and self.spill_dims:
147
144
  # real Excel (formulas' test.xlsx): the array is as big as its biggest range, one row (column)
148
145
  # repeating; a block cell beyond it is #N/A whatever the formula does with errors -
@@ -411,10 +408,46 @@ def compile_book(book: Book, regions: bool = True) -> Compiled:
411
408
  notes.append(f"{r.sheet} rows {r.r1}-{r.r2}, columns {letters(r.cols)} stay in cell mode: they could not "
412
409
  f"be ordered as one block ({why})")
413
410
  used_models: set[str] = set()
411
+ spilled: set[Key] = set()
412
+
413
+ def add_spill(cell) -> bool:
414
+ """FILTER/SORT/UNIQUE/... spilled into a block: one batch for the whole block (fn/dynamic.spill_batch)."""
415
+ nonlocal weight
416
+ if not cell.array or cell.key in book.unread: # compile_cell gives the reason
417
+ return False
418
+ try:
419
+ tree = parse(cell.formula)
420
+ except Unsupported:
421
+ return False
422
+ if not (isinstance(tree, Call) and tree.name in DYNAMIC):
423
+ return False
424
+ c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
425
+ block = [k for k in ((cell.sheet, r, c) for r in range(r1, r2 + 1) for c in range(c1, c2 + 1))
426
+ if k in book.cells and book.cells[k].array and book.cells[k].ref == cell.ref
427
+ and book.cells[k].formula == cell.formula]
428
+ spilled.update(block)
429
+ _close(comp, batches, current)
430
+ weight = 0
431
+ comp.at = (cell.sheet, r1, c1)
432
+ try:
433
+ _finish(comp, batches, spill_batch(comp, tree, cell.sheet, block, (r1, c1, r2, c2), cell.dynamic))
434
+ except Unsupported as err:
435
+ for k in block:
436
+ unresolved[k] = str(err)
437
+ if err.dep is not None:
438
+ chains[k] = err.dep
439
+ except RecursionError:
440
+ for k in block:
441
+ unresolved[k] = DEEP
442
+ finally:
443
+ comp.at = None
444
+ return True
414
445
 
415
446
  def add(key) -> None:
416
447
  nonlocal weight
417
448
  cell = book.cells[key]
449
+ if key in spilled or add_spill(cell):
450
+ return
418
451
  if cell.kind == "blank":
419
452
  unresolved[key] = NO_VALUE
420
453
  return
@@ -140,6 +140,10 @@ def write_project(book: Book, compiled: Compiled, out: Path, stem: str) -> None:
140
140
  (batches_dir / f"{model_name(k)}.sql").write_text(dbt_region_cells(batch) + "\n", encoding="utf-8")
141
141
  schema += (f' - name: {batch.model}\n description: "Region: rows of one sheet that repeat one formula down their rows, '
142
142
  f'computed in one statement. Wide and readable; its {model_name(k)} twin holds the same values as long rows."\n')
143
+ elif hasattr(batch, "render_rows"): # a spill: its one statement, reading earlier cells as all_cells
144
+ body = _escape(batch.render_rows("all_cells"))
145
+ body = body.replace("WITH\n", f"WITH\nall_cells AS (\n{_union(sources_for(k))}\n),\n", 1)
146
+ (batches_dir / f"{model_name(k)}.sql").write_text(body + "\n", encoding="utf-8")
143
147
  else:
144
148
  (batches_dir / f"{model_name(k)}.sql").write_text(dbt_sql(batch, sources_for(k)) + "\n", encoding="utf-8")
145
149
  if k % CHECKPOINT == 0 or k == n:
@@ -286,9 +286,12 @@ def _rng(c, node, sheet: str, skip_subtotals: bool = False) -> Rng:
286
286
  def _split(c, args, sheet: str, fname: str, skip_subtotals: bool = False) -> tuple[list, list, list]:
287
287
  """The arguments as (ranges, arrays, values). A range arithmetic or an IF over ranges is an array; a
288
288
  calculation over plain values stays a value, evaluated before the cells are read (MAX(H3, 1/0) is #DIV/0!)."""
289
+ from cellproof.fn.dynamic import DYNAMIC, as_array
289
290
  ranges, arrays, values = [], [], []
290
291
  for a in args:
291
- if isinstance(a, RefNode):
292
+ if isinstance(a, Call) and a.name in DYNAMIC: # SUM(FILTER(...)): the relation, read like an array
293
+ arrays.append(as_array(c, a, sheet))
294
+ elif isinstance(a, RefNode):
292
295
  ranges.append(_rng(c, a, sheet, skip_subtotals))
293
296
  elif isinstance(a, Binary) or (isinstance(a, Unary) and a.op == "-") or (
294
297
  isinstance(a, Call) and a.name in ("IF", "IFERROR", "IFNA")):
@@ -363,6 +366,9 @@ def _plain(name, how):
363
366
  result = how(c, ranges, arrays, values)
364
367
  if name not in ("COUNT", "COUNTA", "SUM"): # SUM's error order is already the rule inside _sum
365
368
  result = replace(result, err=errs(_errs(c, ranges, arrays, values), result.err))
369
+ markers = [a.unchecked() for a in arrays if hasattr(a, "unchecked")] # FILTER/SORT/UNIQUE: never hidden
370
+ if markers:
371
+ result = replace(result, err=errs(*markers, result.err))
366
372
  return result
367
373
  register(name)(fn)
368
374
 
@@ -13,7 +13,7 @@ from __future__ import annotations
13
13
 
14
14
  from dataclasses import dataclass
15
15
 
16
- from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str
16
+ from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str, num
17
17
  from cellproof.fn.aggregate import _array, _base, _truth
18
18
  from cellproof.fn.lookup import int_literal
19
19
  from cellproof.functions import register
@@ -21,7 +21,8 @@ from cellproof.model import Unsupported
21
21
  from cellproof.parser import Bool, Call, RefNode
22
22
  from cellproof.refs import MAX_ROW, resolve
23
23
 
24
- DYNAMIC = {"FILTER", "SORT", "UNIQUE"}
24
+ DYNAMIC = {"FILTER", "SORT", "SORTBY", "UNIQUE", "SEQUENCE"}
25
+ MAX_ELEMENTS = 1_000_000 # SEQUENCE bigger than this is UNCHECKED and builds nothing (a crafted file's 1E9 x 1E9)
25
26
 
26
27
 
27
28
  @dataclass(frozen=True)
@@ -110,7 +111,9 @@ def filter_(c, args, sheet: str) -> Rel:
110
111
  f"{'dense_rank() OVER (ORDER BY a.j)' if other == 'j' else 'a.j'} AS j, a.kind, a.num, a.txt, a.err "
111
112
  f"FROM {u}_a a JOIN {u}_f f ON f.k = a.{other} WHERE f.keep)")
112
113
  empty = None
113
- if len(args) == 3 and args[2] is not None:
114
+ if len(args) == 3 and isinstance(args[2], RefNode) and not resolve(args[2].text, sheet, c.book).is_single:
115
+ empty = f"SELECT * FROM {_range(c, args[2], sheet).sql} ie" # a range spills whole (formulas' test.xlsx)
116
+ elif len(args) == 3 and args[2] is not None:
114
117
  e = as_any(c.expr(args[2], sheet))
115
118
  empty = f"SELECT 1, 1, {e.kind}, {e.sql}, {e.txt}, {e.err}"
116
119
  else: # real Excel disagrees with itself: #CALC! in Microsoft's docs, #VALUE! in formulas' test.xlsx
@@ -170,31 +173,172 @@ def unique_(c, args, sheet: str) -> Rel:
170
173
  return Rel(_whole(u, ctes, sql, odd, empty), None, a.cols)
171
174
 
172
175
 
173
- BUILDERS = {"FILTER": filter_, "SORT": sort_, "UNIQUE": unique_}
176
+ def _keys_safe(table: str) -> str:
177
+ return (f"(SELECT CASE WHEN EXISTS (SELECT 1 FROM {table} WHERE kind IS NULL OR kind = 'e' OR "
178
+ f"(kind = 's' AND NOT regexp_matches(COALESCE(txt, ''), {SAFE_TEXT}))) THEN '{UNCHECKED}' END)")
174
179
 
175
180
 
176
- def element(c, node, sheet: str, i: int, j: int, block: tuple[int, int], dynamic: bool) -> Expr:
177
- """This cell's element (i, j) of the array, 0-based. A result one row (column) high repeats down (across), as in
178
- a legacy block; past the result a cell is #N/A. The anchor of a dynamic spill is #SPILL! when the result is not
179
- exactly as big as the block Excel spilled it into (the translation disagrees with Excel's own result)."""
180
- rel = relation(c, node, sheet).sql
181
+ RANK = "CASE kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END"
182
+
183
+
184
+ def sortby(c, args, sheet: str) -> Rel:
185
+ """SORTBY(array, by1, [order1], by2, [order2], ...): rows ordered by one-column ranges as high as the array, or
186
+ columns by one-row ranges as wide (real Excel, formulas' test.xlsx); the first key first, equal keys keep their
187
+ order. Keys follow SORT's rules (and what is not checked)."""
188
+ if len(args) < 2:
189
+ raise Unsupported("SORTBY needs an array and a by_array")
190
+ a = relation(c, args[0], sheet)
191
+ pairs, rest = [], list(args[1:])
192
+ while rest:
193
+ by = rest.pop(0)
194
+ order = 1 if not rest or rest[0] is None else int_literal(rest[0])
195
+ if rest:
196
+ rest.pop(0)
197
+ if order is None:
198
+ raise Unsupported("SORTBY with a sort order that is not one whole number not supported yet")
199
+ if not isinstance(by, RefNode):
200
+ raise Unsupported("SORTBY by something other than a range not supported yet")
201
+ pairs.append((_range(c, by, sheet), order))
202
+ if a.rows is None or a.cols is None:
203
+ raise Unsupported("SORTBY of an array whose size is only known at run time not supported yet")
204
+ if all(by.cols == 1 and by.rows == a.rows for by, _ in pairs):
205
+ axis, across = "i", "j"
206
+ elif all(by.rows == 1 and by.cols == a.cols for by, _ in pairs):
207
+ axis, across = "j", "i"
208
+ else:
209
+ return Rel("(SELECT 1 AS i, 1 AS j, 'e' AS kind, NULL AS num, NULL AS txt, '#VALUE!' AS err)", 1, 1)
181
210
  u = _uid(c)
182
- # ponytail: the relation runs once per value column (4x); one struct per cell if big spills get slow
183
- head = (f"WITH {u}_r AS MATERIALIZED {rel}, {u}_d AS (SELECT max(i) AS h, max(j) AS w FROM {u}_r), "
184
- f"{u}_e AS (SELECT r.* FROM {u}_r r, {u}_d d WHERE r.i = (CASE WHEN d.h = 1 THEN 1 ELSE {i + 1} END) "
185
- f"AND r.j = (CASE WHEN d.w = 1 THEN 1 ELSE {j + 1} END))")
186
-
187
- def get(col: str) -> str:
188
- return f"({head} SELECT {col} FROM {u}_e)"
189
- kind = f"COALESCE(NULLIF({get('kind')}, 'e'), 'n')" # an empty element spills as 0
190
- num = f"COALESCE({get('num')}, CAST(0 AS DOUBLE))"
191
- txt = f"COALESCE({get('txt')}, '')"
192
- size = f"(SELECT h <> {block[0]} OR w <> {block[1]} FROM {u}_d)" if dynamic and (i, j) == (0, 0) else "FALSE"
211
+ bad = [f"CASE WHEN {order} NOT IN (1, -1) THEN '#VALUE!' END" for _, order in pairs]
212
+ ctes = [f"{u}_a AS MATERIALIZED {a.sql}"]
213
+ order_by, joins = [], []
214
+ for n, (by, order) in enumerate(pairs):
215
+ ctes.append(f"{u}_k{n} AS MATERIALIZED (SELECT {axis} AS p, kind, num, txt FROM {by.sql} b WHERE {across} = 1)")
216
+ bad.append(_keys_safe(f"{u}_k{n}"))
217
+ d = "ASC" if order == 1 else "DESC"
218
+ k = f"k{n}"
219
+ joins.append(f"JOIN {u}_k{n} {k} ON {k}.p = a.p")
220
+ order_by += [f"(CASE {k}.kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END) {d}", f"{k}.num {d}",
221
+ f"lower(COALESCE({k}.txt, '')) {d}"]
222
+ new = (f"(SELECT a.p AS op, row_number() OVER (ORDER BY {', '.join(order_by)}, a.p) AS np "
223
+ f"FROM (SELECT DISTINCT {axis} AS p FROM {u}_a) a {' '.join(joins)})")
224
+ i_sql, j_sql = ("n.np", "a.j") if axis == "i" else ("a.i", "n.np")
225
+ sql = (f"(SELECT {i_sql} AS i, {j_sql} AS j, a.kind, a.num, a.txt, a.err FROM {u}_a a "
226
+ f"JOIN {new} n ON n.op = a.{axis})")
227
+ first_bad = "COALESCE(" + ", ".join(f"({b})" for b in bad) + ")"
228
+ return Rel(_whole(u, ctes, sql, first_bad), a.rows, a.cols)
229
+
230
+
231
+ def sequence(c, args, sheet: str) -> Rel:
232
+ """SEQUENCE(rows, [columns], [start], [step]): start, start + step, ... row by row; rows and columns are
233
+ truncated. Fewer than one row or column, or more than MAX_ELEMENTS elements, is UNCHECKED."""
234
+ if not 1 <= len(args) <= 4:
235
+ raise Unsupported("SEQUENCE needs one to four arguments")
236
+ one = Expr("CAST(1 AS DOUBLE)", "num")
237
+ rows, cols, start, step = (num(c.expr(a, sheet)) if a is not None else one
238
+ for a in list(args) + [None] * (4 - len(args)))
239
+ h, w = f"trunc({rows.sql})", f"trunc({cols.sql})"
240
+ ok = f"({h} >= 1 AND {w} >= 1 AND {h} * {w} <= {MAX_ELEMENTS})"
241
+ n = lambda x: f"(CASE WHEN {ok} THEN CAST({x} AS BIGINT) ELSE 0 END)" # noqa: E731 - builds nothing when not ok
242
+ body = (f"(SELECT gi.i AS i, gj.j AS j, 'n' AS kind, {start.sql} + (({'gi.i'} - 1) * {w} + gj.j - 1) * "
243
+ f"{step.sql} AS num, CAST(NULL AS VARCHAR) AS txt, CAST(NULL AS VARCHAR) AS err "
244
+ f"FROM range(1, {n(h)} + 1) gi(i) CROSS JOIN range(1, {n(w)} + 1) gj(j))")
245
+ u = _uid(c)
246
+ bad = (f"COALESCE({rows.err}, {cols.err}, {start.err}, {step.err}, "
247
+ f"CASE WHEN NOT {ok} THEN '{UNCHECKED}' END)")
248
+ return Rel(_whole(u, [], body, bad), None, None)
249
+
250
+
251
+ BUILDERS = {"FILTER": filter_, "SORT": sort_, "SORTBY": sortby, "UNIQUE": unique_, "SEQUENCE": sequence}
252
+
253
+
254
+ class SpillBatch(list):
255
+ """The Step of every cell of a spilled block, and one set-based statement computing them all: the relation once,
256
+ each cell joined to its element (like region.RegionBatch)."""
257
+
258
+ def __init__(self, steps, sql: str):
259
+ super().__init__(steps)
260
+ self.sql = sql
261
+
262
+ def render_rows(self, prev: str) -> str:
263
+ return self.sql.replace(CELLS, prev)
264
+
265
+
266
+ def spill_batch(c, node, sheet: str, block: list, box: tuple, dynamic: bool) -> SpillBatch:
267
+ """A result one row (column) high repeats down (across), as in a legacy block; past the result a cell is #N/A.
268
+ The anchor of a dynamic spill is #SPILL! when the result is not exactly as big as the block Excel spilled it
269
+ into (the translation disagrees with Excel's own result); an UNRESOLVED marker wins over that check."""
270
+ from cellproof.compiler import Step
271
+ r1, c1, r2, c2 = box
272
+ rel = relation(c, node, sheet).sql
193
273
  markers = ", ".join(f"'{m}'" for m in SETTINGS)
194
- first = f"(SELECT err FROM {u}_r WHERE i = 1 AND j = 1)" # an UNRESOLVED marker wins over the size check
195
- err = (f"({head} SELECT CASE WHEN {first} IN ({markers}) THEN {first} WHEN {size} THEN '#SPILL!' WHEN NOT EXISTS (SELECT 1 FROM {u}_e) THEN '#N/A' "
196
- f"ELSE (SELECT err FROM {u}_e) END)") # no FROM: a missing element still gives one row
197
- return Expr(num, "any", err, txt, kind)
274
+ pos = ", ".join(f"({r}, {col}, {r - r1 + 1}, {col - c1 + 1})" for _, r, col in block)
275
+ size = (f"(p.pi = 1 AND p.pj = 1 AND (d.h <> {r2 - r1 + 1} OR d.w <> {c2 - c1 + 1}))" if dynamic else "FALSE")
276
+ err = (f"CASE WHEN f.e IN ({markers}) THEN f.e WHEN {size} THEN '#SPILL!' WHEN s.i IS NULL THEN '#N/A' "
277
+ f"ELSE s.err END")
278
+ sql = (f"WITH\nspill AS MATERIALIZED {rel},\n"
279
+ f"d AS (SELECT max(i) AS h, max(j) AS w FROM spill),\n"
280
+ f"f AS (SELECT (SELECT err FROM spill WHERE i = 1 AND j = 1) AS e),\n"
281
+ f"cellpos AS (SELECT * FROM (VALUES {pos}) v(row_num, col_num, pi, pj)),\n"
282
+ f"picked AS (SELECT p.row_num, p.col_num, s.kind, s.num, s.txt, {err} AS e FROM cellpos p CROSS JOIN d "
283
+ f"CROSS JOIN f LEFT JOIN spill s ON s.i = (CASE WHEN d.h = 1 THEN 1 ELSE p.pi END) "
284
+ f"AND s.j = (CASE WHEN d.w = 1 THEN 1 ELSE p.pj END))\n"
285
+ # an empty element spills as 0; an error is kind 'e' with its code as text, as in the cells table
286
+ f"SELECT {lit_str(sheet).sql} AS sheet, row_num, col_num, "
287
+ f"CASE WHEN e IS NOT NULL THEN 'e' WHEN kind IS NULL THEN 'n' ELSE kind END AS kind, "
288
+ f"CASE WHEN e IS NOT NULL THEN NULL WHEN kind IS NULL THEN CAST(0 AS DOUBLE) ELSE num END AS num, "
289
+ f"CASE WHEN e IS NOT NULL THEN e WHEN kind = 's' THEN COALESCE(txt, '') END AS txt FROM picked")
290
+ steps = [Step(k, Expr("NULL", "any", "'#N/A'"), f"s{n}") for n, k in enumerate(block, 1)]
291
+ return SpillBatch(steps, sql)
292
+
293
+
294
+ @dataclass(frozen=True)
295
+ class RelArray:
296
+ """A FILTER/SORT/UNIQUE result as the aggregates read an array (aggregate.Arr's agg/err_sql/rel)."""
297
+ sql: str
298
+ srcs: tuple = ("relation",)
299
+ shape: None = None
300
+
301
+ def rel(self) -> str:
302
+ # what an aggregate does with an empty element (skip it, or take it as 0) was not checked: UNCHECKED
303
+ return (f"(SELECT i AS row_num, j AS col_num, CASE WHEN kind IS NULL THEN 'e' ELSE kind END AS kind, num, "
304
+ f"txt, CASE WHEN kind IS NULL THEN '{UNCHECKED}' ELSE err END AS err FROM {self.sql} ra)")
305
+
306
+ def agg(self, fn: str, kinds: str = "('n')") -> str:
307
+ return f"(SELECT {fn} FROM {self.rel()} WHERE kind IN {kinds})"
308
+
309
+ def err_sql(self) -> str:
310
+ return f"(SELECT err FROM {self.rel()} WHERE err IS NOT NULL ORDER BY row_num, col_num LIMIT 1)"
311
+
312
+ def unchecked(self) -> str:
313
+ """An UNRESOLVED marker in the relation, which even COUNT (that skips errors) must not hide."""
314
+ markers = ", ".join(f"'{m}'" for m in SETTINGS)
315
+ return f"(SELECT err FROM {self.rel()} WHERE err IN ({markers}) LIMIT 1)"
316
+
317
+
318
+ def as_array(c, node, sheet: str) -> RelArray:
319
+ return RelArray(relation(c, node, sheet).sql)
320
+
321
+
322
+ def _size(name: str, col: str):
323
+ def fn(c, args, sheet):
324
+ if len(args) != 1:
325
+ raise Unsupported(f"{name} needs one argument")
326
+ a = args[0]
327
+ if isinstance(a, RefNode):
328
+ area = resolve(a.text, sheet, c.book)
329
+ return Expr(f"CAST({area.r2 - area.r1 + 1 if col == 'i' else area.c2 - area.c1 + 1} AS DOUBLE)", "num")
330
+ if isinstance(a, Call) and a.name in DYNAMIC:
331
+ rel = relation(c, a, sheet).sql
332
+ # an error result (FILTER's #CALC!, an UNCHECKED marker) is the answer, not a size of one
333
+ first = f"(SELECT err FROM {rel} rs WHERE i = 1 AND j = 1 AND kind = 'e' AND "
334
+ whole = f"{first} (SELECT COUNT(*) FROM {rel} rc) = 1)"
335
+ return Expr(f"(SELECT CAST(max({col}) AS DOUBLE) FROM {rel} rm)", "num", whole)
336
+ raise Unsupported(f"{name} of something other than a range, FILTER, SORT or UNIQUE not supported yet")
337
+ register(name)(fn)
338
+
339
+
340
+ _size("ROWS", "i")
341
+ _size("COLUMNS", "j")
198
342
 
199
343
 
200
344
  def _inside(name):
@@ -101,3 +101,66 @@ def npv(c, args, sheet):
101
101
  sql = (f"(SELECT COALESCE(SUM(num / POWER(1 + {r.sql}, p)), CAST(0 AS DOUBLE)) FROM ({periods}) npv_periods)")
102
102
  return Expr(sql, "num", errs(r.err, *(v.err for v in typed), *(g.err(c) for g in ranges),
103
103
  _unchecked(f"{r.sql} = -1")))
104
+
105
+
106
+ NEWTON_STEPS = 100
107
+
108
+
109
+ def _newton(ctes: str, f: str, fp: str, guess: str) -> tuple[str, str]:
110
+ """(root SQL, converged SQL) of Newton's method from guess; f and fp are SQL of the column r (and the CTEs).
111
+ Excel stops at 0.00001 percent; this runs to the last digits, which differ from Excel's last iterate by far less
112
+ than the verifier's 1e-9. Not converging, or leaving r > -1, is #NUM!, as in Excel."""
113
+ it = (f"WITH RECURSIVE {ctes}, it(n, r, d) AS (SELECT 0, CAST({guess} AS DOUBLE), CAST(NULL AS DOUBLE) "
114
+ f"UNION ALL SELECT n + 1, r - fv / fpv, fv / fpv FROM (SELECT it.n, it.r, {f} AS fv, {fp} AS fpv FROM it, "
115
+ f"args WHERE it.n < {NEWTON_STEPS} AND it.r > -1 AND (it.d IS NULL OR abs(it.d) > 1e-13 * greatest(1, "
116
+ f"abs(it.r)))) s) SELECT {{}} FROM it ORDER BY n DESC LIMIT 1")
117
+ ok = "(d IS NOT NULL AND isfinite(r) AND r > -1 AND abs(d) <= 1e-10 * greatest(1, abs(r)))"
118
+ return f"({it.format('CASE WHEN ' + ok + ' THEN r END')})", f"({it.format(ok)})"
119
+
120
+
121
+ @register("IRR")
122
+ def irr(c, args, sheet):
123
+ """IRR(values, [guess]): the rate where the NPV of the values (the first one now) is 0. Text, TRUE/FALSE and empty
124
+ cells are skipped. Several sign changes can have several roots, and which one Excel's iteration lands on was
125
+ not checked: UNCHECKED."""
126
+ arity("IRR", args, 1, 2)
127
+ if not isinstance(args[0], RefNode):
128
+ raise Unsupported("IRR of values typed into the formula not supported yet")
129
+ rng = _rng(c, args[0], sheet)
130
+ guess = num(c.expr(args[1], sheet)) if len(args) == 2 and args[1] is not None else Expr("CAST(0.1 AS DOUBLE)", "num")
131
+ where = f"FROM {CELLS} WHERE {rng.where()} AND kind = 'n'"
132
+ ctes = f"args AS (SELECT list(num ORDER BY row_num, col_num) AS v {where})"
133
+ f = "list_sum(list_transform(v, lambda x, i: x / pow(1 + it.r, i - 1)))"
134
+ fp = "list_sum(list_transform(v, lambda x, i: -(i - 1) * x / pow(1 + it.r, i)))"
135
+ root, ok = _newton(ctes, f, fp, guess.sql)
136
+ signs = f"(SELECT COUNT(*) FILTER (WHERE num > 0) > 0 AND COUNT(*) FILTER (WHERE num < 0) > 0 {where})"
137
+ flips = (f"(SELECT COUNT(*) FROM (SELECT sign(num) AS s, lag(sign(num)) OVER (ORDER BY row_num, col_num) AS p "
138
+ f"{where} AND num <> 0) irr_signs WHERE s <> p)")
139
+ # real Excel (formulas' test.xlsx): an error in an array constant is #VALUE!; in cells, not checked
140
+ error_in = rng.err(c)
141
+ if error_in != "NULL":
142
+ error_in = (f"(CASE WHEN {error_in} IS NOT NULL THEN "
143
+ f"'{'#VALUE!' if rng.area.sheet in c.book.arrays else UNCHECKED}' END)")
144
+ return Expr(root, "num", errs(error_in, guess.err, f"(CASE WHEN NOT {signs} THEN '#NUM!' END)",
145
+ _unchecked(f"{flips} > 1"), f"(CASE WHEN NOT {ok} THEN '#NUM!' END)"))
146
+
147
+
148
+ @register("RATE")
149
+ def rate(c, args, sheet):
150
+ """RATE(nper, pmt, pv, [fv], [type], [guess]): the rate solving PMT's equation, by Newton's method."""
151
+ arity("RATE", args, 3, 6)
152
+ a = _args(c, args[:5], sheet, "RATE", 3, 5)
153
+ n, pmt, pv, fv, t = (x.sql for x in a)
154
+ guess = num(c.expr(args[5], sheet)) if len(args) == 6 and args[5] is not None else Expr("CAST(0.1 AS DOUBLE)", "num")
155
+ ctes = f"args AS (SELECT {n} AS n_, {pmt} AS pmt, {pv} AS pv, {fv} AS fv, {t} AS t)"
156
+ g, gp = "pow(1 + it.r, n_)", "(n_ * pow(1 + it.r, n_ - 1))"
157
+ f = f"(pv * {g} + pmt * (1 + it.r * t) * ({g} - 1) / it.r + fv)"
158
+ fp = (f"(pv * {gp} + pmt * (t * ({g} - 1) / it.r + (1 + it.r * t) * ({gp} * it.r - ({g} - 1)) / "
159
+ f"(it.r * it.r)))")
160
+ root, ok = _newton(ctes, f, fp, guess.sql)
161
+ return Expr(root, "num", errs(*(x.err for x in a), guess.err, _type(a[4]),
162
+ # a rate of 0 solves it (the payments only repay): Excel's answer there unchecked
163
+ _unchecked(f"{n} <= 0 OR {n} <> floor({n}) OR abs({pv} + {pmt} * {n} + {fv}) "
164
+ f"<= 1e-9 * greatest(abs({pv}), abs({pmt} * {n}), abs({fv}), 1) "
165
+ f"OR abs({root}) < 1e-9"),
166
+ f"(CASE WHEN NOT {ok} THEN '#NUM!' END)"))