cellproof 0.4.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. {cellproof-0.4.0 → cellproof-0.5.0}/CHANGELOG.md +14 -0
  2. {cellproof-0.4.0 → cellproof-0.5.0}/PKG-INFO +9 -8
  3. {cellproof-0.4.0 → cellproof-0.5.0}/PROGRESS.md +1 -1
  4. {cellproof-0.4.0 → cellproof-0.5.0}/README.md +8 -7
  5. {cellproof-0.4.0 → cellproof-0.5.0}/pyproject.toml +1 -1
  6. cellproof-0.5.0/src/cellproof/__init__.py +1 -0
  7. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/compiler.py +38 -6
  8. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/emitter.py +4 -0
  9. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/aggregate.py +7 -1
  10. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/dynamic.py +168 -24
  11. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/lookup.py +40 -1
  12. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_dynamic.py +44 -3
  13. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_emitter.py +27 -0
  14. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_lookups.py +19 -0
  15. cellproof-0.4.0/src/cellproof/__init__.py +0 -1
  16. {cellproof-0.4.0 → cellproof-0.5.0}/.github/workflows/ci.yml +0 -0
  17. {cellproof-0.4.0 → cellproof-0.5.0}/.github/workflows/publish.yml +0 -0
  18. {cellproof-0.4.0 → cellproof-0.5.0}/.gitignore +0 -0
  19. {cellproof-0.4.0 → cellproof-0.5.0}/AGENTS.md +0 -0
  20. {cellproof-0.4.0 → cellproof-0.5.0}/DESIGN.md +0 -0
  21. {cellproof-0.4.0 → cellproof-0.5.0}/INFO.md +0 -0
  22. {cellproof-0.4.0 → cellproof-0.5.0}/LICENSE +0 -0
  23. {cellproof-0.4.0 → cellproof-0.5.0}/PLAN-2a.md +0 -0
  24. {cellproof-0.4.0 → cellproof-0.5.0}/PLAN-2b.md +0 -0
  25. {cellproof-0.4.0 → cellproof-0.5.0}/PLAN.md +0 -0
  26. {cellproof-0.4.0 → cellproof-0.5.0}/SECURITY.md +0 -0
  27. {cellproof-0.4.0 → cellproof-0.5.0}/docs/JOURNEY.md +0 -0
  28. {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
  29. {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
  30. {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
  31. {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
  32. {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
  33. {cellproof-0.4.0 → cellproof-0.5.0}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
  34. {cellproof-0.4.0 → cellproof-0.5.0}/examples/commission.xlsx +0 -0
  35. {cellproof-0.4.0 → cellproof-0.5.0}/examples/errors.xlsx +0 -0
  36. {cellproof-0.4.0 → cellproof-0.5.0}/examples/functions.xlsx +0 -0
  37. {cellproof-0.4.0 → cellproof-0.5.0}/examples/functions_unsaved.xlsx +0 -0
  38. {cellproof-0.4.0 → cellproof-0.5.0}/examples/make_commission.py +0 -0
  39. {cellproof-0.4.0 → cellproof-0.5.0}/examples/make_errors.py +0 -0
  40. {cellproof-0.4.0 → cellproof-0.5.0}/examples/make_functions.py +0 -0
  41. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/__main__.py +0 -0
  42. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/cli.py +0 -0
  43. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/compute.py +0 -0
  44. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/expr.py +0 -0
  45. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fenwick.py +0 -0
  46. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/__init__.py +0 -0
  47. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/criteria.py +0 -0
  48. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/dates.py +0 -0
  49. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/finance.py +0 -0
  50. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/logic.py +0 -0
  51. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/maths.py +0 -0
  52. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/stats.py +0 -0
  53. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/fn/text.py +0 -0
  54. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/functions.py +0 -0
  55. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/graph.py +0 -0
  56. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/impact.py +0 -0
  57. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/model.py +0 -0
  58. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/parser.py +0 -0
  59. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/r1c1.py +0 -0
  60. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/rangeindex.py +0 -0
  61. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/reader.py +0 -0
  62. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/refs.py +0 -0
  63. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/region.py +0 -0
  64. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/regions.py +0 -0
  65. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/report.py +0 -0
  66. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/tables.py +0 -0
  67. {cellproof-0.4.0 → cellproof-0.5.0}/src/cellproof/verifier.py +0 -0
  68. {cellproof-0.4.0 → cellproof-0.5.0}/tests/bench_shapes.py +0 -0
  69. {cellproof-0.4.0 → cellproof-0.5.0}/tests/corpus.py +0 -0
  70. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_aggregate.py +0 -0
  71. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_arrays.py +0 -0
  72. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_batches.py +0 -0
  73. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_cli.py +0 -0
  74. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_compiler.py +0 -0
  75. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_compute.py +0 -0
  76. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_corpus_findings.py +0 -0
  77. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_criteria.py +0 -0
  78. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_dates.py +0 -0
  79. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_errors.py +0 -0
  80. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_excel_findings.py +0 -0
  81. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_excel_saved.py +0 -0
  82. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_fenwick.py +0 -0
  83. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_finance_stats.py +0 -0
  84. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_functions.py +0 -0
  85. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_graph.py +0 -0
  86. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_impact.py +0 -0
  87. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_implicit.py +0 -0
  88. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_index.py +0 -0
  89. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_lexed.py +0 -0
  90. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_limits.py +0 -0
  91. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_logic.py +0 -0
  92. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_lookup.py +0 -0
  93. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_maths.py +0 -0
  94. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_parser.py +0 -0
  95. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_perf.py +0 -0
  96. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_r1c1.py +0 -0
  97. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_rangeindex.py +0 -0
  98. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_reader.py +0 -0
  99. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_refs.py +0 -0
  100. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_region.py +0 -0
  101. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_regions.py +0 -0
  102. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_regions_differential.py +0 -0
  103. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_regions_split.py +0 -0
  104. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_report.py +0 -0
  105. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_review_regressions.py +0 -0
  106. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_running.py +0 -0
  107. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_shapes.py +0 -0
  108. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_spill.py +0 -0
  109. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_tables.py +0 -0
  110. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_text.py +0 -0
  111. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_types.py +0 -0
  112. {cellproof-0.4.0 → cellproof-0.5.0}/tests/test_verifier.py +0 -0
  113. {cellproof-0.4.0 → cellproof-0.5.0}/tests/xlsxkit.py +0 -0
@@ -1,5 +1,19 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.5.0 (2026-10-31)
4
+
5
+ - **FILTER, SORT, UNIQUE inside other calculations:** `SUM(FILTER(...))`, `COUNTA(UNIQUE(...))`, AVERAGE, MIN, MAX,
6
+ PRODUCT, STDEV, VAR of them; **ROWS** and **COLUMNS** (of a range or of these). An empty element reaching an
7
+ aggregate is UNRESOLVED (whether Excel skips it or takes 0 was not checked), even through COUNT.
8
+ - **XMATCH** (exact, next smaller/larger, wildcards; first-to-last or last-to-first), **SORTBY** (several keys; by
9
+ rows, or by columns when the keys are one row: real Excel), **SEQUENCE** (more than a million elements is
10
+ UNRESOLVED and builds nothing). FILTER's if_empty can be a range, which spills whole.
11
+ - **Spills are one statement:** a spilled block is one batch computing its relation once (verify 1,334 cells
12
+ 31.0 s -> 0.03 s; 33,334 cells in 0.7 s), and one dbt model.
13
+ - **Not guessed:** XLOOKUP/XMATCH next smaller/larger across types or over text with punctuation (Excel ranks numbers
14
+ < text < TRUE/FALSE and orders text by its own collation), and exact text matches with a German sharp s (Excel
15
+ matched "ss" to it). Some of these used to PASS by luck; they are UNRESOLVED now.
16
+
3
17
  ## 0.4.0 (2026-10-31)
4
18
 
5
19
  - **Array formulas over a block** (legacy Ctrl+Shift+Enter blocks and dynamic-array spills) compile cell by cell:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cellproof
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
5
5
  Project-URL: Homepage, https://github.com/Nikhil414/cellproof
6
6
  Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
@@ -111,13 +111,13 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
111
111
  code.
112
112
  6. **Emit** the dbt project and the report.
113
113
 
114
- 125 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
115
- COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
114
+ 130 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
115
+ COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, COLUMNS, EXACT, EXP, FILTER, FIND, FLOOR, FV,
116
116
  HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
117
117
  LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
118
118
  NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
119
- RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
120
- SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
119
+ RIGHT, ROUND(UP/DOWN), ROWS, SEQUENCE, SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
120
+ SORT, SORTBY, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR,
121
121
  YEAR, YEARFRAC.
122
122
 
123
123
  **Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
@@ -142,9 +142,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
142
142
  and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
143
143
  result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
144
144
  computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
145
- Not yet: SORTBY, SEQUENCE, XMATCH, TAKE/DROP/CHOOSECOLS and the other array functions, FILTER/SORT/UNIQUE inside
146
- another calculation (`ROWS(UNIQUE(...))`), SORT by several keys or by columns, SORT keys that are empty, errors or
147
- text with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
145
+ SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
146
+ Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
147
+ calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
148
+ with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
148
149
  - **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
149
150
  `SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
150
151
  empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
@@ -29,7 +29,7 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
29
29
  | Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
30
30
 
31
31
  Tests: 492 (+10 skipped without dbt/Excel files), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
32
- purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1 and 0.4.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
32
+ purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.x, 0.3.0, 0.3.1, 0.4.0 and 0.5.0 (2026-10-31) are on PyPI as `cellproof` (see CHANGELOG.md).
33
33
  A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
34
34
 
35
35
  ## Public test workbooks (2026-10-31)
@@ -77,13 +77,13 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
77
77
  code.
78
78
  6. **Emit** the dbt project and the report.
79
79
 
80
- 125 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
81
- COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FILTER, FIND, FLOOR, FV,
80
+ 130 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
81
+ COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, COLUMNS, EXACT, EXP, FILTER, FIND, FLOOR, FV,
82
82
  HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LARGE, LEFT, LEN, LET,
83
83
  LN, LOG, LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NETWORKDAYS,
84
84
  NOT, NPER, NPV, OR, PERCENTILE(.INC), PI, PMT, POWER, PRODUCT, PROPER, PV, QUARTILE(.INC), RANK(.EQ), REPLACE, REPT,
85
- RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
86
- SORT, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XOR,
85
+ RIGHT, ROUND(UP/DOWN), ROWS, SEQUENCE, SEARCH, SIGN, SMALL, SQRT, STDEV(.S/.P/P), SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
86
+ SORT, SORTBY, SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UNIQUE, UPPER, VALUE, VAR(.S/.P/P), VLOOKUP, WEEKDAY, WORKDAY, XLOOKUP, XMATCH, XOR,
87
87
  YEAR, YEARFRAC.
88
88
 
89
89
  **Excel tables:** structured references (`Sales[Amount]`, `Sales[@Qty]`, `Sales[[#This Row],[Qty]]`, `Sales[#Totals]`,
@@ -108,9 +108,10 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
108
108
  and `@` are read. **FILTER, SORT and UNIQUE** spill their results and nest in each other; a dynamic spill whose
109
109
  result is not exactly as big as the block Excel saved shows #SPILL! in its first cell (a FAIL). The generated SQL
110
110
  computes the block Excel saved: if the data later changes the size of a FILTER's result, re-run cellproof.
111
- Not yet: SORTBY, SEQUENCE, XMATCH, TAKE/DROP/CHOOSECOLS and the other array functions, FILTER/SORT/UNIQUE inside
112
- another calculation (`ROWS(UNIQUE(...))`), SORT by several keys or by columns, SORT keys that are empty, errors or
113
- text with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
111
+ SORTBY, SEQUENCE and XMATCH too, and all of them inside aggregates (`SUM(FILTER(...))`, `ROWS(UNIQUE(...))`).
112
+ Not yet: TAKE/DROP/CHOOSECOLS and the other array functions, these functions inside other non-aggregate
113
+ calculations, SORT by several keys or by columns (SORTBY does both), SORT/SORTBY keys that are empty, errors or text
114
+ with punctuation (Excel's own text order), UNIQUE over empty cells or errors - all UNRESOLVED.
114
115
  - **Array arithmetic in aggregates** (`SUM(A1:A10*B1:B10)`, `SUMPRODUCT((A1:A9>5)*C1:C9)`, `MAX(IF(A1:A9="x",B1:B9))`,
115
116
  `SUM(IFERROR(A1:A9/B1:B9,0))`, `COUNTA(A1:A9&B1:B9)`, `SUM(A1:A9^2)`) follows modern Excel: element by element, an
116
117
  empty element is 0, text converts when it is a number ("3" -> 3) and is #VALUE! otherwise, comparisons rank
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cellproof"
7
- version = "0.4.0"
7
+ version = "0.5.0"
8
8
  description = "Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -0,0 +1 @@
1
+ __version__ = "0.5.0"
@@ -19,7 +19,7 @@ from openpyxl.utils import range_boundaries
19
19
 
20
20
  from cellproof.expr import (BLANK, CELLS, NO_ERR, Expr, any_of, compare, error, errs, finalize, lit_num, lit_str, num,
21
21
  txt, where_cell)
22
- from cellproof.fn.dynamic import DYNAMIC, element
22
+ from cellproof.fn.dynamic import DYNAMIC, spill_batch
23
23
  from cellproof.functions import FUNCS
24
24
  from cellproof.graph import formula_order
25
25
  from cellproof.model import Book, Cell, Key, Unsupported, a1
@@ -138,11 +138,7 @@ class Compiler:
138
138
  c1, r1, _, _ = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
139
139
  self.spill = (cell.row - r1, cell.col - c1)
140
140
  try:
141
- tree = parse(cell.formula)
142
- if self.spill is not None and isinstance(tree, Call) and tree.name in DYNAMIC:
143
- c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
144
- return element(self, tree, cell.sheet, *self.spill, (r2 - r1 + 1, c2 - c1 + 1), cell.dynamic)
145
- e = finalize(self.expr(tree, cell.sheet))
141
+ e = finalize(self.expr(parse(cell.formula), cell.sheet))
146
142
  if self.spill is not None and self.spill_dims:
147
143
  # real Excel (formulas' test.xlsx): the array is as big as its biggest range, one row (column)
148
144
  # repeating; a block cell beyond it is #N/A whatever the formula does with errors -
@@ -411,10 +407,46 @@ def compile_book(book: Book, regions: bool = True) -> Compiled:
411
407
  notes.append(f"{r.sheet} rows {r.r1}-{r.r2}, columns {letters(r.cols)} stay in cell mode: they could not "
412
408
  f"be ordered as one block ({why})")
413
409
  used_models: set[str] = set()
410
+ spilled: set[Key] = set()
411
+
412
+ def add_spill(cell) -> bool:
413
+ """FILTER/SORT/UNIQUE/... spilled into a block: one batch for the whole block (fn/dynamic.spill_batch)."""
414
+ nonlocal weight
415
+ if not cell.array:
416
+ return False
417
+ try:
418
+ tree = parse(cell.formula)
419
+ except Unsupported:
420
+ return False
421
+ if not (isinstance(tree, Call) and tree.name in DYNAMIC):
422
+ return False
423
+ c1, r1, c2, r2 = range_boundaries(cell.ref or a1(cell.key).split("!")[-1])
424
+ block = [k for k in ((cell.sheet, r, c) for r in range(r1, r2 + 1) for c in range(c1, c2 + 1))
425
+ if k in book.cells and book.cells[k].array and book.cells[k].ref == cell.ref
426
+ and book.cells[k].formula == cell.formula]
427
+ spilled.update(block)
428
+ _close(comp, batches, current)
429
+ weight = 0
430
+ comp.at = (cell.sheet, r1, c1)
431
+ try:
432
+ _finish(comp, batches, spill_batch(comp, tree, cell.sheet, block, (r1, c1, r2, c2), cell.dynamic))
433
+ except Unsupported as err:
434
+ for k in block:
435
+ unresolved[k] = str(err)
436
+ if err.dep is not None:
437
+ chains[k] = err.dep
438
+ except RecursionError:
439
+ for k in block:
440
+ unresolved[k] = DEEP
441
+ finally:
442
+ comp.at = None
443
+ return True
414
444
 
415
445
  def add(key) -> None:
416
446
  nonlocal weight
417
447
  cell = book.cells[key]
448
+ if key in spilled or add_spill(cell):
449
+ return
418
450
  if cell.kind == "blank":
419
451
  unresolved[key] = NO_VALUE
420
452
  return
@@ -140,6 +140,10 @@ def write_project(book: Book, compiled: Compiled, out: Path, stem: str) -> None:
140
140
  (batches_dir / f"{model_name(k)}.sql").write_text(dbt_region_cells(batch) + "\n", encoding="utf-8")
141
141
  schema += (f' - name: {batch.model}\n description: "Region: rows of one sheet that repeat one formula down their rows, '
142
142
  f'computed in one statement. Wide and readable; its {model_name(k)} twin holds the same values as long rows."\n')
143
+ elif hasattr(batch, "render_rows"): # a spill: its one statement, reading earlier cells as all_cells
144
+ body = _escape(batch.render_rows("all_cells"))
145
+ body = body.replace("WITH\n", f"WITH\nall_cells AS (\n{_union(sources_for(k))}\n),\n", 1)
146
+ (batches_dir / f"{model_name(k)}.sql").write_text(body + "\n", encoding="utf-8")
143
147
  else:
144
148
  (batches_dir / f"{model_name(k)}.sql").write_text(dbt_sql(batch, sources_for(k)) + "\n", encoding="utf-8")
145
149
  if k % CHECKPOINT == 0 or k == n:
@@ -286,9 +286,12 @@ def _rng(c, node, sheet: str, skip_subtotals: bool = False) -> Rng:
286
286
  def _split(c, args, sheet: str, fname: str, skip_subtotals: bool = False) -> tuple[list, list, list]:
287
287
  """The arguments as (ranges, arrays, values). A range arithmetic or an IF over ranges is an array; a
288
288
  calculation over plain values stays a value, evaluated before the cells are read (MAX(H3, 1/0) is #DIV/0!)."""
289
+ from cellproof.fn.dynamic import DYNAMIC, as_array
289
290
  ranges, arrays, values = [], [], []
290
291
  for a in args:
291
- if isinstance(a, RefNode):
292
+ if isinstance(a, Call) and a.name in DYNAMIC: # SUM(FILTER(...)): the relation, read like an array
293
+ arrays.append(as_array(c, a, sheet))
294
+ elif isinstance(a, RefNode):
292
295
  ranges.append(_rng(c, a, sheet, skip_subtotals))
293
296
  elif isinstance(a, Binary) or (isinstance(a, Unary) and a.op == "-") or (
294
297
  isinstance(a, Call) and a.name in ("IF", "IFERROR", "IFNA")):
@@ -363,6 +366,9 @@ def _plain(name, how):
363
366
  result = how(c, ranges, arrays, values)
364
367
  if name not in ("COUNT", "COUNTA", "SUM"): # SUM's error order is already the rule inside _sum
365
368
  result = replace(result, err=errs(_errs(c, ranges, arrays, values), result.err))
369
+ markers = [a.unchecked() for a in arrays if hasattr(a, "unchecked")] # FILTER/SORT/UNIQUE: never hidden
370
+ if markers:
371
+ result = replace(result, err=errs(*markers, result.err))
366
372
  return result
367
373
  register(name)(fn)
368
374
 
@@ -13,7 +13,7 @@ from __future__ import annotations
13
13
 
14
14
  from dataclasses import dataclass
15
15
 
16
- from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str
16
+ from cellproof.expr import CELLS, SETTINGS, UNCHECKED, Expr, as_any, lit_str, num
17
17
  from cellproof.fn.aggregate import _array, _base, _truth
18
18
  from cellproof.fn.lookup import int_literal
19
19
  from cellproof.functions import register
@@ -21,7 +21,8 @@ from cellproof.model import Unsupported
21
21
  from cellproof.parser import Bool, Call, RefNode
22
22
  from cellproof.refs import MAX_ROW, resolve
23
23
 
24
- DYNAMIC = {"FILTER", "SORT", "UNIQUE"}
24
+ DYNAMIC = {"FILTER", "SORT", "SORTBY", "UNIQUE", "SEQUENCE"}
25
+ MAX_ELEMENTS = 1_000_000 # SEQUENCE bigger than this is UNCHECKED and builds nothing (a crafted file's 1E9 x 1E9)
25
26
 
26
27
 
27
28
  @dataclass(frozen=True)
@@ -110,7 +111,9 @@ def filter_(c, args, sheet: str) -> Rel:
110
111
  f"{'dense_rank() OVER (ORDER BY a.j)' if other == 'j' else 'a.j'} AS j, a.kind, a.num, a.txt, a.err "
111
112
  f"FROM {u}_a a JOIN {u}_f f ON f.k = a.{other} WHERE f.keep)")
112
113
  empty = None
113
- if len(args) == 3 and args[2] is not None:
114
+ if len(args) == 3 and isinstance(args[2], RefNode) and not resolve(args[2].text, sheet, c.book).is_single:
115
+ empty = f"SELECT * FROM {_range(c, args[2], sheet).sql} ie" # a range spills whole (formulas' test.xlsx)
116
+ elif len(args) == 3 and args[2] is not None:
114
117
  e = as_any(c.expr(args[2], sheet))
115
118
  empty = f"SELECT 1, 1, {e.kind}, {e.sql}, {e.txt}, {e.err}"
116
119
  else: # real Excel disagrees with itself: #CALC! in Microsoft's docs, #VALUE! in formulas' test.xlsx
@@ -170,31 +173,172 @@ def unique_(c, args, sheet: str) -> Rel:
170
173
  return Rel(_whole(u, ctes, sql, odd, empty), None, a.cols)
171
174
 
172
175
 
173
- BUILDERS = {"FILTER": filter_, "SORT": sort_, "UNIQUE": unique_}
176
+ def _keys_safe(table: str) -> str:
177
+ return (f"(SELECT CASE WHEN EXISTS (SELECT 1 FROM {table} WHERE kind IS NULL OR kind = 'e' OR "
178
+ f"(kind = 's' AND NOT regexp_matches(COALESCE(txt, ''), {SAFE_TEXT}))) THEN '{UNCHECKED}' END)")
174
179
 
175
180
 
176
- def element(c, node, sheet: str, i: int, j: int, block: tuple[int, int], dynamic: bool) -> Expr:
177
- """This cell's element (i, j) of the array, 0-based. A result one row (column) high repeats down (across), as in
178
- a legacy block; past the result a cell is #N/A. The anchor of a dynamic spill is #SPILL! when the result is not
179
- exactly as big as the block Excel spilled it into (the translation disagrees with Excel's own result)."""
180
- rel = relation(c, node, sheet).sql
181
+ RANK = "CASE kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END"
182
+
183
+
184
+ def sortby(c, args, sheet: str) -> Rel:
185
+ """SORTBY(array, by1, [order1], by2, [order2], ...): rows ordered by one-column ranges as high as the array, or
186
+ columns by one-row ranges as wide (real Excel, formulas' test.xlsx); the first key first, equal keys keep their
187
+ order. Keys follow SORT's rules (and what is not checked)."""
188
+ if len(args) < 2:
189
+ raise Unsupported("SORTBY needs an array and a by_array")
190
+ a = relation(c, args[0], sheet)
191
+ pairs, rest = [], list(args[1:])
192
+ while rest:
193
+ by = rest.pop(0)
194
+ order = 1 if not rest or rest[0] is None else int_literal(rest[0])
195
+ if rest:
196
+ rest.pop(0)
197
+ if order is None:
198
+ raise Unsupported("SORTBY with a sort order that is not one whole number not supported yet")
199
+ if not isinstance(by, RefNode):
200
+ raise Unsupported("SORTBY by something other than a range not supported yet")
201
+ pairs.append((_range(c, by, sheet), order))
202
+ if a.rows is None or a.cols is None:
203
+ raise Unsupported("SORTBY of an array whose size is only known at run time not supported yet")
204
+ if all(by.cols == 1 and by.rows == a.rows for by, _ in pairs):
205
+ axis, across = "i", "j"
206
+ elif all(by.rows == 1 and by.cols == a.cols for by, _ in pairs):
207
+ axis, across = "j", "i"
208
+ else:
209
+ return Rel("(SELECT 1 AS i, 1 AS j, 'e' AS kind, NULL AS num, NULL AS txt, '#VALUE!' AS err)", 1, 1)
181
210
  u = _uid(c)
182
- # ponytail: the relation runs once per value column (4x); one struct per cell if big spills get slow
183
- head = (f"WITH {u}_r AS MATERIALIZED {rel}, {u}_d AS (SELECT max(i) AS h, max(j) AS w FROM {u}_r), "
184
- f"{u}_e AS (SELECT r.* FROM {u}_r r, {u}_d d WHERE r.i = (CASE WHEN d.h = 1 THEN 1 ELSE {i + 1} END) "
185
- f"AND r.j = (CASE WHEN d.w = 1 THEN 1 ELSE {j + 1} END))")
186
-
187
- def get(col: str) -> str:
188
- return f"({head} SELECT {col} FROM {u}_e)"
189
- kind = f"COALESCE(NULLIF({get('kind')}, 'e'), 'n')" # an empty element spills as 0
190
- num = f"COALESCE({get('num')}, CAST(0 AS DOUBLE))"
191
- txt = f"COALESCE({get('txt')}, '')"
192
- size = f"(SELECT h <> {block[0]} OR w <> {block[1]} FROM {u}_d)" if dynamic and (i, j) == (0, 0) else "FALSE"
211
+ bad = [f"CASE WHEN {order} NOT IN (1, -1) THEN '#VALUE!' END" for _, order in pairs]
212
+ ctes = [f"{u}_a AS MATERIALIZED {a.sql}"]
213
+ order_by, joins = [], []
214
+ for n, (by, order) in enumerate(pairs):
215
+ ctes.append(f"{u}_k{n} AS MATERIALIZED (SELECT {axis} AS p, kind, num, txt FROM {by.sql} b WHERE {across} = 1)")
216
+ bad.append(_keys_safe(f"{u}_k{n}"))
217
+ d = "ASC" if order == 1 else "DESC"
218
+ k = f"k{n}"
219
+ joins.append(f"JOIN {u}_k{n} {k} ON {k}.p = a.p")
220
+ order_by += [f"(CASE {k}.kind WHEN 'n' THEN 0 WHEN 's' THEN 1 ELSE 2 END) {d}", f"{k}.num {d}",
221
+ f"lower(COALESCE({k}.txt, '')) {d}"]
222
+ new = (f"(SELECT a.p AS op, row_number() OVER (ORDER BY {', '.join(order_by)}, a.p) AS np "
223
+ f"FROM (SELECT DISTINCT {axis} AS p FROM {u}_a) a {' '.join(joins)})")
224
+ i_sql, j_sql = ("n.np", "a.j") if axis == "i" else ("a.i", "n.np")
225
+ sql = (f"(SELECT {i_sql} AS i, {j_sql} AS j, a.kind, a.num, a.txt, a.err FROM {u}_a a "
226
+ f"JOIN {new} n ON n.op = a.{axis})")
227
+ first_bad = "COALESCE(" + ", ".join(f"({b})" for b in bad) + ")"
228
+ return Rel(_whole(u, ctes, sql, first_bad), a.rows, a.cols)
229
+
230
+
231
+ def sequence(c, args, sheet: str) -> Rel:
232
+ """SEQUENCE(rows, [columns], [start], [step]): start, start + step, ... row by row; rows and columns are
233
+ truncated. Fewer than one row or column, or more than MAX_ELEMENTS elements, is UNCHECKED."""
234
+ if not 1 <= len(args) <= 4:
235
+ raise Unsupported("SEQUENCE needs one to four arguments")
236
+ one = Expr("CAST(1 AS DOUBLE)", "num")
237
+ rows, cols, start, step = (num(c.expr(a, sheet)) if a is not None else one
238
+ for a in list(args) + [None] * (4 - len(args)))
239
+ h, w = f"trunc({rows.sql})", f"trunc({cols.sql})"
240
+ ok = f"({h} >= 1 AND {w} >= 1 AND {h} * {w} <= {MAX_ELEMENTS})"
241
+ n = lambda x: f"(CASE WHEN {ok} THEN CAST({x} AS BIGINT) ELSE 0 END)" # noqa: E731 - builds nothing when not ok
242
+ body = (f"(SELECT gi.i AS i, gj.j AS j, 'n' AS kind, {start.sql} + (({'gi.i'} - 1) * {w} + gj.j - 1) * "
243
+ f"{step.sql} AS num, CAST(NULL AS VARCHAR) AS txt, CAST(NULL AS VARCHAR) AS err "
244
+ f"FROM range(1, {n(h)} + 1) gi(i) CROSS JOIN range(1, {n(w)} + 1) gj(j))")
245
+ u = _uid(c)
246
+ bad = (f"COALESCE({rows.err}, {cols.err}, {start.err}, {step.err}, "
247
+ f"CASE WHEN NOT {ok} THEN '{UNCHECKED}' END)")
248
+ return Rel(_whole(u, [], body, bad), None, None)
249
+
250
+
251
+ BUILDERS = {"FILTER": filter_, "SORT": sort_, "SORTBY": sortby, "UNIQUE": unique_, "SEQUENCE": sequence}
252
+
253
+
254
+ class SpillBatch(list):
255
+ """The Step of every cell of a spilled block, and one set-based statement computing them all: the relation once,
256
+ each cell joined to its element (like region.RegionBatch)."""
257
+
258
+ def __init__(self, steps, sql: str):
259
+ super().__init__(steps)
260
+ self.sql = sql
261
+
262
+ def render_rows(self, prev: str) -> str:
263
+ return self.sql.replace(CELLS, prev)
264
+
265
+
266
+ def spill_batch(c, node, sheet: str, block: list, box: tuple, dynamic: bool) -> SpillBatch:
267
+ """A result one row (column) high repeats down (across), as in a legacy block; past the result a cell is #N/A.
268
+ The anchor of a dynamic spill is #SPILL! when the result is not exactly as big as the block Excel spilled it
269
+ into (the translation disagrees with Excel's own result); an UNRESOLVED marker wins over that check."""
270
+ from cellproof.compiler import Step
271
+ r1, c1, r2, c2 = box
272
+ rel = relation(c, node, sheet).sql
193
273
  markers = ", ".join(f"'{m}'" for m in SETTINGS)
194
- first = f"(SELECT err FROM {u}_r WHERE i = 1 AND j = 1)" # an UNRESOLVED marker wins over the size check
195
- err = (f"({head} SELECT CASE WHEN {first} IN ({markers}) THEN {first} WHEN {size} THEN '#SPILL!' WHEN NOT EXISTS (SELECT 1 FROM {u}_e) THEN '#N/A' "
196
- f"ELSE (SELECT err FROM {u}_e) END)") # no FROM: a missing element still gives one row
197
- return Expr(num, "any", err, txt, kind)
274
+ pos = ", ".join(f"({r}, {col}, {r - r1 + 1}, {col - c1 + 1})" for _, r, col in block)
275
+ size = (f"(p.pi = 1 AND p.pj = 1 AND (d.h <> {r2 - r1 + 1} OR d.w <> {c2 - c1 + 1}))" if dynamic else "FALSE")
276
+ err = (f"CASE WHEN f.e IN ({markers}) THEN f.e WHEN {size} THEN '#SPILL!' WHEN s.i IS NULL THEN '#N/A' "
277
+ f"ELSE s.err END")
278
+ sql = (f"WITH\nspill AS MATERIALIZED {rel},\n"
279
+ f"d AS (SELECT max(i) AS h, max(j) AS w FROM spill),\n"
280
+ f"f AS (SELECT (SELECT err FROM spill WHERE i = 1 AND j = 1) AS e),\n"
281
+ f"cellpos AS (SELECT * FROM (VALUES {pos}) v(row_num, col_num, pi, pj)),\n"
282
+ f"picked AS (SELECT p.row_num, p.col_num, s.kind, s.num, s.txt, {err} AS e FROM cellpos p CROSS JOIN d "
283
+ f"CROSS JOIN f LEFT JOIN spill s ON s.i = (CASE WHEN d.h = 1 THEN 1 ELSE p.pi END) "
284
+ f"AND s.j = (CASE WHEN d.w = 1 THEN 1 ELSE p.pj END))\n"
285
+ # an empty element spills as 0; an error is kind 'e' with its code as text, as in the cells table
286
+ f"SELECT {lit_str(sheet).sql} AS sheet, row_num, col_num, "
287
+ f"CASE WHEN e IS NOT NULL THEN 'e' WHEN kind IS NULL THEN 'n' ELSE kind END AS kind, "
288
+ f"CASE WHEN e IS NOT NULL THEN NULL WHEN kind IS NULL THEN CAST(0 AS DOUBLE) ELSE num END AS num, "
289
+ f"CASE WHEN e IS NOT NULL THEN e WHEN kind = 's' THEN COALESCE(txt, '') END AS txt FROM picked")
290
+ steps = [Step(k, Expr("NULL", "any", "'#N/A'"), f"s{n}") for n, k in enumerate(block, 1)]
291
+ return SpillBatch(steps, sql)
292
+
293
+
294
+ @dataclass(frozen=True)
295
+ class RelArray:
296
+ """A FILTER/SORT/UNIQUE result as the aggregates read an array (aggregate.Arr's agg/err_sql/rel)."""
297
+ sql: str
298
+ srcs: tuple = ("relation",)
299
+ shape: None = None
300
+
301
+ def rel(self) -> str:
302
+ # what an aggregate does with an empty element (skip it, or take it as 0) was not checked: UNCHECKED
303
+ return (f"(SELECT i AS row_num, j AS col_num, CASE WHEN kind IS NULL THEN 'e' ELSE kind END AS kind, num, "
304
+ f"txt, CASE WHEN kind IS NULL THEN '{UNCHECKED}' ELSE err END AS err FROM {self.sql} ra)")
305
+
306
+ def agg(self, fn: str, kinds: str = "('n')") -> str:
307
+ return f"(SELECT {fn} FROM {self.rel()} WHERE kind IN {kinds})"
308
+
309
+ def err_sql(self) -> str:
310
+ return f"(SELECT err FROM {self.rel()} WHERE err IS NOT NULL ORDER BY row_num, col_num LIMIT 1)"
311
+
312
+ def unchecked(self) -> str:
313
+ """An UNRESOLVED marker in the relation, which even COUNT (that skips errors) must not hide."""
314
+ markers = ", ".join(f"'{m}'" for m in SETTINGS)
315
+ return f"(SELECT err FROM {self.rel()} WHERE err IN ({markers}) LIMIT 1)"
316
+
317
+
318
+ def as_array(c, node, sheet: str) -> RelArray:
319
+ return RelArray(relation(c, node, sheet).sql)
320
+
321
+
322
+ def _size(name: str, col: str):
323
+ def fn(c, args, sheet):
324
+ if len(args) != 1:
325
+ raise Unsupported(f"{name} needs one argument")
326
+ a = args[0]
327
+ if isinstance(a, RefNode):
328
+ area = resolve(a.text, sheet, c.book)
329
+ return Expr(f"CAST({area.r2 - area.r1 + 1 if col == 'i' else area.c2 - area.c1 + 1} AS DOUBLE)", "num")
330
+ if isinstance(a, Call) and a.name in DYNAMIC:
331
+ rel = relation(c, a, sheet).sql
332
+ # an error result (FILTER's #CALC!, an UNCHECKED marker) is the answer, not a size of one
333
+ first = f"(SELECT err FROM {rel} rs WHERE i = 1 AND j = 1 AND kind = 'e' AND "
334
+ whole = f"{first} (SELECT COUNT(*) FROM {rel} rc) = 1)"
335
+ return Expr(f"(SELECT CAST(max({col}) AS DOUBLE) FROM {rel} rm)", "num", whole)
336
+ raise Unsupported(f"{name} of something other than a range, FILTER, SORT or UNIQUE not supported yet")
337
+ register(name)(fn)
338
+
339
+
340
+ _size("ROWS", "i")
341
+ _size("COLUMNS", "j")
198
342
 
199
343
 
200
344
  def _inside(name):
@@ -241,6 +241,19 @@ class Search:
241
241
  cond, key = _near("k", x, ">=")
242
242
  self.agg, self.key = "arg_min", f"({key}, {'-' if last else ''}{self.pos})"
243
243
  self.src = f"FROM {table} WHERE {where_area(line, 'k')} AND {cond}"
244
+ # Not guessed (UNCHECKED): next smaller/larger across types or over text with punctuation (real Excel
245
+ # ranks numbers < text < TRUE/FALSE and orders text by its own collation: formulas' test.xlsx), and an
246
+ # exact text match with a German sharp s (Excel matched "ss" to it there)
247
+ kind = {"num": "'n'", "txt": "'s'", "bool": "'b'"}.get(x.type, x.kind)
248
+ xt = x.txt if x.type == "any" else x.sql if x.type == "txt" else "NULL"
249
+ cells = f"FROM {CELLS} k WHERE {where_area(line, 'k')}"
250
+ if mode == "exact":
251
+ odd = f"(strpos(COALESCE({xt}, ''), 'ß') > 0 OR EXISTS (SELECT 1 {cells} AND strpos(COALESCE(k.txt, ''), 'ß') > 0))"
252
+ else:
253
+ safe = "regexp_matches(COALESCE({t}, ''), '^[A-Za-z0-9 ]*$')"
254
+ odd = (f"(NOT {safe.format(t=xt)} OR EXISTS (SELECT 1 {cells} AND (k.kind <> {kind} OR "
255
+ f"(k.kind = 's' AND NOT {safe.format(t='k.txt')}))))")
256
+ self.unchecked = f"(CASE WHEN {odd} THEN '{UNCHECKED}' END)"
244
257
 
245
258
  def pick(self, value: str, default: str | None = None) -> str:
246
259
  if default is not None: # arg_min skips a NULL value: the cell would look not found
@@ -363,13 +376,15 @@ def xlookup(c, args, sheet):
363
376
  else f"r.col_num = k.col_num + {back.c1 - look.c1} AND r.row_num = {back.r1}")
364
377
  search = Search(x, look, mode, wild=match_mode == 2, last=search_mode == -1, back=(lit_str(back.sheet).sql, hit))
365
378
  keys = c.ready(back)
366
- found = _read(search.pick, _type_of(c, back, keys), errs(x.err, _found_error(search.pick, "#N/A")))
379
+ found = _read(search.pick, _type_of(c, back, keys),
380
+ errs(x.err, search.unchecked, _found_error(search.pick, "#N/A")))
367
381
  if len(args) < 4 or args[3] is None:
368
382
  return found
369
383
  missing = c.expr(args[3], sheet)
370
384
  found, missing = unify_all([found, missing])
371
385
  hit_sql = search.hit()
372
386
  pick_err = f"(CASE WHEN {hit_sql} THEN {found.err} ELSE {missing.err} END)"
387
+ pick_err = errs(search.unchecked, pick_err)
373
388
 
374
389
  def case(a: str, b: str) -> str:
375
390
  return f"(CASE WHEN {hit_sql} THEN {a} ELSE {b} END)"
@@ -458,3 +473,27 @@ def choose(c, args, sheet):
458
473
  err = errs(i.err, bad, case([v.err for v in values]) if any(v.err != NO_ERR for v in values) else NO_ERR)
459
474
  return Expr(case([v.sql for v in values]), values[0].type, err, case([v.txt for v in values]),
460
475
  case([v.kind for v in values]))
476
+
477
+
478
+ @register("XMATCH")
479
+ def xmatch(c, args, sheet):
480
+ """XMATCH(lookup, array, [match_mode], [search_mode]): the position XLOOKUP's search finds (exact, next smaller
481
+ -1, next larger 1, wildcards 2; first to last 1 or last to first -1)."""
482
+ arity("XMATCH", args, 2, 4)
483
+ modes = []
484
+ for i, default in ((2, 0), (3, 1)):
485
+ node = args[i] if len(args) > i else None
486
+ mode = default if node is None else int_literal(node)
487
+ if mode is None:
488
+ raise Unsupported("XMATCH modes must be numbers written in the formula")
489
+ modes.append(mode)
490
+ match_mode, search_mode = modes
491
+ if match_mode not in (0, -1, 1, 2) or search_mode not in (1, -1):
492
+ raise Unsupported("XMATCH binary search modes not supported yet")
493
+ line = _line(c, args[1], sheet, "XMATCH")
494
+ x = _xlookup_x(c, args, sheet)
495
+ found = Search(x, line, {0: "exact", 2: "exact", -1: "le", 1: "ge"}[match_mode], wild=match_mode == 2,
496
+ last=search_mode == -1)
497
+ first = line.r1 if line.c1 == line.c2 else line.c1
498
+ return Expr(found.pick(f"CAST({found.pos} - {first} + 1 AS DOUBLE)"), "num",
499
+ errs(x.err, found.unchecked, f"(CASE WHEN NOT {found.hit()} THEN '#N/A' END)"))
@@ -84,9 +84,50 @@ def test_what_excel_was_not_asked_is_not_guessed(tmp_path):
84
84
  **spill("K6", "_xlfn._xlws.FILTER(A2:A5,C2:C5>100)", [["#CALC!"]]),
85
85
  # empty sort keys, in a block bigger than the one element: UNRESOLVED, not a #SPILL! FAIL
86
86
  **spill("L6", "_xlfn._xlws.SORT(G1:G3)", [[0.0], [0.0], [1.0]]),
87
- "J8": ("=ROWS(_xlfn.UNIQUE(B2:B5))", 3.0),
88
87
  "J9": ("=_xlfn._xlws.FILTER(A2:A5,C2:C5>50)", "Asha"), # not an array formula
89
88
  })
90
- assert set(bad) == {("S", 6, 10), ("S", 6, 11), ("S", 6, 12), ("S", 7, 12), ("S", 8, 12), ("S", 8, 10),
91
- ("S", 9, 10)}
89
+ assert set(bad) == {("S", 6, 10), ("S", 6, 11), ("S", 6, 12), ("S", 7, 12), ("S", 8, 12), ("S", 9, 10)}
92
90
  assert all(v[0] == "UNRESOLVED" for v in bad.values())
91
+
92
+
93
+ def test_filter_sort_unique_inside_aggregates(tmp_path):
94
+ _, bad = check(tmp_path, {
95
+ "J1": ('=SUM(_xlfn._xlws.FILTER(C2:C5,B2:B5="West"))', 140.0),
96
+ "J2": ("=COUNTA(_xlfn.UNIQUE(B2:B5))", 3.0),
97
+ "J3": ("=ROWS(_xlfn.UNIQUE(C2:C5))", 3.0), "J4": ("=COLUMNS(A1:C5)", 3.0), "J5": ("=ROWS(A2:C5)", 4.0),
98
+ "J6": ("=MAX(_xlfn._xlws.SORT(C2:C5))", 80.0),
99
+ "J7": ('=AVERAGE(_xlfn._xlws.FILTER(C2:C5,C2:C5<70))', 140 / 3),
100
+ "J8": ('=SUM(_xlfn._xlws.FILTER(C2:C5,C2:C5>100))', "#CALC!"), # nothing kept, no if_empty
101
+ "J9": ('=COUNT(_xlfn._xlws.FILTER(G1:G3,G1:G3<5))', 1.0), # keeps two empty cells: not checked
102
+ })
103
+ assert set(bad) == {("S", 8, 10), ("S", 9, 10)} and all(v[0] == "UNRESOLVED" for v in bad.values())
104
+
105
+
106
+ def test_sortby_and_sequence(tmp_path):
107
+ _, bad = check(tmp_path, {
108
+ **spill("J1", "_xlfn.SORTBY(A2:A5,C2:C5,-1)", [["Asha"], ["Mina"], ["Ravi"], ["Vikram"]]),
109
+ **spill("K1", "_xlfn.SORTBY(A2:A5,C2:C5,1,A2:A5,-1)", [["Vikram"], ["Ravi"], ["Mina"], ["Asha"]]),
110
+ **spill("L1", "_xlfn.SORTBY(A2:A5,C2:C3)", [["#VALUE!"]]), # by_array of another height
111
+ **spill("M1", "_xlfn.SEQUENCE(2,3)", [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]),
112
+ **spill("M4", "_xlfn.SEQUENCE(3,1,10,-5)", [[10.0], [5.0], [0.0]]),
113
+ **spill("M8", "_xlfn.SEQUENCE(2.9)", [[1.0], [2.0]]), # truncated
114
+ "M11": ("=SUM(_xlfn.SEQUENCE(100))", 5050.0),
115
+ **spill("Q1", "_xlfn.SEQUENCE(0)", [["#CALC!"]]), # not checked
116
+ **spill("Q2", "_xlfn.SEQUENCE(1E9,1E9)", [["#NUM!"]]), # never built
117
+ })
118
+ assert set(bad) == {("S", 1, 17), ("S", 2, 17)} and all(v[0] == "UNRESOLVED" for v in bad.values())
119
+
120
+
121
+ def test_corpus_findings_for_sortby_filter_and_xmatch(tmp_path):
122
+ # formulas' test.xlsx (Excel-saved): SORTBY by one-row arrays sorts the columns; FILTER's if_empty range
123
+ # spills whole; XMATCH next-smaller across types or over punctuation, and a sharp s, are not guessed
124
+ _, bad = check(tmp_path, {
125
+ **spill("J1", "_xlfn.SORTBY(C2:C3,{2},1)", [[80.0], [40.0]]),
126
+ **spill("K1", "_xlfn.SORTBY(E1:G1,{3,1,2},1)", [[0.0, 1.0, 1.0]]),
127
+ **spill("K3", "_xlfn.SORTBY({1,2,3;4,5,6},{3,2,2},1,{5,1,4},-1)", [[3.0, 2.0, 1.0], [6.0, 5.0, 4.0]]),
128
+ **spill("K6", '_xlfn._xlws.FILTER(A2:A5,C2:C5>100,B2:B3)', [["West"], ["East"]]),
129
+ "N1": ('=_xlfn.XMATCH("*b?u*",{-1.1,2.1,3.1,4.1},-1)', 4.0),
130
+ "N2": ('=_xlfn.XMATCH("Mina",A2:A5,-1)', 3.0), # same type, plain text: compiled
131
+ "N3": ('=_xlfn.XMATCH("ss",{1,"ß"},0)', 2.0),
132
+ })
133
+ assert set(bad) == {("S", 1, 14), ("S", 3, 14)} and all(v[0] == "UNRESOLVED" for v in bad.values())
@@ -238,3 +238,30 @@ def test_array_constants_reach_the_dbt_models(tmp_path):
238
238
  write_project(book, compiled, tmp_path, "arrays")
239
239
  got = {(s, r, c): (k, n, t) for s, r, c, k, n, t in _run_models(tmp_path)}
240
240
  assert got == execute(book, compiled)
241
+
242
+
243
+ def _spill_book(tmp_path):
244
+ from test_dynamic import DATA, spill
245
+ from xlsxkit import write_xlsx
246
+ from cellproof.reader import load
247
+ wb = tmp_path / "spill.xlsx"
248
+ write_xlsx(wb, {"S": {**DATA, **spill("J1", "_xlfn._xlws.SORT(_xlfn.UNIQUE(B2:B5))", [["East"], ["North"], ["West"]]),
249
+ **spill("K1", "_xlfn._xlws.FILTER(A2:C5,C2:C5>50)", [["Asha", "West", 80.0],
250
+ ["Mina", "west", 60.0]]),
251
+ "M1": ("=J2&K2", "NorthMina")}}) # a cell reading spilled cells
252
+ return wb, load(wb)
253
+
254
+
255
+ def test_spill_models_equal_the_verifier(tmp_path):
256
+ _, book = _spill_book(tmp_path)
257
+ compiled = compile_book(book)
258
+ assert compiled.unresolved == {}
259
+ write_project(book, compiled, tmp_path / "out", "spill")
260
+ got = {(s, r, c): (k, n, t) for s, r, c, k, n, t in _run_models(tmp_path / "out")}
261
+ assert got == execute(book, compiled)
262
+
263
+
264
+ @pytest.mark.skipif(shutil.which("dbt") is None, reason="needs: pip install -e .[dbt]")
265
+ def test_dbt_build_with_spills(tmp_path):
266
+ wb, _ = _spill_book(tmp_path)
267
+ assert main(["convert", str(wb), "--out", str(tmp_path / "out"), "--dbt"]) == 0
@@ -134,3 +134,22 @@ def test_the_walk_matches_excel_s_bisection_on_random_tables():
134
134
  want = bisection(vec, x)
135
135
  kind, num, txt = got[("F", i, 1)]
136
136
  assert (kind == "e" and txt == "#N/A") if want is None else num == want + 1
137
+
138
+
139
+ def test_xmatch_modes():
140
+ # Microsoft's XMATCH examples: exact, next smaller/larger, wildcard, last to first
141
+ from cellproof.compiler import compile_book
142
+ from cellproof.verifier import execute
143
+ from xlsxkit import book_of
144
+ s = {"A1": "Apple", "A2": "Grape", "A3": "Pear", "A4": "Banana", "A5": "Cherry", "A6": "Grape",
145
+ "B1": 10.0, "B2": 20.0, "B3": 30.0, "B4": 40.0,
146
+ "C1": ('=_xlfn.XMATCH("Grape",A1:A6)', 2.0), "C2": ('=_xlfn.XMATCH("Grape",A1:A6,0,-1)', 6.0),
147
+ "C3": ('=_xlfn.XMATCH("gra*",A1:A6,2)', 2.0), "C4": ("=_xlfn.XMATCH(25,B1:B4,-1)", 2.0),
148
+ "C5": ("=_xlfn.XMATCH(25,B1:B4,1)", 3.0), "C6": ('=_xlfn.XMATCH("Kiwi",A1:A6)', "#N/A"),
149
+ "C7": ("=_xlfn.XMATCH(25,B1:B4,0,2)", 0.0)}
150
+ book = book_of({"S": s})
151
+ compiled = compile_book(book)
152
+ got = execute(book, compiled)
153
+ assert [got[("S", r, 3)] for r in range(1, 7)] == [("n", 2.0, None), ("n", 6.0, None), ("n", 2.0, None),
154
+ ("n", 2.0, None), ("n", 3.0, None), ("e", None, "#N/A")]
155
+ assert "binary search" in compiled.unresolved[("S", 7, 3)]
@@ -1 +0,0 @@
1
- __version__ = "0.4.0"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes