cellproof 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. {cellproof-0.2.1 → cellproof-0.2.2}/AGENTS.md +1 -1
  2. {cellproof-0.2.1 → cellproof-0.2.2}/CHANGELOG.md +8 -0
  3. cellproof-0.2.2/INFO.md +153 -0
  4. {cellproof-0.2.1 → cellproof-0.2.2}/PKG-INFO +8 -6
  5. {cellproof-0.2.1 → cellproof-0.2.2}/PROGRESS.md +5 -5
  6. {cellproof-0.2.1 → cellproof-0.2.2}/README.md +6 -4
  7. {cellproof-0.2.1 → cellproof-0.2.2}/pyproject.toml +2 -2
  8. cellproof-0.2.2/src/cellproof/__init__.py +1 -0
  9. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/compiler.py +23 -1
  10. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/expr.py +3 -1
  11. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/dates.py +55 -3
  12. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/lookup.py +29 -1
  13. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/text.py +2 -2
  14. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/model.py +1 -0
  15. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/parser.py +69 -6
  16. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/refs.py +1 -1
  17. cellproof-0.2.2/tests/test_arrays.py +45 -0
  18. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_dates.py +26 -0
  19. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_emitter.py +11 -0
  20. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_lookup.py +29 -0
  21. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_parser.py +1 -1
  22. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_region.py +5 -0
  23. cellproof-0.2.1/src/cellproof/__init__.py +0 -1
  24. {cellproof-0.2.1 → cellproof-0.2.2}/.github/workflows/ci.yml +0 -0
  25. {cellproof-0.2.1 → cellproof-0.2.2}/.github/workflows/publish.yml +0 -0
  26. {cellproof-0.2.1 → cellproof-0.2.2}/.gitignore +0 -0
  27. {cellproof-0.2.1 → cellproof-0.2.2}/DESIGN.md +0 -0
  28. {cellproof-0.2.1 → cellproof-0.2.2}/LICENSE +0 -0
  29. {cellproof-0.2.1 → cellproof-0.2.2}/PLAN-2a.md +0 -0
  30. {cellproof-0.2.1 → cellproof-0.2.2}/PLAN-2b.md +0 -0
  31. {cellproof-0.2.1 → cellproof-0.2.2}/PLAN.md +0 -0
  32. {cellproof-0.2.1 → cellproof-0.2.2}/SECURITY.md +0 -0
  33. {cellproof-0.2.1 → cellproof-0.2.2}/docs/JOURNEY.md +0 -0
  34. {cellproof-0.2.1 → cellproof-0.2.2}/docs/superpowers/plans/2026-10-14-plan-2c-speed.md +0 -0
  35. {cellproof-0.2.1 → cellproof-0.2.2}/docs/superpowers/plans/2026-10-14-plan-2d-table-mode.md +0 -0
  36. {cellproof-0.2.1 → cellproof-0.2.2}/docs/superpowers/specs/2026-10-14-plan-2c-speed-design.md +0 -0
  37. {cellproof-0.2.1 → cellproof-0.2.2}/docs/superpowers/specs/2026-10-14-plan-2d-table-mode-design.md +0 -0
  38. {cellproof-0.2.1 → cellproof-0.2.2}/docs/superpowers/specs/2026-10-18-running-totals-design.md +0 -0
  39. {cellproof-0.2.1 → cellproof-0.2.2}/docs/superpowers/specs/2026-10-19-lookups-row-wise-design.md +0 -0
  40. {cellproof-0.2.1 → cellproof-0.2.2}/examples/commission.xlsx +0 -0
  41. {cellproof-0.2.1 → cellproof-0.2.2}/examples/errors.xlsx +0 -0
  42. {cellproof-0.2.1 → cellproof-0.2.2}/examples/functions.xlsx +0 -0
  43. {cellproof-0.2.1 → cellproof-0.2.2}/examples/functions_unsaved.xlsx +0 -0
  44. {cellproof-0.2.1 → cellproof-0.2.2}/examples/make_commission.py +0 -0
  45. {cellproof-0.2.1 → cellproof-0.2.2}/examples/make_errors.py +0 -0
  46. {cellproof-0.2.1 → cellproof-0.2.2}/examples/make_functions.py +0 -0
  47. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/__main__.py +0 -0
  48. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/cli.py +0 -0
  49. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/compute.py +0 -0
  50. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/emitter.py +0 -0
  51. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fenwick.py +0 -0
  52. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/__init__.py +0 -0
  53. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/aggregate.py +0 -0
  54. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/criteria.py +0 -0
  55. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/logic.py +0 -0
  56. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/fn/maths.py +0 -0
  57. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/functions.py +0 -0
  58. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/graph.py +0 -0
  59. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/r1c1.py +0 -0
  60. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/rangeindex.py +0 -0
  61. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/reader.py +0 -0
  62. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/region.py +0 -0
  63. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/regions.py +0 -0
  64. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/report.py +0 -0
  65. {cellproof-0.2.1 → cellproof-0.2.2}/src/cellproof/verifier.py +0 -0
  66. {cellproof-0.2.1 → cellproof-0.2.2}/tests/bench_shapes.py +0 -0
  67. {cellproof-0.2.1 → cellproof-0.2.2}/tests/corpus.py +0 -0
  68. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_aggregate.py +0 -0
  69. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_batches.py +0 -0
  70. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_cli.py +0 -0
  71. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_compiler.py +0 -0
  72. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_compute.py +0 -0
  73. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_corpus_findings.py +0 -0
  74. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_criteria.py +0 -0
  75. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_errors.py +0 -0
  76. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_excel_findings.py +0 -0
  77. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_excel_saved.py +0 -0
  78. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_fenwick.py +0 -0
  79. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_functions.py +0 -0
  80. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_graph.py +0 -0
  81. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_implicit.py +0 -0
  82. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_index.py +0 -0
  83. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_lexed.py +0 -0
  84. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_limits.py +0 -0
  85. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_logic.py +0 -0
  86. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_lookups.py +0 -0
  87. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_maths.py +0 -0
  88. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_perf.py +0 -0
  89. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_r1c1.py +0 -0
  90. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_rangeindex.py +0 -0
  91. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_reader.py +0 -0
  92. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_refs.py +0 -0
  93. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_regions.py +0 -0
  94. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_regions_differential.py +0 -0
  95. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_regions_split.py +0 -0
  96. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_report.py +0 -0
  97. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_review_regressions.py +0 -0
  98. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_running.py +0 -0
  99. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_shapes.py +0 -0
  100. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_text.py +0 -0
  101. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_types.py +0 -0
  102. {cellproof-0.2.1 → cellproof-0.2.2}/tests/test_verifier.py +0 -0
  103. {cellproof-0.2.1 → cellproof-0.2.2}/tests/xlsxkit.py +0 -0
@@ -28,7 +28,7 @@ CELLPROOF_PERF=1 python -m pytest tests/test_perf.py -s # 200k-formula benchmar
28
28
  | `expr.py` | `Expr` (SQL value + SQL error code), Excel coercions |
29
29
  | `compiler.py` | cell mode: formula -> SQL in batches |
30
30
  | `region.py` | a region -> one set-based statement (window functions for running totals) |
31
- | `functions.py`, `fn/` | the 94 Excel functions |
31
+ | `functions.py`, `fn/` | the 96 Excel functions |
32
32
  | `verifier.py` | runs everything on DuckDB and compares with Excel's saved values |
33
33
  | `emitter.py` | writes the dbt project; `--dbt` runs `dbt build` |
34
34
  | `report.py`, `cli.py`, `compute.py` | REPORT.md/report.json, the CLI, the optional `formulas` answer key |
@@ -1,5 +1,13 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.2 (2026-10-27)
4
+
5
+ - **LOOKUP** (vector and array form). Like the other approximate lookups, UNRESOLVED on data out of order.
6
+ - **YEARFRAC**, all five bases, the basis written in the formula or computed. The 1,992 YEARFRAC cells of pycel's
7
+ test workbook all PASS; a date before 1 March 1900 (Excel's 29 February 1900) is UNRESOLVED.
8
+ - **Array constants** (`{0,50,100}`, `{1,"a";2,"b"}`) wherever a range is taken: `LOOKUP(x,{0,50,100},{"F","C","A"})`,
9
+ `MATCH(x,{...},0)`, `SUM({...})`. Used as one value (`A1*{1,2}`) they stay UNRESOLVED.
10
+
3
11
  ## 0.2.1 (2026-10-20)
4
12
 
5
13
  Found by running all 85 public test workbooks of pycel, formulas and xlcalculator (`python tests/corpus.py DIR`):
@@ -0,0 +1,153 @@
1
+ # INFO: handoff for the next coding agent
2
+
3
+ You are taking over development of **cellproof**. Read this file first, then `AGENTS.md` (rules), `README.md` (what the
4
+ tool does and its limits) and `PROGRESS.md` (state and next steps). This file is the short version of all of them plus
5
+ the things that are not written down elsewhere.
6
+
7
+ Last updated: 2026-10-27. Current release: 0.2.2 (PyPI: `cellproof`). Repo: github.com/Nikhil414/cellproof (public).
8
+
9
+ ## What we are building
10
+
11
+ cellproof turns an Excel workbook (`.xlsx`/`.xlsm`) into SQL and **proves every number matches**.
12
+
13
+ 1. Read the formulas and the values Excel saved in the file (Excel's saved values are a free answer key).
14
+ 2. Translate every formula to SQL (DuckDB dialect), with Excel's exact rules: type coercions, error codes, error order.
15
+ 3. Run the SQL on DuckDB and compare each result with Excel's saved value.
16
+ 4. Emit a dbt project that recomputes the workbook, plus `REPORT.md` / `report.json`.
17
+
18
+ Every formula cell gets one verdict:
19
+
20
+ - **PASS**: SQL result equals Excel's saved value.
21
+ - **FAIL**: SQL result differs; the report shows both values.
22
+ - **UNRESOLVED**: cellproof cannot translate it exactly; the report says why (the text of an `Unsupported` exception).
23
+
24
+ ## End goal
25
+
26
+ A finance or analytics person who has business logic trapped in a spreadsheet can move it to a database and **show
27
+ evidence** that nothing changed. The product is trust, not coverage: an honest UNRESOLVED beats a guessed PASS.
28
+
29
+ Longer term:
30
+
31
+ - Cover the Excel features real business workbooks use (array formulas are the biggest missing piece).
32
+ - Stay fast on big workbooks (200k formulas verifies in about 26 s today).
33
+ - Plan 3 (not started): an AI retry loop for cells that come out UNRESOLVED or FAIL. It may see **formula structure
34
+ only, never cell values**.
35
+ - Other warehouses besides DuckDB for the dbt output (not started).
36
+
37
+ ## Non-negotiable rules (from AGENTS.md)
38
+
39
+ - **Never guess a value.** Anything not translated exactly raises `Unsupported("<reason a user understands>")`.
40
+ - **A false PASS is the worst bug.** When unsure, produce an honest FAIL or UNRESOLVED.
41
+ - **No DuckDB exception may escape from a supported function.** Guard every domain with `CASE`.
42
+ - **Regions must equal cell mode.** `compile_book(book, regions=False)` is the reference; region tests compare to it.
43
+ - **Runtime dependencies stay `openpyxl` and `duckdb`.** Anything else goes in an optional extra.
44
+ - **Cell values never leave the machine.** No network calls with workbook data.
45
+ - **Tests first.** Console output is ASCII-only. Measure before claiming speed.
46
+ - **Real Excel is the ground truth.** The `formulas` library is only a labelled fallback answer key.
47
+ - Commits made by an agent end with a `Co-Authored-By:` trailer naming the agent.
48
+
49
+ ## Where we are (2026-10-27)
50
+
51
+ Shipped and on `main` / PyPI 0.2.2:
52
+
53
+ - Full pipeline: reader, parser, dependency graph, cell-mode compiler, region (table) mode, verifier, dbt emitter, report, CLI.
54
+ - 96 Excel functions (list in README). Latest additions: LOOKUP, YEARFRAC (all 5 bases, dynamic basis), array constants
55
+ such as `{0,50,100}` wherever a range is taken.
56
+ - Region mode: a column repeating one formula compiles to one set-based statement; running totals become window
57
+ functions; VLOOKUP/HLOOKUP/MATCH/INDEX/XLOOKUP and the *IF(S) family pasted down compile to joins.
58
+ - Speed: 200k formulas in 26 s; 50k-row VLOOKUP in 4.6 s.
59
+ - Tests: about 430 pass, 8 skip (dbt tests skip when `dbt` is not on PATH). Whole suite takes 4 to 8 minutes.
60
+ - Validation: 3 workbooks saved by real Excel all PASS. 85 public test workbooks (pycel, formulas, xlcalculator):
61
+ 6,874 PASS, 10 FAIL (all in LibreOffice-saved files, flagged in their reports), 16,764 UNRESOLVED, no crashes.
62
+ Run it with `python tests/corpus.py DIR`.
63
+ - CI: Windows, macOS and Linux x Python 3.10 to 3.13. Release = GitHub release, which triggers the PyPI publish workflow.
64
+
65
+ ## What to do next (in order)
66
+
67
+ 1. **Array formulas and array evaluation.** `SUM(A1:A10*B1:B10)`, `MAX(IF(c, A1:A9, 0))`, `A1*{1,2}`. Today these are
68
+ UNRESOLVED. This is the largest remaining source of UNRESOLVED cells in business workbooks. It needs element-wise
69
+ evaluation of ranges, which is a new design piece: write a short spec in `docs/superpowers/specs/` first, then a plan
70
+ in `docs/superpowers/plans/`, then build test-first. Existing building blocks you can reuse:
71
+ - Array constants are parsed into hidden-sheet `RefNode`s; the values live in an `ARRAYS` dict keyed by content hash.
72
+ The hidden sheet name is `#array-<hash>` (see gotchas for why).
73
+ - Regions already compile column-wise statements; element-wise range math may fit there.
74
+ 2. **Test on more real analyst workbooks.** One user workbook found 4 gaps the 85 public files did not. Every real
75
+ workbook is worth more than another synthetic test. Turn each finding into a test in
76
+ `tests/test_corpus_findings.py`.
77
+ 3. **Read time.** openpyxl loads the workbook twice (8.3 s of the 26 s). Also the per-cell dependency loop (11.3 s compile).
78
+ 4. **Plan 3**: AI retry loop (structure only, never values).
79
+
80
+ Known smaller gaps are listed in `PROGRESS.md` under "Known gaps".
81
+
82
+ ## Code map (`src/cellproof/`)
83
+
84
+ | Module | Job |
85
+ |---|---|
86
+ | `model.py` | `Cell`, `Book`, `Verdict`, `Unsupported` (its message becomes the UNRESOLVED reason) |
87
+ | `reader.py` | loads the workbook twice with openpyxl (formulas, saved values); refuses zip bombs |
88
+ | `parser.py`, `refs.py` | formula text to AST with Excel precedence; reference text to `Area` |
89
+ | `r1c1.py` | how a reference behaves when a formula is pasted down |
90
+ | `regions.py` | finds regions, running totals, splits on cycles |
91
+ | `graph.py`, `fenwick.py`, `rangeindex.py` | dependency order without expanding ranges; cycle detection |
92
+ | `expr.py` | `Expr` (SQL value + SQL error code), Excel coercions, sentinels such as `#1900` |
93
+ | `compiler.py` | cell mode: formula to SQL in batches |
94
+ | `region.py` | a region to one set-based statement |
95
+ | `functions.py`, `fn/` | the Excel functions (`fn/lookup.py`, `fn/dates.py`, ...) |
96
+ | `verifier.py` | runs everything on DuckDB and compares with Excel's saved values |
97
+ | `emitter.py` | writes the dbt project; `--dbt` runs `dbt build` |
98
+ | `report.py`, `cli.py`, `compute.py` | REPORT.md/report.json, the CLI, the optional `formulas` answer key |
99
+
100
+ ## How to work here
101
+
102
+ Setup (Mac/Linux):
103
+
104
+ ```bash
105
+ python -m venv .venv && . .venv/bin/activate
106
+ pip install -e ".[dev,dbt,compute]"
107
+ python -m pytest tests -q # dbt must be on PATH (venv active) or the dbt tests skip
108
+ ```
109
+
110
+ Workflow that has worked for every piece so far:
111
+
112
+ 1. Write the failing tests first (TDD). Add a region-mode test next to every new function; it must equal cell mode.
113
+ 2. Implement the smallest thing that is exactly right. If it cannot be exact, raise `Unsupported` with a plain reason.
114
+ 3. Run the whole suite. After a change that can affect results, re-run the corpus and compare PASS/FAIL/UNRESOLVED
115
+ counts. FAIL must not go up.
116
+ 4. Update `README.md` (function list and limitations), `CHANGELOG.md` (under "Unreleased") and `PROGRESS.md`.
117
+ 5. When Excel's behaviour is uncertain, **do not guess from memory**. Remove the unverified test case, or find an
118
+ Excel-saved workbook that proves it. (Example: a YEARFRAC test with an unverified expected value was dropped.)
119
+
120
+ ## Gotchas learned the hard way
121
+
122
+ - **Braces in sheet names break things.** Python `.format()` and dbt's Jinja both treat `{...}` as placeholders. The
123
+ array sheet used to be `{cellproof-array-hash}` and failed in region mode and in dbt; it is now `#array-<hash>`. Never
124
+ put `{` or `}` in generated names. Detect array sheets by membership in the `ARRAYS` dict, not by a string prefix.
125
+ - **Error order matters.** Excel reports errors left to right, and errors typed into a formula come before errors inside
126
+ referenced cells. YEARFRAC needed the basis error checked before date errors (52 corpus FAILs until fixed).
127
+ - **Parser tests that expect "unparseable" input go stale** when the parser learns new syntax (`{1,2}` was one). Pick a
128
+ form that is still invalid (`{1,A1}`).
129
+ - **Excel facts verified against real Excel** are in `PROGRESS.md` ("What Excel taught us") with tests in
130
+ `tests/test_excel_findings.py`. Read them before touching COUNTIF, DAYS360, TEXT, MOD, POWER or date code.
131
+ - **Approximate lookups on unsorted data** (LOOKUP, MATCH/VLOOKUP with type 1/-1) stay UNRESOLVED. Excel's answer there
132
+ depends on its binary search.
133
+ - **Text to number** only handles plain numbers. Locale-dependent text (`"1,000"`, `"$5"`, `"50%"`) is UNRESOLVED.
134
+ - Console output must stay ASCII (the tool is also used on Windows consoles).
135
+ - No linter is installed in the dev environment (ruff and pyflakes are missing). Keep the existing style by hand.
136
+
137
+ ## Repo and release facts
138
+
139
+ - Public repo `Nikhil414/cellproof` (remote `origin`), single-commit-history start. **Never push the old history
140
+ publicly.** A private backup `Nikhil414/unexcel-dev` exists (remote `dev`, branch `public-main` mirrors `main`).
141
+ The project was called `unexcel` until 0.2.0; older plans and specs still use that name.
142
+ - Release: bump `version` in `pyproject.toml` and `src/cellproof/__init__.py`, date the CHANGELOG, run the tests,
143
+ commit, push `main`, then `gh release create vX.Y.Z`. The release triggers the PyPI publish workflow. Confirm CI is
144
+ green first.
145
+ - Only push, tag or release when the owner asks.
146
+ - The owner prefers decisions to questions: decide, build, test, security-check, and explain what you did.
147
+
148
+ ## Safety checklist before every commit
149
+
150
+ - No workbook data, secrets or tokens in the repo, tests or logs.
151
+ - No new runtime dependency.
152
+ - No network call that carries workbook data.
153
+ - A false PASS is possible? Make it an UNRESOLVED instead.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cellproof
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches.
5
5
  Project-URL: Homepage, https://github.com/Nikhil414/cellproof
6
6
  Project-URL: Issues, https://github.com/Nikhil414/cellproof/issues
@@ -21,7 +21,7 @@ Classifier: Programming Language :: Python :: 3.13
21
21
  Classifier: Topic :: Database
22
22
  Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
23
23
  Requires-Python: >=3.10
24
- Requires-Dist: duckdb>=1.1
24
+ Requires-Dist: duckdb>=1.3
25
25
  Requires-Dist: openpyxl>=3.1
26
26
  Provides-Extra: compute
27
27
  Requires-Dist: formulas<1.4,>=1.3.4; extra == 'compute'
@@ -110,12 +110,12 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
110
110
  code.
111
111
  6. **Emit** the dbt project and the report.
112
112
 
113
- 94 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
113
+ 96 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
114
114
  COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR,
115
115
  HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LEFT, LEN, LN, LOG,
116
- LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
116
+ LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
117
117
  PROPER, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SQRT, SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
118
- SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VLOOKUP, WEEKDAY, XLOOKUP, XOR, YEAR.
118
+ SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VLOOKUP, WEEKDAY, XLOOKUP, XOR, YEAR, YEARFRAC.
119
119
 
120
120
  **Speed** (Windows 11 laptop, measured): a 200,000-formula workbook (four formula columns over 50,000 rows, one of them
121
121
  a running total) converts and verifies in 26 s, all PASS. 50,000 rows of VLOOKUP into a 1,000-row table: 4.6 s.
@@ -126,6 +126,8 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
126
126
  cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
127
127
 
128
128
  - **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
129
+ - **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...);
130
+ where one value is expected (`A1*{1,2}`) they need array evaluation and are UNRESOLVED.
129
131
  - **Not supported:** array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
130
132
  `INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
131
133
  external workbooks, structured table references (`Table1[@Col]`), INDEX used as a reference, the 1904 date system in
@@ -133,7 +135,7 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
133
135
  - **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
134
136
  Excel does (implicit intersection). A range that IF/IFERROR/CHOOSE hands to another function
135
137
  (`MAX(IF(c,A1:A9,0))`, where Excel uses the whole range) is UNRESOLVED.
136
- - **Approximate lookups on unsorted data** (MATCH/VLOOKUP/HLOOKUP with match type 1/-1): UNRESOLVED, because Excel's
138
+ - **Approximate lookups on unsorted data** (LOOKUP, and MATCH/VLOOKUP/HLOOKUP with match type 1/-1): UNRESOLVED, because Excel's
137
139
  answer then depends on its binary search.
138
140
  - **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Text Excel reads with the computer's language
139
141
  settings (`"1,000"`, `"$5"`, `"50%"`, `"1/2/2020"`, `"2:00 PM"`, `"Jan 5"`) makes the cell UNRESOLVED, because the
@@ -20,15 +20,15 @@ Everything below is on `main`, built test-first, each piece reviewed by a fresh
20
20
  | Criteria row-wise | the *IF(S) family with a criterion from the row (`=COUNTIFS($E:$E,">"&A2,...)`) compiles into the region's statement too |
21
21
  | Open source | MIT license, README with limits, SECURITY.md, CI on Windows/macOS/Linux x Python 3.10-3.13 |
22
22
 
23
- Tests: about 425 (dbt build included), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
23
+ Tests: about 437 (dbt build included), plus 3 workbooks saved by real Excel that all PASS (4 cells UNRESOLVED on
24
24
  purpose). Public on GitHub (one-commit history start). 0.1.0 was released as `unexcel`; 0.2.0 (2026-10-19) and 0.2.1 (2026-10-20) are on PyPI as `cellproof` (see CHANGELOG.md).
25
25
  A user's practice workbook: 3 PASS / 47 UNRESOLVED -> 48 PASS / 0 FAIL / 2 UNRESOLVED.
26
26
 
27
27
  ## Public test workbooks (2026-10-19)
28
28
 
29
29
  `python tests/corpus.py DIR` runs the 85 .xlsx/.xlsm test files of pycel, formulas and xlcalculator (Excel-saved,
30
- except 8 LibreOffice ones): 4,693 PASS, 10 FAIL (all in LibreOffice files, flagged in their reports), 18,945
31
- UNRESOLVED, no crashes. The UNRESOLVED are mostly functions those files test on purpose: CONVERT, YEARFRAC,
30
+ except 8 LibreOffice ones): 6,874 PASS, 10 FAIL (all in LibreOffice files, flagged in their reports), 16,764
31
+ UNRESOLVED, no crashes (was 4,693 PASS before LOOKUP, YEARFRAC and array constants). The UNRESOLVED are mostly functions those files test on purpose: CONVERT, YEARFRAC,
32
32
  DEC2HEX/BIN/OCT, array formulas and `{1,2}` constants, CODE/UNICODE, LOOKUP. For business workbooks the ones that
33
33
  matter are array formulas, array constants, LOOKUP and YEARFRAC. Findings are tests in `tests/test_corpus_findings.py`.
34
34
 
@@ -80,8 +80,8 @@ What Excel taught us (each has a test in `tests/test_excel_findings.py`):
80
80
 
81
81
  ## Next
82
82
 
83
- 1. Close the UNRESOLVED gaps business workbooks hit: LOOKUP, YEARFRAC, array constants (`{1,2,3}`), simple array
84
- formulas (`SUM(A1:A10*B1:B10)`, `MAX(IF(...))`).
83
+ 1. Array formulas and array evaluation (`SUM(A1:A10*B1:B10)`, `MAX(IF(...))`, `A1*{1,2}`). LOOKUP, YEARFRAC and
84
+ array constants where a range is taken are done (unreleased).
85
85
  2. Test on more real analyst workbooks (one user workbook found 4 gaps the 85 public files did not).
86
86
  3. Read time (openpyxl loads the workbook twice) and the per-cell dependency loop.
87
87
  4. Plan 3: AI retry loop for failed cells (it sees formula structure only, never values).
@@ -76,12 +76,12 @@ reader -> regions -> dependency graph -> compiler -> verifier -> dbt emitter ->
76
76
  code.
77
77
  6. **Emit** the dbt project and the report.
78
78
 
79
- 94 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
79
+ 96 Excel functions are supported: ABS, AND, AVERAGE, AVERAGEIF(S), CEILING, CHAR, CHOOSE, CLEAN, CONCAT(ENATE),
80
80
  COUNT, COUNTA, COUNTBLANK, COUNTIF(S), DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, EXACT, EXP, FIND, FLOOR,
81
81
  HLOOKUP, IF, IFERROR, IFNA, IFS, INDEX, INT, IS* (BLANK, ERR, ERROR, LOGICAL, NA, NUMBER, TEXT), LEFT, LEN, LN, LOG,
82
- LOG10, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
82
+ LOG10, LOOKUP, LOWER, MATCH, MAX, MAXIFS, MEDIAN, MID, MIN, MINIFS, MOD, MODE, MONTH, MROUND, NA, NOT, OR, PI, POWER, PRODUCT,
83
83
  PROPER, REPLACE, REPT, RIGHT, ROUND(UP/DOWN), SEARCH, SIGN, SQRT, SUBSTITUTE, SUBTOTAL, SUM, SUMIF(S), SUMPRODUCT,
84
- SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VLOOKUP, WEEKDAY, XLOOKUP, XOR, YEAR.
84
+ SWITCH, TEXT, TEXTJOIN, TRIM, TRUE/FALSE, TRUNC, UPPER, VALUE, VLOOKUP, WEEKDAY, XLOOKUP, XOR, YEAR, YEARFRAC.
85
85
 
86
86
  **Speed** (Windows 11 laptop, measured): a 200,000-formula workbook (four formula columns over 50,000 rows, one of them
87
87
  a running total) converts and verifies in 26 s, all PASS. 50,000 rows of VLOOKUP into a 1,000-row table: 4.6 s.
@@ -92,6 +92,8 @@ COUNTIFS/SUMIFS with a criterion from the row run at the same speed.
92
92
  cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each reported with a reason:
93
93
 
94
94
  - **File formats:** `.xlsx` and `.xlsm` only (no `.xls`, `.xlsb`, `.ods`). Macros (VBA) are ignored.
95
+ - **Array constants** (`{0,50,100}`) work wherever a range is taken (`SUM`, `MATCH`, `LOOKUP`, `VLOOKUP`, `INDEX`...);
96
+ where one value is expected (`A1*{1,2}`) they need array evaluation and are UNRESOLVED.
95
97
  - **Not supported:** array formulas and dynamic arrays, volatile functions (`NOW`, `TODAY`, `RAND`, `OFFSET`, and
96
98
  `INDIRECT` of text built while the sheet runs; `INDIRECT("Sales")` is fine), functions outside the list above,
97
99
  external workbooks, structured table references (`Table1[@Col]`), INDEX used as a reference, the 1904 date system in
@@ -99,7 +101,7 @@ cellproof prefers an honest UNRESOLVED or FAIL to a guess. Known limits, each re
99
101
  - **Ranges where one value is expected** (`=IF(E$3:E$41="F",B$3:B$41)`) take the formula's own row or column, as
100
102
  Excel does (implicit intersection). A range that IF/IFERROR/CHOOSE hands to another function
101
103
  (`MAX(IF(c,A1:A9,0))`, where Excel uses the whole range) is UNRESOLVED.
102
- - **Approximate lookups on unsorted data** (MATCH/VLOOKUP/HLOOKUP with match type 1/-1): UNRESOLVED, because Excel's
104
+ - **Approximate lookups on unsorted data** (LOOKUP, and MATCH/VLOOKUP/HLOOKUP with match type 1/-1): UNRESOLVED, because Excel's
103
105
  answer then depends on its binary search.
104
106
  - **Text to number:** only plain numbers convert (`"3"`, `" -1.5e3 "`). Text Excel reads with the computer's language
105
107
  settings (`"1,000"`, `"$5"`, `"50%"`, `"1/2/2020"`, `"2:00 PM"`, `"Jan 5"`) makes the cell UNRESOLVED, because the
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cellproof"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "Turn an Excel workbook into SQL (a dbt project on DuckDB) and prove every number matches."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -26,7 +26,7 @@ classifiers = [
26
26
  "Topic :: Database",
27
27
  "Topic :: Office/Business :: Financial :: Spreadsheet",
28
28
  ]
29
- dependencies = ["openpyxl>=3.1", "duckdb>=1.1"]
29
+ dependencies = ["openpyxl>=3.1", "duckdb>=1.3"]
30
30
 
31
31
  [project.optional-dependencies]
32
32
  dev = ["pytest>=8", "jinja2>=3.1"]
@@ -0,0 +1 @@
1
+ __version__ = "0.2.2"
@@ -20,7 +20,7 @@ from cellproof.expr import (BLANK, CELLS, NO_ERR, Expr, any_of, compare, error,
20
20
  from cellproof.functions import FUNCS
21
21
  from cellproof.graph import formula_order
22
22
  from cellproof.model import Book, Cell, Key, Unsupported, a1
23
- from cellproof.parser import Binary, Bool, Call, ErrLit, Num, Postfix, RefNode, Str, Unary, parse
23
+ from cellproof.parser import ARRAYS, Binary, Bool, Call, ErrLit, Num, Postfix, RefNode, Str, Unary, arrays, parse
24
24
  from cellproof.rangeindex import RangeIndex, field
25
25
  from cellproof.refs import Area, resolve
26
26
 
@@ -103,6 +103,7 @@ class Keys:
103
103
 
104
104
  class Compiler:
105
105
  def __init__(self, book: Book):
106
+ add_arrays(book)
106
107
  self.book = book
107
108
  self.done: dict[Key, tuple[str, bool]] = {} # formula cell in a closed batch -> (type, may be an error)
108
109
  self.open: dict[Key, tuple[str, str, bool]] = {} # formula cell in the open batch -> (cte, type, may err)
@@ -124,6 +125,8 @@ class Compiler:
124
125
  def _intersect(self, area, text: str) -> Expr:
125
126
  """A range where one value is expected: Excel's implicit intersection takes the cell in the formula's own row
126
127
  (a one-column range) or column (a one-row range), and is #VALUE! when there is none."""
128
+ if area.sheet in self.book.arrays:
129
+ raise Unsupported("array constant used where one value is expected (needs array evaluation) not supported yet")
127
130
  if self.at is None or area.sheet != self.at[0] or (area.r1 != area.r2 and area.c1 != area.c2):
128
131
  raise Unsupported(f"range {text} used as a single value")
129
132
  # real Excel: in MAX(IF(c, A1:A9, 0)) IF hands MAX the whole range; only an operator, a function's own
@@ -277,6 +280,24 @@ class Compiler:
277
280
  return self.done[key][1] if cell.formula is not None else cell.kind == "e"
278
281
 
279
282
 
283
+ def add_arrays(book: Book) -> None:
284
+ """The values of the book's array constants as cells of their hidden sheets (see parser.ARRAY_SHEET)."""
285
+ for cell in list(book.cells.values()):
286
+ if cell.formula is None or "{" not in cell.formula:
287
+ continue
288
+ try:
289
+ names = set(arrays(parse(cell.formula)))
290
+ except (Unsupported, RecursionError):
291
+ continue
292
+ for name in names - book.arrays:
293
+ book.arrays.add(name)
294
+ for r, row in enumerate(ARRAYS[name], 1):
295
+ for c, (kind, value) in enumerate(row, 1):
296
+ book.cells[(name, r, c)] = Cell(name, r, c, None, value, kind)
297
+ book.max_row[name] = len(ARRAYS[name])
298
+ book.__dict__.pop("_rows_by_col", None) # cached from the cells
299
+
300
+
280
301
  def _finish(comp: Compiler, batches: list, batch) -> None:
281
302
  """A batch is finished: later cells read its results from the table. `batch` is a list of Steps or a region batch."""
282
303
  batches.append(batch)
@@ -299,6 +320,7 @@ def compile_book(book: Book, regions: bool = True) -> Compiled:
299
320
  from cellproof.region import compile_region, region_note
300
321
  from cellproof.regions import RegionScan, column_order, find_regions, order_with_regions
301
322
 
323
+ add_arrays(book)
302
324
  scan = find_regions(book) if regions else RegionScan([], [])
303
325
  order, unresolved, kept = order_with_regions(book, scan.regions)
304
326
  comp = Compiler(book)
@@ -16,7 +16,9 @@ LOCALE_REASON = ("reads text as a number, date or time (like \"1/2/2020\" or \"1
16
16
  "computer's language settings, which the file does not record")
17
17
  PLATFORM = "#PLATFORM" # not an Excel error either: Excel for Mac answers differently (CHAR above 127)
18
18
  PLATFORM_REASON = "CHAR above 127 gives a different character in Excel for Mac, which last calculated this file"
19
- SETTINGS = {LOCALE: LOCALE_REASON, PLATFORM: PLATFORM_REASON} # markers the verifier reports as UNRESOLVED
19
+ EARLY1900 = "#1900" # nor this: a date before 1 March 1900, where Excel counts a 29 February that never existed
20
+ EARLY1900_REASON = "YEARFRAC of a date before 1 March 1900, around Excel's 29 February 1900 that never existed"
21
+ SETTINGS = {LOCALE: LOCALE_REASON, PLATFORM: PLATFORM_REASON, EARLY1900: EARLY1900_REASON} # reported as UNRESOLVED
20
22
  Type = Literal["num", "txt", "bool", "blank", "err", "any"]
21
23
 
22
24
 
@@ -1,4 +1,4 @@
1
- """DATE, YEAR, MONTH, DAY, WEEKDAY, DAYS, EDATE, EOMONTH, DAYS360, DATEDIF, and TEXT's date formats.
1
+ """DATE, YEAR, MONTH, DAY, WEEKDAY, DAYS, EDATE, EOMONTH, DAYS360, YEARFRAC, DATEDIF, and TEXT's date formats.
2
2
 
3
3
  Excel stores a date as a serial number: 1 = 1900-01-01, and it counts a 29 February 1900 that never existed
4
4
  (serial 60), so serials from 61 on are days since 1899-12-30. Serial 0 is "1900-01-00". Workbooks using the
@@ -6,7 +6,7 @@ Excel stores a date as a serial number: 1 = 1900-01-01, and it counts a 29 Febru
6
6
  """
7
7
  from __future__ import annotations
8
8
 
9
- from cellproof.expr import Expr, errs, num, num_strict
9
+ from cellproof.expr import EARLY1900, NO_ERR, Expr, errs, num, num_strict
10
10
  from cellproof.fn.lookup import int_literal
11
11
  from cellproof.fn.text import FORMATS
12
12
  from cellproof.functions import arity, register
@@ -101,7 +101,9 @@ def _shift(name: str, last: bool):
101
101
  # real Excel: TRUE/FALSE in either argument is #VALUE!
102
102
  s, m = _serial(c, args[0], sheet, strict=True), num_strict(c.expr(args[1], sheet))
103
103
  start = f"greatest({s.sql}, 1)" if last else s.sql # serial 0 is "1900-01-00": EOMONTH(0,0) is 31
104
- d = f"CAST({to_date(start)} + to_months(CAST(trunc({m.sql}) AS INTEGER)) AS DATE)"
104
+ # clamp before the cast: DuckDB 2.0 folds the cast of a huge literal and fails even where the guard below
105
+ # would have returned #NUM!
106
+ d = f"CAST({to_date(start)} + to_months(CAST(greatest(least(trunc({m.sql}), 120001), -120001) AS INTEGER)) AS DATE)"
105
107
  if last:
106
108
  d = f"last_day({d})"
107
109
  ok = f"({_ok(s)} AND ({m.err}) IS NULL AND trunc({m.sql}) BETWEEN -120000 AND 120000)"
@@ -141,6 +143,56 @@ def days360(c, args, sheet):
141
143
  return Expr(f"(CASE WHEN {_ok(a)} AND {_ok(b)} THEN {sql} END)", "num", errs(a.err, b.err))
142
144
 
143
145
 
146
+ def _leap(y: str) -> str:
147
+ return f"(({y}) % 4 = 0 AND (({y}) % 100 <> 0 OR ({y}) % 400 = 0))"
148
+
149
+
150
+ @register("YEARFRAC")
151
+ def yearfrac(c, args, sheet):
152
+ """The day counts as Excel computes them (D. A. Wheeler's analysis of Excel's YEARFRAC, which matches it on every
153
+ pair of dates he tried); dates in either order."""
154
+ arity("YEARFRAC", args, 2, 3)
155
+ a, b = _serial(c, args[0], sheet, strict=True), _serial(c, args[1], sheet, strict=True)
156
+ lo, hi = f"least({a.sql}, {b.sql})", f"greatest({a.sql}, {b.sql})"
157
+ d1, d2 = to_date(lo), to_date(hi)
158
+ y1, m1, dd1, y2, m2, dd2 = (f"{f}({d})" for d in (d1, d2) for f in ("year", "month", "day"))
159
+ days = f"({hi} - {lo})"
160
+
161
+ def frac(basis: int) -> str:
162
+ if basis in (0, 4):
163
+ if basis == 4:
164
+ day1, day2 = f"least({dd1}, 30)", f"least({dd2}, 30)"
165
+ else: # U.S. 30/360: unlike DAYS360, a 31st end stays 31 unless the start is the 30th or 31st
166
+ feb1, feb2 = f"({m1} = 2 AND {d1} = last_day({d1}))", f"({m2} = 2 AND {d2} = last_day({d2}))"
167
+ day1 = f"(CASE WHEN {dd1} = 31 OR {feb1} THEN 30 ELSE {dd1} END)"
168
+ day2 = (f"(CASE WHEN {dd2} = 31 AND {dd1} >= 30 THEN 30 WHEN {dd1} < 30 AND {feb1} AND {feb2} THEN 30 "
169
+ f"ELSE {dd2} END)")
170
+ return f"((({y2}) - ({y1})) * 360 + (({m2}) - ({m1})) * 30 + ({day2} - {day1})) / 360.0"
171
+ if basis == 1: # actual/actual
172
+ within = f"({y1} = {y2} OR ({y1} + 1 = {y2} AND ({m1} > {m2} OR ({m1} = {m2} AND {dd1} >= {dd2}))))"
173
+ mar1, mar2 = f"make_date({y1}, 3, 1)", f"make_date({y2}, 3, 1)"
174
+ feb29 = (f"(({_leap(y1)} AND {d1} < {mar1} AND {d2} >= {mar1}) OR ({_leap(y2)} AND {d2} >= {mar2} AND "
175
+ f"{d1} < {mar2}) OR ({m2} = 2 AND {dd2} = 29))")
176
+ year = f"(CASE WHEN ({y1} = {y2} AND {_leap(y1)}) OR {feb29} THEN 366.0 ELSE 365.0 END)"
177
+ average = f"((make_date({y2} + 1, 1, 1) - make_date({y1}, 1, 1)) / ({y2} - {y1} + 1.0))"
178
+ return f"(CASE WHEN {within} THEN {days} / {year} ELSE {days} / {average} END)"
179
+ return f"({days} / {360.0 if basis == 2 else 365.0})"
180
+
181
+ literal = 0 if len(args) == 2 or args[2] is None else int_literal(args[2])
182
+ if literal is not None:
183
+ if literal not in range(5):
184
+ return Expr("NULL", "num", "'#NUM!'")
185
+ sql, bad = frac(literal), NO_ERR
186
+ else: # a basis computed at run time: truncated, #NUM! outside 0-4. Real Excel checks it before the dates
187
+ basis = num_strict(c.expr(args[2], sheet))
188
+ n = f"trunc({basis.sql})"
189
+ sql = "(CASE " + " ".join(f"WHEN {n} = {i} THEN {frac(i)}" for i in range(5)) + " END)"
190
+ bad = errs(basis.err, f"(CASE WHEN {n} NOT BETWEEN 0 AND 4 THEN '#NUM!' END)")
191
+ ok = f"({_ok(a)} AND {_ok(b)} AND ({bad}) IS NULL AND {lo} >= 61)"
192
+ return Expr(f"(CASE WHEN {ok} THEN CAST({sql} AS DOUBLE) END)", "num",
193
+ errs(bad, a.err, b.err, f"(CASE WHEN {lo} < 61 THEN '{EARLY1900}' END)"))
194
+
195
+
144
196
  @register("DATEDIF")
145
197
  def datedif(c, args, sheet):
146
198
  arity("DATEDIF", args, 3, 3)
@@ -1,4 +1,4 @@
1
- """VLOOKUP, HLOOKUP, MATCH, INDEX, XLOOKUP, CHOOSE.
1
+ """VLOOKUP, HLOOKUP, MATCH, INDEX, XLOOKUP, LOOKUP, CHOOSE.
2
2
 
3
3
  The value found is read from the cells table. Its type comes from the cells the range holds: one type ->
4
4
  that type; mixed, or text with empty cells -> 'any' (known when the SQL runs). An empty cell found reads
@@ -256,6 +256,34 @@ def xlookup(c, args, sheet):
256
256
  case(found.kind, missing.kind))
257
257
 
258
258
 
259
+ @register("LOOKUP")
260
+ def lookup(c, args, sheet):
261
+ """Always approximate: the largest value <= x. Vector form searches args[1] and returns the cell at the same
262
+ position of args[2] (either direction); array form searches the first row of a block wider than tall, else its
263
+ first column, and returns the last row or column."""
264
+ arity("LOOKUP", args, 2, 3)
265
+ if len(args) == 3 and args[2] is not None:
266
+ look, back = _line(c, args[1], sheet, "LOOKUP"), _line(c, args[2], sheet, "LOOKUP")
267
+ if len(list(look.cells())) != len(list(back.cells())):
268
+ raise Unsupported("LOOKUP result range must be the same size as the lookup range")
269
+ else:
270
+ area, _ = _area(c, args[1], sheet, "LOOKUP")
271
+ if area.c2 - area.c1 > area.r2 - area.r1:
272
+ look = Area(area.sheet, area.r1, area.c1, area.r1, area.c2)
273
+ back = Area(area.sheet, area.r2, area.c1, area.r2, area.c2)
274
+ else:
275
+ look = Area(area.sheet, area.r1, area.c1, area.r2, area.c1)
276
+ back = Area(area.sheet, area.r1, area.c2, area.r2, area.c2)
277
+ x = _xlookup_x(c, args, sheet)
278
+ _require_sorted(c, look, False, "LOOKUP")
279
+ step = f"k.row_num - {look.r1}" if look.c1 == look.c2 else f"k.col_num - {look.c1}"
280
+ hit = (f"r.col_num = {back.c1} AND r.row_num = {back.r1} + {step}" if back.c1 == back.c2
281
+ else f"r.row_num = {back.r1} AND r.col_num = {back.c1} + {step}")
282
+ search = Search(x, look, "le", wild=False, back=(lit_str(back.sheet).sql, hit))
283
+ keys = c.ready(back)
284
+ return _read(search.pick, _type_of(c, back, keys), errs(x.err, _found_error(search.pick, "#N/A")))
285
+
286
+
259
287
  @register("INDEX")
260
288
  def index(c, args, sheet):
261
289
  arity("INDEX", args, 2, 3)
@@ -84,7 +84,7 @@ def textjoin(c, args, sheet):
84
84
  lists, err = _pieces(c, args[2:], sheet, keep_blanks=not args[1].value)
85
85
  items = f"list_concat({', '.join(lists)})" if len(lists) > 1 else lists[0]
86
86
  if args[1].value:
87
- items = f"list_filter({items}, x -> x <> '')"
87
+ items = f"list_filter({items}, lambda x: x <> '')"
88
88
  return Expr(f"array_to_string({items}, {delim.sql})", "txt", errs(delim.err, *err))
89
89
 
90
90
 
@@ -131,7 +131,7 @@ _text1("LOWER", lambda t: f"lower({t})")
131
131
  # each character paired with the one before it; t stays outside the lambda (it can be a subquery, which DuckDB does
132
132
  # not allow inside one). \p{L}: any letter, accented ones too (RE2's [[:alpha:]] is ASCII only)
133
133
  _text1("PROPER", lambda t: (f"array_to_string(list_transform(list_zip(string_split({t}, ''), "
134
- f"list_concat([' '], string_split({t}, ''))), x -> CASE WHEN "
134
+ f"list_concat([' '], string_split({t}, ''))), lambda x: CASE WHEN "
135
135
  f"regexp_matches(x[2], '\\p{{L}}') THEN lower(x[1]) ELSE upper(x[1]) END), '')"))
136
136
  _text1("CLEAN", lambda t: f"regexp_replace({t}, '[\\x00-\\x1f]', '', 'g')")
137
137
 
@@ -46,6 +46,7 @@ class Book:
46
46
  max_row: dict[str, int] = field(default_factory=dict)
47
47
  date1904: bool = False # the workbook counts dates from 1904 (old Mac Excel)
48
48
  app: str = "" # the program that last saved the file (docProps/app.xml), "" when the file does not say
49
+ arrays: set[str] = field(default_factory=set) # hidden sheets of array constants (cellproof.parser.ARRAY_SHEET)
49
50
 
50
51
  @cached_property
51
52
  def _rows_by_col(self) -> dict[tuple[str, int], list[int]]:
@@ -1,10 +1,12 @@
1
1
  """Excel formula text -> small AST. Pratt parser over openpyxl's tokenizer."""
2
2
  from __future__ import annotations
3
3
 
4
+ import hashlib
4
5
  from dataclasses import dataclass
5
6
  from typing import Iterator, Union
6
7
 
7
8
  from openpyxl.formula.tokenizer import Token, Tokenizer, TokenizerError
9
+ from openpyxl.utils import get_column_letter
8
10
 
9
11
  from cellproof.model import Unsupported
10
12
 
@@ -61,6 +63,11 @@ class Call:
61
63
 
62
64
  Node = Union[Num, Str, Bool, ErrLit, RefNode, Unary, Postfix, Binary, Call]
63
65
 
66
+ # An array constant ({1,2;3,4}) becomes a reference to a hidden sheet holding its values, so every function that takes a
67
+ # range takes it unchanged. The compiler adds those cells to the book (see add_arrays); used as one value it is refused.
68
+ ARRAY_SHEET = "#array-"
69
+ ARRAYS: dict[str, tuple[tuple[tuple[str, object], ...], ...]] = {} # hidden sheet -> rows of (kind, value)
70
+
64
71
  # Excel precedence, low -> high. Unary minus binds tighter than ^, so -2^2 = 4.
65
72
  BINARY = {"=": 1, "<>": 1, "<": 1, ">": 1, "<=": 1, ">=": 1, "&": 2, "+": 3, "-": 3, "*": 4, "/": 4, "^": 5}
66
73
  PREFIX = 6
@@ -85,19 +92,36 @@ def _fixed_reference(args) -> bool:
85
92
  and (len(args) == 1 or (isinstance(args[1], Bool) and args[1].value)))
86
93
 
87
94
 
88
- def refs(node) -> Iterator[str]:
89
- """Every reference text in the tree, left to right."""
95
+ def _refs(node) -> Iterator[str]:
90
96
  if isinstance(node, RefNode):
91
97
  yield node.text
92
98
  elif isinstance(node, (Unary, Postfix)):
93
- yield from refs(node.operand)
99
+ yield from _refs(node.operand)
94
100
  elif isinstance(node, Binary):
95
- yield from refs(node.left)
96
- yield from refs(node.right)
101
+ yield from _refs(node.left)
102
+ yield from _refs(node.right)
97
103
  elif isinstance(node, Call):
98
104
  for arg in node.args:
99
105
  if arg is not None:
100
- yield from refs(arg)
106
+ yield from _refs(arg)
107
+
108
+
109
+ def _array(text: str) -> str | None:
110
+ """The hidden sheet of an array constant's reference, else None (a real sheet may share the prefix)."""
111
+ if not text.startswith("'" + ARRAY_SHEET):
112
+ return None
113
+ name = text[1:text.find("'!")]
114
+ return name if name in ARRAYS else None
115
+
116
+
117
+ def refs(node) -> Iterator[str]:
118
+ """Every reference text in the tree, left to right; array constants are constants, not dependencies."""
119
+ return (t for t in _refs(node) if _array(t) is None)
120
+
121
+
122
+ def arrays(node) -> Iterator[str]:
123
+ """The hidden sheet of every array constant in the tree."""
124
+ return (name for t in _refs(node) if (name := _array(t)) is not None)
101
125
 
102
126
 
103
127
  class _Parser:
@@ -159,6 +183,8 @@ class _Parser:
159
183
  return inner
160
184
  if tok.type == Token.FUNC and tok.subtype == Token.OPEN:
161
185
  return self.call(tok.value[:-1].upper().removeprefix("_XLFN."))
186
+ if tok.type == Token.ARRAY and tok.subtype == Token.OPEN:
187
+ return self.array()
162
188
  raise Unsupported(f"unexpected {tok.value!r} in formula")
163
189
 
164
190
  def call(self, name):
@@ -178,3 +204,40 @@ class _Parser:
178
204
  return Call(name, tuple(args))
179
205
  if not (tok.type == Token.SEP and tok.subtype == Token.ARG):
180
206
  raise Unsupported(f"unexpected {tok.value!r} in {name}()")
207
+
208
+ def array(self):
209
+ rows, row = [], []
210
+ while True:
211
+ tok = self.take()
212
+ sign = 1
213
+ if tok.type == Token.OP_PRE and tok.value in "+-":
214
+ sign, tok = (-1 if tok.value == "-" else 1), self.take()
215
+ if tok.type != Token.OPERAND or tok.subtype == Token.RANGE or (sign != 1 and tok.subtype != Token.NUMBER):
216
+ raise Unsupported(f"unexpected {tok.value!r} in array constant")
217
+ if tok.subtype == Token.NUMBER:
218
+ row.append(("n", sign * float(tok.value)))
219
+ elif tok.subtype == Token.TEXT:
220
+ if tok.value == '""': # the cells table cannot tell empty text from no text
221
+ raise Unsupported("array constant holding empty text \"\"")
222
+ row.append(("s", tok.value[1:-1].replace('""', '"')))
223
+ elif tok.subtype == Token.LOGICAL:
224
+ row.append(("b", tok.value.upper() == "TRUE"))
225
+ else:
226
+ row.append(("e", tok.value.upper()))
227
+ sep = self.take()
228
+ if sep.type == Token.SEP and sep.subtype == Token.ARG:
229
+ continue
230
+ rows.append(tuple(row))
231
+ row = []
232
+ if sep.type == Token.SEP and sep.subtype == Token.ROW:
233
+ continue
234
+ if sep.type == Token.ARRAY and sep.subtype == Token.CLOSE:
235
+ break
236
+ raise Unsupported(f"unexpected {sep.value!r} in array constant")
237
+ if len({len(r) for r in rows}) != 1:
238
+ raise Unsupported("array constant with rows of different lengths")
239
+ values = tuple(rows)
240
+ name = ARRAY_SHEET + hashlib.sha1(repr(values).encode()).hexdigest()[:16]
241
+ ARRAYS[name] = values
242
+ end = get_column_letter(len(values[0])) + str(len(values))
243
+ return RefNode(f"'{name}'!A1" + ("" if end == "A1" else f":{end}"))
@@ -47,7 +47,7 @@ def resolve(text: str, sheet: str, book: Book) -> Area:
47
47
  raise Unsupported(f"external link {text}")
48
48
  elif ref.upper() in book.names:
49
49
  sheet, ref = book.names[ref.upper()]
50
- if sheet not in book.sheets:
50
+ if sheet not in book.sheets and sheet not in book.arrays:
51
51
  raise Unsupported(f"unknown sheet {sheet!r}")
52
52
  try:
53
53
  if ":" not in ref and not any(ch.isdigit() for ch in ref):
@@ -0,0 +1,45 @@
1
+ """Array constants ({1,2;3,4}) behave like a range of those values wherever a range is taken."""
2
+ from xlsxkit import book_of
3
+
4
+ from cellproof.compiler import compile_book
5
+ from cellproof.parser import parse, refs
6
+ from cellproof.verifier import execute
7
+
8
+
9
+ def run(formulas, data=None):
10
+ book = book_of({"S": formulas} | ({"D": data} if data else {}))
11
+ compiled = compile_book(book)
12
+ return execute(book, compiled), compiled
13
+
14
+
15
+ def N(v):
16
+ return ("n", float(v), None)
17
+
18
+
19
+ def test_array_constants_where_a_range_is_taken():
20
+ got, c = run({
21
+ "A1": ("=SUM({1,2;3,-4})", 2),
22
+ "A2": ('=SUM({1,"2",TRUE})', 1), # like a range: text and TRUE skipped
23
+ "A3": ('=MATCH("b",{"a","b","c"},0)', 2),
24
+ "A4": ('=INDEX({10,20;30,40},2,1)', 30),
25
+ "A5": ('=LOOKUP(75,{0,50,100},{"F","C","A"})', "C"),
26
+ "A6": ('=VLOOKUP(2,{1,"x";2,"y"},2,FALSE)', "y"),
27
+ "A7": ("=COUNTA({1,#N/A,\"x\"})", 3),
28
+ "A8": ("=MAX({3,9,1})+1", 10),
29
+ "A9": ("=SUMPRODUCT({1,2,3},{4,5,6})", 32),
30
+ "A10": ("={7}*2", 14), # one value: just that value
31
+ })
32
+ assert c.unresolved == {}
33
+ assert [got[("S", r, 1)] for r in range(1, 11)] == [
34
+ N(2), N(1), N(2), N(30), ("s", None, "C"), ("s", None, "y"), N(3), N(10), N(32), N(14)]
35
+
36
+
37
+ def test_array_constants_where_one_value_is_expected_are_not_guessed():
38
+ _, c = run({"A1": ("=A2*{1,2}", 0), "A2": 3, "A3": ('=IF({1,0},"a","b")', "a"), "A4": ("={1,2;3}", 0)})
39
+ assert "array constant" in c.unresolved[("S", 1, 1)]
40
+ assert "array constant" in c.unresolved[("S", 3, 1)]
41
+ assert "rows of different lengths" in c.unresolved[("S", 4, 1)]
42
+
43
+
44
+ def test_array_constants_are_not_dependencies():
45
+ assert list(refs(parse('SUM(A1,{1,2})'))) == ["A1"]
@@ -41,3 +41,29 @@ def test_1904_workbooks_and_odd_forms_say_why():
41
41
  assert compile_book(other).unresolved == {
42
42
  ("S", 1, 1): 'DATEDIF unit must be "Y", "M" or "D" written in the formula',
43
43
  ("S", 2, 1): "WEEKDAY return type must be 1, 2 or 3"}
44
+
45
+
46
+ YEARFRAC = [ # Microsoft's documented examples, then the basis rules
47
+ ("=YEARFRAC(DATE(2012,1,1), DATE(2012,7,30))", 0.58055556), ("=YEARFRAC(DATE(2012,1,1), DATE(2012,7,30), 1)", 0.57650273),
48
+ ("=YEARFRAC(DATE(2012,1,1), DATE(2012,7,30), 3)", 0.57808219), ("=YEARFRAC(DATE(2012,7,30), DATE(2012,1,1), 2)", 211 / 360),
49
+ ("=YEARFRAC(DATE(2011,1,31), DATE(2011,3,31), 4)", 60 / 360),
50
+ ("=YEARFRAC(DATE(2011,6,1), DATE(2012,3,1), 1)", 274 / 366), ("=YEARFRAC(DATE(2011,1,1), DATE(2013,1,1), 1)", 731 / (1096 / 3)),
51
+ ("=YEARFRAC(DATE(2026,1,1), DATE(2026,7,1), 5)", "#NUM!"), ("=YEARFRAC(-1, 5)", "#NUM!"), ("=YEARFRAC(DATE(2012,1,1), DATE(2012,7,30), B2)", 0.57650273),
52
+ ("=YEARFRAC(DATE(2012,1,1), DATE(2012,7,30), B3)", "#NUM!"), ("=YEARFRAC(1/0, 100, B3)", "#NUM!"),
53
+ ("=YEARFRAC(-1, 100, TRUE)", "#VALUE!"), ("=YEARFRAC(1/0, 100, 7)", "#NUM!"),
54
+ ]
55
+
56
+
57
+ def test_yearfrac():
58
+ book = book_of({"S": {f"A{i}": (f, v) for i, (f, v) in enumerate(YEARFRAC, 1)} | {"B1": ("=YEARFRAC(10, 100)", 0.25), "B2": 1.9, "B3": 5}})
59
+ compiled = compile_book(book)
60
+ got = execute(book, compiled)
61
+ assert ("S", 1, 2) not in compiled.unresolved
62
+ for i, (formula, want) in enumerate(YEARFRAC, 1):
63
+ kind, n, t = got[("S", i, 1)]
64
+ if isinstance(want, str):
65
+ assert (kind, t) == ("e", want), formula
66
+ else:
67
+ assert kind == "n" and abs(n - want) < 5e-9, (formula, n, want)
68
+ # serials before 1 March 1900 cross Excel's 29 February 1900 that never existed: not guessed
69
+ assert got[("S", 1, 2)][0] == "e" and got[("S", 1, 2)][2] == "#1900"
@@ -227,3 +227,14 @@ def test_dbt_build_with_colliding_region_names(tmp_path):
227
227
  wb = tmp_path / "c.xlsx"
228
228
  write_xlsx(wb, {"My Sheet": _sheet(), "My_Sheet": _sheet()})
229
229
  assert main(["convert", str(wb), "--out", str(tmp_path / "out"), "--dbt"]) == 0
230
+
231
+
232
+ def test_array_constants_reach_the_dbt_models(tmp_path):
233
+ spec = {"A1": ('=MATCH("b",{"a","b"},0)', 2.0)}
234
+ spec.update({f"B{r}": (f"=LOOKUP({r},{{0,3,6}},{{10,20,30}})", 0.0) for r in range(1, 9)})
235
+ book = book_of({"S": spec})
236
+ compiled = compile_book(book)
237
+ assert compiled.unresolved == {}
238
+ write_project(book, compiled, tmp_path, "arrays")
239
+ got = {(s, r, c): (k, n, t) for s, r, c, k, n, t in _run_models(tmp_path)}
240
+ assert got == execute(book, compiled)
@@ -45,3 +45,32 @@ def test_vlookup_exact_with_header_row_and_missing_key():
45
45
  # Region, West, South is not in order, so Excel's approximate match depends on its binary search: not guessed
46
46
  assert "ascending sorted" in c.unresolved[("X", 3, 2)]
47
47
  assert c.unresolved[("X", 4, 2)] == "VLOOKUP column index outside the table"
48
+
49
+
50
+ def test_lookup_vector_form_largest_value_not_above():
51
+ grid = {"A1": 1, "A2": 3, "A3": 3, "A4": 7, "B1": "one", "B2": "three", "B3": "THREE", "B4": "seven",
52
+ "D1": 10, "E1": 30, "F1": 30, "G1": 70}
53
+ got, c = run({"S": grid, "X": {
54
+ "A1": ("=LOOKUP(5, S!A1:A4, S!B1:B4)", "THREE"), # last of equal values, as Excel's binary search
55
+ "A2": ("=LOOKUP(0, S!A1:A4, S!B1:B4)", "#N/A"), # smaller than every value
56
+ "A3": ("=LOOKUP(99, S!A1:A4)", 7), # no result vector: the value found
57
+ "A4": ("=LOOKUP(4, S!A1:A4, S!D1:G1)", 30), # result vector in the other direction
58
+ }})
59
+ assert c.unresolved == {}
60
+ assert got[("X", 1, 1)] == ("s", None, "THREE")
61
+ assert got[("X", 2, 1)] == ("e", None, "#N/A")
62
+ assert got[("X", 3, 1)] == ("n", 7.0, None)
63
+ assert got[("X", 4, 1)] == ("n", 30.0, None)
64
+
65
+
66
+ def test_lookup_array_form_and_refusals():
67
+ wide = {"A1": "a", "B1": "b", "C1": "c", "A2": 1, "B2": 2, "C2": 3,
68
+ "E1": 1, "E2": 5, "E3": 2, "F1": 10}
69
+ got, c = run({"S": wide, "X": {
70
+ "A1": ('=LOOKUP("B", S!A1:C2)', 2), # wider than tall: first row searched, last row returned
71
+ "A2": ("=LOOKUP(3, S!E1:E3, S!F1:F3)", 0), # unsorted: Excel's answer depends on its search
72
+ "A3": ("=LOOKUP(1, S!E1:E2, S!F1:F3)", 10), # result vector of another size
73
+ }})
74
+ assert got[("X", 1, 1)] == ("n", 2.0, None)
75
+ assert "ascending sorted" in c.unresolved[("X", 2, 1)]
76
+ assert "same size" in c.unresolved[("X", 3, 1)]
@@ -25,7 +25,7 @@ def test_functions_and_empty_args():
25
25
  assert parse("NOW()") == Call("NOW", ())
26
26
 
27
27
 
28
- @pytest.mark.parametrize("bad", ["SUM(1,", "{1,2}", "A1:B2 B1:C3", "(1+2"])
28
+ @pytest.mark.parametrize("bad", ["SUM(1,", "{1,A1}", "A1:B2 B1:C3", "(1+2"])
29
29
  def test_unparseable_raises_unsupported(bad):
30
30
  with pytest.raises(Unsupported):
31
31
  parse(bad)
@@ -451,6 +451,11 @@ TEXT_LOOKUPS = {
451
451
  "I": "=VLOOKUP(N{r},$K$1:$L$4,2)", # approximate on sorted numbers
452
452
  "J": "=MATCH(N{r},$K$1:$K$4,1)",
453
453
  "M": "=XLOOKUP(N{r},$K$1:$K$4,$L$1:$L$4,-1,1)", # next larger
454
+ "O": "=LOOKUP(N{r},$K$1:$K$4,$L$1:$L$4)",
455
+ "P": "=LOOKUP(N{r},$K$1:$L$4)", # array form: last column
456
+ "Q": '=MATCH(A{r},{{"apple","pear","kiwi"}},0)', # array constants ({{ }} for str.format)
457
+ "R": '=LOOKUP(N{r},{{0,10,20}},{{"low","mid","high"}})',
458
+ "S": "=N{r}*SUM({{1,2;3,4}})",
454
459
  }
455
460
 
456
461
 
@@ -1 +0,0 @@
1
- __version__ = "0.2.1"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes