cellproof 0.2.0__py3-none-any.whl

cellproof/functions.py

@@ -0,0 +1,114 @@
+"""Excel function -> SQL. Each entry: fn(compiler, args, sheet) -> Expr.
+
+Ranges are read from the cells table (inputs plus the formula cells of earlier batches).
+Compiler.ready() checks every formula cell in a range was computed in an earlier batch.
+Every function returns its Excel error in Expr.err; see expr.errs for the order rule.
+"""
+from __future__ import annotations
+
+from cellproof.expr import CELLS, NO_ERR, ZERO, Expr, any_of, as_bool, errs, lit_str, num, pick, unify, where_area
+from cellproof.model import Unsupported, a1
+from cellproof.parser import Bool, Num, RefNode, Str
+from cellproof.refs import Area, resolve
+
+FUNCS = {}
+
+
+def register(name):
+    def deco(fn):
+        FUNCS[name] = fn
+        return fn
+
+    return deco
+
+
+def arity(name, args, lo, hi):
+    if not lo <= len(args) <= hi:
+        raise Unsupported(f"{name} needs {lo}-{hi} arguments, got {len(args)}")
+
+
+def _looks_numeric(text: str) -> bool:
+    try:
+        float(text.replace(",", "").strip())
+        return True
+    except ValueError:
+        return False
+
+
+def note_text_numbers(c, keys):
+    """Excel skips text in SUM-like ranges, even text that looks like a number. Tell the user."""
+    for key in keys.text_cells():
+        cell = c.book.cells[key]
+        if _looks_numeric(cell.value):
+            c.findings.append(f"{a1(key)}: number stored as text ({cell.value!r}) is skipped by Excel")
+
+
+def first_error(c, area: Area, keys) -> str:
+    """SQL for the first error in a range, row by row, or NO_ERR when no cell in it can hold one."""
+    if not keys.may_err:
+        return NO_ERR
+    return f"(SELECT txt FROM {CELLS} WHERE {where_area(area)} AND kind = 'e' ORDER BY row_num, col_num LIMIT 1)"
+
+
+def range_sum(c, area: Area) -> Expr:
+    """SUM over a range: text, TRUE/FALSE and blanks are skipped; the first error wins."""
+    keys = c.ready(area)
+    note_text_numbers(c, keys)
+    return Expr(f"(SELECT COALESCE(SUM(num), CAST(0 AS DOUBLE)) FROM {CELLS} WHERE {where_area(area)} AND kind = 'n')",
+                "num", first_error(c, area, keys))
+
+
+@register("SUM")
+def sum_(c, args, sheet):
+    parts = [range_sum(c, resolve(arg.text, sheet, c.book)) if isinstance(arg, RefNode) else num(c.expr(arg, sheet))
+             for arg in args]
+    if not parts:
+        return ZERO
+    # Real Excel: SUM(H3,1/0) with H3 = #N/A is #DIV/0!. Typed-in expressions are evaluated first.
+    typed = [p for a, p in zip(args, parts) if not isinstance(a, RefNode)]
+    cells = [p for a, p in zip(args, parts) if isinstance(a, RefNode)]
+    return Expr("(" + " + ".join(p.sql for p in parts) + ")", "num", errs(*(p.err for p in typed + cells)))
+
+
+@register("IF")
+def if_(c, args, sheet):
+    arity("IF", args, 2, 3)
+    cond = as_bool(c.expr(args[0], sheet))
+    yes = c.expr(args[1], sheet)
+    no = c.expr(args[2], sheet) if len(args) == 3 else Expr("FALSE", "bool")
+    yes, no = unify(yes, no)
+    taken = NO_ERR if yes.err == no.err == NO_ERR else f"(CASE WHEN {cond.sql} THEN {yes.err} ELSE {no.err} END)"
+    return pick(cond.sql, yes, no, errs(cond.err, taken))
+
+
+@register("ROUND")
+def round_(c, args, sheet):
+    arity("ROUND", args, 2, 2)
+    x, digits = num(c.expr(args[0], sheet)), num(c.expr(args[1], sheet))
+    # DuckDB ROUND on DOUBLE rounds half away from zero like Excel (2.675 -> 2.68, verified on 1.5.6)
+    return Expr(f"ROUND({x.sql}, CAST(TRUNC({digits.sql}) AS INTEGER))", "num", errs(x.err, digits.err))
+
+
+def _logic(name, joiner):
+    def fn(c, args, sheet):
+        if not args:
+            raise Unsupported(f"{name} needs arguments")
+        parts = [as_bool(c.expr(a, sheet)) for a in args]
+        return Expr("(" + f" {joiner} ".join(p.sql for p in parts) + ")", "bool", errs(*(p.err for p in parts)))
+
+    return fn
+
+
+FUNCS["AND"] = _logic("AND", "AND")
+FUNCS["OR"] = _logic("OR", "OR")
+
+
+@register("IFERROR")
+def iferror(c, args, sheet):
+    arity("IFERROR", args, 2, 2)
+    value, fallback = unify(c.expr(args[0], sheet), c.expr(args[1], sheet))
+    if value.err == NO_ERR:
+        return value
+    ok = f"(({value.err}) IS NULL)"
+    err = NO_ERR if fallback.err == NO_ERR else f"(CASE WHEN {ok} THEN NULL ELSE {fallback.err} END)"
+    return pick(ok, value, fallback, err)

cellproof/graph.py

@@ -0,0 +1,150 @@
+"""Order formula cells so every cell comes after the cells it reads.
+
+Ranges are not expanded into cell sets (that is quadratic for a running SUM($B$1:Bn)); they are O(n log n) when the
+cells they read are ordered in ascending order (see the ponytail comment in the main loop for the bottom-up case). Per
+column, a range is an interval of that column's formula cells, and a Fenwick tree of "not yet ordered" counts says
+whether the interval is empty and which cell to wait for next. Cells are released level by level: a node's level is one
+more than the last node it waited for, so every cell that needs only inputs comes first, then the cells that need those.
+
+A group of cells (a region) is one node: it waits for everything its members read from outside the group, and when it is
+released all its members are ordered at once, consecutively, row-major.
+"""
+from __future__ import annotations
+
+import heapq
+from bisect import bisect_left, bisect_right
+from collections import defaultdict
+
+from cellproof.fenwick import Fenwick
+from cellproof.model import Book, Key, Unsupported
+from cellproof.r1c1 import lexed
+from cellproof.refs import resolve
+
+CYCLE = "circular reference (or depends on one)"
+
+Column = tuple[str, int]  # (sheet, col)
+
+
+def formula_order(book: Book, groups=()) -> tuple[list[Key], dict[Key, str]]:
+    formulas = {k for k, c in book.cells.items() if c.formula is not None}
+    grouped = {k for g in groups for k in g}
+    members = [sorted(g) for g in groups] + [[k] for k in sorted(formulas - grouped)]
+    node_of = {k: i for i, m in enumerate(members) for k in m}
+    frows: dict[Column, list[int]] = defaultdict(list)  # rows holding a formula, per column, sorted
+    for sheet, row, col in sorted(formulas):
+        frows[(sheet, col)].append(row)
+    fcols: dict[str, list[int]] = defaultdict(list)  # columns holding a formula, per sheet, sorted
+    for sheet, col in sorted(frows):
+        fcols[sheet].append(col)
+    unordered = {column: Fenwick([1] * len(rows)) for column, rows in frows.items()}
+    span: dict[tuple[int, Column], tuple[int, int]] = {}  # (group node, column) -> its members' index interval
+    for n, m in enumerate(members[:len(groups)]):
+        per: dict[Column, list[int]] = defaultdict(list)
+        for sheet, row, col in m:
+            per[(sheet, col)].append(bisect_left(frows[(sheet, col)], row))
+        for column, idx in per.items():
+            if max(idx) - min(idx) + 1 == len(idx):  # contiguous: a range over it can be clipped
+                span[(n, column)] = (min(idx), max(idx) + 1)
+
+    singles: dict[Key, set[Key]] = {}  # cell -> formula cells it reads one by one
+    slices: dict[Key, list[tuple[Column, int, int]]] = {}  # cell -> (column, first, past-last) index into frows
+    lex = lexed(book)
+    for key in formulas:
+        cell = book.cells[key]
+        one: set[Key] = set()
+        many: list[tuple[Column, int, int]] = []
+        shape, texts = lex[key]
+        try:
+            for text in (texts if shape is not None and shape.parses() else ()):  # = refs(parse(cell.formula))
+                area = resolve(text, cell.sheet, book)
+                if area.is_single:
+                    # a read of another member of the same group is not a wait; a cell reading itself still is
+                    if area.key in formulas and (node_of[area.key] != node_of[key] or area.key == key):
+                        one.add(area.key)
+                    continue
+                cols = fcols.get(area.sheet, [])
+                for col in cols[bisect_left(cols, area.c1):bisect_right(cols, area.c2)]:
+                    rows = frows[(area.sheet, col)]
+                    i, j = bisect_left(rows, area.r1), bisect_right(rows, area.r2)
+                    own = span.get((node_of[key], (area.sheet, col)))
+                    # a range over the group's own members is not a wait (regions only admit such ranges when they
+                    # read earlier rows of an earlier column: a growing SUM), only the cells outside the group are
+                    parts = [(i, min(j, own[0])), (max(i, own[1]), j)] if own else [(i, j)]
+                    for a, b in parts:
+                        if b > a:
+                            many.append(((area.sheet, col), a, b))
+        except (Unsupported, RecursionError):
+            pass  # the compiler reports the real reason for this cell
+        singles[key], slices[key] = one, many
+
+    users = defaultdict(list)  # cell -> nodes waiting for it, once per edge
+    for key, reads in singles.items():
+        for dep in reads:
+            users[dep].append(node_of[key])
+    blocked_on: dict[tuple[Column, int], list] = defaultdict(list)  # (column, index) -> slices waiting for that cell
+    waiting = [0] * len(members)
+    for key in formulas:
+        node = node_of[key]
+        waiting[node] += len(singles[key]) + len(slices[key])
+        for column, i, j in slices[key]:
+            blocked_on[(column, j - 1)].append((node, column, i, j))
+    level = [0] * len(members)
+    ready = [(0, members[n][0], n) for n in range(len(members)) if waiting[n] == 0]
+    heapq.heapify(ready)
+    order = []
+
+    def release(user: int, after: int) -> None:
+        level[user] = max(level[user], level[after] + 1)
+        waiting[user] -= 1
+        if waiting[user] == 0:
+            heapq.heappush(ready, (level[user], members[user][0], user))
+
+    while ready:
+        _, _, node = heapq.heappop(ready)
+        for key in members[node]:
+            order.append(key)
+            column = (key[0], key[2])
+            pos = bisect_left(frows[column], key[1])
+            unordered[column].add(pos, -1)
+            for user in users[key]:
+                release(user, node)
+            for user, ucol, i, j in blocked_on.pop((column, pos), ()):
+                fw = unordered[ucol]
+                if fw.range(i, j) == 0:
+                    release(user, node)
+                else:  # wait for the last cell of the range that is still unordered
+                    # ponytail: bottom-up chain (B_r = B_{r+1}+1 with C_r = SUM($B$1:B_r)) moves every waiting slice to
+                    # the previous cell one step at a time, so ordering is O(n^2 log n) there (4000 rows: 28.8 s vs 7.9 s
+                    # for the old graph). Upgrade: register each slice on canonical Fenwick/segment nodes with counters,
+                    # or move whole waiter lists in one step.
+                    blocked_on[(ucol, fw.kth(fw.prefix(j)))].append((user, ucol, i, j))
+    return order, {k: CYCLE for k in sorted(formulas - set(order))}
+
+
+def order_with_groups(book: Book, groups: list[list[Key]], internally_acyclic: bool = False):
+    """Order with groups as nodes. A group that creates a cycle the cells do not have is dissolved and ordering
+    repeated; the set of groups only shrinks, so this ends. Returns (order, unresolved, kept groups).
+
+    A group can hide a true cycle only through reads among its own members. Callers whose groups cannot have such a
+    cycle pass internally_acyclic=True, and the extra ungrouped pass then runs only when the grouped order leaves cells
+    unresolved. Regions qualify because find_regions enforces two things: their same-row reads are acyclic
+    (column_order), and no absolute cell, range or name may read the region's own formula cells. The only reads of
+    other rows it admits are a running total's own previous row and a growing SUM($B$k:B{r}) of an earlier column,
+    both of earlier rows only (formula_order clips such ranges to the cells outside the group). Relaxing any of this
+    breaks the invariant."""
+    groups = [list(g) for g in groups]
+    base_bad = None
+    while True:
+        order, bad = formula_order(book, groups)
+        if not groups or (internally_acyclic and not bad):
+            return order, bad, groups
+        if base_bad is None:
+            base_bad = set(formula_order(book)[1])
+        # cycles that grouping created, and true cycles that grouping hid (a node hides a cycle inside itself)
+        extra = (set(bad) - base_bad) | (base_bad - set(bad))
+        if not extra:
+            return order, bad, groups
+        kept = [g for g in groups if not (set(g) & extra)]
+        if len(kept) == len(groups):
+            return order, bad, groups
+        groups = kept

cellproof/model.py

@@ -0,0 +1,72 @@
+"""Data types shared by every stage."""
+from __future__ import annotations
+
+from bisect import bisect_left, bisect_right
+from collections import defaultdict
+from dataclasses import dataclass, field
+from functools import cached_property
+from typing import Literal, Union
+
+from openpyxl.utils.cell import get_column_letter
+
+Key = tuple[str, int, int]  # (sheet, row, col)
+Kind = Literal["n", "s", "b", "e", "blank"]
+Value = Union[float, str, bool, None]
+
+
+class Unsupported(Exception):
+    """A cell cannot be translated. The message becomes the UNRESOLVED reason shown to the user."""
+
+
+def a1(key: Key) -> str:
+    sheet, row, col = key
+    return f"{sheet}!{get_column_letter(col)}{row}"
+
+
+@dataclass(frozen=True)
+class Cell:
+    sheet: str
+    row: int
+    col: int
+    formula: str | None  # formula text without the leading "=", None for input cells
+    value: Value  # Excel-saved value; numbers as float, dates as Excel serial numbers
+    kind: Kind  # kind of the saved value
+    array: bool = False  # array (Ctrl+Shift+Enter) formula
+
+    @property
+    def key(self) -> Key:
+        return (self.sheet, self.row, self.col)
+
+
+@dataclass
+class Book:
+    cells: dict[Key, Cell]
+    sheets: list[str]
+    names: dict[str, tuple[str, str]] = field(default_factory=dict)  # NAME (upper) -> (sheet, ref)
+    max_row: dict[str, int] = field(default_factory=dict)
+    date1904: bool = False  # the workbook counts dates from 1904 (old Mac Excel)
+
+    @cached_property
+    def _rows_by_col(self) -> dict[tuple[str, int], list[int]]:
+        """(sheet, col) -> sorted rows that hold a cell. Built once; build a new Book if cells change."""
+        cols = defaultdict(list)
+        for sheet, row, col in self.cells:
+            cols[(sheet, col)].append(row)
+        return {k: sorted(v) for k, v in cols.items()}
+
+    def present(self, sheet: str, r1: int, c1: int, r2: int, c2: int) -> list[Key]:
+        """Keys of the non-empty cells inside a rectangle, row-major. Fast on whole-column ranges."""
+        found = []
+        for col in range(c1, c2 + 1):
+            rows = self._rows_by_col.get((sheet, col), [])
+            found.extend((sheet, r, col) for r in rows[bisect_left(rows, r1):bisect_right(rows, r2)])
+        return sorted(found)
+
+
+@dataclass(frozen=True)
+class Verdict:
+    key: Key
+    status: Literal["PASS", "FAIL", "UNRESOLVED"]
+    expected: Value = None
+    actual: Value = None
+    reason: str = ""

cellproof/parser.py

@@ -0,0 +1,172 @@
+"""Excel formula text -> small AST. Pratt parser over openpyxl's tokenizer."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterator, Union
+
+from openpyxl.formula.tokenizer import Token, Tokenizer, TokenizerError
+
+from cellproof.model import Unsupported
+
+
+@dataclass(frozen=True)
+class Num:
+    value: float
+
+
+@dataclass(frozen=True)
+class Str:
+    value: str
+
+
+@dataclass(frozen=True)
+class Bool:
+    value: bool
+
+
+@dataclass(frozen=True)
+class ErrLit:
+    value: str
+
+
+@dataclass(frozen=True)
+class RefNode:
+    text: str
+
+
+@dataclass(frozen=True)
+class Unary:
+    op: str
+    operand: "Node"
+
+
+@dataclass(frozen=True)
+class Postfix:
+    op: str
+    operand: "Node"
+
+
+@dataclass(frozen=True)
+class Binary:
+    op: str
+    left: "Node"
+    right: "Node"
+
+
+@dataclass(frozen=True)
+class Call:
+    name: str
+    args: tuple  # Node or None (empty argument)
+
+
+Node = Union[Num, Str, Bool, ErrLit, RefNode, Unary, Postfix, Binary, Call]
+
+# Excel precedence, low -> high. Unary minus binds tighter than ^, so -2^2 = 4.
+BINARY = {"=": 1, "<>": 1, "<": 1, ">": 1, "<=": 1, ">=": 1, "&": 2, "+": 3, "-": 3, "*": 4, "/": 4, "^": 5}
+PREFIX = 6
+
+
+def parse(formula: str) -> Node:
+    """Parse formula text without the leading '='."""
+    try:
+        tokens = [t for t in Tokenizer("=" + formula).items if t.type != Token.WSPACE]
+    except TokenizerError as err:
+        raise Unsupported(f"cannot parse formula: {err}") from None
+    p = _Parser(tokens)
+    node = p.expr(0)
+    if p.peek() is not None:
+        raise Unsupported(f"unexpected {p.peek().value!r} in formula")
+    return node
+
+
+def refs(node) -> Iterator[str]:
+    """Every reference text in the tree, left to right."""
+    if isinstance(node, RefNode):
+        yield node.text
+    elif isinstance(node, (Unary, Postfix)):
+        yield from refs(node.operand)
+    elif isinstance(node, Binary):
+        yield from refs(node.left)
+        yield from refs(node.right)
+    elif isinstance(node, Call):
+        for arg in node.args:
+            if arg is not None:
+                yield from refs(arg)
+
+
+class _Parser:
+    def __init__(self, tokens):
+        self.tokens, self.i = tokens, 0
+
+    def peek(self):
+        return self.tokens[self.i] if self.i < len(self.tokens) else None
+
+    def take(self):
+        tok = self.peek()
+        if tok is None:
+            raise Unsupported("formula ends unexpectedly")
+        self.i += 1
+        return tok
+
+    def at(self, type_, subtype):
+        tok = self.peek()
+        return tok is not None and tok.type == type_ and tok.subtype == subtype
+
+    def expr(self, min_bp):
+        left = self.atom()
+        while (tok := self.peek()) is not None:
+            if tok.type == Token.OP_POST:
+                self.take()
+                left = Postfix(tok.value, left)
+                continue
+            if tok.type != Token.OP_IN:
+                break
+            bp = BINARY.get(tok.value)
+            if bp is None:
+                raise Unsupported(f"operator {tok.value!r} not supported")
+            if bp <= min_bp:
+                break
+            self.take()
+            left = Binary(tok.value, left, self.expr(bp))
+        return left
+
+    def atom(self):
+        tok = self.take()
+        if tok.type == Token.OP_PRE:
+            operand = self.expr(PREFIX)
+            return operand if tok.value == "+" else Unary("-", operand)
+        if tok.type == Token.OPERAND:
+            if tok.subtype == Token.NUMBER:
+                return Num(float(tok.value))
+            if tok.subtype == Token.TEXT:
+                return Str(tok.value[1:-1].replace('""', '"'))
+            if tok.subtype == Token.LOGICAL:
+                return Bool(tok.value.upper() == "TRUE")
+            if tok.subtype == Token.ERROR:
+                return ErrLit(tok.value)
+            return RefNode(tok.value)
+        if tok.type == Token.PAREN and tok.subtype == Token.OPEN:
+            inner = self.expr(0)
+            if not self.at(Token.PAREN, Token.CLOSE):
+                raise Unsupported("missing ')'")
+            self.take()
+            return inner
+        if tok.type == Token.FUNC and tok.subtype == Token.OPEN:
+            return self.call(tok.value[:-1].upper().removeprefix("_XLFN."))
+        raise Unsupported(f"unexpected {tok.value!r} in formula")
+
+    def call(self, name):
+        if self.at(Token.FUNC, Token.CLOSE):
+            self.take()
+            return Call(name, ())
+        args = []
+        while True:
+            if self.at(Token.SEP, Token.ARG) or self.at(Token.FUNC, Token.CLOSE):
+                args.append(None)
+            else:
+                args.append(self.expr(0))
+            tok = self.take()
+            if tok.type == Token.FUNC and tok.subtype == Token.CLOSE:
+                return Call(name, tuple(args))
+            if not (tok.type == Token.SEP and tok.subtype == Token.ARG):
+                raise Unsupported(f"unexpected {tok.value!r} in {name}()")