cellproof 0.2.0__py3-none-any.whl

cellproof/r1c1.py

@@ -0,0 +1,202 @@
+"""Reference classification and R1C1 keys: which formulas are 'the same formula pasted down a column'."""
+from __future__ import annotations
+
+import re
+
+from openpyxl.formula.tokenizer import Token, Tokenizer, TokenizerError
+from openpyxl.utils.cell import column_index_from_string
+
+from cellproof.model import Unsupported
+from cellproof.parser import Binary, Call, Postfix, RefNode, Unary, parse
+
+CELL = re.compile(r"^(\$?)([A-Za-z]{1,3})(\$?)([0-9]+)$")
+ROW_ONLY = re.compile(r"^\$?[0-9]+$")
+COLUMN_ONLY = re.compile(r"^\$?[A-Za-z]{1,3}$")
+NAME = re.compile(r"^[A-Za-z_\\][A-Za-z0-9_.]*$")
+ROWREF = "__ROWREF_{}__"
+
+
+def _split(text: str) -> tuple[str, str]:
+    if "!" in text:
+        sheet, ref = text.rsplit("!", 1)
+        return sheet + "!", ref
+    return "", text
+
+
+def classify(text: str, row: int) -> tuple[str, int]:
+    """How a reference behaves when its formula is copied down.
+    ('same_row', col): one cell of the same sheet and the same row, row part not anchored.
+    ('abs', 0): reads the same cells on every row (all row parts anchored, whole columns, names).
+    ('bad', 0): anything else that moves with the row (previous/next row, relative ranges, other sheets, whole rows)."""
+    if "[" in text or "]" in text:
+        return "bad", 0  # structured (Table1[@Col]) and external ([Book]Sheet!A1) references: fail closed
+    prefix, ref = _split(text)
+    parts = ref.split(":")
+    relative = []
+    for part in parts:
+        m = CELL.match(part)
+        if m:
+            relative.append((not m.group(3), column_index_from_string(m.group(2).upper()), int(m.group(4))))
+        elif ROW_ONLY.match(part):
+            return "bad", 0  # whole-row reference
+        elif COLUMN_ONLY.match(part) or NAME.match(part):
+            continue  # whole columns (A:A) and defined names read the same cells on every row
+        else:
+            return "bad", 0  # anything else is not a reference we can prove is row-independent
+    if not any(is_relative for is_relative, _, _ in relative):
+        return "abs", 0
+    if len(parts) == 1 and not prefix and relative[0][2] == row:
+        return "same_row", relative[0][1]
+    return "bad", 0
+
+
+RUNREF = "__RUN_{}_{}__"  # a growing SUM over column {} from row {}: a window column of the region
+
+
+def prev_self(text: str, row: int, col: int) -> bool:
+    """The formula's own cell one row up (relative row, same sheet): the previous value of a running total."""
+    m = CELL.match(text)
+    return (bool(m) and not m.group(3) and int(m.group(4)) == row - 1 >= 1
+            and column_index_from_string(m.group(2).upper()) == col)
+
+
+def growing(text: str, row: int, col: int):
+    """(source column, start row) of $B$k:B{row} (start row anchored, end row relative on this row, one column other
+    than the formula's own, same sheet), else None."""
+    parts = text.split(":")
+    if len(parts) != 2:
+        return None
+    a, b = CELL.match(parts[0]), CELL.match(parts[1])
+    if not a or not b or not a.group(3) or b.group(3):
+        return None
+    src = column_index_from_string(a.group(2).upper())
+    start, end = int(a.group(4)), int(b.group(4))
+    if column_index_from_string(b.group(2).upper()) != src or src == col or end != row or start > row:
+        return None
+    return src, start
+
+
+def _norm(part: str, row: int, col: int) -> str:
+    m = CELL.match(part)
+    if not m:
+        return part
+    c = column_index_from_string(m.group(2).upper())
+    cs = f"C{c}" if m.group(1) else f"C[{c - col}]"
+    rs = f"R{m.group(4)}" if m.group(3) else f"R[{int(m.group(4)) - row}]"
+    return rs + cs
+
+
+def r1c1_key(formula: str, row: int, col: int) -> str:
+    """The formula with every reference written relative to its own cell: equal keys = same formula pasted down."""
+    out = []
+    for tok in Tokenizer("=" + formula).items:
+        if tok.type == Token.OPERAND and tok.subtype == Token.RANGE:
+            prefix, ref = _split(tok.value)
+            out.append(prefix + ":".join(_norm(p, row, col) for p in ref.split(":")))
+        else:
+            out.append(tok.value)
+    return "".join(out)
+
+
+MAX_ROW = 1048576
+
+
+class Shape:
+    """A formula's tokens with the row numbers of its relative cell references taken out, so the same formula pasted
+    to another row of the column can be predicted as text instead of tokenized again (the tokenizer is most of the
+    compile time). Equal text means equal tokens: only the digits inside reference operands change, and the tokenizer
+    never splits an operand on a digit."""
+
+    __slots__ = ("key", "formula", "segments", "refs", "_parses")
+
+    def __init__(self, formula: str, row: int, col: int):
+        self.formula, self._parses = formula, None
+        key, self.segments, refs = [], [], []  # segments: fixed text, or a reference as [text or row offset, ...]
+        for tok in Tokenizer("=" + formula).items:
+            if tok.type == Token.OPERAND and tok.subtype == Token.RANGE:
+                prefix, ref = _split(tok.value)
+                key.append(prefix + ":".join(_norm(p, row, col) for p in ref.split(":")))
+                refs.append(tok.value)
+                pieces = [prefix]
+                for i, part in enumerate(ref.split(":")):
+                    m = CELL.match(part)
+                    pieces.append(":" if i else "")
+                    if m and not m.group(3):
+                        pieces += [m.group(1) + m.group(2), int(m.group(4)) - row]
+                    else:
+                        pieces.append(part)
+                self.segments.append(pieces)
+            else:
+                key.append(tok.value)
+                self.segments.append(tok.value)
+        self.key, self.refs = "".join(key), tuple(refs)  # refs: this cell's reference texts, as written
+
+    def at(self, row: int) -> tuple[str, tuple[str, ...]] | None:
+        """(formula text, reference texts) of this formula pasted to `row`; None when a reference leaves the sheet."""
+        text, refs = [], []
+        for seg in self.segments:
+            if isinstance(seg, str):
+                text.append(seg)
+                continue
+            out = []
+            for p in seg:
+                if isinstance(p, int):
+                    r = row + p
+                    if not 1 <= r <= MAX_ROW:
+                        return None
+                    out.append(str(r))
+                else:
+                    out.append(p)
+            ref = "".join(out)
+            refs.append(ref)
+            text.append(ref)
+        return "".join(text), tuple(refs)
+
+    def parses(self) -> bool:
+        """Whether the parser accepts it; the same for every cell of the shape (the parser never reads reference text)."""
+        if self._parses is None:
+            try:
+                parse(self.formula)
+                self._parses = True
+            except (Unsupported, RecursionError):
+                self._parses = False
+        return self._parses
+
+
+def lexed(book) -> dict:
+    """Formula cell -> (Shape or None when it does not tokenize, its reference texts left to right). Built once per
+    book; a run of one formula pasted down a column is tokenized once."""
+    got = book.__dict__.get("_lexed")
+    if got is not None:
+        return got
+    got = {}
+    for (sheet, col), rows in book._rows_by_col.items():
+        shape, last = None, None
+        for row in rows:
+            cell = book.cells[(sheet, row, col)]
+            if cell.formula is None:
+                continue
+            seen = shape.at(row) if shape is not None and last == row - 1 else None
+            if seen is None or seen[0] != cell.formula:
+                try:
+                    shape = Shape(cell.formula, row, col)
+                    seen = (cell.formula, shape.refs)
+                except TokenizerError:
+                    shape, seen = None, None
+            got[(sheet, row, col)] = (shape, seen[1] if seen else ())
+            last = row
+    book.__dict__["_lexed"] = got
+    return got
+
+
+def map_refs(node, f):
+    """The tree with every RefNode replaced by f(node)."""
+    if isinstance(node, RefNode):
+        return f(node)
+    if isinstance(node, (Unary, Postfix)):
+        return type(node)(node.op, map_refs(node.operand, f))
+    if isinstance(node, Binary):
+        return Binary(node.op, map_refs(node.left, f), map_refs(node.right, f))
+    if isinstance(node, Call):
+        return Call(node.name, tuple(None if a is None else map_refs(a, f) for a in node.args))
+    return node

cellproof/rangeindex.py

@@ -0,0 +1,97 @@
+"""What the compiler needs to know about a rectangle of cells, in O(columns x log rows) instead of O(cells).
+
+Per (sheet, column) one Fenwick tree holds a packed counter for every present cell: how many cells of each kind
+(n s b e a blank), how many can hold an error, and how many formula cells are finished. Input cells are counted
+when the index is built; a formula cell is counted by complete() when its batch closes.
+"""
+from __future__ import annotations
+
+from bisect import bisect_left, bisect_right
+from collections import defaultdict
+from dataclasses import dataclass
+
+from cellproof.fenwick import Fenwick
+from cellproof.model import Book, Key
+
+FIELDS = ("n", "s", "b", "e", "a", "blank", "merr", "fdone")
+BITS = 40  # summary() adds counters across columns: 1,048,576 rows x 16,384 columns = 2**34 < 2**40, no carry
+SHIFT = {name: BITS * i for i, name in enumerate(FIELDS)}
+MASK = (1 << BITS) - 1
+Column = tuple[str, int]
+
+
+def field(total: int, name: str) -> int:
+    return (total >> SHIFT[name]) & MASK
+
+
+@dataclass(frozen=True)
+class Summary:
+    present: int  # non-empty cells in the rectangle
+    formulas: int  # of those, formula cells
+    total: int  # packed counters, read with field()
+
+
+class RangeIndex:
+    def __init__(self, book: Book):
+        self.book = book
+        self.finished: set[Key] = set()
+        self.rows = book._rows_by_col  # (sheet, col) -> sorted rows that hold a cell
+        self.cols: dict[str, list[int]] = defaultdict(list)  # sheet -> sorted columns that hold a cell
+        for sheet, col in sorted(self.rows):
+            self.cols[sheet].append(col)
+        self.fen: dict[Column, Fenwick] = {}
+        self.formula_prefix: dict[Column, list[int]] = {}  # formula cells among the first i cells of the column
+        self.text_rows: dict[Column, list[int]] = {}  # rows of input text cells
+        for (sheet, col), rows in self.rows.items():
+            values, prefix, text = [], [0], []
+            for row in rows:
+                cell = book.cells[(sheet, row, col)]
+                if cell.formula is not None:
+                    values.append(0)
+                    prefix.append(prefix[-1] + 1)
+                    continue
+                values.append((1 << SHIFT[cell.kind]) + ((1 << SHIFT["merr"]) if cell.kind == "e" else 0))
+                prefix.append(prefix[-1])
+                if cell.kind == "s":
+                    text.append(row)
+            self.fen[(sheet, col)] = Fenwick(values)
+            self.formula_prefix[(sheet, col)] = prefix
+            self.text_rows[(sheet, col)] = text
+
+    def complete(self, key: Key, kind: str, may_err: bool) -> None:
+        """A formula cell is finished: kind is its value kind (n s b e a), may_err whether it can hold an error."""
+        sheet, row, col = key
+        rows = self.rows.get((sheet, col), [])
+        pos = bisect_left(rows, row)
+        cell = self.book.cells.get(key)
+        if pos >= len(rows) or rows[pos] != row or cell is None or cell.formula is None:
+            raise ValueError(f"{key} is not a formula cell in the index")
+        if key in self.finished:
+            raise ValueError(f"{key} was already completed")
+        self.finished.add(key)
+        add = (1 << SHIFT[kind]) + (1 << SHIFT["fdone"]) + ((1 << SHIFT["merr"]) if may_err else 0)
+        self.fen[(sheet, col)].add(pos, add)
+
+    def _slices(self, sheet: str, r1: int, c1: int, r2: int, c2: int):
+        cols = self.cols.get(sheet, [])
+        for col in cols[bisect_left(cols, c1):bisect_right(cols, c2)]:
+            rows = self.rows[(sheet, col)]
+            yield (sheet, col), bisect_left(rows, r1), bisect_right(rows, r2)
+
+    def summary(self, sheet: str, r1: int, c1: int, r2: int, c2: int) -> Summary:
+        present = formulas = total = 0
+        for column, i, j in self._slices(sheet, r1, c1, r2, c2):
+            if j > i:
+                present += j - i
+                prefix = self.formula_prefix[column]
+                formulas += prefix[j] - prefix[i]
+                total += self.fen[column].range(i, j)
+        return Summary(present, formulas, total)
+
+    def text_cells(self, sheet: str, r1: int, c1: int, r2: int, c2: int) -> list[Key]:
+        """Input text cells in the rectangle, row-major."""
+        found = []
+        for (s, col), _, _ in self._slices(sheet, r1, c1, r2, c2):
+            rows = self.text_rows[(s, col)]
+            found.extend((s, r, col) for r in rows[bisect_left(rows, r1):bisect_right(rows, r2)])
+        return sorted(found)

cellproof/reader.py

@@ -0,0 +1,83 @@
+"""Load a workbook twice: once for formulas, once for the values Excel saved."""
+from __future__ import annotations
+
+import datetime
+import zipfile
+from pathlib import Path
+
+import openpyxl
+from openpyxl.utils.datetime import CALENDAR_MAC_1904, to_excel
+from openpyxl.worksheet.formula import ArrayFormula
+
+from cellproof.model import Book, Cell, Kind, Value
+
+ERRORS = {"#NULL!", "#DIV/0!", "#VALUE!", "#REF!", "#NAME?", "#NUM!", "#N/A"}
+SUPPORTED = {".xlsx", ".xlsm"}
+# An .xlsx is a zip of XML. A small file that unzips to gigabytes (a "zip bomb") would exhaust memory in openpyxl.
+# ponytail: one total-size limit; raise it if real workbooks ever need more
+MAX_UNZIPPED = 2 * 1024**3
+
+
+class UnreadableWorkbook(Exception):
+    pass
+
+
+def _saved(cell, epoch) -> tuple[Value, Kind]:
+    v = cell.value
+    if v is None:
+        if cell.data_type == "str":  # Excel's cached empty string is None in openpyxl.
+            return "", "s"
+        return None, "blank"
+    if cell.data_type == "e":
+        return str(v), "e"
+    if isinstance(v, bool):  # before int: bool is an int subclass
+        return v, "b"
+    if isinstance(v, (int, float)):
+        return float(v), "n"
+    if isinstance(v, (datetime.datetime, datetime.date, datetime.time)):
+        return float(to_excel(v, epoch)), "n"  # handles Excel's 1900 leap-year bug and the 1904 system
+    return str(v), "s"
+
+
+def load(path: str | Path) -> Book:
+    path = Path(path)
+    if path.suffix.lower() not in SUPPORTED:
+        raise UnreadableWorkbook(f"{path.suffix or 'these'} files are not supported in v1 - save the workbook as .xlsx")
+    try:
+        with zipfile.ZipFile(path) as z:
+            size = sum(i.file_size for i in z.infolist())
+    except (OSError, zipfile.BadZipFile) as err:
+        raise UnreadableWorkbook(f"cannot open {path.name}: {err}") from err
+    if size > MAX_UNZIPPED:
+        raise UnreadableWorkbook(f"cannot open {path.name}: it unzips to {size / 1024**3:.1f} GB, more than the "
+                                 f"{MAX_UNZIPPED // 1024**3} GB limit")
+    try:
+        fwb = openpyxl.load_workbook(path)
+        vwb = openpyxl.load_workbook(path, data_only=True)
+    except Exception as err:  # openpyxl raises many types for corrupt or encrypted files
+        raise UnreadableWorkbook(f"cannot open {path.name}: {err}") from err
+
+    names = {}
+    for name, dn in fwb.defined_names.items():
+        dests = list(dn.destinations)
+        if len(dests) == 1:
+            sheet, ref = dests[0]
+            names[name.upper()] = (sheet, ref.replace("$", ""))
+
+    cells, max_row = {}, {}
+    for ws in fwb.worksheets:
+        vws = vwb[ws.title]
+        max_row[ws.title] = ws.max_row
+        # ponytail: openpyxl's private _cells holds only the cells the file has; iter_rows() would build
+        # every row x column up to max_row (one styled cell at G1048576 cost 6 s; at Z, 3.8 GB)
+        for (r, col), c in sorted(ws._cells.items()):
+            if c.value is None:
+                continue
+            value, kind = _saved(vws.cell(row=r, column=col), fwb.epoch)
+            formula, array = None, False
+            if isinstance(c.value, ArrayFormula):
+                formula, array = (c.value.text or "").removeprefix("="), True
+            elif c.data_type == "f":
+                formula = c.value.removeprefix("=")
+            cells[(ws.title, c.row, c.column)] = Cell(ws.title, c.row, c.column, formula, value, kind, array)
+    return Book(cells, [ws.title for ws in fwb.worksheets], names, max_row, fwb.epoch == CALENDAR_MAC_1904)

cellproof/refs.py

@@ -0,0 +1,58 @@
+"""Reference text ('Sales'!C:C, $A$1, Rate) -> a concrete rectangle of cells."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterator
+
+from openpyxl.utils.cell import range_boundaries
+
+from cellproof.model import Book, Key, Unsupported
+
+MAX_ROW = 1_048_576
+
+
+@dataclass(frozen=True)
+class Area:
+    sheet: str
+    r1: int
+    c1: int
+    r2: int
+    c2: int
+
+    @property
+    def key(self) -> Key:
+        return (self.sheet, self.r1, self.c1)
+
+    @property
+    def is_single(self) -> bool:
+        return self.r1 == self.r2 and self.c1 == self.c2
+
+    def cells(self) -> Iterator[Key]:
+        for r in range(self.r1, self.r2 + 1):
+            for c in range(self.c1, self.c2 + 1):
+                yield (self.sheet, r, c)
+
+    def present(self, book: Book) -> list[Key]:
+        """Non-empty cells in the area, row-major. Use this, not cells(), to read a range."""
+        return book.present(self.sheet, self.r1, self.c1, self.r2, self.c2)
+
+
+def resolve(text: str, sheet: str, book: Book) -> Area:
+    ref = text
+    if "!" in text:
+        sheet, ref = text.rsplit("!", 1)
+        if sheet.startswith("'"):
+            sheet = sheet[1:-1].replace("''", "'")
+        if "[" in sheet:
+            raise Unsupported(f"external link {text}")
+    elif ref.upper() in book.names:
+        sheet, ref = book.names[ref.upper()]
+    if sheet not in book.sheets:
+        raise Unsupported(f"unknown sheet {sheet!r}")
+    try:
+        c1, r1, c2, r2 = range_boundaries(ref.replace("$", ""))
+    except ValueError:
+        raise Unsupported(f"unknown name {text!r}") from None
+    if c1 is None:
+        raise Unsupported(f"whole-row reference {text}")
+    return Area(sheet, r1 or 1, c1, r2 or MAX_ROW, c2)  # C:C is every row, as in Excel; Area.present skips empties