tariffkit 0.2.0__py3-none-any.whl

tariffkit/providers/pge/statements/__init__.py

@@ -0,0 +1,26 @@
+"""Local PG&E statement import."""
+
+from __future__ import annotations
+
+from .errors import StatementAmbiguityError, StatementError
+from .model import (
+    Section,
+    Statement,
+    StatementAgreement,
+    StatementLine,
+    StatementSection,
+)
+from .parse import normalize_tariff, parse_statement, read_statement
+
+__all__ = [
+    "Section",
+    "Statement",
+    "StatementAgreement",
+    "StatementAmbiguityError",
+    "StatementError",
+    "StatementLine",
+    "StatementSection",
+    "normalize_tariff",
+    "parse_statement",
+    "read_statement",
+]

tariffkit/providers/pge/statements/errors.py

@@ -0,0 +1,20 @@
+"""Errors raised while importing PG&E statements."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from tariffkit.errors import TariffKitError
+
+
+class StatementError(TariffKitError):
+    """A statement could not be read or did not pass its self-check."""
+
+
+class StatementAmbiguityError(StatementError):
+    """The statement does not provide one-to-one agreement evidence."""
+
+    def __init__(self, message: str, *, diagnostics: Iterable[str] = ()) -> None:
+        self.diagnostics = tuple(diagnostics)
+        detail = "; ".join(self.diagnostics)
+        super().__init__(f"{message}: {detail}" if detail else message)

tariffkit/providers/pge/statements/model.py

@@ -0,0 +1,320 @@
+"""What a PG&E statement says, in a shape a computed bill can be compared against.
+
+A statement prints the same money more than once, in different arrangements, and
+that redundancy is the most useful thing about it. The December 2025 / January
+2026 statement says ``$333.87`` twice: once as the sum of time-of-use lines split
+across the rate change on 1 January, and once as an unbundled list of tariff
+components. Those two views have to agree with each other before either is worth
+comparing against anything this library computed, which is what
+:meth:`Statement.self_check` is for.
+
+The four sections are not four pages. They are four *presentations*:
+
+``SUMMARY``
+    What is owed, and the split between the utility and the generation provider.
+``PGE_DELIVERY``
+    Time-of-use lines, one block per sub-period. A cycle spanning a rate change
+    prints two blocks, which is how the statement itself confirms that
+    effective-dated pricing is the right model.
+``PGE_BREAKDOWN``
+    The same total, unbundled into tariff components. This is the presentation
+    that maps onto ``Bill.import_components``.
+``CCA_GENERATION``
+    The generation provider's own page. On a CCA account this is where the state
+    energy surcharge prints, which is how it stayed invisible for so long while
+    every line on the utility's pages reconciled.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from datetime import date
+from enum import StrEnum
+
+from tariffkit.billing import BillingPeriod
+
+#: A printed figure is rounded to the cent, so agreement is judged at half a cent
+#: plus a cent per line that had to be added together to get there.
+CENT = 0.005
+
+
+class Section(StrEnum):
+    SUMMARY = "summary"
+    PGE_DELIVERY = "pge_delivery"
+    PGE_BREAKDOWN = "pge_breakdown"
+    CCA_GENERATION = "cca_generation"
+
+
+#: Lines that restate other lines rather than adding to them. Counting these
+#: would double the section's sum, and the failure is quiet: the section total
+#: comes out plausibly wrong rather than obviously so.
+SUBTOTAL_LABELS = frozenset({"net charges", "subtotal"})
+
+
+@dataclass(frozen=True, slots=True)
+class StatementLine:
+    """One printed row carrying an amount."""
+
+    label: str
+    amount: float
+    section: Section
+    page: int
+    #: Present on a metered row: "982.126000 kWh @ $0.13500", and also on a
+    #: per-day one: "29 days @ $0.79343", which is how the Base Services Charge
+    #: prints. Both are quantity-times-rate, so both are kept the same way.
+    quantity: float | None = None
+    unit: str = ""
+    rate: float | None = None
+    #: Which sub-period block this row appeared under, when the section has them.
+    subperiod: tuple[date, date] | None = None
+    #: The sub-heading this row sits under, where one distinguishes otherwise
+    #: identical labels. On the Solar Billing Plan the same three time-of-use
+    #: labels print twice, once under "Energy Produced" and once under "Energy
+    #: Delivered" -- export and import. Without this they look like the section
+    #: boundaries overlapping, and either the credit or the charge is dropped.
+    block: str = ""
+    #: Which service agreement this row belongs to, 1-based. A statement issued
+    #: in a month the account changed tariff carries two of everything, with the
+    #: same labels in each, and they are different charges under different
+    #: schedules rather than a section read twice.
+    agreement: int = 1
+    raw: str = ""
+
+    @property
+    def is_subtotal(self) -> bool:
+        return self.label.strip().lower() in SUBTOTAL_LABELS
+
+    @property
+    def kwh(self) -> float | None:
+        """The quantity, when it is energy rather than days."""
+        return self.quantity if self.unit.lower() == "kwh" else None
+
+
+@dataclass(frozen=True, slots=True)
+class StatementSection:
+    name: Section
+    lines: tuple[StatementLine, ...] = ()
+    #: The section's own printed total, when it prints one.
+    printed_total: float | None = None
+    total_label: str = ""
+
+    @property
+    def charged(self) -> tuple[StatementLine, ...]:
+        """The rows that add up, excluding restatements of other rows."""
+        return tuple(line for line in self.lines if not line.is_subtotal)
+
+    def total(self) -> float:
+        return sum(line.amount for line in self.charged)
+
+    def find(self, label: str) -> tuple[StatementLine, ...]:
+        wanted = label.strip().lower()
+        return tuple(line for line in self.lines if line.label.strip().lower() == wanted)
+
+
+@dataclass(frozen=True, slots=True)
+class StatementAgreement:
+    """One service-agreement span and the tariff evidence printed with it."""
+
+    period: BillingPeriod
+    printed_schedule: str
+    tariff: str
+    page: int
+    account_masked: str = ""
+    baseline_territory: str = ""
+    pcia_vintage: int | None = None
+
+    def __post_init__(self) -> None:
+        if not self.printed_schedule.strip():
+            raise ValueError("statement agreement needs a printed schedule")
+        if not self.tariff.strip():
+            raise ValueError("statement agreement needs a normalized tariff")
+        if self.page < 0:
+            raise ValueError("statement agreement page must be non-negative")
+        if self.account_masked and (
+            len(self.account_masked) != 4 or not self.account_masked.isdigit()
+        ):
+            raise ValueError("statement agreement account suffix must be four digits")
+
+    @property
+    def start(self) -> date:
+        """The inclusive first date printed for this agreement."""
+        return self.period.start
+
+    @property
+    def end(self) -> date:
+        """The inclusive last date printed for this agreement."""
+        return self.period.end
+
+
+@dataclass(frozen=True, slots=True)
+class Statement:
+    """One issued statement, parsed."""
+
+    statement_date: date
+    period: BillingPeriod
+    amount_due: float
+    #: Last four digits only. The full number is read to find the masked form and
+    #: then discarded: it never reaches a fixture, a log line, or JSON output.
+    account_masked: str = ""
+    billed_days: int | None = None
+    billed_kwh: float | None = None
+    #: How many service agreements the statement covers. More than one means the
+    #: account changed tariff mid-cycle and the utility priced each part
+    #: separately -- a count, never the identifiers themselves. Priced, not
+    #: refused: ``compute_segments`` handles it.
+    service_agreements: int = 1
+    #: Exact service-agreement spans paired with tariff evidence from the same
+    #: delivery page. An empty tuple means the statement did not provide enough
+    #: evidence to establish that mapping.
+    agreements: tuple[StatementAgreement, ...] = ()
+    #: Gas, on a combined statement. This account burned no therms and was still
+    #: billed a minimum transportation charge, so a zero-usage service is not a
+    #: zero-money one. Recorded to make the amount due add up and then ignored:
+    #: nothing here prices gas, so no computed component may claim it.
+    gas_charges: float | None = None
+    #: Summary-level electric adjustments, e.g. the California Climate Credit,
+    #: which belong to no detail section and are not per-cycle charges.
+    electric_adjustments: float | None = None
+    sections: tuple[StatementSection, ...] = ()
+    #: What the statement says about itself, used to catch a stale account
+    #: configuration before it can produce a confident, fabricated finding.
+    rate_schedule: str = ""
+    #: Every rate schedule the statement prints, in order and deduplicated. More
+    #: than one means the cycle was billed under more than one tariff.
+    printed_schedules: tuple[str, ...] = ()
+    cca_name: str = ""
+    cca_rate_schedule: str = ""
+    baseline_territory: str = ""
+    pcia_vintage: int | None = None
+    source: str = ""
+    #: True when the text came from recognising rendered pages rather than from
+    #: the PDF's own text layer. Recorded because a misread digit is a plausible
+    #: amount rather than an obvious failure, so a reader deserves to know which
+    #: figures were read and which were recognised.
+    recognised: bool = False
+    warnings: tuple[str, ...] = field(default_factory=tuple)
+
+    def section(self, name: Section) -> StatementSection | None:
+        for candidate in self.sections:
+            if candidate.name is name:
+                return candidate
+        return None
+
+    def lines(self) -> Iterator[StatementLine]:
+        for section in self.sections:
+            yield from section.lines
+
+    @property
+    def subperiods(self) -> tuple[tuple[date, date], ...]:
+        """The rate-change blocks the utility itself split the cycle into."""
+        seen: list[tuple[date, date]] = []
+        for line in self.lines():
+            if line.subperiod and line.subperiod not in seen:
+                seen.append(line.subperiod)
+        return tuple(seen)
+
+    @property
+    def electric_charges(self) -> float:
+        """The part of the bill this library is actually pricing.
+
+        The amount due is not comparable to a computed total: it carries gas,
+        and it carries summary-level credits like the California Climate Credit
+        that are not per-cycle charges. On 2025-11 those two make the difference
+        between $213.89 and $268.13 -- a statement that reconciles line by line
+        while the headline figures look $54 apart, which reads as a failure and
+        is not one.
+        """
+        return self.amount_due - (self.gas_charges or 0.0) - (self.electric_adjustments or 0.0)
+
+    def self_check(self) -> list[str]:
+        """Problems with the parse itself, independent of any computed bill.
+
+        Run before reconciliation and treated as fatal, because a dropped row
+        produces a bill-shaped disagreement. Reporting that as a billing defect
+        would destroy the only thing this harness sells, which is that its
+        findings can be trusted.
+        """
+        problems: list[str] = []
+
+        for section in self.sections:
+            # The summary is a running balance -- prior balance, payments
+            # received, this cycle's charges -- not a list of things that add up
+            # to the amount due. Summing it would fail on every statement, and a
+            # check that always fails is one nobody reads.
+            if section.name is Section.SUMMARY:
+                continue
+            if section.printed_total is None or not section.charged:
+                continue
+            summed = section.total()
+            slack = CENT + 0.01 * len(section.charged)
+            if abs(summed - section.printed_total) > slack:
+                problems.append(
+                    f"{section.name}: rows sum to {summed:.2f} but the section prints "
+                    f"{section.printed_total:.2f} (off by {summed - section.printed_total:+.2f})"
+                )
+
+        # The utility states one number twice, in two arrangements. They must
+        # agree, or one of the two presentations was mis-parsed.
+        delivery = self.section(Section.PGE_DELIVERY)
+        breakdown = self.section(Section.PGE_BREAKDOWN)
+        if delivery is None:
+            problems.append("the PG&E delivery section is missing")
+        elif not self.agreements:
+            problems.append("the statement has no service-agreement tariff evidence")
+        if delivery and breakdown and None not in (delivery.printed_total, breakdown.printed_total):
+            assert delivery.printed_total is not None and breakdown.printed_total is not None
+            if abs(delivery.printed_total - breakdown.printed_total) > CENT:
+                problems.append(
+                    f"the two views of the utility's charges disagree: delivery detail prints "
+                    f"{delivery.printed_total:.2f}, unbundled breakdown prints "
+                    f"{breakdown.printed_total:.2f}"
+                )
+
+        parts = [
+            s.printed_total
+            for s in self.sections
+            if s.name in (Section.PGE_DELIVERY, Section.CCA_GENERATION)
+            and s.printed_total is not None
+        ]
+        # The identity is not "the electric sections add up to the bill". A PG&E
+        # statement is a combined one: it can carry gas, and it carries
+        # summary-level adjustments that belong to no detail section. Both are
+        # named on the statement, so both are added here rather than absorbed
+        # into a tolerance -- an unexplained residue should stay visible.
+        expected = sum(parts) + (self.electric_adjustments or 0.0) + (self.gas_charges or 0.0)
+        if parts and abs(expected - self.amount_due) > CENT:
+            extra = []
+            if self.electric_adjustments:
+                extra.append(f"adjustments {self.electric_adjustments:+.2f}")
+            if self.gas_charges:
+                extra.append(f"gas {self.gas_charges:+.2f}")
+            detail = f" (sections {sum(parts):.2f}, {', '.join(extra)})" if extra else ""
+            problems.append(
+                f"the statement's own parts sum to {expected:.2f} but it is due "
+                f"{self.amount_due:.2f}{detail}; a whole section is probably missing"
+            )
+
+        if self.billed_days is not None and self.billed_days != self.period.days:
+            problems.append(
+                f"the statement prints {self.billed_days} billing days but its dates "
+                f"{self.period.start}..{self.period.end} span {self.period.days}"
+            )
+
+        seen: set[tuple[Section, str, tuple[date, date] | None, str, int]] = set()
+        for line in self.lines():
+            key = (
+                line.section,
+                line.label.strip().lower(),
+                line.subperiod,
+                line.block,
+                line.agreement,
+            )
+            if key in seen:
+                problems.append(
+                    f"{line.section}: {line.label!r} appears twice in the same block, "
+                    f"so the section boundaries probably overlap"
+                )
+            seen.add(key)
+
+        return problems

tariffkit/providers/pge/statements/ocr.py

@@ -0,0 +1,193 @@
+"""Reading a statement that carries no usable text.
+
+PG&E's statements before November 2025 are drawn with Type 3 fonts whose
+``ToUnicode`` map declares most glyphs to be U+0020. Extraction is not failing on
+them -- it is faithfully returning the spaces the document asks for -- so no
+choice of PDF library helps, and pypdf and poppler agree. The pixels are the only
+honest source, which means rendering and recognising them.
+
+The parser downstream expects ``extraction_mode="layout"`` text, where a row is a
+label, two or more spaces, and a right-aligned amount. So this does not just
+return words: it rebuilds the page as a character grid from the word boxes
+tesseract reports, which is what makes the same parser work on both kinds of
+statement rather than needing a second one.
+
+**Recognition can be wrong, and that matters more here than usual** -- a misread
+digit is a plausible dollar amount rather than an obvious failure. Nothing here
+tries to be clever about that. It relies instead on the check that already
+exists: a statement prints its totals twice, and :meth:`Statement.self_check`
+refuses to reconcile one whose sections do not sum to what it says is due. A
+digit recognised wrongly almost always breaks that arithmetic.
+"""
+
+from __future__ import annotations
+
+import csv
+import shutil
+import subprocess
+import tempfile
+from collections.abc import Iterator
+from pathlib import Path
+
+from .errors import StatementError
+
+#: Rendering resolution. 300 is not a floor to be raised for better results:
+#: 400 was measurably worse here, losing the statement date entirely, because
+#: tesseract's model is trained around this scale and the reconstructed grid
+#: depends on its word boxes being consistent. Change it only with a statement
+#: in front of you.
+DPI = 300
+
+#: Tesseract's page segmentation mode. 11 is "sparse text": find words, attempt
+#: no layout analysis at all.
+#:
+#: That sounds like the wrong choice and is the right one, because the rows are
+#: reassembled here from word geometry anyway. Tesseract's own analysis fights
+#: that -- on a statement laid out as a table beside a sidebar, mode 6 silently
+#: drops whole rows. It lost the Peak charge from the 2025-09 delivery table,
+#: $104.43 of a $178.64 section, while the chart legend two inches lower
+#: recognised the same figure perfectly. Mode 11 reads that row completely.
+PSM = "11"
+
+#: Modes to try, in order. Recognition quality is not uniform across statements
+#: -- 11 reads rows 6 drops, and 6 occasionally resolves an amount column 11
+#: fragments -- and there is no way to tell in advance which will do better on a
+#: given page. So more than one is attempted and the *statement decides*: a bill
+#: prints its totals twice and its sections have to sum to them, which is an
+#: objective test the document supplies itself rather than a preference for
+#: whichever answer looks nicer.
+PSM_CANDIDATES = ("11", "6", "4")
+
+#: How much of a line's height two words may differ by and still be the same
+#: row. Sparse mode does no line grouping, so rows are clustered by vertical
+#: position here; a fraction rather than a constant because it has to hold at
+#: whatever resolution the page was rendered at.
+ROW_TOLERANCE = 0.6
+
+
+def available() -> bool:
+    """Whether the external tools needed for recognition are installed."""
+    return bool(shutil.which("pdftoppm") and shutil.which("tesseract"))
+
+
+def _render(source: Path, into: Path) -> list[Path]:
+    result = subprocess.run(
+        ["pdftoppm", "-r", str(DPI), "-png", str(source), str(into / "page")],
+        capture_output=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        raise StatementError(
+            f"could not render {source.name} for recognition: "
+            f"{result.stderr.decode('utf-8', 'replace')[:200]}"
+        )
+    return sorted(into.glob("page*.png"))
+
+
+def _words(image: Path, psm: str = PSM) -> list[tuple[int, int, int, int, str]]:
+    """Recognised words as (top, height, left, width, text)."""
+    result = subprocess.run(
+        ["tesseract", str(image), "stdout", "--psm", psm, "tsv"],
+        capture_output=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        raise StatementError(
+            f"recognition failed on {image.name}: {result.stderr.decode('utf-8', 'replace')[:200]}"
+        )
+
+    rows: list[tuple[int, int, int, int, str]] = []
+    reader = csv.DictReader(result.stdout.decode("utf-8", "replace").splitlines(), delimiter="\t")
+    for row in reader:
+        text = (row.get("text") or "").strip()
+        if not text:
+            continue
+        try:
+            # Geometry only. Sparse mode reports no meaningful block, paragraph
+            # or line numbering, and relying on any of it is what reordered rows
+            # across sections before.
+            rows.append(
+                (
+                    int(row["top"]),
+                    int(row["height"]),
+                    int(row["left"]),
+                    int(row["width"]),
+                    text,
+                )
+            )
+        except KeyError, ValueError:
+            continue
+    return rows
+
+
+def _as_layout(words: list[tuple[int, int, int, int, str]]) -> str:
+    """Rebuild a character grid from word boxes.
+
+    The parser reads column positions, not just words, so the horizontal
+    geometry has to survive. Each word is placed at the column its pixel offset
+    implies, using the median character width as the scale -- which is what makes
+    "Distribution" and its right-aligned amount stay in different fields.
+    """
+    if not words:
+        return ""
+
+    widths = sorted(width / len(text) for _, _, _, width, text in words if text)
+    char_width = widths[len(widths) // 2] or 1.0
+    heights = sorted(height for _, height, _, _, _ in words)
+    band = (heights[len(heights) // 2] or 1) * ROW_TOLERANCE
+
+    # Clustered down the page. Ordering by anything tesseract numbers rather
+    # than by position reorders rows across section boundaries, and the symptom
+    # is one section short by a charge while another is over by the same rows.
+    grouped: list[list[tuple[int, str]]] = []
+    anchor = None
+    for top, _, left, _, text in sorted(words, key=lambda w: (w[0], w[2])):
+        if anchor is None or top - anchor > band:
+            grouped.append([])
+            anchor = top
+        grouped[-1].append((round(left / char_width), text))
+
+    out: list[str] = []
+    for group in grouped:
+        row = ""
+        for column, text in sorted(group):
+            if column <= len(row):
+                # The estimate put this word at or inside the end of the last
+                # one. One space, never none: run together, "Off Peak" becomes
+                # "OffPeak" and matches no rule, and "Baseline Allowance" stops
+                # being recognised as the row to skip -- so its kWh quantity is
+                # read as a few hundred dollars of charges.
+                row += " " + text
+            else:
+                row = row.ljust(column) + text
+        out.append(row.rstrip())
+    return "\n".join(out)
+
+
+def readings(path: str | Path) -> Iterator[list[str]]:
+    """Each candidate recognition of a statement's pages, best mode first.
+
+    Yields rather than returns so a caller can stop at the first reading that
+    satisfies the statement's own arithmetic, and pay for the slower modes only
+    when the first one does not.
+    """
+    source = Path(path)
+    if not available():
+        raise StatementError(
+            f"{source.name} carries no readable text and recognition tools are not "
+            f"installed; `brew install tesseract poppler` provides both"
+        )
+
+    with tempfile.TemporaryDirectory(prefix="nem-ocr-") as scratch:
+        images = _render(source, Path(scratch))
+        if not images:
+            raise StatementError(f"{source.name} produced no pages to recognise")
+        for psm in PSM_CANDIDATES:
+            yield [_as_layout(_words(image, psm)) for image in images]
+
+
+def pages_via_ocr(path: str | Path) -> list[str]:
+    """The first recognition of a statement's pages."""
+    for pages in readings(path):
+        return pages
+    raise StatementError(f"{Path(path).name} produced no pages to recognise")