tariffkit 0.2.0__py3-none-any.whl

tariffkit/secrets.py

@@ -0,0 +1,164 @@
+"""Credentials stored in the operating system's keyring."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import re
+from typing import Final, Protocol, cast
+
+from .errors import ConfigError
+
+SERVICE: Final = "tariffkit"
+_CREDENTIAL_SET = re.compile(r"^[a-z0-9](?:[a-z0-9_.-]{0,62}[a-z0-9])?$")
+SECRET_NAMES: Final = (
+    "home_assistant.token",
+    "influxdb.token",
+    "mqtt.password",
+    "mqtt.username",
+    "pge.account_urn",
+    "pge.browser_cookie",
+    "pge.password",
+    "pge.username",
+    "pge.validation_cookie",
+)
+
+
+class _KeyringErrors(Protocol):
+    KeyringError: type[Exception]
+    NoKeyringError: type[Exception]
+
+
+class _Keyring(Protocol):
+    errors: _KeyringErrors
+
+    def get_password(self, service: str, name: str) -> str | None: ...
+
+    def set_password(self, service: str, name: str, value: str) -> None: ...
+
+    def delete_password(self, service: str, name: str) -> None: ...
+
+
+def _keyring() -> _Keyring | None:
+    try:
+        module = importlib.import_module("keyring")
+    except ImportError:
+        return None
+    return cast(_Keyring, module)
+
+
+def get_secret(name: str) -> str | None:
+    """Return a named secret, or ``None`` when keyring support is not installed."""
+    _validate_name(name)
+    if os.environ.get("TARIFFKIT_DISABLE_KEYRING") == "1":
+        return None
+    keyring = _keyring()
+    if keyring is None:
+        return None
+    try:
+        return keyring.get_password(SERVICE, name)
+    except keyring.errors.NoKeyringError:
+        # Headless containers commonly have the package but no OS secret
+        # service. That means "no keyring source", not a failed credential read;
+        # environment injection remains available there.
+        return None
+    except keyring.errors.KeyringError as exc:
+        raise ConfigError(f"could not read {name!r} from the operating-system keyring") from exc
+
+
+def set_secret(name: str, value: str) -> None:
+    """Store a non-empty secret without exposing it in process arguments."""
+    _validate_name(name)
+    if not value:
+        raise ConfigError("secret value must not be empty")
+    keyring = _require_keyring()
+    try:
+        keyring.set_password(SERVICE, name, value)
+    except keyring.errors.KeyringError as exc:
+        raise ConfigError(f"could not store {name!r} in the operating-system keyring") from exc
+
+
+def delete_secret(name: str) -> None:
+    """Delete a named secret, raising when the keyring operation fails."""
+    _validate_name(name)
+    keyring = _require_keyring()
+    try:
+        keyring.delete_password(SERVICE, name)
+    except keyring.errors.KeyringError as exc:
+        raise ConfigError(f"could not delete {name!r} from the operating-system keyring") from exc
+
+
+def configured_secrets() -> tuple[str, ...]:
+    """Names that are present, without ever returning their values."""
+    return tuple(name for name in SECRET_NAMES if get_secret(name) is not None)
+
+
+def get_named_secret(credential_set: str, name: str) -> str | None:
+    """Read a secret from a named keyring set without exposing its value."""
+    _validate_credential_set(credential_set)
+    _validate_name(name)
+    if os.environ.get("TARIFFKIT_DISABLE_KEYRING") == "1":
+        return None
+    keyring = _keyring()
+    if keyring is None:
+        return None
+    try:
+        return keyring.get_password(f"{SERVICE}:credential:{credential_set}", name)
+    except keyring.errors.NoKeyringError:
+        return None
+    except keyring.errors.KeyringError as exc:
+        raise ConfigError(
+            f"could not read {name!r} from credential set {credential_set!r}"
+        ) from exc
+
+
+def set_named_secret(credential_set: str, name: str, value: str) -> None:
+    """Store a secret in a named keyring set."""
+    _validate_credential_set(credential_set)
+    _validate_name(name)
+    if not value:
+        raise ConfigError("secret value must not be empty")
+    keyring = _require_keyring()
+    try:
+        keyring.set_password(f"{SERVICE}:credential:{credential_set}", name, value)
+    except keyring.errors.KeyringError as exc:
+        raise ConfigError(f"could not store {name!r} in credential set {credential_set!r}") from exc
+
+
+def delete_named_secret(credential_set: str, name: str) -> None:
+    """Delete a secret from a named keyring set."""
+    _validate_credential_set(credential_set)
+    _validate_name(name)
+    keyring = _require_keyring()
+    try:
+        keyring.delete_password(f"{SERVICE}:credential:{credential_set}", name)
+    except keyring.errors.KeyringError as exc:
+        raise ConfigError(
+            f"could not delete {name!r} from credential set {credential_set!r}"
+        ) from exc
+
+
+def configured_named_secrets(credential_set: str) -> tuple[str, ...]:
+    """List names present in a named credential set, without values."""
+    return tuple(
+        name for name in SECRET_NAMES if get_named_secret(credential_set, name) is not None
+    )
+
+
+def _require_keyring() -> _Keyring:
+    keyring = _keyring()
+    if keyring is None:
+        raise ConfigError(
+            "credential storage requires the 'secrets' extra: pip install 'tariffkit[secrets]'"
+        )
+    return keyring
+
+
+def _validate_name(name: str) -> None:
+    if name not in SECRET_NAMES:
+        raise ConfigError(f"unknown secret {name!r}; choose one of {', '.join(SECRET_NAMES)}")
+
+
+def _validate_credential_set(name: str) -> None:
+    if _CREDENTIAL_SET.fullmatch(name) is None:
+        raise ConfigError("credential set must be a lowercase safe name")

tariffkit/sources/__init__.py

@@ -0,0 +1,71 @@
+"""Interval-data sources.
+
+Everything here turns some external record of metered energy into
+``IntervalReading`` objects for :mod:`tariffkit.billing`. Sources may pull in
+dependencies -- a WebSocket client, an HTTP client -- which is why they live
+outside the billing package, which stays stdlib-only.
+
+Three sources, with different strengths:
+
+============= ========================= ============================ ===========
+source        reads                     totals                       history
+============= ========================= ============================ ===========
+greenbutton   PG&E's Green Button CSV   2 dp per interval, ~2% low    as supplied
+homeassistant long-term statistics      pre-aggregated buckets        ~5 months
+influx        raw counter samples       exact (endpoint difference)   ~14 months
+============= ========================= ============================ ===========
+
+Totals are not the whole story. Both live sources see the same counter, so they
+agree on how much energy moved; they disagree on *when*, and time-of-use pricing
+cares. Home Assistant credits an advance to the bucket holding the later sample,
+which biases energy forward over every boundary it spans; ``influx`` works from
+the raw samples and can spread it pro rata instead. Measured against a real
+statement that is worth more than the totals are -- see
+:mod:`tariffkit.sources.influx`. Green Button is the opposite trade: its totals
+are the worst of the three and its timing the best, because it is the utility's
+own record at true fifteen-minute metering.
+
+``greenbutton`` is stdlib-only and so needs no extra, unlike its neighbours. It
+lives here anyway: what a module reads is a better organising principle than
+what it happens to import, and putting the one file source somewhere else made
+it hard to find.
+"""
+
+from .greenbutton import GreenButtonLayout, read_green_button
+from .homeassistant import (
+    DEFAULT_EXPORT_ENTITY,
+    DEFAULT_IMPORT_ENTITY,
+    HaSettings,
+    describe_resolution,
+    load_dotenv,
+    read_statistics,
+    read_statistics_async,
+)
+from .influx import InfluxSettings, monotonic, read_counters
+from .pge import (
+    PgeSession,
+    PgeSettings,
+    PortalError,
+    parse_green_button,
+    read_green_button_download,
+)
+
+__all__ = [
+    "DEFAULT_EXPORT_ENTITY",
+    "DEFAULT_IMPORT_ENTITY",
+    "GreenButtonLayout",
+    "HaSettings",
+    "InfluxSettings",
+    "PgeSession",
+    "PgeSettings",
+    "PortalError",
+    "describe_resolution",
+    "load_dotenv",
+    "monotonic",
+    "parse_green_button",
+    "read_counters",
+    "read_green_button",
+    "read_green_button_download",
+    "read_statistics",
+    "read_statistics_async",
+]

tariffkit/sources/greenbutton.py

@@ -0,0 +1,318 @@
+"""Interval readings from a Green Button CSV export.
+
+Green Button is the industry format for handing a customer their own meter data,
+and PG&E exposes it as "Download my data" on the usage page. What comes back is
+a CSV named like ``pge_electric_usage_interval_data_<account>_<dates>.csv``,
+generally at fifteen-minute resolution.
+
+**This reads the CSV form, not the XML one.** Green Button also has an ESPI/XML
+serialisation, which is a different parser and is not implemented here.
+
+It is Green Button first rather than Green Button only. The preamble skipping in
+:func:`_skip_preamble` exists because PG&E's export opens with account metadata,
+and the default column names are the ones PG&E emits -- but every column is
+configurable through :class:`GreenButtonLayout`, so an inverter log or a Home
+Assistant statistics dump with different headers works too.
+
+The one thing to know before trusting it: **PG&E rounds every interval to two
+decimals before exporting**, and on a low-import month that loses about 2% of the
+total. Reading the meter directly through :mod:`tariffkit.sources.influx` or
+:mod:`tariffkit.sources.homeassistant` avoids it. The Green Button file's
+advantage is that it is the utility's own record of what it billed, at true
+fifteen-minute metering, which makes it the better source for how energy divided
+across time-of-use periods even where its totals are worse.
+
+Stdlib-only, unlike its neighbours in this package: there is no download step to
+authenticate and no client library involved, just a file the user already has.
+"""
+
+from __future__ import annotations
+
+import csv
+import re
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta, tzinfo
+from itertools import pairwise
+from pathlib import Path
+from typing import IO, NamedTuple
+
+from ..billing.models import IntervalReading
+from ..errors import DataError
+from ..timeutil import PACIFIC
+
+
+class _Row(NamedTuple):
+    """One parsed CSV row, before interval length is known."""
+
+    start: datetime
+    imported: float | None
+    exported: float | None
+    net: float | None
+
+
+#: Column names seen in the wild, tried in order when none is configured.
+START_CANDIDATES = (
+    "start",
+    "start_time",
+    "timestamp",
+    "datetime",
+    "date_time",
+    "time",
+    "last_reset",
+)
+IMPORT_CANDIDATES = ("imported", "import", "import_kwh", "delivered", "consumption", "usage", "kwh")
+EXPORT_CANDIDATES = ("exported", "export", "export_kwh", "received", "surplus", "production")
+NET_CANDIDATES = ("net", "net_kwh", "net_usage")
+#: Split date/time pairs, as PG&E's interval export uses. A matching pair takes
+#: precedence over a single timestamp column, because PG&E's time column is named
+#: "START TIME" and would otherwise be mistaken for a whole timestamp. A date
+#: column with no time column beside it falls back to being read as one.
+DATE_CANDIDATES = ("date", "usage_date", "read_date", "interval_date")
+TIME_CANDIDATES = ("start_time", "time", "interval_start_time", "hour")
+#: Trailing unit spellings stripped when matching a header against the candidate
+#: lists, so ``"USAGE (kWh)"`` matches the ``usage`` candidate.
+UNIT_SUFFIXES = ("_kwh", "_kw_h", "_wh", "_kilowatt_hours")
+
+
+def _normalize(name: str) -> str:
+    """Fold case, punctuation, and units so header spellings converge.
+
+    ``"IMPORT (kWh)"`` and ``"import_kwh"`` are the same column; so are
+    ``"START TIME"`` and ``"start_time"``. Matching on the raw string means every
+    new export format needs its own candidate entry.
+    """
+    return re.sub(r"[^0-9a-z]+", "_", name.strip().lower()).strip("_")
+
+
+@dataclass(frozen=True, slots=True)
+class GreenButtonLayout:
+    """Which columns hold what.
+
+    Leave fields as ``None`` to auto-detect from the header.
+    """
+
+    start: str | None = None
+    #: Split date/time pair. Setting both takes precedence over ``start``; set
+    #: neither and a pair is still auto-detected ahead of a single column.
+    date: str | None = None
+    time: str | None = None
+    imported: str | None = None
+    exported: str | None = None
+    #: Signed column, positive meaning import. Alternative to imported/exported.
+    net: str | None = None
+    #: Interval length. Inferred from the first two rows when omitted.
+    duration: timedelta | None = None
+    #: Applied to timestamps that carry no UTC offset.
+    assume_tz: tzinfo = PACIFIC
+
+
+def _pick(header: list[str], configured: str | None, candidates: tuple[str, ...]) -> str | None:
+    if configured is not None:
+        if configured not in header:
+            raise DataError(f"column {configured!r} not in header: {header}")
+        return configured
+    # Exact header names first, then unit-stripped aliases, so an alias can never
+    # shadow a column that genuinely carries the candidate's name. Registering
+    # both in one pass also made the answer depend on column order: a file with
+    # "IMPORT (kWh)" before "import" resolved differently from the same two the
+    # other way round.
+    normalized: dict[str, str] = {}
+    for name in header:
+        if name:
+            normalized.setdefault(_normalize(name), name)
+    # A candidate list then does not need an entry per unit spelling. PG&E alone
+    # ships "IMPORT (kWh)" on an exporting account and "USAGE (kWh)" on one
+    # without.
+    for name in header:
+        if not name:
+            continue
+        key = _normalize(name)
+        for suffix in UNIT_SUFFIXES:
+            if key.endswith(suffix) and key != suffix.lstrip("_"):
+                normalized.setdefault(key[: -len(suffix)], name)
+    for candidate in candidates:
+        if candidate in normalized:
+            return normalized[candidate]
+    return None
+
+
+def _skip_preamble(handle: IO[str], layout: GreenButtonLayout) -> Iterator[str]:
+    """Yield lines from the real header row onward.
+
+    PG&E's interval export opens with account metadata (name, address, account
+    and service numbers) and a blank line before the column header, so feeding
+    the file straight to ``DictReader`` picks up ``"Name,EMMANUEL ..."`` as the
+    header. Scan for the first row carrying a recognisable timestamp column
+    instead of assuming a fixed offset, since the preamble's length is not
+    documented and has changed before.
+
+    Configured column names count as recognisable too, so a preamble does not
+    stop ``GreenButtonLayout(start="when")`` from working on a file whose header this
+    module would not otherwise know.
+    """
+    wanted = set(START_CANDIDATES) | set(DATE_CANDIDATES)
+    wanted |= {_normalize(name) for name in (layout.start, layout.date) if name}
+    lines = handle.read().splitlines()
+    for index, row in enumerate(csv.reader(lines)):
+        if any(_normalize(cell) in wanted for cell in row):
+            yield from lines[index:]
+            return
+    # No recognisable header anywhere. Hand back the file unchanged rather than
+    # guessing which row was meant: the caller gets the "no timestamp column"
+    # error naming whatever the first row held, which for a file with a preamble
+    # is the preamble. Nothing here can do better without inventing a rule for
+    # where the header starts, and a wrong guess would mis-parse silently.
+    yield from lines
+
+
+def _parse_timestamp(raw: str, assume_tz: tzinfo) -> datetime:
+    text = raw.strip().replace("Z", "+00:00")
+    try:
+        moment = datetime.fromisoformat(text)
+    except ValueError as exc:
+        raise DataError(f"could not parse timestamp {raw!r} as ISO 8601") from exc
+    if moment.tzinfo is None:
+        # Naive timestamps are ambiguous across DST. Assume the configured zone
+        # and say so, rather than silently treating them as UTC.
+        return moment.replace(tzinfo=assume_tz)
+    return moment
+
+
+def _parse_float(raw: str | None) -> float:
+    if raw is None:
+        return 0.0
+    text = raw.strip().replace("$", "").replace(",", "")
+    if not text:
+        return 0.0
+    try:
+        return float(text)
+    except ValueError as exc:
+        raise DataError(f"could not parse {raw!r} as a number") from exc
+
+
+def read_green_button(
+    source: str | Path | IO[str], layout: GreenButtonLayout | None = None
+) -> list[IntervalReading]:
+    """Read interval readings from a CSV file or open stream."""
+    layout = layout or GreenButtonLayout()
+    if isinstance(source, str | Path):
+        with Path(source).open(encoding="utf-8-sig", newline="") as handle:
+            return list(_read(handle, layout))
+    return list(_read(source, layout))
+
+
+def _read(handle: IO[str], layout: GreenButtonLayout) -> Iterator[IntervalReading]:
+    reader = csv.DictReader(_skip_preamble(handle, layout))
+    header = reader.fieldnames
+    if not header:
+        raise DataError("CSV has no header row")
+    header = list(header)
+
+    date_col = _pick(header, layout.date, DATE_CANDIDATES)
+    time_col = _pick(header, layout.time, TIME_CANDIDATES)
+    start_col = _pick(header, layout.start, START_CANDIDATES)
+
+    # A split date/time pair wins over a single column, because PG&E names its
+    # time column "START TIME" -- which looks like a full timestamp column but
+    # holds "00:15". Falling back to a lone date column keeps files that put a
+    # complete ISO value under "date" working.
+    paired = date_col is not None and time_col is not None and date_col != time_col
+    if not paired:
+        start_col = start_col or date_col
+        date_col = time_col = None
+        if start_col is None:
+            raise DataError(
+                f"no timestamp column found in {header}; set GreenButtonLayout(start=...)"
+            )
+
+    import_col = _pick(header, layout.imported, IMPORT_CANDIDATES)
+    export_col = _pick(header, layout.exported, EXPORT_CANDIDATES)
+    net_col = _pick(header, layout.net, NET_CANDIDATES)
+
+    if import_col is None and export_col is None and net_col is None:
+        raise DataError(
+            f"no energy column found in {header}; set GreenButtonLayout(imported=...), "
+            f"(exported=...), or (net=...)"
+        )
+
+    rows: list[_Row] = []
+    for row in reader:
+        if date_col is not None and time_col is not None:
+            day = (row.get(date_col) or "").strip()
+            clock = (row.get(time_col) or "").strip()
+            if not day or not clock:
+                continue
+            start = _parse_timestamp(f"{day}T{clock}", layout.assume_tz)
+        else:
+            assert start_col is not None
+            if not (row.get(start_col) or "").strip():
+                continue
+            start = _parse_timestamp(row[start_col], layout.assume_tz)
+        if net_col is not None and import_col is None and export_col is None:
+            rows.append(_Row(start, None, None, _parse_float(row.get(net_col))))
+        else:
+            rows.append(
+                _Row(
+                    start,
+                    _parse_float(row.get(import_col) if import_col else None),
+                    _parse_float(row.get(export_col) if export_col else None),
+                    None,
+                )
+            )
+
+    if not rows:
+        raise DataError("CSV contained no data rows")
+
+    rows = _resolve_repeated_hour(rows)
+    duration = layout.duration or _infer_duration([r.start for r in rows])
+
+    for start, imported, exported, net in rows:
+        if net is not None:
+            yield IntervalReading.from_net(start, net, duration)
+        else:
+            yield IntervalReading(
+                start, imported=imported or 0.0, exported=exported or 0.0, duration=duration
+            )
+
+
+def _resolve_repeated_hour(rows: list[_Row]) -> list[_Row]:
+    """Mark the second pass through a repeated local hour as ``fold=1``.
+
+    A naive timestamp on the autumn transition is genuinely ambiguous: 01:00
+    happens twice. ``zoneinfo`` resolves both to ``fold=0``, the earlier PDT
+    hour, so an hour of readings prices at the wrong export rate -- PG&E labels
+    the repeat HS2 -- and coverage reports the file as overlapping itself.
+
+    Meter exports are chronological, which supplies the missing bit: if a row's
+    instant does not advance, and setting ``fold=1`` makes it advance, then it
+    belongs to the second pass. Rows carrying an explicit offset are already
+    unambiguous, and for them ``fold`` does not move the instant, so this is a
+    no-op rather than a special case.
+    """
+    resolved: list[_Row] = []
+    previous: datetime | None = None
+    for row in rows:
+        start = row.start
+        if previous is not None and start.astimezone(UTC) <= previous.astimezone(UTC):
+            shifted = start.replace(fold=1)
+            if shifted.astimezone(UTC) > previous.astimezone(UTC):
+                start = shifted
+        resolved.append(row._replace(start=start))
+        previous = start
+    return resolved
+
+
+def _infer_duration(timestamps: Sequence[datetime]) -> timedelta:
+    """Infer interval length from the two closest consecutive timestamps.
+
+    The minimum gap rather than the first gap, so a leading DST jump or a single
+    missing row does not stretch every interval in the file.
+    """
+    if len(timestamps) < 2:
+        return timedelta(hours=1)
+    starts = sorted(timestamps)
+    gaps = [later - earlier for earlier, later in pairwise(starts) if later > earlier]
+    if not gaps:
+        return timedelta(hours=1)
+    return min(gaps)