focus-data-toolkit 0.11.0__py3-none-any.whl

focus_data_toolkit/schema/registry.py

@@ -0,0 +1,127 @@
+"""Normative FOCUS column sets per (dataset, version).
+
+Everything here is *computed* from the committed FOCUS 1.4 model — each column records
+the ``version`` it was introduced in — plus a small hand-authored table of columns that
+existed in an earlier version but were **removed** by 1.4 (and so are absent from the 1.4
+model). This keeps the registry in lock-step with the model artifact while still being
+able to describe 1.2 and 1.3 headers faithfully.
+
+Source of truth for the removed-column table: the FOCUS changelog and this repository's own
+1.2/1.3 generators. ``ProviderName`` / ``PublisherName`` were superseded by
+``ServiceProviderName`` / ``HostProviderName`` in 1.3 and removed in 1.4.
+
+Sanity check (see ``version_columns``): Cost and Usage yields 57 columns at 1.2, 65 at 1.3
+and 65 at 1.4 — matching the generators (57/65) and the 1.4 model (65).
+"""
+
+from __future__ import annotations
+
+from functools import cache
+
+from focus_data_toolkit.model import FOCUS_1_4_DATASETS, load_model, resolve_dataset
+
+# FOCUS versions this toolkit reasons about for detection/conversion.
+SUPPORTED_VERSIONS: tuple[str, ...] = ("1.2", "1.3", "1.4")
+
+# dataset -> {column: (introduced_in, removed_in, mandatory_before)} for columns removed by
+# 1.4. ``mandatory_before`` is the version at which the column stopped being required because
+# a replacement arrived; a 1.2 Cost and Usage source needs ProviderName / PublisherName to
+# derive the 1.4-Mandatory ServiceProviderName / HostProviderName (which do not exist until
+# 1.3), so they are mandatory for versions < 1.3.
+REMOVED_COLUMNS: dict[str, dict[str, tuple[str, str, str]]] = {
+    "Cost and Usage": {
+        "ProviderName": ("0.5", "1.4", "1.3"),
+        "PublisherName": ("0.5", "1.4", "1.3"),
+    },
+}
+
+
+def version_tuple(version: str) -> tuple[int, int]:
+    """Parse a ``"major.minor"`` (or longer) FOCUS version to a comparable tuple."""
+    parts = version.strip().split(".")
+    try:
+        return (int(parts[0]), int(parts[1]))
+    except (IndexError, ValueError) as exc:  # pragma: no cover - defensive
+        raise ValueError(f"unparseable FOCUS version {version!r}") from exc
+
+
+def normalize_version(version: str) -> str:
+    """Normalise a version string to ``"major.minor"`` (e.g. ``"1.3.0"`` -> ``"1.3"``)."""
+    major, minor = version_tuple(version)
+    return f"{major}.{minor}"
+
+
+@cache
+def version_columns(dataset: str, version: str) -> frozenset[str]:
+    """FOCUS columns of ``dataset`` present at ``version``.
+
+    A model column is present when it was introduced at or before ``version``; a removed
+    column is present when it was introduced at or before ``version`` and removed strictly
+    after it.
+    """
+    cols = load_model()["datasets"][dataset]["columns"]
+    tv = version_tuple(version)
+    present = {c for c, spec in cols.items() if version_tuple(spec["version"]) <= tv}
+    for col, (intro, removed, _mandatory_before) in REMOVED_COLUMNS.get(dataset, {}).items():
+        if version_tuple(intro) <= tv < version_tuple(removed):
+            present.add(col)
+    return frozenset(present)
+
+
+@cache
+def mandatory_columns(dataset: str, version: str) -> frozenset[str]:
+    """Mandatory FOCUS columns of ``dataset`` present at ``version``.
+
+    Feature level is taken from the 1.4 model; removed columns that were required at
+    ``version`` (before a replacement arrived) are added so detection does not accept a source
+    that cannot fill the 1.4-Mandatory columns it derives.
+    """
+    cols = load_model()["datasets"][dataset]["columns"]
+    tv = version_tuple(version)
+    mandatory = {
+        c
+        for c, spec in cols.items()
+        if version_tuple(spec["version"]) <= tv and spec.get("feature_level") == "Mandatory"
+    }
+    for col, (intro, _removed, mandatory_before) in REMOVED_COLUMNS.get(dataset, {}).items():
+        if version_tuple(intro) <= tv < version_tuple(mandatory_before):
+            mandatory.add(col)
+    return frozenset(mandatory)
+
+
+@cache
+def dataset_exists_at(dataset: str, version: str) -> bool:
+    """Whether ``dataset`` is defined at all at ``version`` (has any column)."""
+    return bool(version_columns(dataset, version))
+
+
+@cache
+def all_dataset_columns(dataset: str) -> frozenset[str]:
+    """Every FOCUS column of ``dataset`` across all versions, including removed ones."""
+    cols = set(load_model()["datasets"][dataset]["columns"])
+    cols |= set(REMOVED_COLUMNS.get(dataset, {}))
+    return frozenset(cols)
+
+
+@cache
+def all_focus_columns() -> frozenset[str]:
+    """Every FOCUS column name across every dataset and version."""
+    out: set[str] = set()
+    for dataset in FOCUS_1_4_DATASETS:
+        out |= all_dataset_columns(dataset)
+    return frozenset(out)
+
+
+def candidate_schemas() -> list[tuple[str, str]]:
+    """All ``(dataset, version)`` pairs that actually exist, in canonical order."""
+    return [
+        (dataset, version)
+        for dataset in FOCUS_1_4_DATASETS
+        for version in SUPPORTED_VERSIONS
+        if dataset_exists_at(dataset, version)
+    ]
+
+
+def resolve_dataset_name(name: str) -> str:
+    """Resolve a dataset alias (``"cau"``, ``"cost-and-usage"``, ...) to its canonical name."""
+    return resolve_dataset(name.replace("-", " "))

focus_data_toolkit/storage/__init__.py

@@ -0,0 +1 @@
+"""Bounded-memory external state for streaming conversion (sqlite3, stdlib)."""

focus_data_toolkit/storage/external_index.py

@@ -0,0 +1,99 @@
+"""Disk-backed aggregation/dedup for streaming conversion, using stdlib ``sqlite3``.
+
+Streaming the huge Cost and Usage file still needs a little global state — the Invoice Detail
+sum per business grain and the distinct Billing Periods. Holding that in Python dicts would
+scale with the number of *groups*; here it lives in a throwaway SQLite database inside the
+atomic staging directory, so memory stays bounded.
+
+Exactness and determinism:
+
+* Costs are stored as **TEXT** and summed with Python ``Decimal`` during the ordered scan —
+  never ``SUM()`` in SQL — so the streamed sum is bit-for-bit the eager sum.
+* Every finalize scan is ``ORDER BY <keys>`` under the default **BINARY** collation, which
+  compares UTF-8 bytes and so matches Python ``sorted()`` on the same string tuples.
+
+The scratch database is disposable (``journal_mode=OFF``, ``synchronous=OFF``); durability
+comes from the atomic writer's fsync of the finished data files, not from this DB.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from collections.abc import Iterator
+from decimal import Decimal
+from pathlib import Path
+
+from focus_data_toolkit.convert.invoice_detail import GrainKey
+
+# Invoice Detail business-grain columns, in key order (matches GRAIN_FIELDS).
+_GRAIN_COLS = (
+    "issuer",
+    "invoice_id",
+    "account",
+    "currency",
+    "bp_start",
+    "bp_end",
+    "charge_category",
+)
+
+
+class ExternalIndex:
+    """SQLite-backed staging for Invoice Detail aggregation and Billing Period dedup."""
+
+    def __init__(self, db_path: str | Path) -> None:
+        self._conn = sqlite3.connect(str(db_path))
+        for pragma in ("journal_mode=OFF", "synchronous=OFF", "temp_store=FILE", "cache_size=-20000"):
+            self._conn.execute(f"PRAGMA {pragma}")
+        self._conn.execute(
+            "CREATE TABLE id_stage (n INTEGER PRIMARY KEY, "
+            + ", ".join(f"{col} TEXT" for col in _GRAIN_COLS)
+            + ", billed_cost TEXT)"
+        )
+        self._conn.execute(
+            "CREATE TABLE bp (start TEXT, end TEXT, issuer TEXT, "
+            "PRIMARY KEY (start, end, issuer)) WITHOUT ROWID"
+        )
+        self._insert_line = (
+            "INSERT INTO id_stage (" + ", ".join(_GRAIN_COLS) + ", billed_cost) VALUES ("
+            + ", ".join("?" * (len(_GRAIN_COLS) + 1)) + ")"
+        )
+
+    def stage_invoice_line(self, grain_key: GrainKey, billed_cost: str) -> None:
+        """Record one Cost and Usage line's contribution to its invoice-detail group."""
+        self._conn.execute(self._insert_line, (*grain_key, billed_cost))
+
+    def stage_billing_period(self, start: str, end: str, issuer: str) -> None:
+        """Record a (start, end, issuer) billing period (first occurrence wins)."""
+        self._conn.execute(
+            "INSERT OR IGNORE INTO bp (start, end, issuer) VALUES (?, ?, ?)", (start, end, issuer)
+        )
+
+    def finalize_invoice_groups(self) -> Iterator[tuple[GrainKey, Decimal]]:
+        """Yield ``(grain_key, summed_billed_cost)`` per group, in sorted grain order."""
+        self._conn.commit()
+        order = ", ".join(_GRAIN_COLS) + ", n"
+        cursor = self._conn.execute(
+            f"SELECT {', '.join(_GRAIN_COLS)}, billed_cost FROM id_stage ORDER BY {order}"
+        )
+        current: GrainKey | None = None
+        total = Decimal(0)
+        for row in cursor:
+            key: GrainKey = tuple(row[: len(_GRAIN_COLS)])
+            cost = row[len(_GRAIN_COLS)]
+            if current is None:
+                current = key
+            if key != current:
+                yield current, total
+                current = key
+                total = Decimal(0)
+            total += Decimal(cost or "0")
+        if current is not None:
+            yield current, total
+
+    def finalize_billing_periods(self) -> Iterator[tuple[str, str, str]]:
+        """Yield distinct ``(start, end, issuer)`` billing periods, in sorted order."""
+        self._conn.commit()
+        yield from self._conn.execute("SELECT start, end, issuer FROM bp ORDER BY start, end, issuer")
+
+    def close(self) -> None:
+        self._conn.close()

focus_data_toolkit/storage/spill.py

@@ -0,0 +1,150 @@
+"""Threshold-spilling string maps for streaming cross-dataset validation.
+
+The bundle validator's checks keep per-key state (seen ids, foreign-key targets, running
+sums) whose cardinality scales with the *large* Cost and Usage dataset. A
+:class:`SpillableIndexPool` hands out ``str -> str`` mutable mappings that live in an
+ordinary in-memory ``dict`` until a size threshold, then migrate transparently into a shared
+throwaway SQLite database — so validating a bundle far larger than RAM stays bounded.
+
+Like :mod:`focus_data_toolkit.storage.external_index`, the database is disposable scratch
+state (``journal_mode=OFF``, ``synchronous=OFF``): durability comes from the atomic writer's
+fsync of the published files, never from this DB. The file is created lazily on the first
+spill, so small bundles touch the disk not at all.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from collections.abc import Iterator, MutableMapping
+from pathlib import Path
+
+# Keys held in memory per map before spilling to SQLite. At ~100 bytes per key/value pair
+# this bounds each map's resident size to roughly 20 MB worst-case before it moves to disk.
+DEFAULT_SPILL_THRESHOLD = 200_000
+
+
+class SpillableIndexPool:
+    """Factory of :class:`SpillableMap` instances sharing one lazy scratch SQLite database."""
+
+    def __init__(self, db_path: str | Path, *, threshold: int = DEFAULT_SPILL_THRESHOLD) -> None:
+        if threshold < 1:
+            raise ValueError(f"spill threshold must be >= 1, got {threshold}")
+        self._db_path = Path(db_path)
+        self._threshold = threshold
+        self._conn: sqlite3.Connection | None = None
+        self._tables = 0
+        self._spilled = False
+
+    @property
+    def threshold(self) -> int:
+        return self._threshold
+
+    @property
+    def spilled(self) -> bool:
+        """Whether any map has ever spilled (i.e. the scratch database was created)."""
+        return self._spilled
+
+    def make_map(self) -> SpillableMap:
+        """Return a fresh empty ``str -> str`` mapping backed by this pool."""
+        self._tables += 1
+        return SpillableMap(self, f"kv{self._tables}")
+
+    def _connection(self) -> sqlite3.Connection:
+        if self._conn is None:
+            self._conn = sqlite3.connect(str(self._db_path))
+            self._spilled = True
+            for pragma in (
+                "journal_mode=OFF",
+                "synchronous=OFF",
+                "temp_store=FILE",
+                "cache_size=-20000",
+            ):
+                self._conn.execute(f"PRAGMA {pragma}")
+        return self._conn
+
+    def close(self) -> None:
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+
+class SpillableMap(MutableMapping[str, str]):
+    """A ``str -> str`` mapping that spills from a dict to the pool's SQLite past a threshold."""
+
+    def __init__(self, pool: SpillableIndexPool, table: str) -> None:
+        self._pool = pool
+        self._table = table
+        self._mem: dict[str, str] | None = {}
+
+    def _spill(self) -> None:
+        assert self._mem is not None
+        conn = self._pool._connection()
+        conn.execute(f"CREATE TABLE {self._table} (k TEXT PRIMARY KEY, v TEXT) WITHOUT ROWID")
+        conn.executemany(
+            f"INSERT INTO {self._table} (k, v) VALUES (?, ?)", self._mem.items()
+        )
+        self._mem = None
+
+    def __setitem__(self, key: str, value: str) -> None:
+        if self._mem is not None:
+            self._mem[key] = value
+            if len(self._mem) > self._pool.threshold:
+                self._spill()
+            return
+        self._pool._connection().execute(
+            f"INSERT OR REPLACE INTO {self._table} (k, v) VALUES (?, ?)", (key, value)
+        )
+
+    def __getitem__(self, key: str) -> str:
+        if self._mem is not None:
+            return self._mem[key]
+        row = (
+            self._pool._connection()
+            .execute(f"SELECT v FROM {self._table} WHERE k = ?", (key,))
+            .fetchone()
+        )
+        if row is None:
+            raise KeyError(key)
+        return row[0]
+
+    def __delitem__(self, key: str) -> None:
+        if self._mem is not None:
+            del self._mem[key]
+            return
+        cursor = self._pool._connection().execute(
+            f"DELETE FROM {self._table} WHERE k = ?", (key,)
+        )
+        if cursor.rowcount == 0:
+            raise KeyError(key)
+
+    def __contains__(self, key: object) -> bool:
+        if self._mem is not None:
+            return key in self._mem
+        if not isinstance(key, str):
+            return False
+        row = (
+            self._pool._connection()
+            .execute(f"SELECT 1 FROM {self._table} WHERE k = ?", (key,))
+            .fetchone()
+        )
+        return row is not None
+
+    def __len__(self) -> int:
+        if self._mem is not None:
+            return len(self._mem)
+        return self._pool._connection().execute(
+            f"SELECT COUNT(*) FROM {self._table}"
+        ).fetchone()[0]
+
+    def __iter__(self) -> Iterator[str]:
+        if self._mem is not None:
+            yield from self._mem
+            return
+        # Sorted (BINARY collation = UTF-8 byte order) for a deterministic iteration order.
+        for (key,) in self._pool._connection().execute(
+            f"SELECT k FROM {self._table} ORDER BY k"
+        ):
+            yield key
+
+
+__all__ = ["DEFAULT_SPILL_THRESHOLD", "SpillableIndexPool", "SpillableMap"]

focus_data_toolkit/studio/__init__.py

@@ -0,0 +1,19 @@
+"""Focus Data Toolkit Studio — a LOCAL web UI over the same Core.
+
+The Studio never reimplements FOCUS logic: it drives the exact SDK the CLI and Runner use
+(``detect_focus_schema``, ``convert_files``, ``validate_dataset_bundle``, the generators), so its
+manifests, diagnostics and checksums are identical to a CLI run. It is designed for **local,
+single-user** use: it binds to loopback by default, requires a per-start token, validates
+Host/Origin headers, confines file access to an allowlisted root, and processes on the bounded
+streaming path — data never leaves the machine.
+
+This subpackage lives behind the optional ``[studio]`` extra (FastAPI + uvicorn). The CLI's
+``focus-toolkit ui`` command imports it lazily, so a core install without the extra still works.
+"""
+
+from __future__ import annotations
+
+from focus_data_toolkit.studio.config import StudioConfig
+from focus_data_toolkit.studio.server import run
+
+__all__ = ["StudioConfig", "run"]