mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/governors.py

@@ -0,0 +1,1163 @@
+"""The caps that bound what one tenant can spend, and the window they are counted over.
+
+Cloud Run jobs are per-execution and scale to zero, so there is no autoscaler to build here.
+Scale is free; caps are the work. What needs building is the thing that stops a single tenant
+from spending without bound — a monthly run allowance, a limit on how many runs may be in flight at
+once, and a monthly spend ceiling.
+
+A cap breach is a normal, expected, quiet outcome, not a defect. It is the product working:
+:class:`CapExceeded` names the cap that was reached and the numbers behind it, in the same idiom
+as "no new run is not an error". Nothing here logs, retries, or escalates.
+
+Two couplings in this module are invisible from either side alone:
+
+* :func:`utc_calendar_month_bounds` must agree with ``mostlyright-cloud/dashboard/lib/usage.ts``,
+  which counts usage over the current UTC calendar month.
+* :class:`UsageRecord` carries an opaque 16-hex tenant coordinate, never a credential. Its shape is
+  cloud's ``hashKeyId`` shape — the first 8 bytes of a SHA-256 digest, hex-encoded — and the shape
+  is all this module knows: **what the coordinate stands for is the caller's choice, and this
+  module cannot tell one choice from another.** On the hosted admission path, the only production
+  caller, the subject is the job's ``workspace_id`` (``hosted_bootstrap._tenant_coordinate``), NOT
+  an API key, so these counters are not cloud's per-key billing ledger and do not join against
+  ``usage_events.key_id`` row for row. A caller declares its subject through
+  :class:`CapLedger`'s ``subject`` argument, which is what a refusal a customer reads then names.
+
+Both are stated in prose on the definitions below, because nothing in either language checks them.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import os
+import re
+import tempfile
+import threading
+from collections.abc import Iterator, Mapping
+from contextlib import AbstractContextManager, contextmanager
+from dataclasses import dataclass, replace
+from datetime import UTC, datetime
+from pathlib import Path
+from types import TracebackType
+from typing import Any, Protocol
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    parse_canonical_json,
+)
+
+# Policy maxima: the most any caller may ever ask for. These are ceilings, NOT defaults — the
+# defaults below sit well under them, and a caller may tighten further but never widen past these.
+# They exist so that a mistaken or hostile configuration cannot raise a cap to infinity: the
+# ceiling is enforced in ``CapPolicy.__post_init__``, where no caller can reach around it.
+MAX_RUNS_PER_MONTH = 10_000
+MAX_CONCURRENT_RUNS = 16
+MAX_MONTHLY_COST_UNITS = 1_000_000
+
+#: Conservative starting caps. A new tenant gets these until someone deliberately chooses
+#: otherwise.
+DEFAULT_RUNS_PER_MONTH = 100
+DEFAULT_CONCURRENT_RUNS = 2
+DEFAULT_MONTHLY_COST_UNITS = 10_000
+
+#: One cost unit is one whole cent of billable spend. Integer units only: money is never counted
+#: in floats, because two runs that each cost 0.1 must sum to exactly 0.2 or the ledger lies.
+COST_UNIT_DESCRIPTION = "one cent of billable spend"
+
+#: The only shape a tenant coordinate may take here: cloud's ``hashKeyId`` shape, the first 8 bytes
+#: of a SHA-256 digest, hex-encoded — 16 lowercase hex characters. Enforcing the shape is what makes
+#: it impossible to pass a raw credential where an identifier belongs: no credential this product
+#: mints is 16 lowercase hex characters, so a mistaken raw key is refused before it can be recorded.
+#: What the shape CANNOT do is say what was hashed. A workspace hash and an API key id are the same
+#: 16 characters here, which is why the subject is declared and never inferred — see
+#: :data:`CAP_SUBJECTS`.
+KEY_ID_PATTERN = re.compile(r"\A[0-9a-f]{16}\Z")
+
+#: The only accepted persisted cap-store schema. Other schema labels are refused.
+CAP_STORE_SCHEMA = "mostlyright-cap-store.v1"
+
+#: The sidecar a file-backed store takes its writer lock on, appended to the store's own name. It
+#: is a separate file because the store file's inode is replaced on every write and a lock on a
+#: replaced inode excludes nobody.
+LOCK_FILE_SUFFIX = ".lock"
+
+#: What a refusal may call the thing whose caps were counted. A ledger counts by an opaque
+#: coordinate (:data:`KEY_ID_PATTERN`) and cannot recover what was hashed into it, so the caller
+#: declares the subject and the sentence a customer reads names the right thing. ``"workspace"``
+#: leads because the hosted admission path is the only production caller and files a workspace
+#: coordinate; ``"key"`` is here for a caller that files genuine API key ids, and none does today.
+CAP_SUBJECTS = ("workspace", "key")
+
+#: What a refusal may call the work item that was counted. The subject above says WHOSE work was
+#: counted; this says WHAT was counted, and neither can be inferred from a 16-hex coordinate — the
+#: counters have exactly the same shape either way. ``"run"`` leads and is the default, because the
+#: build-side admission path is what these counters were written for and every existing refusal is
+#: worded for it. ``"read"`` exists because the serving surface counts data reads, and a customer
+#: refused a read who is told they have used their monthly RUN allowance is being sent to a
+#: different product surface to look for a number that will not be there. That is the same failure
+#: :data:`CAP_SUBJECTS` exists to prevent, one noun over. Both nouns end up in a sentence a customer
+#: reads, so both are bound by ``docs/VOCABULARY.md`` the way every other customer-facing term is:
+#: this is a closed set for the same reason that file is a table and not a suggestion.
+CAP_WORK_UNITS = ("run", "read")
+
+#: The refusal sentences, per work unit, keyed by the ``cap_name`` each one belongs to. Both
+#: wordings live here side by side, in one place, so that neither can be edited without the other
+#: being on screen — the alternative, re-wording at the calling edge, is the second wording of one
+#: fact that ``deploy.plan_go_live`` refuses to create. The ``cap_name`` keys are machine handles
+#: that callers branch on and are the same for every work unit; only the prose differs.
+_REFUSAL_SENTENCES: Mapping[str, Mapping[str, str]] = {
+    "run": {
+        "runs_per_month": (
+            "this {subject} has used its monthly run allowance: "
+            "{observed} runs this month against a limit of {limit}"
+        ),
+        "concurrent_runs": (
+            "this {subject} is already running as much at once as it is allowed: "
+            "{observed} runs at once against a limit of {limit}"
+        ),
+        "monthly_cost_units": (
+            "this run would pass the monthly spend limit for this {subject}: "
+            "{observed} units this month against a limit of {limit}"
+        ),
+    },
+    "read": {
+        "runs_per_month": (
+            "this {subject} has used its monthly read allowance: "
+            "{observed} reads this month against a limit of {limit}"
+        ),
+        "concurrent_runs": (
+            "this {subject} is already serving as much at once as it is allowed: "
+            "{observed} reads at once against a limit of {limit}"
+        ),
+        "monthly_cost_units": (
+            "this read would pass the monthly spend limit for this {subject}: "
+            "{observed} units this month against a limit of {limit}"
+        ),
+    },
+}
+
+
+@dataclass(frozen=True)
+class CapPolicy:
+    """Trusted governor caps; callers may tighten but never raise policy maxima.
+
+    Same shape and same promise as :class:`pipeline.ResourceLimits`: a frozen dataclass whose
+    ``__post_init__`` refuses anything outside ``1..ceiling``, naming the field it refused. The
+    type check is ``type(value) is not int`` rather than ``isinstance`` on purpose — ``bool`` is a
+    subclass of ``int``, and ``max_concurrent_runs=True`` is a configuration mistake, not a cap.
+
+    ``max_runs_per_month`` and ``max_monthly_cost_units`` are counted from the durable store, so
+    they bind across processes wherever one is configured. ``max_concurrent_runs`` does not, and
+    the reason is structural: in-flight slots are held in memory and are deliberately never
+    persisted (:class:`CapStore`), so a ledger can only see what its own process holds. Under a
+    topology where each run is its own process — a Cloud Run job execution, which is where this
+    product's admission gate runs — every ledger sees one run in flight, and a limit of at least
+    one can never be reached. Treat this field as a per-process limit, not a platform-wide limit.
+    """
+
+    max_runs_per_month: int = DEFAULT_RUNS_PER_MONTH
+    max_concurrent_runs: int = DEFAULT_CONCURRENT_RUNS
+    max_monthly_cost_units: int = DEFAULT_MONTHLY_COST_UNITS
+
+    def __post_init__(self) -> None:
+        values = {
+            "max_runs_per_month": (self.max_runs_per_month, MAX_RUNS_PER_MONTH),
+            "max_concurrent_runs": (self.max_concurrent_runs, MAX_CONCURRENT_RUNS),
+            "max_monthly_cost_units": (self.max_monthly_cost_units, MAX_MONTHLY_COST_UNITS),
+        }
+        for name, (value, maximum) in values.items():
+            if type(value) is not int or value < 1 or value > maximum:
+                raise ValueError(f"{name} must be between 1 and {maximum}")
+
+    def tighten(
+        self,
+        *,
+        max_runs_per_month: int | None = None,
+        max_concurrent_runs: int | None = None,
+        max_monthly_cost_units: int | None = None,
+    ) -> CapPolicy:
+        """Return a policy no wider than this one; refuse any override that widens a field.
+
+        Tightening is the only direction a caller may move a cap. An override equal to the current
+        value is accepted (it is not a widening); an override above it raises, naming the field and
+        the value it would have exceeded.
+        """
+
+        overrides = {
+            "max_runs_per_month": max_runs_per_month,
+            "max_concurrent_runs": max_concurrent_runs,
+            "max_monthly_cost_units": max_monthly_cost_units,
+        }
+        chosen: dict[str, int] = {}
+        for name, value in overrides.items():
+            if value is None:
+                continue
+            current: int = getattr(self, name)
+            if type(value) is not int or value < 1:
+                raise ValueError(f"{name} must be an integer of at least 1")
+            if value > current:
+                raise ValueError(f"{name} may be tightened but never raised above {current}")
+            chosen[name] = value
+        return replace(self, **chosen)
+
+
+def utc_calendar_month_bounds(now: datetime) -> tuple[datetime, datetime]:
+    """Return the first instant of ``now``'s UTC month and the first instant of the next one.
+
+    The coupling this function exists to hold: ``mostlyright-cloud/dashboard/lib/usage.ts`` counts
+    a key's usage over the current UTC calendar month — never a rolling 30-day window, never the
+    server's local zone — so that the same organization gets the same answer queried from anywhere.
+    This function must produce that same window. If it drifts, nothing raises on either side: the
+    harness and the dashboard simply report different numbers for the same key, and the first
+    person to notice is a customer disputing a bill. The other half of this contract lives in
+    ``mostlyright-cloud/dashboard/lib/usage.ts``; change neither without changing both.
+
+    The returned bounds are half-open: ``start <= instant < end``. ``now`` must be timezone-aware,
+    so that a local time can never be silently read as UTC and shift the window by up to a day.
+    """
+
+    if not isinstance(now, datetime):
+        raise ValueError("a usage window is computed from a datetime")
+    if now.tzinfo is None or now.tzinfo.utcoffset(now) is None:
+        raise ValueError("a usage window needs a timezone-aware datetime, not a local time")
+
+    instant = now.astimezone(UTC)
+    start = datetime(instant.year, instant.month, 1, tzinfo=UTC)
+    if instant.month == 12:
+        end = datetime(instant.year + 1, 1, 1, tzinfo=UTC)
+    else:
+        end = datetime(instant.year, instant.month + 1, 1, tzinfo=UTC)
+    return start, end
+
+
+class CapExceeded(RuntimeError):
+    """A cap was reached, so the run was not started. This is the product working.
+
+    A cap breach is an expected operating condition. Report it without retrying or paging.
+
+    The message is plain language, safe to show the person holding the key. It names the cap that
+    was reached and the two numbers behind it, and nothing else: no key, no internal identifier.
+    """
+
+    def __init__(self, cap_name: str, *, observed: int, limit: int, message: str) -> None:
+        super().__init__(message)
+        self.cap_name = cap_name
+        self.observed = observed
+        self.limit = limit
+
+
+class CapLedgerError(RuntimeError):
+    """A reservation was used wrongly — committed twice, released twice, or left open.
+
+    Unlike :class:`CapExceeded`, this one IS a defect: it means calling code lost track of a slot
+    it was holding. It is raised loudly so that the leak is found in a test rather than in a month
+    of quietly shrinking capacity.
+    """
+
+
+class CapStoreError(RuntimeError):
+    """The durable counters could not be trusted, so nothing may be decided from them.
+
+    A store that cannot be read is not an empty store. Treating an unreadable, truncated, or
+    non-canonical file as "no usage yet" would hand every key a fresh monthly allowance at the
+    moment the file broke, which is the failure a cap exists to prevent. Callers on an admission
+    path must turn this into a refusal, never into an admission.
+    """
+
+
+def _checked_key_id(value: Any) -> str:
+    if not isinstance(value, str) or KEY_ID_PATTERN.match(value) is None:
+        raise ValueError(
+            "a key id must be 16 lowercase hex characters, the value cloud's hashKeyId returns, "
+            "and never a raw key"
+        )
+    return value
+
+
+def _checked_units(value: Any, name: str) -> int:
+    if type(value) is not int or value < 0:
+        raise ValueError(f"{name} must be a whole number of cost units, zero or more")
+    return value
+
+
+@dataclass(frozen=True)
+class UsageRecord:
+    """What one tenant used in one window: a coordinate, never a credential.
+
+    ``key_id`` is a 16-lowercase-hex tenant coordinate, and the field name is historical rather
+    than descriptive — it is kept because it is inside the persisted :data:`CAP_STORE_SCHEMA` and
+    renaming it would orphan every store already written. Its derivation is the one
+    :func:`key_seam.hash_key_id` uses — the first 8 bytes of a SHA-256 digest, hex-encoded, which
+    is also the shape cloud stores in ``usage_events.key_id`` — but **the shape is not the
+    subject.** What was hashed is the caller's choice and nothing here can recover it: the shape
+    check in :data:`KEY_ID_PATTERN` accepts a workspace hash and an API key id equally, which is
+    exactly why the subject is stated in prose and declared by :class:`CapLedger`'s ``subject``.
+
+    On the hosted admission path — the only production writer — the subject is the job's
+    ``workspace_id`` (``hosted_bootstrap._tenant_coordinate``), so the records in a production cap
+    store are per WORKSPACE. They are a spend control, not the billing attribution record, and they
+    do not line up with cloud's ``usage_events.key_id`` rows, which are per API key. Comparing a
+    number here with the cloud dashboard's per-key usage page holds only for a caller that files
+    genuine key ids, and no caller in this repository does.
+
+    A raw credential must never be stored, logged, or serialized here. ``key_seam`` is deliberately
+    NOT imported for this: the two modules stay independent, and :data:`KEY_ID_PATTERN` refuses a
+    raw key outright because no credential this product mints is 16 lowercase hex characters.
+
+    ``window_start`` is the first instant of the record's UTC calendar month, from
+    :func:`utc_calendar_month_bounds`, which is the same window
+    ``mostlyright-cloud/dashboard/lib/usage.ts`` counts over. The window matches; whether the
+    subjects match is the paragraph above.
+
+    ``runs`` and ``cost_units`` mean different things in the two places this record appears, and
+    the difference is stated rather than left to be inferred. Returned by :meth:`CapLedger.usage`
+    they are COMMITTED: work that finished. Read from or written to a :class:`CapStore` they are
+    CHARGED: every run admitted this month, including runs still in flight, because that is the
+    number an admission decision has to be made against.
+    """
+
+    key_id: str
+    window_start: datetime
+    runs: int
+    cost_units: int
+
+    def __post_init__(self) -> None:
+        _checked_key_id(self.key_id)
+        if not isinstance(self.window_start, datetime) or self.window_start.tzinfo is None:
+            raise ValueError("window_start must be a timezone-aware datetime")
+        _checked_units(self.runs, "runs")
+        _checked_units(self.cost_units, "cost_units")
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return the record as plain data. Contains the key id; never contains a key."""
+
+        return {
+            "key_id": self.key_id,
+            "window_start": self.window_start.astimezone(UTC).isoformat(),
+            "runs": self.runs,
+            "cost_units": self.cost_units,
+        }
+
+
+class CapStore(Protocol):
+    """Durable backing for a ledger's charged counters, keyed by key id.
+
+    What crosses this seam is a key's CHARGED total for the month: every run that has been admitted,
+    whether or not it has finished. What does not cross it is the concurrency slot — the fact that
+    some run is in flight right now. The two are separated on purpose, because they fail in
+    opposite directions:
+
+    * A monthly charge that outlives the process that made it is CONSERVATIVE. An execution that
+      dies between admission and its final accounting has still consumed a run, and leaving that
+      run counted is the safe reading. The charge is also self-limiting: it expires with the month.
+    * A concurrency slot that outlives its process is UNSAFE in the other direction. Nothing would
+      be left to give it back, so a crashed run would consume capacity forever.
+
+    So charges persist and slots do not, and the honest consequence of the second half is stated on
+    :attr:`CapPolicy.max_concurrent_runs` and in :class:`CapLedger`: concurrency is bounded inside
+    one process, never across processes.
+
+    Charging at ADMISSION rather than at completion is what makes a monthly allowance mean anything
+    when executions overlap. A ledger that decided from the store and only wrote when the run
+    finished would admit every execution that started before the first one finished: each reads the
+    same under-limit number, each is admitted, and the file afterwards records more runs than the
+    limit allows. Reserve the worst case, admit only if it fits, correct the number afterwards.
+
+    :meth:`load` must raise :class:`CapStoreError` rather than return empty counters whenever the
+    stored state cannot be trusted.
+
+    :meth:`lock` is not optional and not an optimization. A ledger's commit is a read-modify-write
+    over shared state, and the deployment this store exists for — every execution of one Cloud Run
+    job pointed at one shared path — runs that sequence from several processes at once. Without a
+    lock that excludes other WRITERS, not merely other threads, two executions each read the same
+    counters, each add their own run, and the second write erases the first: the run allowance
+    silently stops counting, which is the one thing this module exists to do. The lock must be held
+    across the whole load-mutate-save sequence, so it is a context manager rather than a pair of
+    calls.
+    """
+
+    def load(self) -> dict[str, UsageRecord]: ...
+
+    def save(self, records: Mapping[str, UsageRecord]) -> None: ...
+
+    def lock(self) -> AbstractContextManager[None]: ...
+
+
+class FileCapStore:
+    """A cap store backed by one file, written whole and replaced atomically.
+
+    The file is serialized with :func:`canonical.canonical_json_bytes` and read back with
+    :func:`canonical.parse_canonical_json`, so a file whose bytes are not the canonical
+    representation of their own content — a hand-edit, a partial write from a different writer, a
+    re-serialization by another tool — is refused rather than acted on.
+
+    Writing goes to a sibling temporary file in the same directory and then :func:`os.replace`,
+    which is atomic within a filesystem: a concurrent reader observes either the whole previous
+    state or the whole new one, never a half-written ledger. The temporary file is a sibling, not a
+    file in the system temporary directory, because a rename across filesystems is not atomic.
+
+    An atomic write is not enough on its own, and the difference is the whole reason :meth:`lock`
+    exists. :func:`os.replace` makes one write indivisible; it does nothing for a read-modify-write
+    performed by two processes at once, where both read the same counters and the second write
+    replaces rather than extends the first. :meth:`lock` takes an exclusive :func:`fcntl.flock` on a
+    SIDECAR file — ``<store>.lock`` — and the sidecar is load-bearing: this store replaces its own
+    file on every write, so a lock taken on the store file itself would be held on an inode that the
+    next writer no longer opens. The sidecar is created once and never replaced.
+
+    **What the storage has to be.** Both mechanisms are POSIX filesystem behaviours, so the shared
+    volume this store is pointed at has to be a POSIX filesystem that honours them — a Filestore or
+    other NFS volume with locking, or a persistent disk. A Cloud Storage bucket mounted with gcsfuse
+    is NOT such a volume: it implements neither an atomic rename nor advisory locking, so both the
+    ``os.replace`` reasoning above and the exclusion below are void there. That is a deployment
+    requirement, stated here because nothing in this file can detect the difference at runtime.
+
+    :meth:`save` on its own takes no lock: it is the whole-file write half of a sequence whose
+    caller (:class:`CapLedger`) holds the lock across the load and the mutation too. Calling it
+    directly is a deliberate act — seeding an empty store before any execution is pointed at it —
+    and there is nothing to exclude at that moment.
+
+    A path that does not exist is refused unless the caller constructs the store with
+    ``create_if_missing=True``, which is how a new store is deliberately seeded::
+
+        FileCapStore(path, create_if_missing=True).save({})
+
+    The default is the strict one on purpose. On an admission path, "the counter file is gone" and
+    "this tenant has used nothing this month" must not be the same answer: a store that resets
+    itself when its file disappears gives an unbounded allowance to whoever can delete it.
+    """
+
+    def __init__(self, path: str | os.PathLike[str], *, create_if_missing: bool = False) -> None:
+        self._path = Path(path)
+        self._create_if_missing = bool(create_if_missing)
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    @property
+    def lock_path(self) -> Path:
+        """The sidecar this store's writer lock is taken on. Never replaced, never read."""
+
+        return self._path.with_name(self._path.name + LOCK_FILE_SUFFIX)
+
+    @contextmanager
+    def lock(self) -> Iterator[None]:
+        """Hold this store against every other writer for one read-modify-write.
+
+        Exclusion is by :func:`fcntl.flock` on :attr:`lock_path`, which excludes other PROCESSES —
+        the thing a :class:`threading.Lock` cannot do and the thing a shared cap store needs, since
+        every execution of a Cloud Run job is its own process. The lock is advisory, so it binds
+        only writers that take it; every writer inside this product does, through
+        :class:`CapLedger`.
+
+        The wait is unbounded on purpose. The critical section is one small read, one arithmetic
+        step and one write, and a timeout here would have to choose between refusing a run that is
+        within its allowance and admitting one that is not. A holder that dies releases the lock:
+        the kernel drops it when the descriptor closes.
+        """
+
+        try:
+            descriptor = os.open(self.lock_path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600)
+        except OSError as error:
+            raise CapStoreError("the cap store could not be locked for writing") from error
+        try:
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX)
+            except OSError as error:
+                raise CapStoreError("the cap store could not be locked for writing") from error
+            try:
+                yield
+            finally:
+                fcntl.flock(descriptor, fcntl.LOCK_UN)
+        finally:
+            os.close(descriptor)
+
+    def load(self) -> dict[str, UsageRecord]:
+        """Return the persisted counters, or refuse if they cannot be trusted."""
+
+        try:
+            raw = self._path.read_bytes()
+        except FileNotFoundError as error:
+            if self._create_if_missing:
+                return {}
+            raise CapStoreError(
+                "the cap store file does not exist, so no allowance can be counted from it"
+            ) from error
+        except OSError as error:
+            raise CapStoreError("the cap store file could not be read") from error
+
+        try:
+            value = parse_canonical_json(raw)
+        except CanonicalJSONError as error:
+            raise CapStoreError("the cap store file is not canonical JSON") from error
+        return self._records(value)
+
+    def save(self, records: Mapping[str, UsageRecord]) -> None:
+        """Replace the stored counters with ``records`` in one atomic step."""
+
+        payload = {
+            "schema_version": CAP_STORE_SCHEMA,
+            "keys": {
+                _checked_key_id(key): self._checked_record(key, record).to_dict()
+                for key, record in records.items()
+            },
+        }
+        raw = canonical_json_bytes(payload)
+        directory = self._path.parent
+        try:
+            descriptor, temporary = tempfile.mkstemp(
+                dir=str(directory),
+                prefix=f".{self._path.name}.",
+                suffix=".partial",
+            )
+        except OSError as error:
+            raise CapStoreError("the cap store file could not be written") from error
+        try:
+            with os.fdopen(descriptor, "wb") as handle:
+                handle.write(raw)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, self._path)
+        except OSError as error:
+            Path(temporary).unlink(missing_ok=True)
+            raise CapStoreError("the cap store file could not be written") from error
+
+    def _records(self, value: Any) -> dict[str, UsageRecord]:
+        if not isinstance(value, dict) or set(value) != {"schema_version", "keys"}:
+            raise CapStoreError("the cap store file is not a cap store")
+        if value["schema_version"] != CAP_STORE_SCHEMA:
+            raise CapStoreError("the cap store file was written by a different store version")
+        keys = value["keys"]
+        if not isinstance(keys, dict):
+            raise CapStoreError("the cap store file does not hold per-key counters")
+
+        records: dict[str, UsageRecord] = {}
+        for key, item in keys.items():
+            if not isinstance(item, dict) or set(item) != {
+                "key_id",
+                "window_start",
+                "runs",
+                "cost_units",
+            }:
+                raise CapStoreError("a cap store entry does not have the exact counter fields")
+            try:
+                record = UsageRecord(
+                    key_id=item["key_id"],
+                    window_start=_stored_window_start(item["window_start"]),
+                    runs=item["runs"],
+                    cost_units=item["cost_units"],
+                )
+            except ValueError as error:
+                raise CapStoreError("a cap store entry is not a usable usage record") from error
+            records[key] = self._checked_record(key, record)
+        return records
+
+    @staticmethod
+    def _checked_record(key: Any, record: Any) -> UsageRecord:
+        if not isinstance(record, UsageRecord) or record.key_id != key:
+            raise CapStoreError("a cap store entry is filed under a different key id than it names")
+        return record
+
+
+def _stored_window_start(value: Any) -> datetime:
+    if not isinstance(value, str):
+        raise CapStoreError("a stored window start must be text")
+    try:
+        parsed = datetime.fromisoformat(value)
+    except ValueError as error:
+        raise CapStoreError("a stored window start is not an ISO 8601 instant") from error
+    if parsed.tzinfo is None or parsed.tzinfo.utcoffset(parsed) is None:
+        raise CapStoreError("a stored window start must carry a timezone")
+    start, _ = utc_calendar_month_bounds(parsed)
+    if parsed.astimezone(UTC) != start:
+        raise CapStoreError("a stored window start is not the first instant of a UTC month")
+    return start
+
+
+class Reservation:
+    """A held slot: one run's claim on a key's allowance, until it is committed or released.
+
+    Use it as a context manager. Leaving the block with an exception releases the slot, so a run
+    that raises can never leak capacity; leaving it without committing is a defect and raises
+    :class:`CapLedgerError` after releasing the slot, so the mistake is loud but not expensive.
+    """
+
+    __slots__ = (
+        "_ledger",
+        "_settled",
+        "estimated_cost_units",
+        "key_id",
+        "reservation_id",
+        "window_start",
+    )
+
+    def __init__(
+        self,
+        ledger: CapLedger,
+        *,
+        reservation_id: int,
+        key_id: str,
+        window_start: datetime,
+        estimated_cost_units: int,
+    ) -> None:
+        self._ledger = ledger
+        self.reservation_id = reservation_id
+        self.key_id = key_id
+        self.window_start = window_start
+        self.estimated_cost_units = estimated_cost_units
+        self._settled = False
+
+    @property
+    def settled(self) -> bool:
+        """True once this reservation has been committed or released."""
+
+        return self._settled
+
+    def commit(self, *, actual_cost_units: int) -> None:
+        """Record the run and what it actually cost, then give the slot back."""
+
+        self._ledger.commit(self, actual_cost_units=actual_cost_units)
+
+    def release(self) -> None:
+        """Give the slot back without recording a run."""
+
+        self._ledger.release(self)
+
+    def __enter__(self) -> Reservation:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> bool:
+        if self._settled:
+            return False
+        self._ledger.release(self)
+        if exc_type is None:
+            raise CapLedgerError(
+                "a reservation must be committed before its block ends; "
+                "the slot has been released and no run was recorded"
+            )
+        return False
+
+
+@dataclass
+class _KeyWindow:
+    """One key's charged counters for one UTC month, plus the runs it has in flight now.
+
+    ``runs`` and ``cost_units`` count what has been ADMITTED, not what has finished: a run is
+    charged the moment its reservation is granted (see :class:`CapLedger`). ``reservations`` holds
+    the subset of that charge belonging to runs this process still has open.
+    """
+
+    window_start: datetime
+    runs: int = 0
+    cost_units: int = 0
+    reservations: dict[int, Reservation] = None  # type: ignore[assignment]
+
+    def __post_init__(self) -> None:
+        if self.reservations is None:
+            self.reservations = {}
+
+
+class CapLedger:
+    """Concurrency-safe reservation and final accounting for one tenant's caps.
+
+    Modeled on :class:`agent_runtime.BudgetLedger`: reserve first against the worst case, do the
+    work, then account for what actually happened. Every counter read and every counter write
+    happens under one lock, so two threads racing to take the last slot cannot both win.
+
+    Counters are per coordinate and per UTC calendar month. When a reserve arrives whose ``now``
+    falls in a later month than the stored window, the monthly counters roll over rather than
+    accumulating across months. Runs already in flight are carried across that roll, because
+    concurrency is an instantaneous limit and not a monthly allowance — a run that started in
+    August still occupies a slot at one second past midnight in September. The roll only ever goes
+    forward: a ``now`` that falls in an EARLIER month than the window already adopted is decided and
+    charged against that adopted window, because zeroing a month whose runs are already committed
+    would hand its allowance out twice (:meth:`_window_locked`).
+
+    Counters live in memory unless a ``store`` is supplied. With one, every decision and every
+    mutation is a read-modify-write performed under the store's own inter-process lock: the
+    persisted counters are re-read inside that lock, the decision or the mutation is made on what
+    was just read, and the result is written back before the lock is dropped. That is what makes
+    the monthly allowance hold across the executions of one Cloud Run job pointed at one shared
+    path. A ledger that read its counters once at construction and wrote the whole file back from
+    that snapshot would lose every commit another execution made in between — including commits for
+    OTHER tenants, whose counters live in the same file — so the snapshot is deliberately never the
+    thing written from.
+
+    Reading under the lock is necessary and is not sufficient, and the difference is where an
+    allowance is actually won or lost. The run is charged at ADMISSION — inside the same lock that
+    read the counters the admission was decided from — not when it finishes. Deciding under the
+    lock but writing at completion still admits every execution that starts before the first one
+    ends: each takes the lock in turn, each reads the same under-limit number because no charge has
+    landed yet, and each is admitted. The overshoot is exactly the number of executions running at
+    once, silently, with the file afterwards recording more runs than the limit allows.
+
+    So ``runs`` and ``cost_units`` on a window, and in the store, are CHARGED totals: every run
+    admitted this month, finished or not. :meth:`commit` moves no run count — the run was already
+    counted — and only replaces the held estimate with the real spend. :meth:`release` refunds the
+    charge, which is what keeps a fault between admission and exec from spending a month's
+    allowance. :meth:`usage` reports committed work, subtracting this ledger's open reservations.
+
+    A process that dies between admission and either outcome leaves its run charged. That is the
+    intended direction: the run was admitted, and a charge that outlives its process expires with
+    the month rather than lasting forever the way a leaked concurrency slot would.
+
+    Without a store, this ledger counts only what this process did: that bounds concurrency within
+    a process but cannot enforce a monthly allowance across processes, and a caller relying on it
+    has to say so plainly rather than imply an enforcement that is not there.
+
+    Concurrency is the cap this class cannot make durable, and the reason is structural rather than
+    unfinished: in-flight reservations are deliberately not persisted (:class:`CapStore`), because a
+    process that dies holding a slot would otherwise consume it forever. ``max_concurrent_runs``
+    therefore bounds what one PROCESS holds at once. In a topology where each run is its own
+    process, it does not bind across requests.
+
+    **What a coordinate stands for, and why a refusal has to be told.** The ``key_id`` argument on
+    every method here is a 16-lowercase-hex coordinate whose SHAPE is checked and whose SUBJECT is
+    not: a workspace hash and an API key id are indistinguishable to :data:`KEY_ID_PATTERN`. The
+    hosted admission path files a WORKSPACE — ``hosted_bootstrap._tenant_coordinate`` hashes the
+    validated job's ``workspace_id``, and the customer API key never reaches that process at all —
+    so the argument's name is historical and does not describe what production puts in it. Because
+    a :class:`CapExceeded` message is read by a customer, it must name what was actually counted,
+    and this class cannot work that out from the coordinate. The caller declares it with
+    ``subject``, one of :data:`CAP_SUBJECTS`, and every refusal below says "this ``subject``".
+    Pointing a refused customer at a per-key usage page for counters that are per workspace is how
+    a refusal turns into a misdiagnosis.
+
+    **What was counted, and what this noun does not fix.** ``work_unit``, one of
+    :data:`CAP_WORK_UNITS`, is the same declaration one noun over: it says whether the counted work
+    item was a build run or a data read, so a refused read is refused in the words a read deserves.
+    It changes the WORDING and never the NUMBER. The ceiling is still
+    ``MAX_RUNS_PER_MONTH = 10_000`` work items per calendar month per coordinate, whatever the work
+    item is called, and for a production query API 10,000 reads per month is low. Raising it is not
+    a wording decision and is not taken here: the same constant bounds the build-side allowance, so
+    raising it weakens that cap too. Separate read counters require a :data:`CAP_STORE_SCHEMA`
+    migration. The current limit is 10,000 reads per key per month.
+    """
+
+    def __init__(
+        self,
+        policy: CapPolicy | None = None,
+        *,
+        store: CapStore | None = None,
+        subject: str = "workspace",
+        work_unit: str = "run",
+    ) -> None:
+        if policy is None:
+            policy = CapPolicy()
+        if not isinstance(policy, CapPolicy):
+            raise ValueError("a cap ledger needs a CapPolicy")
+        if subject not in CAP_SUBJECTS:
+            raise ValueError(
+                "a cap ledger's subject must name what its coordinates stand for, one of "
+                + ", ".join(CAP_SUBJECTS)
+            )
+        if work_unit not in CAP_WORK_UNITS:
+            raise ValueError(
+                "a cap ledger's work unit must name what its counters count, one of "
+                + ", ".join(CAP_WORK_UNITS)
+            )
+        if store is not None and not callable(getattr(store, "lock", None)):
+            # Refused here rather than degraded to "no locking": a store that cannot exclude other
+            # writers cannot hold an allowance, and the failure of one that silently does not is
+            # invisible until a tenant's counters are already gone.
+            raise ValueError("a cap store must offer an inter-process lock")
+        self._policy = policy
+        self._subject = subject
+        self._work_unit = work_unit
+        self._sentences = _REFUSAL_SENTENCES[work_unit]
+        self._lock = threading.Lock()
+        self._next_id = 1
+        self._windows: dict[str, _KeyWindow] = {}
+        self._reservations: dict[int, Reservation] = {}
+        self._store = store
+        if store is not None:
+            # A load failure propagates as CapStoreError. Constructing a ledger over counters that
+            # cannot be read has to fail here, where the caller can still refuse the run, rather
+            # than succeed with empty counters and admit everything.
+            with store.lock():
+                persisted = store.load()
+            for key, record in persisted.items():
+                self._windows[key] = _KeyWindow(
+                    window_start=record.window_start,
+                    runs=record.runs,
+                    cost_units=record.cost_units,
+                )
+
+    @property
+    def policy(self) -> CapPolicy:
+        return self._policy
+
+    @property
+    def subject(self) -> str:
+        """What this ledger's coordinates stand for, and what its refusals name."""
+
+        return self._subject
+
+    @property
+    def work_unit(self) -> str:
+        """What this ledger's counters count, and the noun its refusals use for it."""
+
+        return self._work_unit
+
+    def reserve(
+        self,
+        key_id: str,
+        *,
+        now: datetime,
+        estimated_cost_units: int = 0,
+    ) -> Reservation:
+        """Hold a slot for one run, or refuse with the cap that stopped it named.
+
+        The estimate is what the run may cost at worst; it is held against the monthly spend
+        limit until :meth:`commit` replaces it with the real number.
+
+        With a store, the persisted counters are re-read inside the store's writer lock before any
+        cap is evaluated, and this run's charge is written back before that lock is dropped. Both
+        halves are needed: the read makes the decision current, and the write makes it binding on
+        the next execution to ask. See :class:`CapLedger` for why deciding under the lock and
+        writing at completion would still admit every execution that overlaps the first.
+        """
+
+        key = _checked_key_id(key_id)
+        estimate = _checked_units(estimated_cost_units, "estimated_cost_units")
+        window_start, _ = utc_calendar_month_bounds(now)
+
+        with self._store_lock(), self._lock:
+            self._refresh_charged_locked()
+            window = self._window_locked(key, window_start)
+            in_flight = len(window.reservations)
+
+            # ``window.runs`` and ``window.cost_units`` are CHARGED totals: every run admitted this
+            # month, this ledger's open reservations included, and — with a store — every other
+            # process's too, because the charge is written at admission. So this run is the only
+            # thing added here. Adding the in-flight count again would charge this process's own
+            # open runs twice.
+            runs_observed = window.runs + 1
+            if runs_observed > self._policy.max_runs_per_month:
+                raise CapExceeded(
+                    "runs_per_month",
+                    observed=runs_observed,
+                    limit=self._policy.max_runs_per_month,
+                    message=self._sentences["runs_per_month"].format(
+                        subject=self._subject,
+                        observed=runs_observed,
+                        limit=self._policy.max_runs_per_month,
+                    ),
+                )
+
+            concurrent_observed = in_flight + 1
+            if concurrent_observed > self._policy.max_concurrent_runs:
+                raise CapExceeded(
+                    "concurrent_runs",
+                    observed=concurrent_observed,
+                    limit=self._policy.max_concurrent_runs,
+                    message=self._sentences["concurrent_runs"].format(
+                        subject=self._subject,
+                        observed=concurrent_observed,
+                        limit=self._policy.max_concurrent_runs,
+                    ),
+                )
+
+            cost_observed = window.cost_units + estimate
+            if cost_observed > self._policy.max_monthly_cost_units:
+                raise CapExceeded(
+                    "monthly_cost_units",
+                    observed=cost_observed,
+                    limit=self._policy.max_monthly_cost_units,
+                    message=self._sentences["monthly_cost_units"].format(
+                        subject=self._subject,
+                        observed=cost_observed,
+                        limit=self._policy.max_monthly_cost_units,
+                    ),
+                )
+
+            # The charge lands HERE, inside the same store lock that read the counters this
+            # decision was made from, and before any caller can act on the admission. That is the
+            # whole of what makes the allowance hold when executions overlap: the next process to
+            # take this lock reads a number that already includes this run, so it cannot be handed
+            # the same slot. Written at commit instead, the charge would arrive after every
+            # concurrent execution had already been admitted against the number it replaced.
+            window.runs += 1
+            window.cost_units += estimate
+            try:
+                self._persist_locked()
+            except BaseException:
+                # An admission that could not be recorded is not an admission. The charge is backed
+                # out and the refusal reaches the caller, who fails closed on it.
+                window.runs -= 1
+                window.cost_units -= estimate
+                raise
+
+            reservation = Reservation(
+                self,
+                reservation_id=self._next_id,
+                key_id=key,
+                # The window the charge LANDED in, which is the window above and not necessarily
+                # the one this ``now`` names: a reading behind the month the store already holds is
+                # decided and charged against that later window (see :meth:`_window_locked`).
+                # :meth:`commit` and :meth:`release` reach for the charge by this field, so a
+                # reservation that named the caller's month instead would refund nothing and would
+                # count its run a second time.
+                window_start=window.window_start,
+                estimated_cost_units=estimate,
+            )
+            self._next_id += 1
+            window.reservations[reservation.reservation_id] = reservation
+            self._reservations[reservation.reservation_id] = reservation
+            return reservation
+
+    def commit(self, reservation: Reservation, *, actual_cost_units: int) -> None:
+        """Record one finished run at what it actually cost and give its slot back.
+
+        The monthly totals move on committed cost, not on the estimate: an estimate that turned
+        out generous returns to the allowance the moment the run finishes.
+
+        If the month rolled over while the run was in flight, the run is recorded against the
+        window the ledger is now in. Counting it against a month that has already been reported
+        would let a long run land in a closed window; counting it now can only be conservative.
+
+        With a store, the whole sequence — re-read, add this run, write back — happens inside the
+        store's writer lock, so this run is added to what other executions have already committed
+        instead of replacing it.
+        """
+
+        actual = _checked_units(actual_cost_units, "actual_cost_units")
+        with self._store_lock(), self._lock:
+            self._refresh_charged_locked()
+            self._settle_locked(reservation)
+            window = self._windows[reservation.key_id]
+            if window.window_start == reservation.window_start:
+                # The run itself was charged at admission, so nothing is added to the run count
+                # here. Only the difference between what was held and what was really spent moves,
+                # which is how a generous estimate returns to the allowance the moment it is known.
+                window.cost_units = max(
+                    0, window.cost_units - reservation.estimated_cost_units + actual
+                )
+            else:
+                # The month rolled while the run was in flight. Its admission charge belongs to a
+                # window that is now closed and must not be reached back into, so the run is
+                # recorded in the window this ledger is in. That can only be conservative.
+                window.runs += 1
+                window.cost_units += actual
+            self._persist_locked()
+
+    def release(self, reservation: Reservation) -> None:
+        """Give a slot back without recording a run. The refused or failed run costs nothing.
+
+        This DOES write the store, because :meth:`reserve` charged the run there. A release is the
+        refund of that charge: the run count and the held estimate go back, so a run that was
+        admitted and then never happened does not spend a month's allowance. Without the refund the
+        allowance would ratchet down on every fault between admission and exec.
+
+        A refund only happens where the charge is still reachable — the same key, the same window.
+        If the month rolled while the run was in flight, the charge sits in a window that is closed
+        and is left alone rather than deducted from the new month's allowance.
+        """
+
+        with self._store_lock(), self._lock:
+            self._refresh_charged_locked()
+            self._settle_locked(reservation)
+            window = self._windows[reservation.key_id]
+            if window.window_start == reservation.window_start:
+                window.runs = max(0, window.runs - 1)
+                window.cost_units = max(0, window.cost_units - reservation.estimated_cost_units)
+            self._persist_locked()
+
+    def usage(self, key_id: str, *, now: datetime) -> UsageRecord:
+        """Return what this coordinate has committed in the UTC month containing ``now``.
+
+        A key nobody has used yet is an empty record, not an error: there is nothing exceptional
+        about a key that has done no work.
+
+        This reports committed work, while cap counters are charged
+        totals that also include runs still in flight. The difference is this ledger's own open
+        reservations, and they are subtracted here: a run that has been admitted but has not
+        finished is not a run this coordinate has used. Charges held by other processes cannot
+        be told apart from committed work in a shared file and are therefore included — the
+        conservative direction, and the only one available.
+        """
+
+        key = _checked_key_id(key_id)
+        window_start, _ = utc_calendar_month_bounds(now)
+        with self._store_lock(), self._lock:
+            self._refresh_charged_locked()
+            window = self._windows.get(key)
+            if window is None or window.window_start != window_start:
+                return UsageRecord(key_id=key, window_start=window_start, runs=0, cost_units=0)
+            held_runs = len(window.reservations)
+            held_cost = sum(held.estimated_cost_units for held in window.reservations.values())
+            return UsageRecord(
+                key_id=key,
+                window_start=window_start,
+                runs=max(0, window.runs - held_runs),
+                cost_units=max(0, window.cost_units - held_cost),
+            )
+
+    def runs_in_flight(self, key_id: str, *, now: datetime) -> int:
+        """How many runs this coordinate is holding slots for right now."""
+
+        key = _checked_key_id(key_id)
+        utc_calendar_month_bounds(now)
+        with self._lock:
+            window = self._windows.get(key)
+            return 0 if window is None else len(window.reservations)
+
+    def _window_locked(self, key_id: str, window_start: datetime) -> _KeyWindow:
+        """Return the window this decision is made in, rolling the month forward and never back.
+
+        The roll is one-directional on purpose, and the reason is the same one that makes
+        :meth:`_refresh_charged_locked` skip a stored window older than this ledger's. Every
+        execution of the job reads its own clock, and those clocks are independent. Around a UTC
+        month boundary one of them reads the month that has just ended while the store already holds
+        the new one, and a roll to that older reading would zero the counters, admit the run against
+        an allowance that is already spent, and then write the zeroed window over the file — where
+        it is what every other execution reads. The month's committed runs would be gone, silently,
+        and its allowance handed out again.
+
+        So a reading older than the window already adopted decides against that adopted window
+        instead: nothing is zeroed, nothing older is persisted, and the run is charged where the
+        month's other runs are. A reading in a later month rolls the window, carrying the
+        in-flight reservations across, because concurrency is an instantaneous limit and not a
+        monthly allowance.
+        """
+
+        window = self._windows.get(key_id)
+        if window is None:
+            window = _KeyWindow(window_start=window_start)
+            self._windows[key_id] = window
+            return window
+        if window_start < window.window_start:
+            return window
+        if window.window_start != window_start:
+            rolled = _KeyWindow(window_start=window_start)
+            rolled.reservations = window.reservations
+            self._windows[key_id] = rolled
+            return rolled
+        return window
+
+    @contextmanager
+    def _store_lock(self) -> Iterator[None]:
+        """Hold the store against other writers, or do nothing when there is no store.
+
+        Always entered BEFORE this ledger's own :class:`threading.Lock`, in every method that takes
+        both. One order, everywhere, is what keeps two threads of the same process from each
+        holding one lock and waiting for the other.
+        """
+
+        if self._store is None:
+            yield
+            return
+        with self._store.lock():
+            yield
+
+    def _refresh_charged_locked(self) -> None:
+        """Adopt the persisted charged counters, keeping this process's in-flight slots.
+
+        Called inside the store's writer lock, immediately before a cap is evaluated or a counter is
+        moved, so that both act on what is on disk now rather than on what was there when this
+        ledger was constructed. Only the counters are adopted; ``reservations`` belongs to this
+        process and is left exactly as it is. The adopted numbers already include the charges this
+        ledger made for its own open reservations, which is why nothing adds the in-flight count
+        back on top of them.
+
+        Two cases are decided here rather than left implicit:
+
+        * A stored window OLDER than the one this ledger has already rolled into is ignored. Its
+          counters belong to a month that is closed, and carrying them forward would spend a new
+          month's allowance on last month's runs.
+        * A key this ledger knows and the store does not keeps its in-memory counters. The
+          conservative direction is the only safe one on an admission path: treating a key that
+          vanished from the file as a key that has used nothing would hand out a fresh allowance to
+          whoever can edit the file.
+        """
+
+        if self._store is None:
+            return
+        for key, record in self._store.load().items():
+            window = self._windows.get(key)
+            if window is None:
+                self._windows[key] = _KeyWindow(
+                    window_start=record.window_start,
+                    runs=record.runs,
+                    cost_units=record.cost_units,
+                )
+                continue
+            if record.window_start < window.window_start:
+                continue
+            window.window_start = record.window_start
+            window.runs = record.runs
+            window.cost_units = record.cost_units
+
+    def _persist_locked(self) -> None:
+        """Write the committed counters back, inside the lock that produced them.
+
+        Persisting under the same lock is what makes the file a snapshot of a real state rather
+        than of a state that never existed: no other thread can move a counter between the last
+        mutation and the write.
+
+        The whole map is written, which is safe only because :meth:`_refresh_charged_locked` has
+        just replaced it with what the file holds, under the store's writer lock. Written from a
+        stale map instead, this same call would delete other tenants' committed runs.
+
+        A key whose charged counters are both zero is left out rather than written as a row of
+        zeroes. Absent and zero say the same thing to every reader of this file, and not writing
+        the row keeps a store from growing a permanent entry for every key that was ever refunded.
+        """
+
+        if self._store is None:
+            return
+        self._store.save(
+            {
+                key: UsageRecord(
+                    key_id=key,
+                    window_start=window.window_start,
+                    runs=window.runs,
+                    cost_units=window.cost_units,
+                )
+                for key, window in self._windows.items()
+                if window.runs or window.cost_units
+            }
+        )
+
+    def _settle_locked(self, reservation: Reservation) -> None:
+        if not isinstance(reservation, Reservation):
+            raise CapLedgerError("a cap ledger settles reservations it issued, nothing else")
+        held = self._reservations.get(reservation.reservation_id)
+        if held is not reservation or reservation.settled:
+            raise CapLedgerError("this reservation is not open on this ledger")
+        del self._reservations[reservation.reservation_id]
+        window = self._windows.get(reservation.key_id)
+        if window is not None:
+            window.reservations.pop(reservation.reservation_id, None)
+        reservation._settled = True
+
+
+__all__ = [
+    "CAP_STORE_SCHEMA",
+    "CAP_SUBJECTS",
+    "CAP_WORK_UNITS",
+    "COST_UNIT_DESCRIPTION",
+    "DEFAULT_CONCURRENT_RUNS",
+    "DEFAULT_MONTHLY_COST_UNITS",
+    "DEFAULT_RUNS_PER_MONTH",
+    "KEY_ID_PATTERN",
+    "LOCK_FILE_SUFFIX",
+    "MAX_CONCURRENT_RUNS",
+    "MAX_MONTHLY_COST_UNITS",
+    "MAX_RUNS_PER_MONTH",
+    "CapExceeded",
+    "CapLedger",
+    "CapLedgerError",
+    "CapPolicy",
+    "CapStore",
+    "CapStoreError",
+    "FileCapStore",
+    "Reservation",
+    "UsageRecord",
+    "utc_calendar_month_bounds",
+]