il-audit 0.1.0__py3-none-any.whl

il_audit/__init__.py

@@ -0,0 +1,20 @@
+"""Cross-app audit and analytics logging for Locaria Intelligence Layer."""
+
+from il_audit.config import AuditSettings, get_audit_settings
+from il_audit.context import AuditContext
+from il_audit.emit import emit, emit_api_usage, emit_batch, emit_resource_usage, schedule_beacon_rows
+from il_audit.registry import register_builder
+
+__all__ = [
+    "AuditContext",
+    "AuditSettings",
+    "emit",
+    "emit_api_usage",
+    "emit_batch",
+    "emit_resource_usage",
+    "get_audit_settings",
+    "register_builder",
+    "schedule_beacon_rows",
+]
+
+__version__ = "0.1.0"

il_audit/beacon_router.py

@@ -0,0 +1,60 @@
+"""FastAPI router for client UI beacons."""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+from typing import Callable
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+
+from il_audit.beacon_schemas import BeaconRequest
+from il_audit.context import AuditContext
+from il_audit.emit import schedule_beacon_rows
+
+_beacon_last_flush: dict[str, float] = defaultdict(float)
+_BEACON_MIN_INTERVAL_SECONDS = 5.0
+
+
+def create_beacon_router(
+    *,
+    app_name: str,
+    get_context: Callable[[Request], AuditContext],
+) -> APIRouter:
+    router = APIRouter()
+
+    @router.post("/api/audit/beacon")
+    def audit_beacon(request: Request, body: BeaconRequest) -> dict[str, str | int]:
+        session_key = request.client.host if request.client else "unknown"
+        now = time.monotonic()
+        if now - _beacon_last_flush[session_key] < _BEACON_MIN_INTERVAL_SECONDS:
+            raise HTTPException(status_code=429, detail="Beacon rate limit exceeded")
+        _beacon_last_flush[session_key] = now
+        if not body.events:
+            return {"status": "ok", "accepted": 0}
+        context = get_context(request)
+        if context.app != app_name:
+            context = AuditContext(
+                app=app_name,
+                actor_uid=context.actor_uid,
+                actor_email=context.actor_email,
+                trace_id=context.trace_id,
+                request_id=context.request_id,
+                source="beacon",
+            )
+        else:
+            context = AuditContext(
+                app=context.app,
+                actor_uid=context.actor_uid,
+                actor_email=context.actor_email,
+                trace_id=context.trace_id,
+                request_id=context.request_id,
+                source="beacon",
+            )
+        schedule_beacon_rows(
+            context=context,
+            events=[event.model_dump() for event in body.events],
+        )
+        return {"status": "ok", "accepted": len(body.events)}
+
+    return router

il_audit/beacon_schemas.py

@@ -0,0 +1,22 @@
+"""Pydantic models for POST /api/audit/beacon."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class BeaconEventItem(BaseModel):
+    event_type: str
+    surface: str | None = None
+    element_id: str | None = None
+    interaction: str | None = None
+    duration_ms: int | None = None
+    result_count: int | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    client_ts: str | None = None
+
+
+class BeaconRequest(BaseModel):
+    events: list[BeaconEventItem] = Field(default_factory=list, max_length=50)

il_audit/bq_client.py

@@ -0,0 +1,119 @@
+"""BigQuery client helpers for audit log writes."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from google.api_core.exceptions import NotFound
+
+_logger = logging.getLogger(__name__)
+
+_shared_clients: dict[str, Any] = {}
+_client_lock = threading.Lock()
+
+WRITE_APPEND = "WRITE_APPEND"
+
+
+def _import_bigquery() -> Any:
+    try:
+        from google.cloud import bigquery
+    except ImportError as exc:
+        raise RuntimeError(
+            "google-cloud-bigquery is required — pip install google-cloud-bigquery",
+        ) from exc
+    return bigquery
+
+
+def get_bigquery_client(*, project: str) -> Any:
+    project_id = project.strip()
+    if not project_id:
+        raise ValueError("BigQuery project id is required")
+    with _client_lock:
+        cached = _shared_clients.get(project_id)
+        if cached is not None:
+            return cached
+        bigquery = _import_bigquery()
+        client = bigquery.Client(project=project_id)
+        _shared_clients[project_id] = client
+        return client
+
+
+def run_bigquery_query(*, sql: str, project: str) -> list[dict[str, Any]]:
+    bigquery = _import_bigquery()
+    client = get_bigquery_client(project=project)
+    rows = client.query(sql).result()
+    return [dict(row.items()) for row in rows]
+
+
+def write_bigquery_rows(
+    *,
+    project: str,
+    dataset: str,
+    table: str,
+    rows: Sequence[Mapping[str, Any]],
+    schema: Sequence[Any] | None = None,
+) -> int:
+    if not rows:
+        return 0
+    bigquery = _import_bigquery()
+    client = get_bigquery_client(project=project)
+    table_ref = f"{project.strip()}.{dataset.strip()}.{table.strip()}"
+    payload = [dict(row) for row in rows]
+    job_config = bigquery.LoadJobConfig(write_disposition=WRITE_APPEND)
+    if schema:
+        job_config.schema = list(schema)
+    job = client.load_table_from_json(payload, table_ref, job_config=job_config)
+    job.result()
+    return len(payload)
+
+
+class BigQueryTableManager:
+    def __init__(self, *, project_id: str) -> None:
+        project = project_id.strip()
+        if not project:
+            raise ValueError("BigQuery project id is required")
+        self.project_id = project
+        self._client: Any | None = None
+
+    @property
+    def client(self) -> Any:
+        if self._client is None:
+            self._client = get_bigquery_client(project=self.project_id)
+        return self._client
+
+    def ensure_dataset_exists(self, dataset_id: str) -> None:
+        bigquery = _import_bigquery()
+        dataset_ref = f"{self.project_id}.{dataset_id.strip()}"
+        try:
+            self.client.get_dataset(dataset_ref)
+        except NotFound:
+            _logger.info("Creating BigQuery dataset %s", dataset_ref)
+            self.client.create_dataset(bigquery.Dataset(dataset_ref))
+
+    def ensure_table_exists(
+        self,
+        *,
+        dataset_id: str,
+        table_id: str,
+        schema: Sequence[Any],
+    ) -> bool:
+        bigquery = _import_bigquery()
+        table_ref = f"{self.project_id}.{dataset_id.strip()}.{table_id.strip()}"
+        schema_list = list(schema)
+        try:
+            table = self.client.get_table(table_ref)
+            existing_names = {field.name for field in table.schema}
+            missing = [field for field in schema_list if field.name not in existing_names]
+            if missing:
+                table.schema = list(table.schema) + missing
+                self.client.update_table(table, ["schema"])
+            return False
+        except NotFound:
+            self.ensure_dataset_exists(dataset_id)
+            _logger.info("Creating BigQuery table %s", table_ref)
+            table = bigquery.Table(table_ref, schema=schema_list)
+            self.client.create_table(table)
+            return True

il_audit/builders/__init__.py

@@ -0,0 +1,5 @@
+"""Shared row builders for envelope-style tables."""
+
+from il_audit.builders.shared import build_envelope_row
+
+__all__ = ["build_envelope_row"]

il_audit/builders/shared.py

@@ -0,0 +1,45 @@
+"""Shared envelope row builder for domain audit tables."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from il_audit.context import AuditContext
+from il_audit.envelope import build_envelope
+
+
+def _module_for_event(event_type: str) -> str:
+    prefix = event_type.split(".", 1)[0]
+    return prefix
+
+
+def build_envelope_row(
+    *,
+    context: AuditContext,
+    event_type: str,
+    metadata: dict[str, Any] | None = None,
+    module: str | None = None,
+    action: str | None = None,
+    status: str | None = None,
+    resource_type: str | None = None,
+    resource_id: str | None = None,
+    batch_id: str | None = None,
+    before: dict[str, Any] | None = None,
+    after: dict[str, Any] | None = None,
+    **extra: Any,
+) -> dict[str, Any]:
+    row = build_envelope(
+        context=context,
+        module=module or _module_for_event(event_type),
+        event_type=event_type,
+        action=action,
+        status=status,
+        resource_type=resource_type,
+        resource_id=resource_id,
+        batch_id=batch_id,
+        before=before,
+        after=after,
+        metadata=metadata,
+    )
+    row.update({key: value for key, value in extra.items() if value is not None})
+    return row

il_audit/config.py

@@ -0,0 +1,117 @@
+"""Runtime configuration for il_audit — defaults from bundled JSON, overrides via env."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any
+
+from il_audit.cost_catalog import COST_RATES_SOURCE_SEED
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.getenv(name, "").strip().lower()
+    if not raw:
+        return default
+    return raw in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.getenv(name, "").strip()
+    if not raw:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        return default
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = os.getenv(name, "").strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def _resolve_bq_project() -> str:
+    for key in ("IL_BQ_LOGS_PROJECT", "GOOGLE_CLOUD_PROJECT", "GCP_PROJECT"):
+        value = os.getenv(key, "").strip()
+        if value:
+            return value
+    return ""
+
+
+@lru_cache(maxsize=1)
+def _bundled_defaults() -> dict[str, Any]:
+    from il_audit.cost_rates_loader import load_audit_defaults
+
+    return load_audit_defaults()
+
+
+@dataclass(frozen=True)
+class AuditSettings:
+    """BigQuery audit logging settings."""
+
+    bq_project: str
+    bq_dataset: str
+    enable_audit_logs: bool
+    store_query_text: bool
+    flush_batch_size: int
+    flush_interval_seconds: float
+    cost_rates_source: str
+    cost_rates_cache_seconds: int
+
+    @classmethod
+    def from_env(cls) -> AuditSettings:
+        defaults = _bundled_defaults()
+        bq_project = _resolve_bq_project()
+        bq_dataset = (
+            os.getenv("IL_BQ_AUDIT_DATASET", "").strip()
+            or str(defaults.get("bq_audit_dataset") or "")
+        )
+        enable = _env_bool("IL_ENABLE_AUDIT_LOGS", bool(defaults.get("enable_audit_logs", True)))
+        if not _env_bool("IL_ENABLE_BQ_LOGS", True):
+            enable = False
+        cost_source = (
+            os.getenv("IL_COST_RATES_SOURCE", "").strip().lower()
+            or str(defaults.get("cost_rates_source") or COST_RATES_SOURCE_SEED)
+        )
+        return cls(
+            bq_project=bq_project,
+            bq_dataset=bq_dataset,
+            enable_audit_logs=enable,
+            store_query_text=_env_bool(
+                "IL_AUDIT_STORE_QUERY_TEXT",
+                bool(defaults.get("store_query_text", True)),
+            ),
+            flush_batch_size=_env_int(
+                "IL_AUDIT_FLUSH_BATCH_SIZE",
+                int(defaults.get("flush_batch_size") or 50),
+            ),
+            flush_interval_seconds=_env_float(
+                "IL_AUDIT_FLUSH_INTERVAL_SECONDS",
+                float(defaults.get("flush_interval_seconds") or 1.0),
+            ),
+            cost_rates_source=cost_source,
+            cost_rates_cache_seconds=_env_int(
+                "IL_COST_RATES_CACHE_SECONDS",
+                int(defaults.get("cost_rates_cache_seconds") or 3600),
+            ),
+        )
+
+
+@lru_cache(maxsize=1)
+def get_audit_settings() -> AuditSettings:
+    return AuditSettings.from_env()
+
+
+def reset_audit_settings_cache() -> None:
+    get_audit_settings.cache_clear()
+    _bundled_defaults.cache_clear()
+    from il_audit.cost_rates_loader import reset_cost_rates_cache
+
+    reset_cost_rates_cache()

il_audit/context.py

@@ -0,0 +1,17 @@
+"""Audit context passed to every emit call."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class AuditContext:
+    """Who performed an action and how to correlate it."""
+
+    app: str
+    actor_uid: str | None = None
+    actor_email: str | None = None
+    trace_id: str | None = None
+    request_id: str | None = None
+    source: str = "api"

il_audit/cost_calculator.py

@@ -0,0 +1,50 @@
+"""USD estimation from the active cost_rates catalog (BQ or seed JSON)."""
+
+from __future__ import annotations
+
+from il_audit.cost_rates_loader import RateEntry, get_active_cost_rates
+
+
+def _lookup_entry(*, service: str, model_id: str) -> RateEntry | None:
+    catalog = get_active_cost_rates()
+    return catalog.index().get((service, model_id))
+
+
+def estimate_api_cost_usd(
+    *,
+    service: str,
+    model_id: str,
+    input_tokens: int = 0,
+    output_tokens: int = 0,
+) -> tuple[float, str, str]:
+    """
+    Estimate API cost from the active catalog.
+
+    Returns:
+        (cost_usd, pricing_version, pricing_source)
+    """
+    catalog = get_active_cost_rates()
+    entry = catalog.index().get((service, model_id))
+    if entry is None:
+        return 0.0, catalog.rate_version, catalog.pricing_source
+    cost = (
+        (max(0, input_tokens) * entry.input_price_per_1m_tokens)
+        + (max(0, output_tokens) * entry.output_price_per_1m_tokens)
+    ) / 1_000_000
+    version = entry.rate_version or catalog.rate_version
+    return round(cost, 8), version, catalog.pricing_source
+
+
+def estimate_unit_cost_usd(
+    *,
+    service: str,
+    model_id: str,
+    quantity: float,
+) -> tuple[float, str, str]:
+    """Estimate resource cost from unit-priced catalog entry."""
+    catalog = get_active_cost_rates()
+    entry = catalog.index().get((service, model_id))
+    if entry is None or entry.unit_price is None:
+        return 0.0, catalog.rate_version, catalog.pricing_source
+    version = entry.rate_version or catalog.rate_version
+    return round(max(0.0, quantity) * entry.unit_price, 8), version, catalog.pricing_source

il_audit/cost_catalog.py

@@ -0,0 +1,48 @@
+"""Stable identifiers for cost attribution — single source, no magic strings elsewhere."""
+
+from __future__ import annotations
+
+# Billable API services (maps to cost_rates.service)
+SERVICE_VERTEX_GEMINI = "vertex_gemini"
+SERVICE_VERTEX_EMBEDDING = "vertex_embedding"
+SERVICE_VERTEX_RERANK = "vertex_rerank"
+SERVICE_DRIVE_API = "drive_api"
+SERVICE_CLOUD_RUN = "cloud_run"
+SERVICE_GCS = "gcs"
+
+# API operations (maps to api_usage_events.operation)
+OPERATION_GENERATE_CONTENT = "generate_content"
+OPERATION_EMBED_CONTENT = "embed_content"
+OPERATION_SEMANTIC_RANKER = "semantic_ranker"
+OPERATION_FILES_GET = "files.get"
+
+# Ingest token categories (maps to api_usage_events.category)
+API_CATEGORY_VISION = "vision"
+API_CATEGORY_TAGGING = "tagging"
+API_CATEGORY_SUMMARY = "summary"
+API_CATEGORY_EMBEDDING = "embedding"
+
+# Correlation types (maps to api_usage_events.correlation_type)
+CORRELATION_INGEST_JOB = "ingest_job"
+CORRELATION_INGEST_FILE = "ingest_file"
+CORRELATION_CHAT_TURN = "chat_turn"
+CORRELATION_DRIVE_PIPELINE = "drive_pipeline"
+CORRELATION_CLIENT_RECONCILE = "client_reconcile"
+
+# Resource usage types (maps to resource_usage_events.resource_type)
+RESOURCE_CLOUD_RUN_CPU_SECONDS = "cloud_run_cpu_seconds"
+RESOURCE_GCS_STAGING_BYTES = "gcs_staging_bytes"
+RESOURCE_POSTGRES_CHUNK_WRITES = "postgres_chunk_writes"
+RESOURCE_DRIVE_API_CALLS = "drive_api_calls"
+
+# Resource rate keys (maps to cost_rates model_id for unit-priced resources)
+RESOURCE_RATE_CPU_SECOND = "cpu_second"
+RESOURCE_RATE_STORAGE_BYTE = "storage_byte"
+
+# Pricing source values stored on usage rows until BQ catalog is authoritative
+PRICING_SOURCE_SEED_CATALOG = "seed_catalog"
+PRICING_SOURCE_BIGQUERY_CATALOG = "bigquery_catalog"
+
+# cost_rates loader source (AuditSettings.cost_rates_source)
+COST_RATES_SOURCE_SEED = "seed"
+COST_RATES_SOURCE_BIGQUERY = "bigquery"

il_audit/cost_rates_loader.py

@@ -0,0 +1,179 @@
+"""Load versioned cost rates from BigQuery catalog or bundled seed JSON."""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import time
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from il_audit.cost_catalog import (
+    COST_RATES_SOURCE_BIGQUERY,
+    COST_RATES_SOURCE_SEED,
+    PRICING_SOURCE_BIGQUERY_CATALOG,
+    PRICING_SOURCE_SEED_CATALOG,
+)
+
+_logger = logging.getLogger(__name__)
+
+_CACHE_LOCK = threading.Lock()
+_CACHE: "_RatesCache | None" = None
+
+
+@dataclass(frozen=True, slots=True)
+class RateEntry:
+    service: str
+    model_id: str
+    input_price_per_1m_tokens: float
+    output_price_per_1m_tokens: float
+    unit_price: float | None = None
+    unit: str = "tokens"
+    rate_version: str = ""
+    source_url: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class LoadedCostRates:
+    rate_version: str
+    pricing_source: str
+    entries: tuple[RateEntry, ...]
+
+    def index(self) -> dict[tuple[str, str], RateEntry]:
+        return {(entry.service, entry.model_id): entry for entry in self.entries}
+
+
+@dataclass
+class _RatesCache:
+    loaded_at: float
+    catalog: LoadedCostRates
+
+
+def _load_json(name: str) -> dict[str, Any]:
+    path = Path(__file__).resolve().parent / "data" / name
+    with path.open(encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def load_audit_defaults() -> dict[str, Any]:
+    """Bundled defaults — overridden by environment in config.AuditSettings."""
+    return _load_json("audit_defaults.json")
+
+
+def _parse_rate_entry(raw: dict[str, Any], *, default_version: str) -> RateEntry:
+    return RateEntry(
+        service=str(raw["service"]),
+        model_id=str(raw["model_id"]),
+        input_price_per_1m_tokens=float(raw.get("input_price_per_1m_tokens") or 0.0),
+        output_price_per_1m_tokens=float(raw.get("output_price_per_1m_tokens") or 0.0),
+        unit_price=(float(raw["unit_price"]) if raw.get("unit_price") is not None else None),
+        unit=str(raw.get("unit") or "tokens"),
+        rate_version=str(raw.get("rate_version") or default_version),
+        source_url=(str(raw["source_url"]) if raw.get("source_url") else None),
+    )
+
+
+def load_seed_cost_rates() -> LoadedCostRates:
+    payload = _load_json("cost_rates_seed.json")
+    version = str(payload.get("rate_version") or "")
+    entries = tuple(
+        _parse_rate_entry(item, default_version=version) for item in payload.get("rates") or []
+    )
+    return LoadedCostRates(
+        rate_version=version,
+        pricing_source=PRICING_SOURCE_SEED_CATALOG,
+        entries=entries,
+    )
+
+
+def load_bigquery_cost_rates(*, project: str, dataset: str) -> LoadedCostRates | None:
+    if not project.strip() or not dataset.strip():
+        return None
+    try:
+        from il_audit.bq_client import run_bigquery_query
+    except ImportError:
+        return None
+    sql = f"""
+        SELECT
+          rate_version,
+          service,
+          model_id,
+          input_price_per_1m_tokens,
+          output_price_per_1m_tokens,
+          unit_price,
+          unit,
+          source_url
+        FROM `{project.strip()}.{dataset.strip()}.cost_rates`
+        WHERE effective_from <= CURRENT_DATE()
+        QUALIFY ROW_NUMBER() OVER (
+          PARTITION BY service, model_id
+          ORDER BY effective_from DESC, rate_version DESC
+        ) = 1
+    """
+    try:
+        rows = run_bigquery_query(project=project, sql=sql)
+    except Exception:
+        _logger.exception("Failed to load cost_rates from BigQuery")
+        return None
+    if not rows:
+        return None
+    version = str(rows[0].get("rate_version") or "")
+    entries = tuple(
+        RateEntry(
+            service=str(row["service"]),
+            model_id=str(row["model_id"]),
+            input_price_per_1m_tokens=float(row.get("input_price_per_1m_tokens") or 0.0),
+            output_price_per_1m_tokens=float(row.get("output_price_per_1m_tokens") or 0.0),
+            unit_price=(float(row["unit_price"]) if row.get("unit_price") is not None else None),
+            unit=str(row.get("unit") or "tokens"),
+            rate_version=str(row.get("rate_version") or version),
+            source_url=(str(row["source_url"]) if row.get("source_url") else None),
+        )
+        for row in rows
+    )
+    return LoadedCostRates(
+        rate_version=version,
+        pricing_source=PRICING_SOURCE_BIGQUERY_CATALOG,
+        entries=entries,
+    )
+
+
+def get_active_cost_rates(*, force_refresh: bool = False) -> LoadedCostRates:
+    """Return cached cost rates from configured source (BQ when available, else seed)."""
+    from il_audit.config import get_audit_settings
+
+    settings = get_audit_settings()
+    now = time.monotonic()
+    global _CACHE
+    with _CACHE_LOCK:
+        if (
+            not force_refresh
+            and _CACHE is not None
+            and (now - _CACHE.loaded_at) < settings.cost_rates_cache_seconds
+        ):
+            return _CACHE.catalog
+
+        catalog: LoadedCostRates | None = None
+        if settings.cost_rates_source == COST_RATES_SOURCE_BIGQUERY:
+            catalog = load_bigquery_cost_rates(
+                project=settings.bq_project,
+                dataset=settings.bq_dataset,
+            )
+        if catalog is None:
+            catalog = load_seed_cost_rates()
+        _CACHE = _RatesCache(loaded_at=now, catalog=catalog)
+        return catalog
+
+
+def reset_cost_rates_cache() -> None:
+    global _CACHE
+    with _CACHE_LOCK:
+        _CACHE = None
+
+
+def package_data_path(relative: str) -> Path:
+    """Return path to bundled data file (for tests/docs)."""
+    return Path(__file__).resolve().parent / "data" / relative