pf-core 0.1.0__py3-none-any.whl

pf_core/__init__.py

@@ -0,0 +1,20 @@
+"""pf-core: a dependency-light Python foundation.
+
+The base install (``pip install pf-core``) is the architectural foundation
+only: structured logging, an exception hierarchy, config + env resolvers,
+utils, and the ``Service`` base class — five small deps (structlog,
+python-dotenv, pyyaml, nanoid, rich), no httpx/pydantic/LLM stack.
+
+Everything else ships as opt-in, orthogonally-composable extras: anti-slop
+output guards (``[validate]``), LLM clients (``[llm]`` ⊇ ``[validate]``), HTTP
+utils (``[http]``), CLI scaffolding (``[cli]``), and the FastAPI + SQLAlchemy
+app framework (``[db]``, ``[web]``, ``[jobs]``, ``[tracking]``, ``[eval]``,
+``[admin]``). See ``docs/INSTALLATION.md`` for the extras matrix.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("pf-core")
+except PackageNotFoundError:  # running from a source tree with no install
+    __version__ = "0.0.0+unknown"

pf_core/_extras.py

@@ -0,0 +1,64 @@
+"""Friendly errors for missing optional-dependency extras.
+
+The foundation install (``pip install pf-core``) is dependency-light: it does
+not ship httpx, pydantic, json-repair, tenacity, or typer. Modules that need
+those live behind opt-in extras ([http], [llm], [cli], [jobs], ...). When such
+a module is imported without its extra installed, the bare third-party
+``ImportError`` ("No module named 'json_repair'") is opaque. This helper turns
+it into a message that names the extra and the exact pip command.
+
+Usage at the top of a gated leaf module::
+
+    try:
+        import httpx
+    except ImportError as e:  # pragma: no cover - exercised by bare-install CI
+        from pf_core._extras import extra_import_error
+
+        raise extra_import_error(
+            "llm", "httpx", feature="pf_core.clients.openrouter"
+        ) from e
+"""
+
+from __future__ import annotations
+
+# Extra name -> the pip target a user should install. Anything not listed
+# falls back to ``pf-core[<extra>]``.
+_INSTALL: dict[str, str] = {
+    "http": "pf-core[http]",
+    "cli": "pf-core[cli]",
+    "validate": "pf-core[validate]",
+    "llm": "pf-core[llm]",
+    "db": "pf-core[db]",
+    "web": "pf-core[web]",
+    "jobs": "pf-core[jobs]",
+    "tracking": "pf-core[tracking]",
+    "eval": "pf-core[eval]",
+    "admin": "pf-core[admin]",
+    "articles": "pf-core[articles]",
+    "jsonschema": "pf-core[jsonschema]",
+    "redis": "pf-core[redis]",
+    "ratelimit": "pf-core[ratelimit]",
+}
+
+
+def install_target(extra: str) -> str:
+    """Return the ``pip install`` target for an extra (e.g. ``pf-core[llm]``)."""
+    return _INSTALL.get(extra, f"pf-core[{extra}]")
+
+
+def extra_import_error(extra: str, package: str, *, feature: str) -> ImportError:
+    """Build an ``ImportError`` that names the missing extra and pip command.
+
+    Args:
+        extra: The optional-dependency extra that ships ``package`` (e.g. ``"llm"``).
+        package: The third-party import name that failed (e.g. ``"json_repair"``).
+        feature: The pf-core module or capability the caller was importing, used
+            in the message (e.g. ``"pf_core.llm.parse"``).
+
+    Returns:
+        An ``ImportError`` to ``raise ... from`` the original failure.
+    """
+    return ImportError(
+        f"{feature} requires the '{extra}' extra; '{package}' is not installed. "
+        f"Install it with:  pip install {install_target(extra)}"
+    )

pf_core/alembic.py

@@ -0,0 +1,72 @@
+"""
+Alembic migration helper — shared env.py logic for all projects.
+
+Supports SQLite (batch mode), MySQL/MariaDB, and PostgreSQL. Uses pf_core.db
+for engine management so migrations share the same connection config as the app.
+
+Usage in a project's ``alembic/env.py``::
+
+    from pf_core.alembic import run_migrations_online
+
+    run_migrations_online()
+
+With SQLite fallback (e.g., an essay-grading app on SQLite)::
+
+    from pf_core.alembic import run_migrations_online
+
+    run_migrations_online(fallback_sqlite="grading.db")
+
+With explicit metadata (for autogenerate)::
+
+    from myapp.models import Base
+    from pf_core.alembic import run_migrations_online
+
+    run_migrations_online(target_metadata=Base.metadata)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from alembic import context
+
+from pf_core.db import db_url, get_engine, is_sqlite
+
+
+def run_migrations_online(
+    *,
+    fallback_sqlite: str = "",
+    target_metadata: Any = None,
+    compare_type: bool = False,
+) -> None:
+    """Run Alembic migrations in online mode.
+
+    Call this from your project's ``alembic/env.py``. Handles:
+    - SQLite batch mode (``render_as_batch=True``)
+    - MySQL/MariaDB and PostgreSQL standard mode
+    - Engine reuse via ``pf_core.db.get_engine()``
+
+    Args:
+        fallback_sqlite: Path to SQLite file if DATABASE_URL is not set.
+        target_metadata: SQLAlchemy MetaData for autogenerate support.
+            Pass ``None`` (default) for raw-SQL migrations.
+        compare_type: Whether Alembic should detect column type changes.
+    """
+    if context.is_offline_mode():
+        raise RuntimeError(
+            "Offline mode is not supported. Set DATABASE_URL and run in online mode."
+        )
+
+    url = db_url(fallback_sqlite=fallback_sqlite)
+    sqlite = is_sqlite(url)
+    engine = get_engine(url)
+
+    with engine.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=compare_type,
+            render_as_batch=sqlite,
+        )
+        with context.begin_transaction():
+            context.run_migrations()

pf_core/budget/__init__.py

@@ -0,0 +1,147 @@
+"""
+Pre-call cost guardrails for LLM spending.
+
+The kernel-safe surface (importable without ``pf-core[db]``) covers the
+budget guard itself and YAML config loading:
+
+    from pf_core.budget import (
+        check_budget, project_cost, CostBudgetExceeded,
+        compute_period_start, compute_period_end,
+        load_yaml, clear_config_cache,
+    )
+
+The DB-backed surface (requires ``pf-core[db]``) is loaded lazily — the
+import only triggers SQLAlchemy when an attribute is first accessed:
+
+    from pf_core.budget import (
+        BudgetRepo, BudgetSnapshotRepo, CostRateRepo, aggregate_spent,
+        sync_budgets_from_yaml,
+        refresh_snapshots, start_budget_refresh_loop,
+        record_blocked_run, record_override,
+        ALL_BUDGET_TABLES, llm_budgets, llm_budget_snapshots, llm_cost_rates,
+    )
+
+In a kernel-only install (no ``[db]`` extra), ``check_budget()`` and
+``project_cost()`` still import and run, but ``project_cost()`` falls back
+to ``0.0`` when no DB-backed cost rates are reachable, and any code path
+that calls into the repos will surface ``ModuleNotFoundError: sqlalchemy``.
+
+See ``docs/cost-budget.md``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+# Eager imports — kernel-safe modules with no top-level sqlalchemy coupling.
+# These remain importable in a base ``pip install pf-core`` (no extras).
+from pf_core.budget.check import (  # noqa: F401
+    CostBudgetExceeded,
+    check_budget,
+    compute_period_end,
+    compute_period_start,
+    project_cost,
+)
+from pf_core.budget.config import (  # noqa: F401
+    clear_config_cache,
+    load_yaml,
+)
+
+
+# Lazy imports — these submodules pull SQLAlchemy at module top, so we
+# defer them via PEP 562 ``__getattr__`` to keep the kernel install clean.
+# The first attribute access triggers the import; subsequent accesses are
+# free because we cache into ``globals()``.
+_LAZY: dict[str, str] = {
+    # Repos (DB-required)
+    "BudgetRepo":           "pf_core.budget.repo",
+    "BudgetSnapshotRepo":   "pf_core.budget.repo",
+    "CostRateRepo":         "pf_core.budget.repo",
+    "aggregate_spent":      "pf_core.budget.repo",
+    # Audit logging (DB-required)
+    "record_blocked_run":   "pf_core.budget.audit",
+    "record_override":      "pf_core.budget.audit",
+    # Snapshot / scheduler jobs (DB-required)
+    "refresh_snapshots":    "pf_core.budget.snapshot_job",
+    "start_budget_refresh_loop": "pf_core.budget.scheduler",
+    # YAML→DB sync (DB-required path within the otherwise-kernel config module)
+    "sync_budgets_from_yaml": "pf_core.budget.config",
+    # Schema tables (DB-required — registers on shared MetaData on first access)
+    "ALL_BUDGET_TABLES":    "pf_core.budget._schema",
+    "llm_budgets":          "pf_core.budget._schema",
+    "llm_budget_snapshots": "pf_core.budget._schema",
+    "llm_cost_rates":       "pf_core.budget._schema",
+}
+
+
+def __getattr__(name: str):
+    """Lazy import for DB-required attributes (PEP 562)."""
+    target = _LAZY.get(name)
+    if target is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    import importlib
+
+    mod = importlib.import_module(target)
+    value = getattr(mod, name)
+    globals()[name] = value  # cache so subsequent access skips __getattr__
+    return value
+
+
+def __dir__() -> list[str]:
+    """Expose lazy attributes to ``dir()`` and IDE autocomplete."""
+    return sorted(set(globals()) | set(_LAZY))
+
+
+# Type-checker view of the lazy attributes — keeps ``mypy`` / IDEs happy
+# without paying the import cost at runtime.
+if TYPE_CHECKING:
+    from pf_core.budget._schema import (  # noqa: F401
+        ALL_BUDGET_TABLES,
+        llm_budget_snapshots,
+        llm_budgets,
+        llm_cost_rates,
+    )
+    from pf_core.budget.audit import (  # noqa: F401
+        record_blocked_run,
+        record_override,
+    )
+    from pf_core.budget.config import sync_budgets_from_yaml  # noqa: F401
+    from pf_core.budget.repo import (  # noqa: F401
+        BudgetRepo,
+        BudgetSnapshotRepo,
+        CostRateRepo,
+        aggregate_spent,
+    )
+    from pf_core.budget.scheduler import start_budget_refresh_loop  # noqa: F401
+    from pf_core.budget.snapshot_job import refresh_snapshots  # noqa: F401
+
+
+__all__ = [
+    # Guard (eager)
+    "check_budget",
+    "project_cost",
+    "CostBudgetExceeded",
+    "compute_period_start",
+    "compute_period_end",
+    # Config — kernel-safe (eager)
+    "load_yaml",
+    "clear_config_cache",
+    # Config — DB-backed (lazy)
+    "sync_budgets_from_yaml",
+    # Repos (lazy)
+    "BudgetRepo",
+    "BudgetSnapshotRepo",
+    "CostRateRepo",
+    "aggregate_spent",
+    # Snapshot / scheduler jobs (lazy)
+    "refresh_snapshots",
+    "start_budget_refresh_loop",
+    # Audit (lazy)
+    "record_blocked_run",
+    "record_override",
+    # Schema (lazy)
+    "ALL_BUDGET_TABLES",
+    "llm_budgets",
+    "llm_budget_snapshots",
+    "llm_cost_rates",
+]

pf_core/budget/_schema.py

@@ -0,0 +1,122 @@
+"""
+SQLAlchemy schema for pf-core cost budgets.
+
+Defines three framework-owned tables that enforce per-agent, per-job, and
+per-tag cost caps on LLM calls:
+
+- ``llm_budgets`` — authoritative budget definitions (one row per scope+period)
+- ``llm_budget_snapshots`` — periodic aggregate cache for fast pre-call checks
+- ``llm_cost_rates`` — per-model price list for projecting call cost
+
+Shares the ``metadata`` object from ``pf_core.llm.tracking.schema`` so a
+single ``metadata.create_all()`` creates tracking, jobs, cache, and budget
+tables in one pass.
+
+See ``docs/cost-budget.md`` for the full reference.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy import (
+    Boolean,
+    Column,
+    Date,
+    ForeignKey,
+    Index,
+    Integer,
+    Numeric,
+    PrimaryKeyConstraint,
+    String,
+    Table,
+    UniqueConstraint,
+)
+
+from pf_core.llm.tracking.schema import (
+    _JSON,
+    _PK_SMALL,
+    _FK_SMALL,
+    _TIMESTAMP_US,
+    _server_now,
+    metadata,
+)
+
+# ---------------------------------------------------------------------------
+# Tables
+# ---------------------------------------------------------------------------
+
+llm_budgets = Table(
+    "llm_budgets",
+    metadata,
+    Column("id", _PK_SMALL, primary_key=True, autoincrement=True),
+    Column("scope_kind", String(32), nullable=False),
+    Column("scope_value", String(128), nullable=True),
+    Column("period", String(16), nullable=False),
+    Column("limit_usd", Numeric(12, 4), nullable=False),
+    Column("soft_thresholds", _JSON, nullable=True),
+    Column("hard_cap", Boolean, nullable=False, server_default="1"),
+    Column("action", String(32), nullable=False, server_default="block"),
+    Column("enabled", Boolean, nullable=False, server_default="1"),
+    Column("created_at", _TIMESTAMP_US, nullable=False, server_default=_server_now()),
+    Column("updated_at", _TIMESTAMP_US, nullable=False, server_default=_server_now()),
+    UniqueConstraint(
+        "scope_kind", "scope_value", "period", name="uq_llm_budgets_scope_period"
+    ),
+    Index("idx_llm_budgets_enabled", "enabled"),
+)
+"""Budget definitions. scope_kind ∈ {global, agent, job_kind, job_id, tag}."""
+
+
+llm_budget_snapshots = Table(
+    "llm_budget_snapshots",
+    metadata,
+    Column(
+        "budget_id",
+        _FK_SMALL,
+        ForeignKey("llm_budgets.id", ondelete="CASCADE"),
+        nullable=False,
+    ),
+    Column("period_start", Date, nullable=False),
+    Column("spent_usd", Numeric(12, 4), nullable=False, server_default="0"),
+    Column("run_count", Integer, nullable=False, server_default="0"),
+    Column(
+        "last_updated", _TIMESTAMP_US, nullable=False, server_default=_server_now()
+    ),
+    PrimaryKeyConstraint("budget_id", "period_start", name="pk_llm_budget_snapshots"),
+    Index("idx_llm_budget_snapshots_period", "period_start"),
+)
+"""Periodic aggregate cache. Not source of truth — rebuilt from llm_runs."""
+
+
+llm_cost_rates = Table(
+    "llm_cost_rates",
+    metadata,
+    Column(
+        "model_id",
+        _FK_SMALL,
+        ForeignKey("llm_models.id", ondelete="CASCADE"),
+        nullable=False,
+    ),
+    Column("input_per_1k", Numeric(8, 6), nullable=False),
+    Column("output_per_1k", Numeric(8, 6), nullable=False),
+    Column("cache_read_per_1k", Numeric(8, 6), nullable=True),
+    Column("cache_write_per_1k", Numeric(8, 6), nullable=True),
+    Column("reasoning_per_1k", Numeric(8, 6), nullable=True),
+    Column("effective_from", Date, nullable=False),
+    Column("effective_to", Date, nullable=True),
+    PrimaryKeyConstraint(
+        "model_id", "effective_from", name="pk_llm_cost_rates"
+    ),
+    Index("idx_llm_cost_rates_model_eff", "model_id", "effective_from"),
+)
+"""Per-model price list. Multiple rows per model allowed (versioned pricing)."""
+
+
+# ---------------------------------------------------------------------------
+# Public table list (in dependency order)
+# ---------------------------------------------------------------------------
+
+ALL_BUDGET_TABLES = (
+    llm_budgets,
+    llm_budget_snapshots,
+    llm_cost_rates,
+)

pf_core/budget/audit.py

@@ -0,0 +1,73 @@
+"""
+Audit helpers for blocked and override budget events.
+
+``record_blocked_run()`` writes a zero-cost ``llm_runs`` row with
+``status='budget_blocked'`` plus ``budget:blocked`` + ``budget:scope=...``
+tags so analytics can answer "how many calls did the budget stop?".
+
+``record_override()`` attaches a ``budget:override`` tag and an
+``llm_run_outcomes`` row to an existing run.
+"""
+
+from __future__ import annotations
+
+from pf_core.budget.check import CostBudgetExceeded
+from pf_core.llm.tracking.repo import LlmRunRepo
+from pf_core.llm.tracking.schema import llm_run_tags
+from pf_core.llm.tracking.subrepos import LlmRunOutcomeRepo
+
+
+def record_blocked_run(
+    *,
+    agent_type: str,
+    model: str,
+    exc: CostBudgetExceeded,
+    job_id: int | None = None,
+) -> int:
+    """Insert a ``status='budget_blocked'`` run with scope tags.
+
+    Returns the new ``llm_runs.id``.
+    """
+    descriptor = (
+        f"{exc.scope_kind}:{exc.scope_value}:{exc.period}"
+        if exc.scope_value
+        else f"{exc.scope_kind}:{exc.period}"
+    )
+    tags = ["budget:blocked", f"budget:scope={descriptor}"]
+
+    run_id = LlmRunRepo().record(
+        agent_type=agent_type,
+        model=model,
+        status="budget_blocked",
+        usage={
+            "cost_usd": 0.0,
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "duration_ms": 1,
+        },
+        error=str(exc),
+        error_class="CostBudgetExceeded",
+        job_id=job_id,
+        tags=tags,
+    )
+    return int(run_id)
+
+
+def record_override(
+    *,
+    run_id: int,
+    reason: str,
+    operator: str | None = None,
+) -> None:
+    """Tag *run_id* as a budget override and write an outcome row."""
+    from pf_core.db.connection import transaction
+
+    with transaction() as conn:
+        conn.execute(
+            llm_run_tags.insert().values(llm_run_id=run_id, tag="budget:override")
+        )
+    LlmRunOutcomeRepo().record(
+        run_id,
+        outcome_kind="budget_override",
+        notes=reason if not operator else f"{reason} (operator: {operator})",
+    )