prompture 0.0.39.dev1__tar.gz → 0.0.40__tar.gz

{prompture-0.0.39.dev1/prompture.egg-info → prompture-0.0.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prompture
-Version: 0.0.39.dev1
+Version: 0.0.40
 Summary: Ask LLMs to return structured JSON and run cross-model tests. API-first.
 Author-email: Juan Denis <juan@vene.co>
 License-Expression: MIT

prompture-0.0.40/VERSION

@@ -0,0 +1 @@
+0.0.40

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture/__init__.py

@@ -110,6 +110,7 @@ from .image import (
     image_from_url,
     make_image,
 )
+from .ledger import ModelUsageLedger, get_recently_used_models
 from .logging import JSONFormatter, configure_logging
 from .model_rates import (
     ModelCapabilities,
@@ -221,6 +222,7 @@ __all__ = [
     "MemoryCacheBackend",
     "ModelCapabilities",
     "ModelRetry",
+    "ModelUsageLedger",
     "OllamaDriver",
     "OpenAIDriver",
     "OpenRouterDriver",
@@ -268,6 +270,7 @@ __all__ = [
     "get_persona",
     "get_persona_names",
     "get_persona_registry_snapshot",
+    "get_recently_used_models",
     "get_registry_snapshot",
     "get_required_fields",
     "get_trait",

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.39.dev1'
-__version_tuple__ = version_tuple = (0, 0, 39, 'dev1')
+__version__ = version = '0.0.40'
+__version_tuple__ = version_tuple = (0, 0, 40)
 
 __commit_id__ = commit_id = None

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture/async_conversation.py

@@ -304,6 +304,15 @@ class AsyncConversation:
         self._usage["turns"] += 1
         self._maybe_auto_save()
 
+        from .ledger import _resolve_api_key_hash, record_model_usage
+
+        record_model_usage(
+            self._model_name,
+            api_key_hash=_resolve_api_key_hash(self._model_name),
+            tokens=meta.get("total_tokens", 0),
+            cost=meta.get("cost", 0.0),
+        )
+
     async def ask(
         self,
         content: str,

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture/async_core.py

@@ -35,6 +35,18 @@ from .tools import (
 logger = logging.getLogger("prompture.async_core")
 
 
+def _record_usage_to_ledger(model_name: str, meta: dict[str, Any]) -> None:
+    """Fire-and-forget ledger recording for standalone async core functions."""
+    from .ledger import _resolve_api_key_hash, record_model_usage
+
+    record_model_usage(
+        model_name,
+        api_key_hash=_resolve_api_key_hash(model_name),
+        tokens=meta.get("total_tokens", 0),
+        cost=meta.get("cost", 0.0),
+    )
+
+
 async def clean_json_text_with_ai(
     driver: AsyncDriver, text: str, model_name: str = "", options: dict[str, Any] | None = None
 ) -> str:
@@ -117,6 +129,8 @@ async def render_output(
         "model_name": model_name or getattr(driver, "model", ""),
     }
 
+    _record_usage_to_ledger(model_name, resp.get("meta", {}))
+
     return {"text": raw, "usage": usage, "output_format": output_format}
 
 
@@ -211,6 +225,8 @@ async def ask_for_json(
     raw = resp.get("text", "")
     cleaned = clean_json_text(raw)
 
+    _record_usage_to_ledger(model_name, resp.get("meta", {}))
+
     try:
         json_obj = json.loads(cleaned)
         json_string = cleaned

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture/conversation.py

@@ -311,6 +311,15 @@ class Conversation:
         self._usage["turns"] += 1
         self._maybe_auto_save()
 
+        from .ledger import _resolve_api_key_hash, record_model_usage
+
+        record_model_usage(
+            self._model_name,
+            api_key_hash=_resolve_api_key_hash(self._model_name),
+            tokens=meta.get("total_tokens", 0),
+            cost=meta.get("cost", 0.0),
+        )
+
     def ask(
         self,
         content: str,

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture/core.py

@@ -31,6 +31,18 @@ from .tools import (
 logger = logging.getLogger("prompture.core")
 
 
+def _record_usage_to_ledger(model_name: str, meta: dict[str, Any]) -> None:
+    """Fire-and-forget ledger recording for standalone core functions."""
+    from .ledger import _resolve_api_key_hash, record_model_usage
+
+    record_model_usage(
+        model_name,
+        api_key_hash=_resolve_api_key_hash(model_name),
+        tokens=meta.get("total_tokens", 0),
+        cost=meta.get("cost", 0.0),
+    )
+
+
 def _build_content_with_images(text: str, images: list[ImageInput] | None = None) -> str | list[dict[str, Any]]:
     """Return plain string when no images, or a list of content blocks."""
     if not images:
@@ -231,6 +243,8 @@ def render_output(
         "model_name": model_name or getattr(driver, "model", ""),
     }
 
+    _record_usage_to_ledger(model_name, resp.get("meta", {}))
+
     return {"text": raw, "usage": usage, "output_format": output_format}
 
 
@@ -353,6 +367,8 @@ def ask_for_json(
     raw = resp.get("text", "")
     cleaned = clean_json_text(raw)
 
+    _record_usage_to_ledger(model_name, resp.get("meta", {}))
+
     try:
         json_obj = json.loads(cleaned)
         json_string = cleaned

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture/discovery.py

@@ -27,14 +27,18 @@ logger = logging.getLogger(__name__)
 
 
 @overload
-def get_available_models(*, include_capabilities: bool = False) -> list[str]: ...
+def get_available_models(*, include_capabilities: bool = False, verified_only: bool = False) -> list[str]: ...
 
 
 @overload
-def get_available_models(*, include_capabilities: bool = True) -> list[dict[str, Any]]: ...
+def get_available_models(*, include_capabilities: bool = True, verified_only: bool = False) -> list[dict[str, Any]]: ...
 
 
-def get_available_models(*, include_capabilities: bool = False) -> list[str] | list[dict[str, Any]]:
+def get_available_models(
+    *,
+    include_capabilities: bool = False,
+    verified_only: bool = False,
+) -> list[str] | list[dict[str, Any]]:
     """Auto-detect available models based on configured drivers and environment variables.
 
     Iterates through supported providers and checks if they are configured
@@ -46,6 +50,8 @@ def get_available_models(*, include_capabilities: bool = False) -> list[str] | l
         include_capabilities: When ``True``, return enriched dicts with
             ``model``, ``provider``, ``model_id``, and ``capabilities``
             fields instead of plain ``"provider/model_id"`` strings.
+        verified_only: When ``True``, only return models that have been
+            successfully used (as recorded by the usage ledger).
 
     Returns:
         A sorted list of unique model strings (default) or enriched dicts.
@@ -96,11 +102,11 @@ def get_available_models(*, include_capabilities: bool = False) -> list[str] | l
             elif provider == "grok":
                 if settings.grok_api_key or os.getenv("GROK_API_KEY"):
                     is_configured = True
-            elif provider == "ollama":
-                is_configured = True
-            elif provider == "lmstudio":
-                is_configured = True
-            elif provider == "local_http" and os.getenv("LOCAL_HTTP_ENDPOINT"):
+            elif (
+                provider == "ollama"
+                or provider == "lmstudio"
+                or (provider == "local_http" and os.getenv("LOCAL_HTTP_ENDPOINT"))
+            ):
                 is_configured = True
 
             if not is_configured:
@@ -175,12 +181,47 @@ def get_available_models(*, include_capabilities: bool = False) -> list[str] | l
 
     sorted_models = sorted(available_models)
 
+    # --- verified_only filtering ---
+    verified_set: set[str] | None = None
+    if verified_only or include_capabilities:
+        try:
+            from .ledger import _get_ledger
+
+            ledger = _get_ledger()
+            verified_set = ledger.get_verified_models()
+        except Exception:
+            logger.debug("Could not load ledger for verified models", exc_info=True)
+            verified_set = set()
+
+    if verified_only and verified_set is not None:
+        sorted_models = [m for m in sorted_models if m in verified_set]
+
     if not include_capabilities:
         return sorted_models
 
     # Build enriched dicts with capabilities from models.dev
     from .model_rates import get_model_capabilities
 
+    # Fetch all ledger stats for annotation (keyed by model_name)
+    ledger_stats: dict[str, dict[str, Any]] = {}
+    try:
+        from .ledger import _get_ledger
+
+        for row in _get_ledger().get_all_stats():
+            name = row["model_name"]
+            if name not in ledger_stats:
+                ledger_stats[name] = row
+            else:
+                # Aggregate across API key hashes
+                existing = ledger_stats[name]
+                existing["use_count"] += row["use_count"]
+                existing["total_tokens"] += row["total_tokens"]
+                existing["total_cost"] += row["total_cost"]
+                if row["last_used"] > existing["last_used"]:
+                    existing["last_used"] = row["last_used"]
+    except Exception:
+        logger.debug("Could not load ledger stats for enrichment", exc_info=True)
+
     enriched: list[dict[str, Any]] = []
     for model_str in sorted_models:
         parts = model_str.split("/", 1)
@@ -190,13 +231,22 @@ def get_available_models(*, include_capabilities: bool = False) -> list[str] | l
         caps = get_model_capabilities(provider, model_id)
         caps_dict = dataclasses.asdict(caps) if caps is not None else None
 
-        enriched.append(
-            {
-                "model": model_str,
-                "provider": provider,
-                "model_id": model_id,
-                "capabilities": caps_dict,
-            }
-        )
+        entry: dict[str, Any] = {
+            "model": model_str,
+            "provider": provider,
+            "model_id": model_id,
+            "capabilities": caps_dict,
+            "verified": verified_set is not None and model_str in verified_set,
+        }
+
+        stats = ledger_stats.get(model_str)
+        if stats:
+            entry["last_used"] = stats["last_used"]
+            entry["use_count"] = stats["use_count"]
+        else:
+            entry["last_used"] = None
+            entry["use_count"] = 0
+
+        enriched.append(entry)
 
     return enriched

prompture-0.0.40/prompture/ledger.py

@@ -0,0 +1,252 @@
+"""Persistent model usage ledger — tracks which LLM models have been used.
+
+Stores per-model usage stats (call count, tokens, cost, timestamps) in a
+SQLite database at ``~/.prompture/usage/model_ledger.db``.  The public
+convenience functions are fire-and-forget: they never raise exceptions so
+they cannot break existing extraction/conversation flows.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("prompture.ledger")
+
+_DEFAULT_DB_DIR = Path.home() / ".prompture" / "usage"
+_DEFAULT_DB_PATH = _DEFAULT_DB_DIR / "model_ledger.db"
+
+_SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS model_usage (
+    model_name  TEXT NOT NULL,
+    api_key_hash TEXT NOT NULL,
+    use_count   INTEGER NOT NULL DEFAULT 1,
+    total_tokens INTEGER NOT NULL DEFAULT 0,
+    total_cost  REAL NOT NULL DEFAULT 0.0,
+    first_used  TEXT NOT NULL,
+    last_used   TEXT NOT NULL,
+    last_status TEXT NOT NULL DEFAULT 'success',
+    PRIMARY KEY (model_name, api_key_hash)
+);
+"""
+
+
+class ModelUsageLedger:
+    """SQLite-backed model usage tracker.
+
+    Thread-safe via an internal :class:`threading.Lock`.
+
+    Args:
+        db_path: Path to the SQLite database file.  Defaults to
+            ``~/.prompture/usage/model_ledger.db``.
+    """
+
+    def __init__(self, db_path: str | Path | None = None) -> None:
+        self._db_path = Path(db_path) if db_path else _DEFAULT_DB_PATH
+        self._db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+        self._init_db()
+
+    def _init_db(self) -> None:
+        with self._lock:
+            conn = sqlite3.connect(str(self._db_path))
+            try:
+                conn.executescript(_SCHEMA_SQL)
+                conn.commit()
+            finally:
+                conn.close()
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(str(self._db_path))
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    # ------------------------------------------------------------------ #
+    # Recording
+    # ------------------------------------------------------------------ #
+
+    def record_usage(
+        self,
+        model_name: str,
+        *,
+        api_key_hash: str = "",
+        tokens: int = 0,
+        cost: float = 0.0,
+        status: str = "success",
+    ) -> None:
+        """Record a model usage event (upsert).
+
+        On conflict the row's counters are incremented and ``last_used``
+        is updated.
+        """
+        now = datetime.now(timezone.utc).isoformat()
+        with self._lock:
+            conn = self._connect()
+            try:
+                conn.execute(
+                    """
+                    INSERT INTO model_usage
+                        (model_name, api_key_hash, use_count, total_tokens, total_cost,
+                         first_used, last_used, last_status)
+                    VALUES (?, ?, 1, ?, ?, ?, ?, ?)
+                    ON CONFLICT(model_name, api_key_hash) DO UPDATE SET
+                        use_count    = use_count + 1,
+                        total_tokens = total_tokens + excluded.total_tokens,
+                        total_cost   = total_cost + excluded.total_cost,
+                        last_used    = excluded.last_used,
+                        last_status  = excluded.last_status
+                    """,
+                    (model_name, api_key_hash, tokens, cost, now, now, status),
+                )
+                conn.commit()
+            finally:
+                conn.close()
+
+    # ------------------------------------------------------------------ #
+    # Queries
+    # ------------------------------------------------------------------ #
+
+    def get_model_stats(self, model_name: str, api_key_hash: str = "") -> dict[str, Any] | None:
+        """Return stats for a specific model + key combination, or ``None``."""
+        with self._lock:
+            conn = self._connect()
+            try:
+                row = conn.execute(
+                    "SELECT * FROM model_usage WHERE model_name = ? AND api_key_hash = ?",
+                    (model_name, api_key_hash),
+                ).fetchone()
+                if row is None:
+                    return None
+                return dict(row)
+            finally:
+                conn.close()
+
+    def get_verified_models(self) -> set[str]:
+        """Return model names that have at least one successful usage."""
+        with self._lock:
+            conn = self._connect()
+            try:
+                rows = conn.execute(
+                    "SELECT DISTINCT model_name FROM model_usage WHERE last_status = 'success'"
+                ).fetchall()
+                return {r["model_name"] for r in rows}
+            finally:
+                conn.close()
+
+    def get_recently_used(self, limit: int = 10) -> list[dict[str, Any]]:
+        """Return recent model usage rows ordered by ``last_used`` descending."""
+        with self._lock:
+            conn = self._connect()
+            try:
+                rows = conn.execute(
+                    "SELECT * FROM model_usage ORDER BY last_used DESC LIMIT ?",
+                    (limit,),
+                ).fetchall()
+                return [dict(r) for r in rows]
+            finally:
+                conn.close()
+
+    def get_all_stats(self) -> list[dict[str, Any]]:
+        """Return all usage rows."""
+        with self._lock:
+            conn = self._connect()
+            try:
+                rows = conn.execute("SELECT * FROM model_usage ORDER BY last_used DESC").fetchall()
+                return [dict(r) for r in rows]
+            finally:
+                conn.close()
+
+
+# ------------------------------------------------------------------
+# Module-level singleton
+# ------------------------------------------------------------------
+
+_ledger: ModelUsageLedger | None = None
+_ledger_lock = threading.Lock()
+
+
+def _get_ledger() -> ModelUsageLedger:
+    """Return (and lazily create) the module-level singleton ledger."""
+    global _ledger
+    if _ledger is None:
+        with _ledger_lock:
+            if _ledger is None:
+                _ledger = ModelUsageLedger()
+    return _ledger
+
+
+# ------------------------------------------------------------------
+# Public convenience functions (fire-and-forget)
+# ------------------------------------------------------------------
+
+
+def record_model_usage(
+    model_name: str,
+    *,
+    api_key_hash: str = "",
+    tokens: int = 0,
+    cost: float = 0.0,
+    status: str = "success",
+) -> None:
+    """Record a model usage event.  Never raises — all exceptions are swallowed."""
+    try:
+        _get_ledger().record_usage(
+            model_name,
+            api_key_hash=api_key_hash,
+            tokens=tokens,
+            cost=cost,
+            status=status,
+        )
+    except Exception:
+        logger.debug("Failed to record model usage for %s", model_name, exc_info=True)
+
+
+def get_recently_used_models(limit: int = 10) -> list[dict[str, Any]]:
+    """Return recently used models.  Returns empty list on error."""
+    try:
+        return _get_ledger().get_recently_used(limit)
+    except Exception:
+        logger.debug("Failed to get recently used models", exc_info=True)
+        return []
+
+
+# ------------------------------------------------------------------
+# API key hash helper
+# ------------------------------------------------------------------
+
+_LOCAL_PROVIDERS = frozenset({"ollama", "lmstudio", "local_http", "airllm"})
+
+
+def _resolve_api_key_hash(model_name: str) -> str:
+    """Derive an 8-char hex hash of the API key for the given model's provider.
+
+    Local providers (ollama, lmstudio, etc.) return ``""``.
+    """
+    try:
+        provider = model_name.split("/", 1)[0].lower() if "/" in model_name else model_name.lower()
+        if provider in _LOCAL_PROVIDERS:
+            return ""
+
+        from .settings import settings
+
+        key_map: dict[str, str | None] = {
+            "openai": settings.openai_api_key,
+            "claude": settings.claude_api_key,
+            "google": settings.google_api_key,
+            "groq": settings.groq_api_key,
+            "grok": settings.grok_api_key,
+            "openrouter": settings.openrouter_api_key,
+            "azure": settings.azure_api_key,
+            "huggingface": settings.hf_token,
+        }
+        api_key = key_map.get(provider)
+        if not api_key:
+            return ""
+        return hashlib.sha256(api_key.encode()).hexdigest()[:8]
+    except Exception:
+        return ""

{prompture-0.0.39.dev1 → prompture-0.0.40/prompture.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prompture
-Version: 0.0.39.dev1
+Version: 0.0.40
 Summary: Ask LLMs to return structured JSON and run cross-model tests. API-first.
 Author-email: Juan Denis <juan@vene.co>
 License-Expression: MIT

{prompture-0.0.39.dev1 → prompture-0.0.40}/prompture.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ LICENSE
 MANIFEST.in
 README.md
 ROADMAP.md
+VERSION
 pyproject.toml
 requirements.txt
 test.py
@@ -69,6 +70,7 @@ prompture/field_definitions.py
 prompture/group_types.py
 prompture/groups.py
 prompture/image.py
+prompture/ledger.py
 prompture/logging.py
 prompture/model_rates.py
 prompture/persistence.py