icyloom 1.0.0__py3-none-any.whl

icyloom/__init__.py

@@ -0,0 +1,10 @@
+"""IcyLoom — passerelle locale unifiée (cache LLM + tiroir de contexte).
+
+Projet autonome : aucun import, aucun lien, aucun chemin vers un autre projet.
+Tout est piloté par un unique fichier de configuration YAML.
+"""
+
+__all__ = ["__version__", "PROJECT_NAME"]
+
+__version__ = "1.0.0"
+PROJECT_NAME = "icyloom"

icyloom/__main__.py

@@ -0,0 +1,8 @@
+"""Point d'entrée ``python -m icyloom``."""
+
+from __future__ import annotations
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

icyloom/admin.py

@@ -0,0 +1,225 @@
+"""API d'administration et métriques (8153).
+
+Séparée de la passerelle LLM : elle ne relaie aucun trafic de modèle, elle
+expose l'état du système (cache, tiroir, ledger, relais d'outils).
+
+Accès distant : si `server.remote_access_token` est défini, les routes
+protégées exigent l'en-tête `Authorization: Bearer <token>`. Sinon l'API
+reste ouverte sur la boucle locale.
+"""
+
+from __future__ import annotations
+
+import logging
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
+
+from .cache import CacheStore
+from .config import Config
+from .context.store import ContextStore
+from .dashboard import SnapshotCache, dashboard_html, snapshot
+from .db import Database
+from .ledger import Ledger
+from .toolproxy import ToolProxy
+
+log = logging.getLogger("icyloom.admin")
+
+#: Routes qui exigent un jeton quand un jeton est configuré.
+def create_app(config: Config) -> FastAPI:
+    """Construit l'API d'administration."""
+    db = Database(config)
+    db.ensure_schema()
+    cache = CacheStore(db, ttl_seconds=config.cache.ttl_seconds,
+                       max_entries=config.cache.max_entries)
+    context = ContextStore(config)
+    tools = ToolProxy(config)
+    ledger = Ledger(db, enabled=config.ledger.enabled)
+
+    # L'instantané est recalculé en tâche de fond : l'API répond en
+    # millisecondes même quand la base est très chargée.
+    snapshots = SnapshotCache(config, hours=24, days=14, interval=2.0)
+
+    @asynccontextmanager
+    async def lifespan(_app: FastAPI):
+        snapshots.start()
+        try:
+            yield
+        finally:
+            snapshots.stop()
+
+    app = FastAPI(title="IcyLoom Admin", version="1.0.0", lifespan=lifespan)
+    app.state.config = config
+    app.state.snapshots = snapshots
+
+    @app.middleware("http")
+    async def _auth(request: Request, call_next):
+        token = config.server.remote_access_token
+        if token and request.url.path != "/health":
+            header = request.headers.get("authorization", "")
+            if header != f"Bearer {token}":
+                return JSONResponse({"error": "unauthorized"}, status_code=401)
+        return await call_next(request)
+
+    @app.get("/health")
+    async def health():
+        return {"status": "healthy", "service": "icyloom-admin"}
+
+    # -- dashboard --------------------------------------------------------
+    @app.get("/", response_class=HTMLResponse)
+    @app.get("/dashboard", response_class=HTMLResponse)
+    async def dashboard():
+        """Console HTML : toutes les métriques et les graphiques."""
+        return HTMLResponse(dashboard_html())
+
+    @app.get("/assets/logo.svg")
+    async def logo():
+        """Logo officiel, servi tel quel depuis le paquet.
+
+        Le même fichier est versionné dans le dépôt : la console, le README et
+        GitHub partagent donc une seule et même marque.
+        """
+        path = Path(__file__).resolve().parent / "static" / "logo.svg"
+        if not path.exists():
+            raise HTTPException(status_code=404, detail="logo absent")
+        return FileResponse(path, media_type="image/svg+xml")
+
+    @app.get("/api/dashboard")
+    async def dashboard_data(hours: int = 24, days: int = 14):
+        """Instantané unique consommé par la console (lecture seule).
+
+        La fenêtre par défaut vient du cache partagé (recalculé en continu
+        en tâche de fond) : la réponse est instantanée. Les autres fenêtres
+        sont calculées à la demande.
+        """
+        if (hours, days) == (snapshots.hours, snapshots.days):
+            return snapshots.get()
+        return snapshot(config, hours=hours, days=days)
+
+    @app.get("/api/gateway")
+    async def gateway_state():
+        """État living de la passerelle (circuits, amonts) : relais vers 8150.
+
+        Le dashboard reste sur une seule origine : la page ne parle jamais
+        directement à un autre port.
+        """
+        url = f"http://{config.server.host}:{config.ports.gateway_http}/api/status"
+        try:
+            import httpx
+
+            async with httpx.AsyncClient(timeout=2.0) as client:
+                response = await client.get(url)
+                return {"ok": response.status_code == 200,
+                        "status_code": response.status_code,
+                        "state": response.json() if response.status_code == 200 else None}
+        except Exception as exc:  # pragma: no cover - passerelle absente
+            return {"ok": False, "error": str(exc), "state": None}
+
+    @app.get("/api/status")
+    async def status():
+        return {
+            "service": "icyloom",
+            "ports": config.ports.model_dump(),
+            "cache": cache.stats(),
+            "context": context.stats(),
+            "tools": tools.stats(),
+            "ledger": ledger.summary(),
+        }
+
+    @app.get("/api/ledger")
+    async def ledger_endpoint():
+        return ledger.summary()
+
+    @app.get("/api/ledger/daily")
+    async def ledger_daily(days: int = 14):
+        return {"series": ledger.daily(days)}
+
+    @app.get("/api/metrics")
+    async def metrics():
+        with db.cursor(commit=False) as cur:
+            row = cur.execute(
+                "SELECT COUNT(*) AS calls,"
+                " SUM(CASE WHEN kind='hit' THEN 1 ELSE 0 END) AS hits,"
+                " COALESCE(SUM(prompt_tokens + completion_tokens),0) AS real_tokens,"
+                " COALESCE(SUM(saved_prompt + saved_completion),0) AS saved_tokens"
+                " FROM llm_events"
+            ).fetchone()
+        calls = int(row["calls"])
+        hits = int(row["hits"] or 0)
+        real = int(row["real_tokens"])
+        saved = int(row["saved_tokens"])
+        return {
+            "calls": calls,
+            "hits": hits,
+            "hit_rate_pct": round(hits / calls * 100.0, 2) if calls else 0.0,
+            "tokens_real": real,
+            "tokens_saved": saved,
+            "saved_pct": round(saved / (saved + real) * 100.0, 2) if (saved + real) else 0.0,
+            "note": "tokens réels (usage amont). Aucune estimation.",
+        }
+
+    @app.get("/api/cache/stats")
+    async def cache_stats():
+        return cache.stats()
+
+    @app.post("/api/cache-clear")
+    async def cache_clear(request: Request):
+        """Purge du cache LLM. Exige un en-tête de confirmation explicite."""
+        if request.headers.get("X-IcyLoom-Confirm", "").lower() not in ("1", "true", "yes"):
+            raise HTTPException(status_code=400, detail={
+                "error": "confirmation_required",
+                "message": "Envoyer X-IcyLoom-Confirm: 1 pour confirmer.",
+            })
+        payload = await request.json() if await request.body() else {}
+        removed = cache.clear(payload.get("key") if isinstance(payload, dict) else None)
+        return {"removed": removed}
+
+    @app.get("/api/context/stats")
+    async def context_stats():
+        return context.stats()
+
+    @app.get("/api/sessions")
+    async def sessions():
+        return {"sessions": [s.__dict__ for s in context.sessions.list()]}
+
+    @app.post("/api/sessions/purge")
+    async def sessions_purge(request: Request):
+        """Purge d'une session. Exige un en-tête de confirmation explicite."""
+        if request.headers.get("X-IcyLoom-Confirm", "").lower() not in ("1", "true", "yes"):
+            raise HTTPException(status_code=400, detail={
+                "error": "confirmation_required",
+                "message": "Envoyer X-IcyLoom-Confirm: 1 pour confirmer.",
+            })
+        payload = await request.json() if await request.body() else {}
+        session_id = payload.get("session_id") if isinstance(payload, dict) else None
+        if not session_id:
+            raise HTTPException(status_code=400, detail={"error": "session_id_required"})
+        return context.sessions.purge(str(session_id))
+
+    @app.get("/api/tools")
+    async def tools_list():
+        return {"servers": tools.server_names(), "stats": tools.stats()}
+
+    @app.post("/api/tools/clear")
+    async def tools_clear(request: Request):
+        if request.headers.get("X-IcyLoom-Confirm", "").lower() not in ("1", "true", "yes"):
+            raise HTTPException(status_code=400, detail={
+                "error": "confirmation_required",
+                "message": "Envoyer X-IcyLoom-Confirm: 1 pour confirmer.",
+            })
+        payload = await request.json() if await request.body() else {}
+        server = payload.get("server") if isinstance(payload, dict) else None
+        return tools.clear_cache(server)
+
+    return app
+
+
+def run_admin(config: Config, host: str | None = None, port: int | None = None) -> None:
+    """Lance l'API d'administration (bloquant)."""
+    import uvicorn
+
+    uvicorn.run(create_app(config), host=host or config.server.host,
+                port=port or config.ports.admin, log_level="warning")

icyloom/breaker.py

@@ -0,0 +1,157 @@
+"""Circuit breaker par amont.
+
+Trois états : *closed* (tout passe), *open* (tout est refusé), *half-open*
+(une sonde est autorisée pour vérifier le retour à la normale).
+
+Le point important : le **réarmement est réel**. Passé le délai
+``reset_seconds``, l'état *open* devient *half-open* et une seule sonde est
+autorisée. Si la sonde réussit, le breaker revient *closed* et le compteur
+d'échecs est remis à zéro. Sans ce réarmement effectif, un amont recoveries
+resterait bloqué indéfiniment.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class BreakerState(str, Enum):
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+
+@dataclass
+class BreakerStatus:
+    """Photographie de l'état d'un breaker."""
+
+    name: str
+    state: BreakerState
+    failures: int
+    opened_at: float | None
+    seconds_until_reset: float
+    probes_in_flight: int = 0
+
+
+@dataclass
+class CircuitBreaker:
+    """Disjoncteur par amont."""
+
+    name: str
+    failure_threshold: int = 3
+    reset_seconds: float = 30.0
+    half_open_probes: int = 1
+    _failures: int = field(default=0, init=False)
+    _state: BreakerState = field(default=BreakerState.CLOSED, init=False)
+    _opened_at: float | None = field(default=None, init=False)
+    _probes: int = field(default=0, init=False)
+    _lock: threading.Lock = field(default_factory=threading.Lock, init=False)
+
+    # -- état -------------------------------------------------------------
+    @property
+    def state(self) -> BreakerState:
+        with self._lock:
+            self._refresh()
+            return self._state
+
+    def _refresh(self) -> None:
+        """Fait expirer l'état *open* une fois le délai écoulé (appel verrouillé)."""
+        if self._state is BreakerState.OPEN and self._opened_at is not None:
+            if time.monotonic() - self._opened_at >= self.reset_seconds:
+                self._state = BreakerState.HALF_OPEN
+                self._probes = 0
+
+    def allows(self) -> bool:
+        """Le breaker autorise-t-il une requête maintenant ?"""
+        with self._lock:
+            self._refresh()
+            if self._state is BreakerState.CLOSED:
+                return True
+            if self._state is BreakerState.HALF_OPEN:
+                if self._probes < self.half_open_probes:
+                    self._probes += 1
+                    return True
+                return False
+            return False
+
+    # -- feedback ---------------------------------------------------------
+    def record_success(self) -> None:
+        with self._lock:
+            self._failures = 0
+            self._state = BreakerState.CLOSED
+            self._opened_at = None
+            self._probes = 0
+
+    def record_failure(self) -> None:
+        with self._lock:
+            self._failures += 1
+            if self._state is BreakerState.HALF_OPEN:
+                # La sonde a échoué : on rouvre immédiatement.
+                self._state = BreakerState.OPEN
+                self._opened_at = time.monotonic()
+                self._probes = 0
+            elif self._failures >= self.failure_threshold:
+                self._state = BreakerState.OPEN
+                self._opened_at = time.monotonic()
+
+    def reset(self) -> None:
+        """Remise à zéro manuelle."""
+        with self._lock:
+            self._failures = 0
+            self._state = BreakerState.CLOSED
+            self._opened_at = None
+            self._probes = 0
+
+    def status(self) -> BreakerStatus:
+        with self._lock:
+            self._refresh()
+            remaining = 0.0
+            if self._state is BreakerState.OPEN and self._opened_at is not None:
+                remaining = max(0.0, self.reset_seconds - (time.monotonic() - self._opened_at))
+            return BreakerStatus(
+                name=self.name,
+                state=self._state,
+                failures=self._failures,
+                opened_at=self._opened_at,
+                seconds_until_reset=remaining,
+                probes_in_flight=self._probes,
+            )
+
+
+class BreakerRegistry:
+    """Un breaker par amont."""
+
+    def __init__(self, failure_threshold: int = 3, reset_seconds: float = 30.0,
+                 half_open_probes: int = 1) -> None:
+        self._breakers: dict[str, CircuitBreaker] = {}
+        self._failure_threshold = failure_threshold
+        self._reset_seconds = reset_seconds
+        self._half_open_probes = half_open_probes
+        self._lock = threading.Lock()
+
+    def for_name(self, name: str) -> CircuitBreaker:
+        with self._lock:
+            breaker = self._breakers.get(name)
+            if breaker is None:
+                breaker = CircuitBreaker(
+                    name=name,
+                    failure_threshold=self._failure_threshold,
+                    reset_seconds=self._reset_seconds,
+                    half_open_probes=self._half_open_probes,
+                )
+                self._breakers[name] = breaker
+            return breaker
+
+    def all_status(self) -> list[BreakerStatus]:
+        with self._lock:
+            breakers = list(self._breakers.values())
+        return [b.status() for b in breakers]
+
+    def reset_all(self) -> None:
+        with self._lock:
+            breakers = list(self._breakers.values())
+        for breaker in breakers:
+            breaker.reset()

icyloom/cache.py

@@ -0,0 +1,237 @@
+"""Cache LLM : stockage SQLite, expiration (TTL), éviction LRU.
+
+Le cache est **une seule table** de la base unifiée. Les entrées sont
+adressées par la clé canonique calculée par le moteur de normalisation :
+deux requêtes équivalentes partagent donc la même entrée.
+
+Politique d'éviction : le plafond d'entrées est respecté en retirant les
+entrées les moins récemment utilisées (``last_accessed``). Une entrée
+expirée est ignorée à la lecture et purgée par ``purge_expired``.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+from dataclasses import dataclass
+from typing import Any
+
+from .db import Database
+
+
+@dataclass(frozen=True)
+class CacheEntry:
+    """Une entrée de cache lue depuis la base."""
+
+    key: str
+    normalized: str
+    original: str | None
+    response: str
+    is_stream: bool
+    provider: str | None
+    model: str | None
+    created_at: str
+    last_accessed: str
+    access_count: int
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+
+    def payload(self) -> Any:
+        """Réponse décodée (JSON si possible, sinon texte brut)."""
+        try:
+            return json.loads(self.response)
+        except (json.JSONDecodeError, TypeError):
+            return self.response
+
+
+class CacheStore:
+    """Interface du cache LLM sur la base unifiée."""
+
+    def __init__(self, database: Database, ttl_seconds: int = 604800,
+                 max_entries: int = 50000) -> None:
+        self.db = database
+        self.ttl_seconds = ttl_seconds
+        self.max_entries = max_entries
+
+    # -- lecture ----------------------------------------------------------
+    def get(self, key: str, bump: bool = True) -> CacheEntry | None:
+        """Entrée non expirée pour `key` (None si absente ou expirée).
+
+        `bump` incrémente le compteur d'accès et l'horodatage de dernier
+        usage. L'appelant qui possède déjà une transaction (la passerelle)
+        passe `bump=False` et appelle `bump_access` dans *sa* transaction :
+        une seule écriture WAL par requête.
+        """
+        with self.db.cursor(commit=False) as cur:
+            row = cur.execute(
+                "SELECT key, normalized, original, response, is_stream, provider, model,"
+                " created_at, last_accessed, access_count, prompt_tokens, completion_tokens"
+                " FROM cache_entries WHERE key = ?",
+                (key,),
+            ).fetchone()
+            if row is None:
+                return None
+            if self._expired(str(row["created_at"])):
+                return None
+        if bump:
+            self.touch(key)
+        return CacheEntry(
+            key=row["key"],
+            normalized=row["normalized"],
+            original=row["original"],
+            response=row["response"],
+            is_stream=bool(row["is_stream"]),
+            provider=row["provider"],
+            model=row["model"],
+            created_at=row["created_at"],
+            last_accessed=row["last_accessed"],
+            access_count=int(row["access_count"]),
+            prompt_tokens=int(row["prompt_tokens"] or 0),
+            completion_tokens=int(row["completion_tokens"] or 0),
+        )
+
+    def touch(self, key: str) -> None:
+        """Compteur d'accès + horodatage, dans une transaction dédiée."""
+        with self.db.transaction() as cur:
+            self.bump_access(cur, key)
+
+    def bump_access(self, cur: Any, key: str) -> None:
+        """Incrémente le compteur d'accès sur un curseur déjà ouvert.
+
+        Utilisé par la passerelle : le journal, le ledger et ce compteur
+        partagent alors **une seule** transaction.
+        """
+        cur.execute(
+            "UPDATE cache_entries SET access_count = access_count + 1,"
+            " last_accessed = datetime('now') WHERE key = ?",
+            (key,),
+        )
+
+    def search(self, query: str, limit: int = 10) -> list[dict[str, Any]]:
+        """Recherche d'entrées par fragment du texte normalisé."""
+        limit = max(1, min(int(limit), 100))
+        with self.db.cursor(commit=False) as cur:
+            rows = cur.execute(
+                "SELECT key, normalized, provider, model, created_at, access_count"
+                " FROM cache_entries WHERE normalized LIKE ?"
+                " ORDER BY access_count DESC, last_accessed DESC LIMIT ?",
+                (f"%{query}%", limit),
+            ).fetchall()
+        return [
+            {
+                "key": r["key"],
+                "normalized": r["normalized"],
+                "provider": r["provider"],
+                "model": r["model"],
+                "created_at": r["created_at"],
+                "access_count": r["access_count"],
+            }
+            for r in rows
+        ]
+
+    # -- écriture ---------------------------------------------------------
+    def set(
+        self,
+        key: str,
+        normalized: str,
+        response: str,
+        original: str | None = None,
+        is_stream: bool = False,
+        provider: str | None = None,
+        model: str | None = None,
+        raw_hash: str | None = None,
+        prompt_tokens: int = 0,
+        completion_tokens: int = 0,
+    ) -> None:
+        """Enregistre une réponse et applique la politique d'éviction."""
+        with self.db.transaction() as cur:
+            cur.execute(
+                "INSERT INTO cache_entries"
+                " (key, normalized, original, response, is_stream, provider, model, raw_hash,"
+                "  prompt_tokens, completion_tokens)"
+                " VALUES (?,?,?,?,?,?,?,?,?,?)"
+                " ON CONFLICT(key) DO UPDATE SET"
+                " normalized=excluded.normalized, original=excluded.original,"
+                " response=excluded.response, is_stream=excluded.is_stream,"
+                " provider=excluded.provider, model=excluded.model,"
+                " prompt_tokens=excluded.prompt_tokens,"
+                " completion_tokens=excluded.completion_tokens,"
+                " created_at=datetime('now'), last_accessed=datetime('now'),"
+                " access_count=cache_entries.access_count + 1",
+                (key, normalized, original, response, int(bool(is_stream)), provider, model,
+                 raw_hash, int(prompt_tokens), int(completion_tokens)),
+            )
+            total = int(cur.execute("SELECT COUNT(*) FROM cache_entries").fetchone()[0])
+            if total > self.max_entries:
+                excess = total - self.max_entries
+                cur.execute(
+                    "DELETE FROM cache_entries WHERE key IN ("
+                    " SELECT key FROM cache_entries ORDER BY last_accessed ASC LIMIT ?)",
+                    (excess,),
+                )
+
+    # -- maintenance ------------------------------------------------------
+    def clear(self, key: str | None = None) -> int:
+        """Vide tout le cache ou une entrée précise. Renvoie le nombre supprimé."""
+        with self.db.transaction() as cur:
+            if key:
+                cur.execute("DELETE FROM cache_entries WHERE key = ?", (key,))
+                return int(cur.rowcount or 0)
+            cur.execute("DELETE FROM cache_entries")
+            return int(cur.rowcount or 0)
+
+    def purge_expired(self) -> int:
+        """Supprime les entrées plus anciennes que le TTL."""
+        with self.db.transaction() as cur:
+            cur.execute(
+                "DELETE FROM cache_entries WHERE created_at < datetime('now', ?)",
+                (f"-{int(self.ttl_seconds)} seconds",),
+            )
+            return int(cur.rowcount or 0)
+
+    def stats(self) -> dict[str, Any]:
+        """Compteurs d'usage (taux de succès = entrées réellement servies)."""
+        with self.db.cursor(commit=False) as cur:
+            total = int(cur.execute("SELECT COUNT(*) FROM cache_entries").fetchone()[0])
+            hits = int(cur.execute(
+                "SELECT COALESCE(SUM(access_count - 1), 0) FROM cache_entries"
+            ).fetchone()[0] or 0)
+            by_provider = {
+                r[0]: r[1]
+                for r in cur.execute(
+                    "SELECT provider, COUNT(*) FROM cache_entries GROUP BY provider"
+                ).fetchall()
+            }
+            by_model = {
+                r[0]: r[1]
+                for r in cur.execute(
+                    "SELECT model, COUNT(*) FROM cache_entries GROUP BY model"
+                    " ORDER BY COUNT(*) DESC LIMIT 20"
+                ).fetchall()
+            }
+            oldest = cur.execute(
+                "SELECT MIN(created_at) FROM cache_entries"
+            ).fetchone()[0]
+        return {
+            "entries": total,
+            "reuse_count": hits,
+            "by_provider": by_provider,
+            "by_model": by_model,
+            "oldest_entry": oldest,
+            "ttl_seconds": self.ttl_seconds,
+            "max_entries": self.max_entries,
+        }
+
+    # -- interne ----------------------------------------------------------
+    def _expired(self, created_at: str) -> bool:
+        try:
+            with self.db.cursor(commit=False) as cur:
+                row = cur.execute(
+                    "SELECT (julianday('now') - julianday(?)) * 86400 AS age FROM ?",
+                    (created_at, created_at),
+                ).fetchone()
+            age = float(row["age"]) if row and row["age"] is not None else 0.0
+        except sqlite3.Error:
+            return False
+        return age > self.ttl_seconds