icycache 0.3.2__py3-none-any.whl

icycache/src/icy_normalize.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""IcyCache - moteur de normalisation autonome.
+
+Port fidele de `server.NormalizationPipeline`, mais SANS effet de bord a l'import :
+ni logging.basicConfig, ni DB, ni app FastAPI, ni client httpx.
+Utilisable donc depuis la gateway (server_new.py) et les tests sans perturber
+le serveur en cours.
+
+Objectif : la cle de cache porte sur le payload NORMALISE, pas sur le payload
+brut. "Bonjour !" et "bonjour" doivent tomber dans la meme entree.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import unicodedata
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+__all__ = [
+    "Normalizer",
+    "normalize_payload",
+    "cache_key",
+    "canonical",
+    "text_blocks",
+    "get_default_normalizer",
+]
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_RULES_PATH = PROJECT_ROOT / "config" / "normalization_rules.json"
+
+# Champs sans effet sur la sortie du modele : exclus de la cle de cache.
+CACHE_KEY_IGNORED = {"stream", "stream_options", "user", "metadata"}
+
+# Bloc volatil injecte par Hermes en fin de message ("Retrieved skills (top-K relevant ...)").
+# Il change a chaque requete et empeche tout HIT : retire de la CLE uniquement.
+_VOLATILE_TAIL = re.compile(r"retrieved skills\W*top\W?k relevant to your query.*$", re.DOTALL)
+
+
+def canonical(obj: Any) -> str:
+    """JSON canonique (cles triees, separateurs compacts) -> hachage stable."""
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str)
+
+
+class Normalizer:
+    """Pipeline : NFKD -> lowercase -> Aho-Corasick (politesse/synonymes) -> accents -> ponctuation."""
+
+    def __init__(self, rules_path: Path = DEFAULT_RULES_PATH):
+        self.rules_path = Path(rules_path)
+        self.config: Dict[str, Any] = {}
+        self.automaton = None
+        self.regex_compiled = []
+        self.unicodedata = unicodedata
+        # Ne supprime que la ponctuation de phrase. +-*/=<>%^&|()[]{} etc. sont conserves :
+        # "12+2" et "1+22" ne doivent PAS partager la meme cle de cache.
+        self.cleanup_regex = re.compile(r"[!?.,;:'\"`\u00b4\u201c\u201d\u2018\u2019\u00ab\u00bb\u00a1\u00bf\u2026]", re.UNICODE)
+        self.space_regex = re.compile(r"\s+")
+
+        if self.rules_path.exists():
+            with open(self.rules_path, "r", encoding="utf-8") as f:
+                self.config = json.load(f)
+        else:  # degrade proprement plutot que de planter
+            self.config = {"rules": []}
+
+        try:
+            import ahocorasick
+            self.automaton = ahocorasick.Automaton()
+            for rule in self.config.get("rules", []):
+                pattern = rule["pattern"].lower()
+                self.automaton.add_word(pattern, (pattern, rule["replace"]))
+            self.automaton.make_automaton()
+        except Exception:
+            self.automaton = None
+            for rule in self.config.get("rules", []):
+                self.regex_compiled.append(
+                    (re.compile(r"\b" + re.escape(rule["pattern"].lower()) + r"\b", re.IGNORECASE), rule["replace"])
+                )
+
+    # -- normalisation de texte brut -------------------------------------
+    def normalize(self, text: str) -> str:
+        if not isinstance(text, str) or not text:
+            return ""
+        text = self.unicodedata.normalize("NFKD", text)
+        text = text.lower()
+        text = self._replace_rules(text)
+        # DELTA-002: normalisation des nombres ("two plus two" → "2+2")
+        text = self._normalize_numbers(text)
+        text = "".join(
+            c for c in self.unicodedata.normalize("NFD", text) if self.unicodedata.category(c) != "Mn"
+        )
+        text = self.cleanup_regex.sub("", text)
+        text = self.space_regex.sub(" ", text).strip()
+        return _VOLATILE_TAIL.sub("", text).strip()
+
+    def _replace_rules(self, text: str) -> str:
+        if self.automaton:
+            def _isw(c: str) -> bool:
+                return c.isalnum() or c == "_" or self.unicodedata.category(c) == "Mn"
+            cands = []
+            for end_idx, (pattern, replacement) in self.automaton.iter(text):
+                s, e = end_idx - len(pattern) + 1, end_idx + 1
+                if s > 0 and _isw(pattern[0]) and _isw(text[s - 1]):
+                    continue
+                if e < len(text) and _isw(pattern[-1]) and _isw(text[e]):
+                    continue
+                cands.append((s, e, replacement))
+            cands.sort(key=lambda t: (t[0], -(t[1] - t[0])))
+            out, pos = [], 0
+            for s, e, rep in cands:
+                if s < pos:
+                    continue
+                out.append(text[pos:s])
+                out.append(rep)
+                pos = e
+            out.append(text[pos:])
+            return "".join(out)
+        for pattern, replacement in self.regex_compiled:
+            text = pattern.sub(replacement, text)
+        return text
+
+    def _normalize_numbers(self, text: str) -> str:
+        """DELTA-002: convertit mots-nombres → chiffres et opérateurs verbaux → symboles."""
+        _NUM_WORDS = {"zero": "0", "one": "1", "two": "2", "three": "3", "four": "4",
+                      "five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9",
+                      "ten": "10"}
+        for w, d in _NUM_WORDS.items():
+            text = re.sub(rf"\b{w}\b", d, text)
+        return (text.replace(" plus ", "+").replace(" minus ", "-")
+                 .replace(" times ", "*").replace(" divided by ", "/"))
+
+    def get_hash(self, normalized_text: str) -> str:
+        return hashlib.sha256((normalized_text or "").encode("utf-8")).hexdigest()
+
+
+_DEFAULT: Optional[Normalizer] = None
+
+
+def get_default_normalizer(rules_path: Path = DEFAULT_RULES_PATH) -> Normalizer:
+    """Instance partagee (construction lazy, thread-safe par GIL sur l'assignation)."""
+    global _DEFAULT
+    if _DEFAULT is None:
+        _DEFAULT = Normalizer(rules_path)
+    return _DEFAULT
+
+
+def text_blocks(content: Any) -> list:
+    """Normalise le contenu textuel d'un message ; les blocs non textuels sont intacts.
+
+    - str  -> texte normalise
+    - list -> chaque bloc {"type": "text", "text": ...} est normalise, les autres
+      (image_url, input_audio, tool_use...) sont conserves tels quels.
+    """
+    if isinstance(content, str):
+        return get_default_normalizer().normalize(content)
+    if isinstance(content, list):
+        out = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str):
+                b = dict(block)
+                b["text"] = get_default_normalizer().normalize(block["text"])
+                out.append(b)
+            else:
+                out.append(block)
+        return out
+    return content
+
+
+def normalize_payload(body: Dict[str, Any]) -> Dict[str, Any]:
+    """Copie du body ou les contenus textuels sont normalises (pour la cle de cache).
+
+    Le body envoye a l'upstream n'est PAS modifie : seule la forme canonique
+    servant de cle de cache est normalisee.
+    """
+    if not isinstance(body, dict):
+        return {}
+    out: Dict[str, Any] = {k: v for k, v in body.items() if k not in CACHE_KEY_IGNORED}
+    for key in ("messages", "system", "prompt", "input"):
+        if key in out:
+            val = out[key]
+            if key == "messages" and isinstance(val, list):
+                out[key] = [
+                    {**m, "content": text_blocks(m.get("content"))} if isinstance(m, dict) else m for m in val
+                ]
+            elif key == "system" and isinstance(val, list):
+                out[key] = text_blocks(val)
+            else:
+                out[key] = text_blocks(val)
+    return out
+
+
+def cache_key(body: Dict[str, Any], scope: Optional[Dict[str, Any]] = None) -> str:
+    """SHA256 du payload normalise (+ perimetre optionnel : upstream, type d'endpoint)."""
+    payload = normalize_payload(body or {})
+    if scope:
+        payload = {"__scope": scope, "body": payload}
+    return hashlib.sha256(canonical(payload).encode("utf-8")).hexdigest()

icycache/src/icy_paths.py

@@ -0,0 +1,76 @@
+"""Chemins IcyCache: racine du projet et repertoire de donnees inscriptible.
+
+Deux modes cohabitent:
+
+- **Mode depot** (checkout source, `.git` present) : `PROJECT_ROOT` est le repo,
+  les donnees (`db/`, `logs/`) restent a cote du code. C'est le comportement
+  historique : `D:\\IcyCache\\db\\cache.db`, `D:\\IcyCache\\logs\\app.log`.
+- **Mode installe** (pip / uvx) : `PROJECT_ROOT` est le repertoire du paquet
+  (site-packages), qui n'est pas inscriptible. Les donnees vont alors dans un
+  repertoire utilisateur : `%LOCALAPPDATA%\\IcyCache` sous Windows,
+  `~/.local/share/IcyCache` ailleurs.
+
+Regle de resolution : un chemin **absolu** dans la config est utilise tel quel ;
+un chemin **relatif** est resolu contre `DATA_DIR`. `ICYCACHE_HOME` force
+`DATA_DIR`, `ICYCACHE_DB` force le chemin de base.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+__all__ = [
+    "PROJECT_ROOT",
+    "IS_SOURCE_CHECKOUT",
+    "DATA_DIR",
+    "default_data_dir",
+    "resolve_data_path",
+]
+
+# parent de src/ : le repo en mode depot, le paquet en mode installe.
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+
+IS_SOURCE_CHECKOUT = (PROJECT_ROOT / ".git").exists()
+
+
+def _writable(path: Path) -> bool:
+    try:
+        path.mkdir(parents=True, exist_ok=True)
+    except OSError:
+        return False
+    return os.access(path, os.W_OK)
+
+
+def _is_in_site_packages(path: Path) -> bool:
+    parts = {p.lower() for p in path.parts}
+    return "site-packages" in parts or "dist-packages" in parts
+
+
+def default_data_dir() -> Path:
+    """Repertoire inscriptible pour `db/` et `logs/`."""
+    env = os.environ.get("ICYCACHE_HOME")
+    if env:
+        return Path(env).expanduser()
+
+    if IS_SOURCE_CHECKOUT:
+        return PROJECT_ROOT
+
+    # Installation editable hors site-packages : on garde la racine du projet.
+    if not _is_in_site_packages(PROJECT_ROOT) and _writable(PROJECT_ROOT):
+        return PROJECT_ROOT
+
+    if os.name == "nt":
+        base = os.environ.get("LOCALAPPDATA") or (Path.home() / "AppData" / "Local")
+    else:
+        base = os.environ.get("XDG_DATA_HOME") or (Path.home() / ".local" / "share")
+    return Path(base) / "IcyCache"
+
+
+DATA_DIR = default_data_dir()
+
+
+def resolve_data_path(value, default_relative: str) -> Path:
+    """Absolu -> tel quel. Relatif (ou vide) -> resolu contre `DATA_DIR`."""
+    raw = str(value) if value else default_relative
+    path = Path(raw)
+    return path if path.is_absolute() else (DATA_DIR / path)

icycache/src/icy_semantic.py

@@ -0,0 +1,197 @@
+"""Cache sémantique hybride — DELTA-002 Volet B.
+
+Qdrant est importé LAZILY: si qdrant-client n'est pas installé ou Qdrant est down,
+tout échec = miss silencieux → upstream. Jamais de panne.
+
+Embedding model: auto-discovery via /v1/models (pas de hardcodage).
+"""
+from __future__ import annotations
+import re
+import json
+import time
+import uuid
+import threading
+import logging
+
+logger = logging.getLogger("icycache.semantic")
+
+# ---- Ancres (grounding anti-faux-positif) -----------------------------------
+
+_HARD = [
+    ("num", r"\b\d+(?:[.,]\d+)?\b"),
+    ("quote", r"[\"'`]([^\"'`]{1,60})[\"'`]"),
+    ("path", r"\b[A-Za-z]:\\[^\s]+|(?:\./|/)[^\s]+"),
+    ("unit", r"\b\d+\s?(?:kg|km|cm|mm|mb|gb|tb|ms|min|h|%|°c|°f|usd|eur)\b"),
+    ("url", r"https?://[^\s]+"),
+]
+
+_NUM_WORDS = {"zero": "0", "one": "1", "two": "2", "three": "3", "four": "4",
+              "five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9",
+              "ten": "10"}
+
+
+def normalize_numbers(t: str) -> str:
+    """Convertit mots-nombres → chiffres et opérateurs verbaux → symboles.
+    À appeler AVANT hash ET avant embedding."""
+    for w, d in _NUM_WORDS.items():
+        t = re.sub(rf"\b{w}\b", d, t)
+    return (t.replace(" plus ", "+").replace(" minus ", "-")
+             .replace(" times ", "*").replace(" divided by ", "/"))
+
+
+def extract_hard(text: str) -> frozenset[str]:
+    low = text.lower()
+    out = set()
+    for _, pat in _HARD:
+        out.update(m.group(0).strip() for m in re.finditer(pat, low))
+    return frozenset(out)
+
+
+def extract_soft(text: str) -> frozenset[str]:
+    return frozenset(
+        m.group(0).lower()
+        for m in re.finditer(r"\b[A-Z][a-zà-ÿ]{2,}\b", text)
+        if m.start() > 0
+    )
+
+
+def accepts(query_norm: str, cand: dict, score: float,
+            HARD_MIN: float = 0.95, RELAX: float = 0.97) -> bool:
+    if score < HARD_MIN:
+        return False
+    qh, ch = extract_hard(query_norm), frozenset(cand.get("anchors_hard", []))
+    if qh != ch:
+        return False
+    qs, cs = extract_soft(query_norm), frozenset(cand.get("anchors_soft", []))
+    if qs == cs:
+        return True
+    # RELAX seulement si ancres hard présentes (grounding numérique/path);
+    # sans hard anchors, soft anchors doivent matcher exactement.
+    if qh and score >= RELAX:
+        return True
+    return False
+
+
+# ---- Embedder (auto-discovery LM Studio) ------------------------------------
+
+class Embedder:
+    def __init__(self, base_url: str, model: str | None = None, timeout: float = 10.0):
+        import httpx
+        self._c = httpx.Client(base_url=base_url, timeout=timeout)
+        self._m = model
+        if not self._m:
+            self._m = self._discover_model()
+
+    def _discover_model(self) -> str:
+        try:
+            r = self._c.get("/v1/models")
+            r.raise_for_status()
+            for m in r.json().get("data", []):
+                mid = m.get("id", "")
+                if "embed" in mid.lower() or "bge" in mid.lower() or "e5" in mid.lower():
+                    return mid
+            # fallback: premier modèle dispo
+            models = r.json().get("data", [])
+            if models:
+                return models[0]["id"]
+        except Exception as e:
+            logger.warning("Embedder auto-discovery failed: %s", e)
+        raise RuntimeError("No embedding model found via /v1/models")
+
+    def embed(self, text: str) -> list[float]:
+        r = self._c.post("/v1/embeddings", json={"model": self._m, "input": text})
+        r.raise_for_status()
+        v = r.json()["data"][0]["embedding"]
+        n = sum(x * x for x in v) ** 0.5 or 1.0
+        return [x / n for x in v]
+
+
+# ---- Store Qdrant (lazy import) ---------------------------------------------
+
+class SemanticStore:
+    COLL = "icycache_semantic"
+    _qdrant = None
+    _client = None
+
+    def __init__(self, url: str):
+        self._url = url
+        self._lock = threading.Lock()
+        self._init_qdrant()
+
+    def _init_qdrant(self):
+        try:
+            from qdrant_client import QdrantClient, models
+            SemanticStore._qdrant = (QdrantClient, models)
+            self._client = QdrantClient(url=self._url)
+        except ImportError:
+            logger.info("qdrant-client not installed — semantic cache disabled (miss silencieux)")
+            self._client = None
+        except Exception as e:
+            logger.warning("Qdrant unreachable: %s — semantic cache disabled", e)
+            self._client = None
+
+    @property
+    def available(self) -> bool:
+        return self._client is not None
+
+    def ensure(self, dim: int):
+        if not self._client:
+            return
+        try:
+            QdrantClient, models = SemanticStore._qdrant
+            if not self._client.collection_exists(self.COLL):
+                self._client.create_collection(
+                    self.COLL,
+                    vectors_config=models.VectorParams(size=dim, distance=models.Distance.COSINE),
+                )
+        except Exception as e:
+            logger.warning("Qdrant ensure collection failed: %s", e)
+            self._client = None
+
+    def search(self, sig: str, vec: list[float], seuil: float, top: int = 1):
+        if not self._client:
+            return None
+        try:
+            QdrantClient, models = SemanticStore._qdrant
+            res = self._client.query_points(
+                collection_name=self.COLL, query=vec, limit=top,
+                score_threshold=seuil,
+                query_filter=models.Filter(must=[models.FieldCondition(
+                    key="sig", match=models.MatchValue(value=sig))]),
+                with_payload=True,
+            ).points
+            return res[0] if res else None
+        except Exception as e:
+            logger.warning("Qdrant search failed: %s — miss silencieux", e)
+            return None
+
+    def upsert(self, vec: list[float], payload: dict):
+        if not self._client:
+            return
+        try:
+            QdrantClient, models = SemanticStore._qdrant
+            with self._lock:
+                self._client.upsert(self.COLL, points=[models.PointStruct(
+                    id=str(uuid.uuid4()), vector=vec, payload=payload)])
+        except Exception as e:
+            logger.warning("Qdrant upsert failed: %s — index skipped", e)
+
+
+# ---- Indexation async (ne bloque jamais) ------------------------------------
+
+def index_async(store: SemanticStore, embedder: Embedder, sig: str,
+                 query_norm: str, resp_json: str):
+    def _job():
+        try:
+            v = embedder.embed(query_norm)
+            store.ensure(len(v))
+            store.upsert(v, {
+                "sig": sig,
+                "response": resp_json,
+                "anchors_hard": sorted(extract_hard(query_norm)),
+                "anchors_soft": sorted(extract_soft(query_norm)),
+                "ts": time.time(),
+            })
+        except Exception:
+            pass  # index échoue → pas grave
+    threading.Thread(target=_job, daemon=True).start()