loom-notes 1.1.0__py3-none-any.whl

loom_notes/__init__.py

@@ -0,0 +1,8 @@
+"""loom-notes — mémoire locale pour Claude Desktop et les agents IA."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("loom-notes")  # source unique : pyproject.toml
+except PackageNotFoundError:  # package non installé (exécution depuis les sources)
+    __version__ = "0.0.0"

loom_notes/cli.py

@@ -0,0 +1,229 @@
+"""CLI d'administration : ingestion, recherche, sauvegarde, réindexation."""
+
+import asyncio
+import json
+import os
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from pydantic import BaseModel
+
+from loom_notes.ingest.extract import ExtractionError
+from loom_notes.ingest.paths import PathDeniedError
+from loom_notes.service import MemoryService, NotFoundError, build_service
+from loom_notes.settings import Settings
+from loom_notes.store import ModelMismatchError, StoreUnavailableError
+
+app = typer.Typer(no_args_is_help=True, add_completion=False, help="loom-notes — mémoire locale.")
+
+_fake = False
+_data_dir: Path | None = None
+
+
+@app.callback()
+def _root(
+    fake: Annotated[
+        bool, typer.Option("--fake", help="Modèles factices (tests, sans GPU).")
+    ] = False,
+    data_dir: Annotated[
+        Path | None, typer.Option("--data-dir", help="Répertoire des données.")
+    ] = None,
+) -> None:
+    global _fake, _data_dir
+    _fake, _data_dir = fake, data_dir
+    os.environ.setdefault("TQDM_DISABLE", "1")  # barres de progression de FlagEmbedding
+    os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
+
+
+def _settings() -> Settings:
+    return Settings(data_dir=_data_dir) if _data_dir else Settings()
+
+
+def _run[T](fn: Callable[[MemoryService], Awaitable[T]], *, check_model: bool = True) -> T:
+    async def go() -> T:
+        svc = build_service(_settings(), fake=True if _fake else None)
+        try:
+            await svc.start(check_model=check_model)
+            return await fn(svc)
+        finally:
+            await svc.close()
+
+    try:
+        return asyncio.run(go())
+    except (
+        NotFoundError,
+        ValueError,
+        ModelMismatchError,
+        StoreUnavailableError,
+        ExtractionError,
+        PathDeniedError,
+    ) as exc:
+        typer.secho(f"erreur : {exc}", fg=typer.colors.RED, err=True)
+        raise typer.Exit(1) from exc
+
+
+def _dump(obj: BaseModel | list[BaseModel] | dict[str, int]) -> None:
+    if isinstance(obj, BaseModel):
+        typer.echo(obj.model_dump_json(indent=2))
+    elif isinstance(obj, dict):
+        typer.echo(json.dumps(obj, indent=2, ensure_ascii=False))
+    else:
+        typer.echo(json.dumps([o.model_dump() for o in obj], indent=2, ensure_ascii=False))
+
+
+Tags = Annotated[list[str] | None, typer.Option("--tag", "-t", help="Tag (répétable).")]
+
+
+@app.command("add-text")
+def add_text(
+    project: str,
+    title: str,
+    text: Annotated[str | None, typer.Argument(help="Texte ; lu sur stdin si absent.")] = None,
+    tags: Tags = None,
+) -> None:
+    """Ajoute un texte brut."""
+    content = text if text is not None else typer.get_text_stream("stdin").read()
+    _dump(_run(lambda s: s.add_text(content, title, project, tags)))
+
+
+@app.command("add-url")
+def add_url(project: str, url: str, tags: Tags = None) -> None:
+    """Télécharge une page web et l'ajoute."""
+    _dump(_run(lambda s: s.add_url(url, project, tags)))
+
+
+@app.command("add-file")
+def add_file(project: str, path: Path, tags: Tags = None) -> None:
+    """Ajoute un fichier markdown local."""
+    _dump(_run(lambda s: s.add_file(path, project, tags)))
+
+
+@app.command()
+def search(
+    query: str,
+    project: Annotated[str | None, typer.Option("--project", "-p")] = None,
+    tags: Tags = None,
+    k: Annotated[int, typer.Option("--k", "-k", min=1, max=10)] = 5,
+) -> None:
+    """Recherche hybride + reranking."""
+    hits = _run(lambda s: s.search(query, project, tags, k))
+    if not hits:
+        typer.echo("aucun résultat")
+        return
+    for h in hits:
+        typer.secho(f"{h.score:.3f}  {h.title}  [{h.project}]  {h.doc_id}", bold=True)
+        typer.echo(f"       {h.heading_path}{'  (extrait)' if h.truncated else ''}")
+        typer.echo("       " + h.snippet.replace("\n", "\n       ") + "\n")
+
+
+@app.command()
+def get(doc_id: str) -> None:
+    """Affiche un document complet."""
+    _dump(_run(lambda s: s.get(doc_id)))
+
+
+@app.command("list")
+def list_docs(
+    project: Annotated[str | None, typer.Option("--project", "-p")] = None,
+    n: Annotated[int, typer.Option("--n", "-n", min=1, max=500)] = 20,
+) -> None:
+    """Derniers documents ajoutés."""
+    for d in _run(lambda s: s.list_docs(project, n)):
+        when = (d.updated_at or d.added_at)[:19]
+        typer.echo(f"{when}  {d.project:<12}  {d.title}  ({d.chars} car.)  {d.doc_id}")
+
+
+@app.command()
+def projects() -> None:
+    """Projets et nombre de documents."""
+    _dump(_run(lambda s: s.projects()))
+
+
+@app.command()
+def delete(doc_id: str) -> None:
+    """Supprime un document et ses chunks."""
+    _dump(_run(lambda s: s.delete(doc_id)))
+
+
+@app.command()
+def export(path: Annotated[Path | None, typer.Argument()] = None) -> None:
+    """Exporte tous les documents en JSONL (sauvegarde de référence)."""
+    target = path or _settings().export_path
+    n = _run(lambda s: s.export(target))
+    typer.echo(f"{n} document(s) exporté(s) vers {target}")
+
+
+@app.command("import")
+def import_(path: Annotated[Path | None, typer.Argument()] = None) -> None:
+    """Réimporte un export JSONL (les doc_id existants sont ignorés)."""
+    source = path or _settings().export_path
+    imported, skipped = _run(lambda s: s.import_(source))
+    typer.echo(f"{imported} importé(s), {skipped} ignoré(s)")
+
+
+@app.command("eval")
+def eval_(
+    path: Annotated[
+        Path | None, typer.Argument(help="Jeu doré JSONL ; défaut data/golden.jsonl")
+    ] = None,
+    k: Annotated[int, typer.Option("--k", min=1, max=10)] = 10,
+    as_json: Annotated[bool, typer.Option("--json", help="Rapport complet en JSON.")] = False,
+) -> None:
+    """Évalue le retrieval sur le jeu doré : recall@1, recall@5, MRR, seuil suggéré."""
+    from loom_notes.evals import load_golden, run_eval
+
+    golden = path or _settings().golden_path
+    cases = load_golden(golden)
+    if not cases:
+        typer.secho(f"aucun cas dans {golden} — ajoute-en avec `loom-notes eval-add`", err=True)
+        raise typer.Exit(1)
+    report = _run(lambda s: run_eval(s, cases, k))
+    if as_json:
+        typer.echo(report.model_dump_json(indent=2))
+        return
+    for r in report.results:
+        mark = "ok " if r.rank == 1 else ("~  " if r.rank else "KO ")
+        rank = f"#{r.rank}" if r.rank else "absent"
+        score = f"{r.expected_score:.3f}" if r.expected_score is not None else "  -  "
+        typer.echo(f"{mark} {rank:>7}  {score}  {r.query}")
+    typer.echo(
+        f"\n{report.cases} cas — recall@1 {report.recall_at_1:.2f}  "
+        f"recall@5 {report.recall_at_5:.2f}  MRR {report.mrr:.2f}"
+    )
+    if report.suggested_min_score is not None:
+        noise = report.noise_removed_at_suggested
+        typer.echo(
+            f"min_score suggéré : {report.suggested_min_score} "
+            f"(aucun cas perdu ; {noise:.0%} du bruit filtré)"
+            if noise is not None
+            else f"min_score suggéré : {report.suggested_min_score}"
+        )
+
+
+@app.command("eval-add")
+def eval_add(
+    query: str,
+    doc_id: str,
+    project: Annotated[str | None, typer.Option("--project", "-p")] = None,
+    path: Annotated[Path | None, typer.Option("--path")] = None,
+) -> None:
+    """Ajoute un cas au jeu doré (le titre est relu depuis la base pour lisibilité)."""
+    from loom_notes.evals import GoldenCase, append_golden
+
+    doc = _run(lambda s: s.get(doc_id))
+    case = GoldenCase(query=query, doc_id=doc.doc_id, title=doc.title, project=project)
+    append_golden(path or _settings().golden_path, case)
+    typer.echo(f"ajouté : « {query} » → {doc.title}")
+
+
+@app.command()
+def reindex() -> None:
+    """Reconstruit les chunks depuis les documents avec le modèle courant."""
+    n = _run(lambda s: s.reindex(), check_model=False)
+    typer.echo(f"{n} document(s) réindexé(s)")
+
+
+if __name__ == "__main__":
+    app()

loom_notes/embed/__init__.py

@@ -0,0 +1,3 @@
+from loom_notes.embed.base import Embedder, Embedding, Reranker, SparseVec
+
+__all__ = ["Embedder", "Embedding", "Reranker", "SparseVec"]

loom_notes/embed/base.py

@@ -0,0 +1,52 @@
+"""Interfaces des modèles. Synchrones : le service les exécute dans un thread."""
+
+import importlib
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+
+@dataclass(frozen=True, slots=True)
+class SparseVec:
+    indices: list[int]
+    values: list[float]
+
+
+@dataclass(frozen=True, slots=True)
+class Embedding:
+    dense: list[float]
+    sparse: SparseVec
+
+
+class Embedder(Protocol):
+    @property
+    def model_name(self) -> str: ...
+
+    @property
+    def dim(self) -> int: ...
+
+    def embed_documents(self, texts: Sequence[str]) -> list[Embedding]: ...
+
+    def embed_query(self, text: str) -> Embedding: ...
+
+
+class Reranker(Protocol):
+    @property
+    def model_name(self) -> str: ...
+
+    def score(self, query: str, passages: Sequence[str]) -> list[float]: ...
+
+
+def resolve_device(device: str) -> str:
+    """Résout `auto` en cuda, sinon mps, sinon cpu ; toute autre valeur est rendue telle quelle.
+
+    torch n'est importé que pour `auto`, au chargement des vrais modèles.
+    """
+    if device != "auto":
+        return device
+    torch: Any = importlib.import_module("torch")
+    if torch.cuda.is_available():
+        return "cuda"
+    if torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"

loom_notes/embed/bge_m3.py

@@ -0,0 +1,66 @@
+"""BGE-M3 via FlagEmbedding : dense + sparse en un seul passage, chargé paresseusement."""
+
+import importlib
+from collections.abc import Sequence
+from typing import Any
+
+from loom_notes.embed.base import Embedding, SparseVec, resolve_device
+from loom_notes.settings import Settings
+
+
+class BgeM3Embedder:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._model: Any = None
+
+    @property
+    def model_name(self) -> str:
+        return self._settings.dense_model
+
+    @property
+    def dim(self) -> int:
+        return self._settings.dense_dim
+
+    def _load(self) -> Any:
+        if self._model is None:
+            try:
+                flag: Any = importlib.import_module("FlagEmbedding")
+            except ImportError as exc:
+                raise RuntimeError(
+                    'FlagEmbedding absent : pip install "loom-notes[models]"'
+                ) from exc
+            device = resolve_device(self._settings.device)
+            self._model = flag.BGEM3FlagModel(
+                self._settings.dense_model,
+                use_fp16=self._settings.use_fp16 and device.startswith("cuda"),
+                devices=device,
+            )
+        return self._model
+
+    def _encode(self, texts: Sequence[str]) -> list[Embedding]:
+        out: Any = self._load().encode(
+            list(texts),
+            batch_size=16,
+            max_length=1024,
+            return_dense=True,
+            return_sparse=True,
+            return_colbert_vecs=False,
+        )
+        dense_vecs: Any = out["dense_vecs"]
+        lexical: Any = out["lexical_weights"]
+        result: list[Embedding] = []
+        for dense, weights in zip(dense_vecs, lexical, strict=True):
+            items = sorted((int(k), float(v)) for k, v in weights.items() if float(v) > 0)
+            result.append(
+                Embedding(
+                    dense=[float(x) for x in dense],
+                    sparse=SparseVec(indices=[i for i, _ in items], values=[v for _, v in items]),
+                )
+            )
+        return result
+
+    def embed_documents(self, texts: Sequence[str]) -> list[Embedding]:
+        return self._encode(texts) if texts else []
+
+    def embed_query(self, text: str) -> Embedding:
+        return self._encode([text])[0]

loom_notes/embed/fake.py

@@ -0,0 +1,69 @@
+"""Modèles factices et déterministes pour les tests : pas de GPU, pas de téléchargement.
+Le dense est un sac de mots haché, le sparse compte les mots — assez pour que la recherche
+lexicale se comporte de façon plausible."""
+
+import hashlib
+import math
+import re
+from collections import Counter
+from collections.abc import Sequence
+
+from loom_notes.embed.base import Embedding, SparseVec
+
+_TOKEN = re.compile(r"\w+", re.UNICODE)
+
+
+def _tokens(text: str) -> list[str]:
+    return [t.casefold() for t in _TOKEN.findall(text) if len(t) > 1]
+
+
+def _bucket(token: str, mod: int) -> int:
+    return int.from_bytes(hashlib.blake2b(token.encode(), digest_size=4).digest(), "big") % mod
+
+
+class FakeEmbedder:
+    def __init__(self, dim: int = 64) -> None:
+        self._dim = dim
+
+    @property
+    def model_name(self) -> str:
+        return f"fake-{self._dim}"
+
+    @property
+    def dim(self) -> int:
+        return self._dim
+
+    def _one(self, text: str) -> Embedding:
+        counts = Counter(_tokens(text))
+        dense = [0.0] * self._dim
+        for tok, n in counts.items():
+            dense[_bucket(tok, self._dim)] += float(n)
+        norm = math.sqrt(sum(x * x for x in dense)) or 1.0
+        dense = [x / norm for x in dense]
+        sparse_counts: dict[int, float] = {}
+        for tok, n in counts.items():
+            idx = _bucket(tok, 1 << 20)
+            sparse_counts[idx] = sparse_counts.get(idx, 0.0) + float(n)
+        items = sorted(sparse_counts.items())
+        return Embedding(
+            dense=dense,
+            sparse=SparseVec(indices=[i for i, _ in items], values=[v for _, v in items]),
+        )
+
+    def embed_documents(self, texts: Sequence[str]) -> list[Embedding]:
+        return [self._one(t) for t in texts]
+
+    def embed_query(self, text: str) -> Embedding:
+        return self._one(text)
+
+
+class FakeReranker:
+    @property
+    def model_name(self) -> str:
+        return "fake-reranker"
+
+    def score(self, query: str, passages: Sequence[str]) -> list[float]:
+        q = set(_tokens(query))
+        if not q:
+            return [0.0] * len(passages)
+        return [len(q & set(_tokens(p))) / len(q) for p in passages]

loom_notes/embed/reranker.py

@@ -0,0 +1,44 @@
+"""Cross-encoder bge-reranker-v2-m3 via FlagEmbedding, chargé paresseusement."""
+
+import importlib
+from collections.abc import Sequence
+from typing import Any
+
+from loom_notes.embed.base import resolve_device
+from loom_notes.settings import Settings
+
+
+class BgeReranker:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._model: Any = None
+
+    @property
+    def model_name(self) -> str:
+        return self._settings.reranker_model
+
+    def _load(self) -> Any:
+        if self._model is None:
+            try:
+                flag: Any = importlib.import_module("FlagEmbedding")
+            except ImportError as exc:
+                raise RuntimeError(
+                    'FlagEmbedding absent : pip install "loom-notes[models]"'
+                ) from exc
+            device = resolve_device(self._settings.device)
+            self._model = flag.FlagReranker(
+                self._settings.reranker_model,
+                use_fp16=self._settings.use_fp16 and device.startswith("cuda"),
+                devices=device,
+            )
+        return self._model
+
+    def score(self, query: str, passages: Sequence[str]) -> list[float]:
+        if not passages:
+            return []
+        scores: Any = self._load().compute_score(
+            [[query, p] for p in passages], normalize=True, batch_size=16
+        )
+        if isinstance(scores, int | float):
+            return [float(scores)]
+        return [float(s) for s in scores]

loom_notes/evals.py

@@ -0,0 +1,108 @@
+"""Évaluation du retrieval sur un jeu doré : requête → document attendu.
+
+Le jeu est un JSONL (`data/golden.jsonl`), une ligne par cas. Les métriques sont celles qu'on
+compare d'une version à l'autre : recall@1, recall@5, MRR. L'analyse de seuil sert à régler
+`min_score` sans perdre de rappel.
+"""
+
+import json
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from loom_notes.models import Hit
+from loom_notes.service import MemoryService
+
+
+class GoldenCase(BaseModel):
+    query: str
+    doc_id: str
+    title: str | None = None  # informatif, pour relire le fichier
+    project: str | None = None  # filtre appliqué à la recherche, comme le ferait Claude
+
+
+class CaseResult(BaseModel):
+    query: str
+    doc_id: str
+    rank: int | None = Field(description="Rang du document attendu (1 = premier), None si absent")
+    expected_score: float | None
+    top_doc_id: str | None
+    top_score: float | None
+
+
+class EvalReport(BaseModel):
+    cases: int
+    recall_at_1: float
+    recall_at_5: float
+    mrr: float
+    suggested_min_score: float | None = Field(
+        description="Plus haut seuil qui ne fait perdre aucun cas trouvé (score min des bons)"
+    )
+    noise_removed_at_suggested: float | None = Field(
+        description="Part des résultats hors document attendu qui tomberaient sous ce seuil"
+    )
+    results: list[CaseResult]
+
+
+def load_golden(path: Path) -> list[GoldenCase]:
+    if not path.is_file():
+        return []
+    cases: list[GoldenCase] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if line.strip():
+            cases.append(GoldenCase.model_validate(json.loads(line)))
+    return cases
+
+
+def append_golden(path: Path, case: GoldenCase) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as f:
+        f.write(case.model_dump_json(exclude_none=True) + "\n")
+
+
+async def run_eval(service: MemoryService, cases: list[GoldenCase], k: int = 10) -> EvalReport:
+    results: list[CaseResult] = []
+    positives: list[float] = []
+    negatives: list[float] = []
+    for case in cases:
+        hits = await service.search(case.query, project=case.project, k=k, min_score=0.0)
+        docs = _doc_order(hits)
+        rank = docs.index(case.doc_id) + 1 if case.doc_id in docs else None
+        expected = next((h.score for h in hits if h.doc_id == case.doc_id), None)
+        if expected is not None:
+            positives.append(expected)
+        negatives.extend(h.score for h in hits if h.doc_id != case.doc_id)
+        results.append(
+            CaseResult(
+                query=case.query,
+                doc_id=case.doc_id,
+                rank=rank,
+                expected_score=expected,
+                top_doc_id=hits[0].doc_id if hits else None,
+                top_score=hits[0].score if hits else None,
+            )
+        )
+    n = len(results)
+    found = [r.rank for r in results if r.rank is not None]
+    suggested = round(min(positives), 3) if positives else None
+    noise = (
+        sum(1 for s in negatives if s < suggested) / len(negatives)
+        if suggested is not None and negatives
+        else None
+    )
+    return EvalReport(
+        cases=n,
+        recall_at_1=(sum(1 for r in found if r == 1) / n) if n else 0.0,
+        recall_at_5=(sum(1 for r in found if r <= 5) / n) if n else 0.0,
+        mrr=(sum(1 / r for r in found) / n) if n else 0.0,
+        suggested_min_score=suggested,
+        noise_removed_at_suggested=round(noise, 3) if noise is not None else None,
+        results=results,
+    )
+
+
+def _doc_order(hits: list[Hit]) -> list[str]:
+    seen: dict[str, None] = {}
+    for h in hits:
+        seen.setdefault(h.doc_id, None)
+    return list(seen)

loom_notes/ids.py

@@ -0,0 +1,13 @@
+"""Identifiants : doc_id aléatoire, point_id déterministe (réécriture sans orphelins)."""
+
+import uuid
+
+_NAMESPACE = uuid.UUID("6f1c3a2e-9b7d-4e5a-8c0f-2d1e4b6a7c93")
+
+
+def new_doc_id() -> str:
+    return str(uuid.uuid4())
+
+
+def point_id(doc_id: str, chunk_index: int) -> str:
+    return str(uuid.uuid5(_NAMESPACE, f"{doc_id}:{chunk_index}"))

loom_notes/ingest/__init__.py

@@ -0,0 +1,12 @@
+from loom_notes.ingest.chunk import chunk_document
+from loom_notes.ingest.dedup import content_hash
+from loom_notes.ingest.extract import Extracted, extract_from_html, fetch_url, read_markdown_file
+
+__all__ = [
+    "Extracted",
+    "chunk_document",
+    "content_hash",
+    "extract_from_html",
+    "fetch_url",
+    "read_markdown_file",
+]