loom-notes 1.1.0__py3-none-any.whl

loom_notes/ingest/chunk.py

@@ -0,0 +1,142 @@
+"""Chunking structurel : découpage par titres markdown, puis retaille des sections trop
+longues en paragraphes avec chevauchement. Chaque chunk connaît son chemin de titres."""
+
+import re
+from dataclasses import dataclass
+
+from loom_notes.models import Chunk
+from loom_notes.settings import Settings
+
+_HEADING = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$")
+_FENCE = re.compile(r"^\s*(```|~~~)")
+_PARA_SPLIT = re.compile(r"\n\s*\n")
+_SENTENCE_END = re.compile(r"(?<=[.!?…])\s+")
+
+
+@dataclass(slots=True)
+class _Section:
+    heading_path: str
+    body: str
+
+
+def chunk_document(doc_id: str, title: str, text: str, settings: Settings) -> list[Chunk]:
+    sections = _split_sections(title, text)
+    pieces: list[tuple[str, str]] = []
+    for section in sections:
+        for piece in _split_body(section.body, settings):
+            pieces.append((section.heading_path, piece))
+    pieces = _merge_tiny(pieces, settings)
+    return [
+        Chunk(doc_id=doc_id, index=i, heading_path=path, text=body)
+        for i, (path, body) in enumerate(pieces)
+    ]
+
+
+def _split_sections(title: str, text: str) -> list[_Section]:
+    """Une section par titre rencontré ; le titre du document est la racine du chemin."""
+    stack: list[tuple[int, str]] = []
+    sections: list[_Section] = []
+    buffer: list[str] = []
+    in_fence = False
+
+    def flush() -> None:
+        body = "\n".join(buffer).strip()
+        if body:
+            sections.append(_Section(_path(title, stack), body))
+        buffer.clear()
+
+    for line in text.splitlines():
+        if _FENCE.match(line):
+            in_fence = not in_fence
+            buffer.append(line)
+            continue
+        m = None if in_fence else _HEADING.match(line)
+        if m is None:
+            buffer.append(line)
+            continue
+        flush()
+        level, heading = len(m.group(1)), m.group(2).strip()
+        while stack and stack[-1][0] >= level:
+            stack.pop()
+        # On ne répète pas le titre du document s'il est aussi le premier H1.
+        if not (level == 1 and not stack and heading == title):
+            stack.append((level, heading))
+    flush()
+    return sections or [_Section(title, "")]
+
+
+def _path(title: str, stack: list[tuple[int, str]]) -> str:
+    return " > ".join([title, *(h for _, h in stack)])
+
+
+def _split_body(body: str, settings: Settings) -> list[str]:
+    if not body:
+        return []
+    if len(body) <= settings.chunk_max_chars:
+        return [body]
+    paragraphs = [p.strip() for p in _PARA_SPLIT.split(body) if p.strip()]
+    # Une unité seule doit tenir dans un chunk avec le chevauchement qui la précède.
+    unit_max = max(settings.chunk_target_chars - settings.chunk_overlap_chars, 1)
+    units: list[str] = []
+    for p in paragraphs:
+        units.extend(_split_long(p, unit_max) if len(p) > unit_max else [p])
+
+    chunks: list[str] = []
+    current: list[str] = []
+    size = 0
+    for unit in units:
+        if current and size + len(unit) + 2 > settings.chunk_target_chars:
+            chunks.append("\n\n".join(current))
+            tail = _tail(chunks[-1], settings.chunk_overlap_chars)
+            current = [tail] if tail else []
+            size = len(tail)
+        current.append(unit)
+        size += len(unit) + 2
+    if current:
+        chunks.append("\n\n".join(current))
+    return chunks
+
+
+def _split_long(paragraph: str, max_chars: int) -> list[str]:
+    """Un paragraphe seul plus long que max : coupe aux phrases, sinon en dur."""
+    out: list[str] = []
+    current = ""
+    for sentence in _SENTENCE_END.split(paragraph):
+        if len(sentence) > max_chars:
+            if current:
+                out.append(current)
+                current = ""
+            out.extend(sentence[i : i + max_chars] for i in range(0, len(sentence), max_chars))
+            continue
+        if current and len(current) + len(sentence) + 1 > max_chars:
+            out.append(current)
+            current = sentence
+        else:
+            current = f"{current} {sentence}".strip()
+    if current:
+        out.append(current)
+    return out
+
+
+def _tail(text: str, overlap: int) -> str:
+    if overlap <= 0 or len(text) <= overlap:
+        return ""
+    tail = text[-overlap:]
+    cut = tail.find(" ")
+    return tail[cut + 1 :] if cut != -1 else tail
+
+
+def _merge_tiny(pieces: list[tuple[str, str]], settings: Settings) -> list[tuple[str, str]]:
+    """Absorbe les morceaux trop courts dans le précédent quand ça tient."""
+    merged: list[tuple[str, str]] = []
+    for path, body in pieces:
+        if (
+            merged
+            and len(body) < settings.chunk_min_chars
+            and len(merged[-1][1]) + len(body) + 2 <= settings.chunk_max_chars
+        ):
+            prev_path, prev_body = merged[-1]
+            merged[-1] = (prev_path, f"{prev_body}\n\n{body}")
+        else:
+            merged.append((path, body))
+    return merged

loom_notes/ingest/dedup.py

@@ -0,0 +1,16 @@
+"""Empreinte de contenu, insensible à la casse et aux espaces, pour la déduplication."""
+
+import hashlib
+import re
+import unicodedata
+
+_WS = re.compile(r"\s+")
+
+
+def normalize(text: str) -> str:
+    text = unicodedata.normalize("NFKC", text).casefold()
+    return _WS.sub(" ", text).strip()
+
+
+def content_hash(text: str) -> str:
+    return hashlib.sha256(normalize(text).encode("utf-8")).hexdigest()

loom_notes/ingest/extract.py

@@ -0,0 +1,207 @@
+"""Extraction du texte selon la source : brut, page web, fichier markdown.
+
+Pages web : conversion structurelle HTML → markdown (titres, listes, tableaux, code conservés)
+après retrait du bruit évident (navigation, pied de page, scripts). On privilégie la structure à
+la détection fine du contenu principal : les pages mémorisées sont choisies par l'utilisateur,
+pas moissonnées — et un titre perdu coûte plus cher au retrieval qu'un menu résiduel.
+"""
+
+import asyncio
+import ipaddress
+import re
+import socket
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+import httpx
+from lxml import etree
+from lxml import html as lxml_html
+from lxml.html import HtmlElement, tostring
+from markdownify import markdownify
+
+from loom_notes.settings import Settings
+
+_H1 = re.compile(r"^#\s+(.+?)\s*$", re.MULTILINE)
+_NOISE_XPATH = (
+    "//nav | //footer | //script | //style | //noscript | //iframe | //svg | //form | //template"
+    " | //*[@aria-hidden='true'] | //*[@role='navigation'] | //*[@role='banner' and .//nav]"
+)
+_INLINE_TAGS = ("span", "strong", "b", "em", "i", "code", "a", "small", "abbr", "time")
+
+
+class ExtractionError(Exception):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class Extracted:
+    title: str
+    text: str
+
+
+def extract_from_html(html: str, url: str) -> Extracted:
+    """Contenu de la page en markdown structuré + titre. Lève ExtractionError si vide."""
+    try:
+        doc = cast(HtmlElement, lxml_html.fromstring(html))  # pyright: ignore[reportUnknownMemberType]
+    except (ValueError, etree.ParserError) as exc:
+        raise ExtractionError(f"HTML illisible : {url} ({exc})") from exc
+    title = _page_title(doc) or url
+    for node in _elements(doc, _NOISE_XPATH):
+        parent = node.getparent()
+        if parent is not None:
+            parent.remove(node)
+    _separate_glued_inlines(doc)
+    roots = _elements(doc, "//main") or _elements(doc, "//article") or _elements(doc, "//body")
+    fragment = cast(str, tostring(roots[0] if roots else doc, encoding="unicode"))
+    text = markdownify(
+        fragment, heading_style="ATX", strip=["img", "a"], bullets="-", code_language=""
+    )
+    text = _tidy(text)
+    if not text:
+        raise ExtractionError(f"aucun contenu exploitable extrait de {url}")
+    return Extracted(title=title, text=text)
+
+
+async def fetch_url(url: str, settings: Settings) -> Extracted:
+    """Télécharge la page puis délègue à extract_from_html. Seul point réseau du projet.
+
+    Chaque requête, redirections comprises, passe par `_guard` : sans `fetch_allow_private`,
+    un hôte qui est ou se résout en adresse non publique est refusé. La page est lue par
+    morceaux et coupée au-delà de `fetch_max_bytes`.
+    """
+    try:
+        scheme = httpx.URL(url).scheme
+    except httpx.InvalidURL as exc:
+        raise ExtractionError(f"URL invalide : {url}") from exc
+    if scheme not in ("http", "https"):
+        raise ExtractionError(f"seules les URL http(s) sont acceptées : {url}")
+
+    async def guard(request: httpx.Request) -> None:
+        if not settings.fetch_allow_private:
+            await _refuse_non_public(request.url)
+
+    headers = {"User-Agent": settings.user_agent, "Accept": "text/html,application/xhtml+xml"}
+    try:
+        async with (
+            httpx.AsyncClient(
+                follow_redirects=True,
+                timeout=settings.fetch_timeout_s,
+                headers=headers,
+                event_hooks={"request": [guard]},
+            ) as client,
+            client.stream("GET", url) as resp,
+        ):
+            resp.raise_for_status()
+            ctype = resp.headers.get("content-type", "")
+            if "html" not in ctype and "xml" not in ctype:
+                raise ExtractionError(f"type de contenu non pris en charge : {ctype or 'inconnu'}")
+            body = await _read_capped(resp, settings.fetch_max_bytes)
+            encoding = resp.encoding or "utf-8"
+            final_url = str(resp.url)
+    except httpx.HTTPError as exc:
+        raise ExtractionError(f"téléchargement impossible : {url} ({exc})") from exc
+    return extract_from_html(body.decode(encoding, errors="replace"), url=final_url)
+
+
+async def _refuse_non_public(url: httpx.URL) -> None:
+    """Refuse un hôte qui est, ou dont une des adresses DNS est, non publique."""
+    host = url.host
+    try:
+        addresses = {ipaddress.ip_address(host)}
+    except ValueError:
+        try:
+            infos = await asyncio.get_running_loop().getaddrinfo(
+                host, url.port, type=socket.SOCK_STREAM
+            )
+        except socket.gaierror as exc:
+            raise ExtractionError(f"hôte introuvable : {host}") from exc
+        addresses = {ipaddress.ip_address(info[4][0]) for info in infos}
+    for address in addresses:
+        if not _is_public(address):
+            raise ExtractionError(
+                f"adresse non publique refusée : {host} ({address}). "
+                "LOOM_NOTES_FETCH_ALLOW_PRIVATE=true pour l'autoriser."
+            )
+
+
+def _is_public(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    """Ni boucle locale, ni réseau privé, ni lien local, ni réservée, ni multicast."""
+    if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:
+        address = address.ipv4_mapped
+    return address.is_global and not address.is_multicast
+
+
+async def _read_capped(resp: httpx.Response, max_bytes: int) -> bytes:
+    """Lit le corps (décompressé) sans dépasser max_bytes."""
+    declared = resp.headers.get("content-length", "")
+    if declared.isdigit() and int(declared) > max_bytes:
+        raise ExtractionError(f"page trop volumineuse : {declared} octets (maximum {max_bytes})")
+    chunks: list[bytes] = []
+    size = 0
+    async for chunk in resp.aiter_bytes():
+        size += len(chunk)
+        if size > max_bytes:
+            raise ExtractionError(f"page trop volumineuse : plus de {max_bytes} octets")
+        chunks.append(chunk)
+    return b"".join(chunks)
+
+
+def read_markdown_file(path: Path) -> Extracted:
+    """Lit un fichier markdown local. Le titre est le premier H1, sinon le nom du fichier."""
+    if not path.is_file():
+        raise ExtractionError(f"fichier introuvable : {path}")
+    text = path.read_text(encoding="utf-8").strip()
+    if not text:
+        raise ExtractionError(f"fichier vide : {path}")
+    title = _first_h1(text) or path.stem
+    return Extracted(title=title, text=text)
+
+
+def _first_h1(text: str) -> str | None:
+    m = _H1.search(text)
+    return m.group(1).strip() if m else None
+
+
+def _elements(doc: HtmlElement, xpath: str) -> list[HtmlElement]:
+    return cast(list[HtmlElement], doc.xpath(xpath))
+
+
+def _page_title(doc: HtmlElement) -> str | None:
+    for xpath in (
+        "//meta[@property='og:title']/@content",
+        "//head/title/text()",
+        "//h1//text()",
+    ):
+        values = cast(list[str], doc.xpath(xpath))
+        joined = " ".join(" ".join(values).split())
+        if joined:
+            return joined
+    return None
+
+
+def _separate_glued_inlines(doc: HtmlElement) -> None:
+    """`<span>Label</span>Texte` ou `<span>Label</span><strong>…` sans espace entre les deux :
+    on insère un espace, sinon le markdown colle les mots (« Ce qui arriveL'agent »)."""
+    for el in doc.iter(*_INLINE_TAGS):
+        text = "".join(el.itertext()).rstrip()
+        if not text:
+            continue
+        ends_word = text[-1].isalnum()
+        ends_sentence = text[-1] in ".!?:;"
+        tail = el.tail or ""
+        if tail:
+            if (ends_word and tail[0].isupper()) or (ends_sentence and tail[0].isalpha()):
+                el.tail = " " + tail
+            continue
+        nxt = el.getnext()
+        if ends_word and nxt is not None and isinstance(nxt.tag, str) and nxt.tag in _INLINE_TAGS:
+            head = "".join(nxt.itertext()).lstrip()
+            if head and head[0].isalnum():
+                el.tail = " "
+
+
+def _tidy(text: str) -> str:
+    text = re.sub(r"[ \t]+\n", "\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()

loom_notes/ingest/paths.py

@@ -0,0 +1,53 @@
+"""Contrôle d'accès des fichiers lus par `add_file`.
+
+Trois contrôles, dans cet ordre, et le message dit toujours lequel a mordu :
+1. le chemin résolu (symlinks compris) est inclus dans une racine autorisée ;
+2. aucun composant du chemin relatif ne correspond à un motif refusé ;
+3. l'extension est celle d'un fichier texte/markdown.
+
+La liste de refus de base est dans le code, volontairement : une garantie qu'un fichier de
+configuration peut retirer n'est plus une garantie. La configuration ne peut qu'y ajouter.
+"""
+
+import fnmatch
+from pathlib import Path
+
+from loom_notes.settings import Settings
+
+BASE_DENY: tuple[str, ...] = (
+    ".env", ".env.*", "*.pem", "*.key", "*.p12", "*.pfx",
+    "id_rsa*", "id_ecdsa*", "id_ed25519*",
+    ".ssh", ".aws", ".gnupg", ".netrc", ".npmrc", ".pypirc",
+    ".git", ".venv", "node_modules", "__pycache__",
+)  # fmt: skip
+
+ALLOWED_SUFFIXES: frozenset[str] = frozenset({".md", ".markdown", ".txt"})
+
+
+class PathDeniedError(Exception):
+    pass
+
+
+def resolve_allowed(path: str | Path, settings: Settings) -> Path:
+    """Renvoie le chemin réel si la lecture est autorisée, sinon lève PathDeniedError."""
+    if not settings.allowed_roots:
+        raise PathDeniedError("add_file désactivé : aucune racine dans LOOM_NOTES_ALLOWED_ROOTS")
+    target = Path(path).expanduser().resolve()
+    root = next((r for r in _resolved_roots(settings) if target.is_relative_to(r)), None)
+    if root is None:
+        raise PathDeniedError(f"chemin hors des racines autorisées : {target}")
+    deny = (*BASE_DENY, *settings.deny_patterns)
+    for part in target.relative_to(root).parts:
+        for pattern in deny:
+            if fnmatch.fnmatchcase(part, pattern):
+                raise PathDeniedError(f"chemin refusé par le motif '{pattern}' : {target}")
+    if target.suffix.lower() not in ALLOWED_SUFFIXES:
+        raise PathDeniedError(
+            f"extension non autorisée ({target.suffix or 'aucune'}) ; "
+            f"attendu : {', '.join(sorted(ALLOWED_SUFFIXES))}"
+        )
+    return target
+
+
+def _resolved_roots(settings: Settings) -> list[Path]:
+    return [Path(r).expanduser().resolve() for r in settings.allowed_roots]

loom_notes/models.py

@@ -0,0 +1,90 @@
+"""Modèles de données partagés entre ingestion, store, CLI et serveur MCP."""
+
+from datetime import UTC, datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+SourceKind = Literal["text", "url", "file"]
+
+
+def utc_now_iso() -> str:
+    # Microsecondes conservées : added_at sert de clé de tri, deux ajouts dans la même seconde
+    # doivent rester ordonnés.
+    return datetime.now(UTC).isoformat()
+
+
+class Document(BaseModel):
+    doc_id: str
+    title: str
+    project: str
+    tags: list[str] = Field(default_factory=list)
+    source_kind: SourceKind
+    source: str | None = None
+    added_at: str = Field(default_factory=utc_now_iso)
+    updated_at: str | None = None
+    content_hash: str
+    text: str
+
+
+class Chunk(BaseModel):
+    doc_id: str
+    index: int
+    heading_path: str
+    text: str
+
+    @property
+    def embed_text(self) -> str:
+        """Texte réellement vectorisé : le chemin de titres préfixe le contenu."""
+        return f"{self.heading_path}\n\n{self.text}" if self.heading_path else self.text
+
+
+class DocSummary(BaseModel):
+    doc_id: str
+    title: str
+    project: str
+    tags: list[str]
+    source_kind: SourceKind
+    source: str | None
+    added_at: str
+    updated_at: str | None
+    chars: int
+
+    @classmethod
+    def from_document(cls, doc: Document) -> "DocSummary":
+        return cls(
+            doc_id=doc.doc_id,
+            title=doc.title,
+            project=doc.project,
+            tags=doc.tags,
+            source_kind=doc.source_kind,
+            source=doc.source,
+            added_at=doc.added_at,
+            updated_at=doc.updated_at,
+            chars=len(doc.text),
+        )
+
+
+class Hit(BaseModel):
+    doc_id: str
+    title: str
+    project: str
+    score: float
+    snippet: str = Field(
+        description="Le chunk entier s'il est court, sinon un extrait (voir `truncated`)."
+    )
+    truncated: bool = Field(description="True si `snippet` est un extrait ; `get` donne le reste.")
+    heading_path: str
+    source: str | None
+    added_at: str
+
+
+class IngestResult(BaseModel):
+    doc_id: str
+    title: str
+    project: str
+    chunks: int
+    updated: bool = Field(default=False, description="True si un document existant a été remplacé.")
+    duplicate_of: str | None = Field(
+        default=None, description="doc_id d'un document au contenu identique ; rien n'a été écrit."
+    )

loom_notes/retrieval.py

@@ -0,0 +1,81 @@
+"""Recherche : embedding de la requête → hybride Qdrant → reranking → regroupement par document."""
+
+import asyncio
+from collections.abc import Sequence
+
+from loom_notes.embed.base import Embedder, Reranker
+from loom_notes.models import Hit
+from loom_notes.settings import Settings
+from loom_notes.store.qdrant import MemoryStore, ScoredChunk
+
+
+class Retriever:
+    def __init__(
+        self, settings: Settings, store: MemoryStore, embedder: Embedder, reranker: Reranker
+    ) -> None:
+        self._s = settings
+        self._store = store
+        self._embedder = embedder
+        self.reranker = reranker
+
+    async def search(
+        self,
+        query: str,
+        project: str | None,
+        tags: Sequence[str] | None,
+        k: int,
+        *,
+        min_score: float | None = None,
+    ) -> list[Hit]:
+        query = query.strip()
+        if not query:
+            return []
+        emb = await asyncio.to_thread(self._embedder.embed_query, query)
+        candidates = await self._store.search(
+            emb, project=project, tags=tags, limit=self._s.rerank_candidates
+        )
+        if not candidates:
+            return []
+        passages = [f"{c.heading_path}\n\n{c.text}" for c in candidates]
+        scores = await asyncio.to_thread(self.reranker.score, query, passages)
+        ranked = sorted(zip(candidates, scores, strict=True), key=lambda cs: cs[1], reverse=True)
+        threshold = self._s.min_score if min_score is None else min_score
+        return self._group(ranked, k, threshold)
+
+    def _group(
+        self, ranked: list[tuple[ScoredChunk, float]], k: int, threshold: float
+    ) -> list[Hit]:
+        per_doc: dict[str, int] = {}
+        hits: list[Hit] = []
+        for chunk, score in ranked:
+            if score < threshold:
+                break
+            n = per_doc.get(chunk.doc_id, 0)
+            if n >= self._s.max_chunks_per_doc:
+                continue
+            per_doc[chunk.doc_id] = n + 1
+            short = len(chunk.text) <= self._s.full_chunk_chars
+            hits.append(
+                Hit(
+                    doc_id=chunk.doc_id,
+                    title=chunk.title,
+                    project=chunk.project,
+                    score=round(score, 4),
+                    snippet=chunk.text if short else _snippet(chunk.text, self._s.snippet_chars),
+                    truncated=not short,
+                    heading_path=chunk.heading_path,
+                    source=chunk.source,
+                    added_at=chunk.added_at,
+                )
+            )
+            if len(hits) >= k:
+                break
+        return hits
+
+
+def _snippet(text: str, limit: int) -> str:
+    flat = " ".join(text.split())
+    if len(flat) <= limit:
+        return flat
+    cut = flat.rfind(" ", 0, limit)
+    return flat[: cut if cut > limit // 2 else limit].rstrip() + "…"