mentar 0.1.0.dev0__py3-none-any.whl

mentar/grounding/cache.py

@@ -0,0 +1,127 @@
+"""Passage cache: memoize resolved passages by anchor URL.
+
+Memoisation strategy:
+    - In-memory dict (primary): ~zero cost per turn after warm; lives for the
+      process lifetime.  Because ZIM files are static per build, the cache is
+      deterministic and never stale within a session.
+    - Optional on-disk cache (``cache.dir``): pickled dict keyed by anchor URL.
+      Enabled via ``cfg.grounding.cache.enabled = true`` + ``cache.dir`` path.
+      On-disk cache is best-effort: any I/O failure is logged and ignored (never
+      crashes a turn — degradation contract).
+
+Spec: docs/design/W7_grounding_reader.md (Cost row in module contract).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import pickle
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# ── In-memory store ───────────────────────────────────────────────────────────
+_MEM_CACHE: dict[str, str] = {}
+
+_DISK_CACHE_VERSION = 1
+
+
+def _cache_key(anchor: str) -> str:
+    """Stable, filesystem-safe cache key derived from the anchor URL."""
+    return hashlib.sha256(anchor.encode()).hexdigest()
+
+
+def _disk_path(cache_dir: str | Path, anchor: str) -> Path:
+    key = _cache_key(anchor)
+    return Path(cache_dir) / f"{key}.pkl"
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+
+def get(anchor: str, cfg: dict) -> Optional[str]:
+    """Return a cached passage for ``anchor``, or ``None`` if not cached.
+
+    Checks in-memory first, then on-disk (if enabled).
+
+    Args:
+        anchor: The anchor URL (cache key).
+        cfg:    The ``grounding:`` config block.
+
+    Returns:
+        Cached passage string, or ``None``.
+    """
+    # 1. In-memory
+    if anchor in _MEM_CACHE:
+        logger.debug("cache.get: HIT (memory) anchor=%r", anchor)
+        return _MEM_CACHE[anchor]
+
+    # 2. On-disk
+    cache_cfg = cfg.get("cache", {})
+    if not cache_cfg.get("enabled", False):
+        return None
+
+    cache_dir = _resolve_cache_dir(cache_cfg)
+    if not cache_dir:
+        return None
+
+    disk_file = _disk_path(cache_dir, anchor)
+    if disk_file.exists():
+        try:
+            with disk_file.open("rb") as f:
+                data = pickle.load(f)
+            if isinstance(data, dict) and data.get("v") == _DISK_CACHE_VERSION:
+                passage = data["passage"]
+                _MEM_CACHE[anchor] = passage  # warm in-memory cache
+                logger.debug("cache.get: HIT (disk) anchor=%r", anchor)
+                return passage
+        except Exception:
+            logger.warning("cache.get: failed to read disk cache %s", disk_file, exc_info=True)
+
+    return None
+
+
+def put(anchor: str, passage: str, cfg: dict) -> None:
+    """Store ``passage`` in the cache for ``anchor``.
+
+    Args:
+        anchor:  The anchor URL (cache key).
+        passage: The resolved passage string.
+        cfg:     The ``grounding:`` config block.
+    """
+    _MEM_CACHE[anchor] = passage
+
+    cache_cfg = cfg.get("cache", {})
+    if not cache_cfg.get("enabled", False):
+        return
+
+    cache_dir = _resolve_cache_dir(cache_cfg)
+    if not cache_dir:
+        return
+
+    try:
+        Path(cache_dir).mkdir(parents=True, exist_ok=True)
+        disk_file = _disk_path(cache_dir, anchor)
+        with disk_file.open("wb") as f:
+            pickle.dump({"v": _DISK_CACHE_VERSION, "passage": passage}, f)
+        logger.debug("cache.put: wrote disk cache %s", disk_file)
+    except Exception:
+        logger.warning("cache.put: failed to write disk cache", exc_info=True)
+
+
+def clear_memory() -> None:
+    """Clear the in-memory cache (useful in tests)."""
+    _MEM_CACHE.clear()
+
+
+def _resolve_cache_dir(cache_cfg: dict) -> Optional[str]:
+    """Expand env-var substitution in cache_dir and return the resolved string."""
+    raw_dir: str = cache_cfg.get("dir", "")
+    if not raw_dir:
+        return None
+    # Expand ${VAR:-default} style references
+    expanded = os.path.expandvars(raw_dir)
+    return expanded if expanded else None

mentar/grounding/reader.py

@@ -0,0 +1,271 @@
+"""Thin owned libzim reader: open a ZIM archive and extract article text.
+
+Responsibilities:
+    - open(zim_path) → ZimReader context
+    - get_by_url(anchor_url) → raw HTML bytes or None
+    - get_section(html_bytes, passage_hint) → plain text, hint-guided
+
+Design notes:
+    - Uses libzim (runtime dep, pinned in pyproject.toml).  No MCP server, no JSON-RPC.
+    - Search / lookup logic adapted from OpenZIM MCP (cameronrye/openzim-mcp, MIT) as
+      reference; re-implemented minimally for our anchor-resolution-only pilot path.
+    - Hermit-AI (AGPL) = ideas only; no code copied.
+    - This module is ~100-200 lines over libzim.  It never interprets passage content —
+      it returns bytes/text verbatim; the prompt layer neutralises injections (SAFETY §1.5).
+
+Spec: docs/design/W7_grounding_reader.md; SPEC §15 (layer-1 RAG).
+"""
+
+from __future__ import annotations
+
+import html
+import logging
+import re
+from pathlib import Path
+from typing import Optional
+from urllib.parse import unquote, urlparse
+
+logger = logging.getLogger(__name__)
+
+# ── HTML tag stripper ─────────────────────────────────────────────────────────
+# Minimal regex-based stripper: we only need plain-text paragraphs, not a full
+# DOM; a proper parser is overkill for this narrow path.
+_TAG_RE = re.compile(r"<[^>]+>")
+_MULTI_BLANK_RE = re.compile(r"\n{3,}")
+_HEADING_RE = re.compile(
+    r"<h[1-6][^>]*>(.*?)</h[1-6]>", re.IGNORECASE | re.DOTALL
+)
+_PARA_RE = re.compile(r"<p[^>]*>(.*?)</p>", re.IGNORECASE | re.DOTALL)
+_SECTION_HEADING_RE = re.compile(
+    r"<(?:h[1-6])[^>]*>(.*?)</(?:h[1-6])>", re.IGNORECASE | re.DOTALL
+)
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+
+def _strip_html(raw: str) -> str:
+    """Remove HTML tags and unescape entities; return plain text."""
+    text = _TAG_RE.sub(" ", raw)
+    text = html.unescape(text)
+    # Collapse whitespace runs to single spaces, but keep paragraph breaks.
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r" *\n *", "\n", text)
+    text = _MULTI_BLANK_RE.sub("\n\n", text)
+    return text.strip()
+
+
+def _anchor_to_zim_path(anchor_url: str) -> str:
+    """Convert a wiki anchor URL to the ZIM A-namespace path.
+
+    Wiki URLs like ``https://en.vikidia.org/wiki/Fraction`` map to ZIM path
+    ``A/Fraction``.  We strip the ``/wiki/`` prefix and keep the article slug.
+    Path components are URL-decoded (spaces → underscores in ZIM convention).
+
+    Returns e.g. ``A/Fraction`` or ``A/Unit_fraction``.
+    """
+    parsed = urlparse(anchor_url)
+    path = unquote(parsed.path)  # "/wiki/Unit_fraction"
+    # Remove leading /wiki/ or /w/ style prefix
+    for prefix in ("/wiki/", "/w/"):
+        if path.startswith(prefix):
+            path = path[len(prefix):]
+            break
+    else:
+        # No recognised prefix — strip leading slash
+        path = path.lstrip("/")
+    # ZIM stores article pages under A/ namespace (OpenZIM convention)
+    return f"A/{path}" if path else ""
+
+
+def _extract_lead_section(html_content: str) -> str:
+    """Extract the lead (opening) paragraphs before the first heading.
+
+    Most wiki articles begin with an untitled lead section followed by headings.
+    We collect all <p> tags that appear before the first <h2>/<h3>/… heading.
+    """
+    # Find position of first section heading
+    first_heading = _SECTION_HEADING_RE.search(html_content)
+    lead_html = html_content[: first_heading.start()] if first_heading else html_content
+
+    # Collect paragraph text
+    paras = [_strip_html(m.group(1)) for m in _PARA_RE.finditer(lead_html)]
+    text = "\n\n".join(p for p in paras if p.strip())
+    if not text:
+        # Fallback: strip all tags from lead HTML
+        text = _strip_html(lead_html)
+    return text
+
+
+def _extract_section_by_hint(html_content: str, passage_hint: str) -> str:
+    """Extract the section whose heading best matches ``passage_hint``.
+
+    Strategy (deterministic, no model call):
+    1. Split on <h2>/<h3> headings.
+    2. Score each section by word overlap with the hint.
+    3. Return the best-matching section's paragraph text.
+    4. Fall back to lead section if nothing matches well.
+    """
+    # Split into (heading_text, section_html) pairs
+    parts = re.split(
+        r"(<h[1-6][^>]*>.*?</h[1-6]>)", html_content, flags=re.IGNORECASE | re.DOTALL
+    )
+    # parts alternates: [pre_first_heading, heading1, body1, heading2, body2, ...]
+    sections: list[tuple[str, str]] = []
+    # Lead section (before first heading)
+    if parts[0].strip():
+        sections.append(("", parts[0]))
+    i = 1
+    while i + 1 < len(parts):
+        heading_html = parts[i]
+        body_html = parts[i + 1] if i + 1 < len(parts) else ""
+        heading_text = _strip_html(heading_html).lower()
+        sections.append((heading_text, body_html))
+        i += 2
+
+    if not sections:
+        return _strip_html(html_content)
+
+    # Score by hint word overlap
+    hint_words = set(re.split(r"\W+", passage_hint.lower())) - {"", "the", "a", "an", "of", "and"}
+
+    best_score = -1
+    best_body = sections[0][1]  # default to lead section
+
+    for heading_text, body_html in sections:
+        if not heading_text:
+            # Lead section — check body text for hints
+            body_text_lower = _strip_html(body_html).lower()
+        else:
+            body_text_lower = heading_text
+
+        score = sum(1 for w in hint_words if w in body_text_lower)
+        if score > best_score:
+            best_score = score
+            best_body = body_html
+
+    # Extract paragraphs from the best section
+    paras = [_strip_html(m.group(1)) for m in _PARA_RE.finditer(best_body)]
+    text = "\n\n".join(p for p in paras if p.strip())
+    if not text:
+        text = _strip_html(best_body)
+    return text
+
+
+# ── Public ZimReader class ────────────────────────────────────────────────────
+
+
+class ZimReader:
+    """Thin owned wrapper around ``libzim.reader.Archive``.
+
+    Usage::
+
+        reader = ZimReader(zim_path)   # raises FileNotFoundError if path absent
+        html_bytes = reader.get_by_url("https://en.vikidia.org/wiki/Fraction")
+        text = reader.get_section(html_bytes, "Opening section — fraction as part")
+    """
+
+    def __init__(self, zim_path: str | Path) -> None:
+        """Open the ZIM archive.
+
+        Args:
+            zim_path: Path to the ``.zim`` file.
+
+        Raises:
+            FileNotFoundError: If ``zim_path`` does not exist.
+            RuntimeError: If libzim cannot open the archive.
+        """
+        from libzim.reader import Archive  # deferred: libzim not in test-time import
+
+        path = Path(zim_path)
+        if not path.exists():
+            raise FileNotFoundError(f"ZIM file not found: {path}")
+        self._archive = Archive(path)
+        self._zim_path = path
+        logger.debug("ZimReader: opened %s (%d entries)", path.name, self._archive.all_entry_count)
+
+    # ── Lookup ────────────────────────────────────────────────────────────────
+
+    def get_by_url(self, anchor_url: str) -> Optional[bytes]:
+        """Resolve a wiki anchor URL to raw HTML bytes from the ZIM archive.
+
+        Tries:
+        1. ``A/<slug>`` path (ZIM A-namespace convention).
+        2. Title lookup as fallback (handles alternative capitalisations).
+
+        Args:
+            anchor_url: Full wiki URL, e.g. ``https://en.vikidia.org/wiki/Fraction``.
+
+        Returns:
+            Raw HTML bytes of the article, or ``None`` if not found.
+        """
+        zim_path = _anchor_to_zim_path(anchor_url)
+        if not zim_path:
+            logger.warning("get_by_url: could not derive ZIM path from anchor %r", anchor_url)
+            return None
+
+        slug = zim_path[2:]  # strip "A/"
+
+        # 1. Direct path lookup. Try the A/ namespace (older ZIM convention) AND the
+        # bare slug (modern libzim 3.x ZIMs store articles at the root, no A/ prefix).
+        entry = self._lookup_path(zim_path) or self._lookup_path(slug)
+
+        # 2. Title-based fallback using the slug as title (handles capitalisation variants)
+        if entry is None:
+            title = slug.replace("_", " ")
+            entry = self._lookup_title(title)
+
+        if entry is None:
+            logger.warning(
+                "get_by_url: anchor %r not found in %s (tried path=%r, %r, title=%r)",
+                anchor_url, self._zim_path.name, zim_path, slug, slug.replace("_", " "),
+            )
+            return None
+
+        # Follow redirects
+        while entry.is_redirect:
+            entry = entry.get_redirect_entry()
+
+        item = entry.get_item()
+        return bytes(item.content)
+
+    def get_section(self, html_bytes: bytes, passage_hint: str = "") -> str:
+        """Extract a plain-text passage from raw HTML bytes.
+
+        The passage is guided by ``passage_hint`` (a human description, e.g.
+        "Opening section — fraction as part of something").
+
+        This method returns the content **verbatim** after stripping HTML tags.
+        It never interprets, executes, or filters passage content — that is the
+        prompt layer's responsibility (SAFETY §1.5 / W2.3).
+
+        Args:
+            html_bytes: Raw HTML bytes from :meth:`get_by_url`.
+            passage_hint: Human hint describing which section to prefer.
+
+        Returns:
+            Plain-text passage (may be empty if HTML contained no text).
+        """
+        html_content = html_bytes.decode("utf-8", errors="replace")
+        if passage_hint.strip():
+            return _extract_section_by_hint(html_content, passage_hint)
+        return _extract_lead_section(html_content)
+
+    # ── Internal helpers ──────────────────────────────────────────────────────
+
+    def _lookup_path(self, zim_path: str):
+        """Return an Entry by ZIM path, or None on KeyError."""
+        try:
+            return self._archive.get_entry_by_path(zim_path)
+        except KeyError:
+            return None
+
+    def _lookup_title(self, title: str):
+        """Return an Entry by title, or None on KeyError."""
+        try:
+            return self._archive.get_entry_by_title(title)
+        except KeyError:
+            return None
+
+    def __repr__(self) -> str:
+        return f"ZimReader({self._zim_path!r})"

mentar/grounding/resolve.py

@@ -0,0 +1,125 @@
+"""Pilot resolution path: node grounding block → passage string.
+
+Pilot scope (anchor-resolution only):
+    node_grounding dict (source, anchor, passage_hint)
+        → scope guard (source_map.resolve_zim)
+        → cache lookup
+        → ZimReader.get_by_url(anchor)
+        → ZimReader.get_section(html_bytes, passage_hint)
+        → cache put
+        → plain-text passage (or "" on any failure)
+
+No LLM title-prediction, no BM25, no embeddings — those are deferred to W7.5.
+
+Degradation contract (SAFETY §1.5 / SPEC §15):
+    Every failure mode returns "" and logs a warning.  This function NEVER raises
+    (the outer __init__.resolve_grounding has a belt-and-braces try/except too).
+
+Spec: docs/design/W7_grounding_reader.md; SPEC §15 (layer-1 RAG).
+"""
+
+from __future__ import annotations
+
+import logging
+
+from mentar.grounding import cache as grounding_cache
+from mentar.grounding.reader import ZimReader
+from mentar.grounding.source_map import resolve_zim
+from mentar.grounding.sources import materialize_zim
+
+logger = logging.getLogger(__name__)
+
+# Module-level ZimReader instances keyed by resolved ZIM path string.
+# Avoids re-opening the same archive per turn (opening an archive is cheap but
+# not free; re-using the instance is the right posture for a hot path).
+_READER_POOL: dict[str, ZimReader] = {}
+
+
+def _get_reader(zim_path) -> ZimReader | None:
+    """Return a cached ZimReader for ``zim_path``, opening it on first use.
+
+    Returns ``None`` (with a warning) if the ZIM file is absent or unreadable.
+    """
+    key = str(zim_path)
+    if key in _READER_POOL:
+        return _READER_POOL[key]
+    try:
+        reader = ZimReader(zim_path)
+        _READER_POOL[key] = reader
+        return reader
+    except FileNotFoundError:
+        logger.warning("resolve: ZIM not found at %s — returning empty passage", zim_path)
+        return None
+    except Exception:
+        logger.warning("resolve: failed to open ZIM %s", zim_path, exc_info=True)
+        return None
+
+
+def clear_reader_pool() -> None:
+    """Clear the ZimReader pool (useful in tests to force re-open)."""
+    _READER_POOL.clear()
+
+
+def resolve_grounding_inner(node_grounding: dict, cfg: dict) -> str:
+    """Core resolution: node grounding block → plain-text passage or "".
+
+    Called by ``mentar.grounding.resolve_grounding``; may raise on truly
+    unexpected errors (the public API wraps this in a try/except).
+
+    Args:
+        node_grounding: Dict with ``source``, ``anchor``, ``passage_hint``.
+        cfg:            The ``grounding:`` section of the runtime config.
+
+    Returns:
+        Plain-text passage string, or "" on any recoverable failure.
+    """
+    source: str = node_grounding.get("source", "")
+    anchor: str = node_grounding.get("anchor", "")
+    passage_hint: str = node_grounding.get("passage_hint", "")
+
+    if not source or not anchor:
+        logger.warning(
+            "resolve: missing source or anchor in node_grounding=%r — returning empty",
+            node_grounding,
+        )
+        return ""
+
+    # ── 1. Scope guard + ZIM location resolution ──────────────────────────────
+    zim_location = resolve_zim(source, anchor, cfg)
+    if zim_location is None:
+        # Logged inside resolve_zim (scope error or unconfigured source)
+        return ""
+
+    # ── 2. Cache lookup ───────────────────────────────────────────────────────
+    # Before materialization, so a cache hit never triggers an SMB copy.
+    cached = grounding_cache.get(anchor, cfg)
+    if cached is not None:
+        logger.debug("resolve: cache hit for anchor=%r", anchor)
+        return cached
+
+    # ── 3. Materialize the ZIM to a local path (copies from SMB if needed) ─────
+    zim_path = materialize_zim(zim_location, cfg)
+    if zim_path is None:
+        # Logged inside materialize_zim (missing file / SMB failure / no smbprotocol)
+        return ""
+
+    # ── 4. Open ZIM reader ────────────────────────────────────────────────────
+    reader = _get_reader(zim_path)
+    if reader is None:
+        return ""
+
+    # ── 5. Fetch article HTML ─────────────────────────────────────────────────
+    html_bytes = reader.get_by_url(anchor)
+    if html_bytes is None:
+        # Logged inside get_by_url
+        return ""
+
+    # ── 6. Extract passage ────────────────────────────────────────────────────
+    passage = reader.get_section(html_bytes, passage_hint)
+    if not passage or not passage.strip():
+        logger.warning("resolve: empty passage for anchor=%r passage_hint=%r", anchor, passage_hint)
+        return ""
+
+    # ── 7. Cache and return ───────────────────────────────────────────────────
+    grounding_cache.put(anchor, passage, cfg)
+    return passage

mentar/grounding/source_map.py

@@ -0,0 +1,120 @@
+"""Source-enum → ZIM-file mapping and anchor-host scope guard.
+
+Responsibilities:
+    - Map a ``source`` enum value (vikidia | wikipedia_simple | wikibooks | …)
+      to the configured ZIM file path.
+    - Enforce the scope guard: a node's ``source`` must match the anchor's
+      hostname AND the configured ZIM for that source.  A ``vikidia`` node must
+      never resolve out of the vikidia ZIM.
+
+Spec: docs/design/W7_grounding_reader.md (Scope guard row in module contract).
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+# ── Canonical host suffixes per source ───────────────────────────────────────
+# We check that the anchor hostname *ends with* the canonical suffix for the
+# declared source.  This blocks cross-source roaming without coupling us to a
+# single subdomain variant.
+_SOURCE_HOST_SUFFIXES: dict[str, tuple[str, ...]] = {
+    "vikidia": ("vikidia.org",),
+    "wikipedia_simple": ("simple.wikipedia.org",),
+    "wikibooks": ("wikibooks.org",),
+    # parent_upload and builtin have no network anchor — guard is relaxed for them.
+    "parent_upload": (),
+    "builtin": (),
+}
+
+
+class ScopeError(ValueError):
+    """Raised when a node's anchor host does not match its declared source."""
+
+
+def get_zim_path(source: str, cfg: dict) -> str | None:
+    """Return the configured ZIM *location* for ``source``, or ``None`` if unconfigured.
+
+    The ``sources`` entry may be a **structured spec** (``{project, lang,
+    selection?, flavour?, pin?}`` — the newest matching file in ``zim_dir`` is
+    chosen automatically, latest ``YYYY-MM`` wins) or a plain **filename string**
+    (used as-is). The resulting location may be a local path, a mounted-NAS path,
+    or an SMB URL/UNC depending on ``zim_dir`` —
+    :func:`mentar.grounding.sources.materialize_zim` turns it into a local path.
+
+    Args:
+        source: Source enum string from the curriculum node (e.g. ``"vikidia"``).
+        cfg:    The ``grounding:`` config block (``zim_dir``, ``sources`` sub-dict).
+
+    Returns:
+        The joined location string if the source resolves to a file, else ``None``.
+    """
+    from mentar.grounding.sources import join_location, resolve_filename
+
+    zim_dir = cfg.get("zim_dir", "") or "."
+    spec = (cfg.get("sources") or {}).get(source)
+    if not spec:
+        logger.debug("get_zim_path: source %r not in config.grounding.sources", source)
+        return None
+    filename = resolve_filename(spec, zim_dir, cfg)
+    if not filename:
+        logger.warning("get_zim_path: no ZIM file resolved for source %r in %r", source, zim_dir)
+        return None
+    return join_location(zim_dir, filename)
+
+
+def check_scope(source: str, anchor: str) -> None:
+    """Verify that ``anchor``'s hostname matches ``source``'s expected host(s).
+
+    Args:
+        source: Declared source enum (e.g. ``"vikidia"``).
+        anchor: The anchor URL from the curriculum node.
+
+    Raises:
+        ScopeError: If the anchor host does not match the expected source hosts.
+    """
+    suffixes = _SOURCE_HOST_SUFFIXES.get(source)
+    if suffixes is None:
+        # Unknown source — reject for safety
+        raise ScopeError(
+            f"Unknown source {source!r}; expected one of {sorted(_SOURCE_HOST_SUFFIXES)}"
+        )
+    if not suffixes:
+        # Sources without a network anchor (parent_upload, builtin) — no URL check needed
+        return
+
+    parsed = urlparse(anchor)
+    host = parsed.netloc.lower()
+    if not any(host == s or host.endswith("." + s) for s in suffixes):
+        raise ScopeError(
+            f"Scope violation: source={source!r} but anchor host={host!r} "
+            f"(expected host matching {suffixes})"
+        )
+
+
+def resolve_zim(source: str, anchor: str, cfg: dict) -> str | None:
+    """Validate scope and return the ZIM *location* for ``source``.
+
+    Combines :func:`check_scope` and :func:`get_zim_path`.  Returns ``None``
+    (with a logged warning) instead of raising on scope errors so callers can
+    apply the degradation contract; the ScopeError is logged but swallowed here.
+
+    Args:
+        source: Source enum string.
+        anchor: Full wiki URL from the curriculum node.
+        cfg:    Grounding config block.
+
+    Returns:
+        The ZIM location string (local / mounted / SMB), or ``None`` on scope
+        error / missing config.
+    """
+    try:
+        check_scope(source, anchor)
+    except ScopeError as exc:
+        logger.warning("resolve_zim: %s — returning None (degradation path)", exc)
+        return None
+    return get_zim_path(source, cfg)