memkernel 0.2.0__py3-none-any.whl

memkernel/wiki/pages.py

@@ -0,0 +1,165 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""Wiki page model plus a dependency-free frontmatter parser/renderer.
+
+The page format follows the schema used by ``code-copilot-team/knowledge/wiki``
+(page_type, slug, title, status, last_reviewed, sources) so that pages compiled
+by MemKernel pass that wiki's structural linter unchanged.
+
+Only the YAML subset those pages use is supported: top-level scalars and a
+``sources:`` list whose items are flat ``key: value`` mappings.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+_FRONTMATTER_RE = re.compile(r"\A---[ \t]*\n(.*?)\n---[ \t]*\n?", re.DOTALL)
+_SLUG_RE = re.compile(r"[^a-z0-9]+")
+
+
+class WikiPage(BaseModel):
+    """A single markdown page with its frontmatter."""
+
+    page_type: str
+    slug: str
+    title: str
+    status: str = "draft"
+    last_reviewed: str = ""
+    sources: list[dict[str, str]] = Field(default_factory=list)
+    body: str = ""
+    extra: dict[str, str] = Field(default_factory=dict)
+
+
+def slugify(text: str, max_len: int = 60) -> str:
+    """Return a kebab-case slug suitable for a wiki filename."""
+    slug = _SLUG_RE.sub("-", text.lower()).strip("-")
+    if len(slug) > max_len:
+        slug = slug[:max_len].rstrip("-")
+    return slug or "untitled"
+
+
+def _unquote(value: str) -> str:
+    value = value.strip()
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
+        return value[1:-1]
+    return value
+
+
+def _parse_frontmatter(block: str) -> tuple[dict[str, str], list[dict[str, str]]]:
+    """Parse the supported YAML subset into (scalars, sources)."""
+    scalars: dict[str, str] = {}
+    sources: list[dict[str, str]] = []
+    in_sources = False
+    current: dict[str, str] | None = None
+    for raw in block.splitlines():
+        if not raw.strip() or raw.lstrip().startswith("#"):
+            continue
+        indented = raw[0] in " \t"
+        line = raw.strip()
+        if not indented:
+            in_sources = False
+            current = None
+            key, sep, value = line.partition(":")
+            if not sep:
+                continue
+            key = key.strip()
+            if key == "sources":
+                in_sources = True
+                continue
+            scalars[key] = _unquote(value)
+            continue
+        if not in_sources:
+            continue
+        if line.startswith("- "):
+            current = {}
+            sources.append(current)
+            line = line[2:].strip()
+        if current is None:
+            continue
+        key, sep, value = line.partition(":")
+        if sep:
+            current[key.strip()] = _unquote(value)
+    return scalars, sources
+
+
+def parse_page(text: str) -> WikiPage | None:
+    """Parse a markdown document with YAML frontmatter. Returns None if it has none."""
+    match = _FRONTMATTER_RE.match(text)
+    if match is None:
+        return None
+    scalars, sources = _parse_frontmatter(match.group(1))
+    body = text[match.end() :]
+    known = {"page_type", "slug", "title", "status", "last_reviewed"}
+    page_type = scalars.get("page_type", "")
+    slug = scalars.get("slug", "")
+    if not page_type or not slug:
+        return None
+    return WikiPage(
+        page_type=page_type,
+        slug=slug,
+        title=scalars.get("title", slug),
+        status=scalars.get("status", "draft"),
+        last_reviewed=scalars.get("last_reviewed", ""),
+        sources=sources,
+        body=body.strip("\n") + "\n" if body.strip() else "",
+        extra={k: v for k, v in scalars.items() if k not in known},
+    )
+
+
+def _quote(value: str) -> str:
+    """Quote a scalar when YAML would otherwise misread it."""
+    if value == "" or any(ch in value for ch in ":#\"'[]{}") or value != value.strip():
+        return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+    return value
+
+
+def render_page(page: WikiPage) -> str:
+    """Render a WikiPage back to markdown with frontmatter."""
+    lines = [
+        "---",
+        f"page_type: {page.page_type}",
+        f"slug: {page.slug}",
+        f"title: {_quote(page.title)}",
+        f"status: {page.status}",
+        f"last_reviewed: {page.last_reviewed}",
+    ]
+    for key, value in page.extra.items():
+        lines.append(f"{key}: {_quote(value)}")
+    if page.sources:
+        lines.append("sources:")
+        for source in page.sources:
+            first = True
+            for key, value in source.items():
+                prefix = "  - " if first else "    "
+                lines.append(f"{prefix}{key}: {_quote(value)}")
+                first = False
+    lines.append("---")
+    lines.append("")
+    return "\n".join(lines) + page.body
+
+
+def body_sections(body: str) -> dict[str, str]:
+    """Split a page body into its ``##`` sections (heading -> text)."""
+    sections: dict[str, str] = {}
+    current: str | None = None
+    buffer: list[str] = []
+    for line in body.splitlines():
+        if line.startswith("## "):
+            if current is not None:
+                sections[current] = "\n".join(buffer).strip()
+            current = line[3:].strip()
+            buffer = []
+        elif current is not None:
+            buffer.append(line)
+    if current is not None:
+        sections[current] = "\n".join(buffer).strip()
+    return sections
+
+
+def page_to_dict(page: WikiPage) -> dict[str, Any]:
+    """JSON-friendly view of a page (used by the MCP tools)."""
+    return page.model_dump()

memkernel/wiki/sync.py

@@ -0,0 +1,228 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""Export long-term memories to wiki pages and import wiki pages as memories.
+
+Direction 1 — ``WikiExporter`` ("compile"): every long-term memory that has no
+wiki page yet becomes a *draft* page in the matching section, with the page
+sections the wiki schema requires. The exporter is deliberately LLM-free: it
+produces a reviewable draft (the memory content in the primary section,
+placeholders elsewhere) and leaves synthesis to the agent or curator, mirroring
+the proposal → promote flow of ``code-copilot-team/scripts/wiki``.
+
+Direction 2 — ``WikiImporter`` ("index"): every curated page becomes a
+long-term memory (type by page_type) so ``recall`` can find it. Re-running the
+importer is idempotent: unchanged pages dedup on content hash, changed pages
+supersede their previous version.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from memkernel.core.models import MemoryTier, MemoryType
+from memkernel.wiki.pages import WikiPage, parse_page, render_page, slugify
+
+if TYPE_CHECKING:
+    from memkernel.core.models import Memory
+    from memkernel.core.services import MemoryService
+
+logger = logging.getLogger(__name__)
+
+WIKI_TAG = "wiki"
+INDEX_FILE = "index.md"
+LOG_FILE = "log.md"
+
+# MemoryType -> (page_type, directory). CODE is not exported: code lives in the repo.
+EXPORT_MAP: dict[MemoryType, tuple[str, str]] = {
+    MemoryType.DECISION: ("decision", "decisions"),
+    MemoryType.CONVENTION: ("concept", "concepts"),
+    MemoryType.FACT: ("concept", "concepts"),
+    MemoryType.PROCEDURE: ("workflow", "workflows"),
+}
+
+# page_type -> MemoryType for import. Pages without a mapping are skipped.
+IMPORT_MAP: dict[str, MemoryType] = {
+    "decision": MemoryType.DECISION,
+    "concept": MemoryType.FACT,
+    "overview": MemoryType.FACT,
+    "glossary": MemoryType.FACT,
+    "incident": MemoryType.FACT,
+    "open-question": MemoryType.FACT,
+    "workflow": MemoryType.PROCEDURE,
+    "playbook": MemoryType.PROCEDURE,
+}
+
+# Required H2 sections per page type (first one receives the memory content).
+PAGE_SECTIONS: dict[str, list[str]] = {
+    "decision": ["Decision", "Context", "Alternatives considered", "Consequences"],
+    "concept": ["Summary", "Key ideas", "Where this shows up", "Related"],
+    "workflow": ["When to use this", "Steps", "Verification", "Related"],
+}
+
+PLACEHOLDER = "_Not recorded in memory — curate before marking this page stable._"
+
+
+def _today() -> str:
+    return datetime.now(UTC).date().isoformat()
+
+
+@dataclass
+class ExportReport:
+    """Outcome of an export run."""
+
+    written: list[str] = field(default_factory=list)
+    skipped: int = 0
+
+
+@dataclass
+class ImportReport:
+    """Outcome of an import run."""
+
+    imported: list[str] = field(default_factory=list)
+    skipped: list[str] = field(default_factory=list)
+
+
+class WikiExporter:
+    """Compile long-term memories into draft wiki pages under ``wiki_dir``."""
+
+    def __init__(self, service: MemoryService, wiki_dir: Path) -> None:
+        self._service = service
+        self._wiki_dir = Path(wiki_dir)
+
+    def export(self, memories: list[Memory] | None = None, update_index: bool = True) -> ExportReport:
+        """Write one page per exportable memory that is not already a wiki page.
+
+        Memories that came from the wiki (tagged ``wiki``), chunk records, or
+        memories a newer memory supersedes are skipped. Existing pages with
+        the same slug are left untouched: the wiki, once written, is curated
+        by humans/agents, not overwritten by the store.
+        """
+        report = ExportReport()
+        candidates = memories if memories is not None else self._service.list_memories(tier=MemoryTier.LONG_TERM)
+        superseded = {m.supersedes for m in candidates if m.supersedes}
+        for memory in candidates:
+            mapping = EXPORT_MAP.get(memory.type)
+            if mapping is None or WIKI_TAG in memory.tags or memory.ref_id in superseded or memory.chunk_refs:
+                report.skipped += 1
+                continue
+            page_type, directory = mapping
+            page = self._page_for(memory, page_type)
+            target = self._wiki_dir / directory / f"{page.slug}.md"
+            if target.exists():
+                report.skipped += 1
+                continue
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(render_page(page), encoding="utf-8")
+            rel = f"{directory}/{page.slug}.md"
+            report.written.append(rel)
+            if update_index:
+                self._append_index(rel, page)
+                self._append_log(page, "created")
+        return report
+
+    def _page_for(self, memory: Memory, page_type: str) -> WikiPage:
+        title = memory.content.strip().splitlines()[0][:80] if memory.content.strip() else memory.ref_id
+        slug = slugify(title)
+        sections = PAGE_SECTIONS[page_type]
+        body_lines = [f"# {title}", ""]
+        for i, heading in enumerate(sections):
+            body_lines.append(f"## {heading}")
+            body_lines.append("")
+            body_lines.append(memory.content.strip() if i == 0 else PLACEHOLDER)
+            body_lines.append("")
+        sources: list[dict[str, str]] = [
+            {"url": f"memkernel://{memory.project_id}/{memory.ref_id}", "retrieved": _today()}
+        ]
+        if memory.source:
+            sources.append({"path": memory.source})
+        return WikiPage(
+            page_type=page_type,
+            slug=slug,
+            title=title,
+            status="draft",
+            last_reviewed=_today(),
+            sources=sources,
+            body="\n".join(body_lines),
+            extra={"memkernel_ref": memory.ref_id, "memory_type": memory.type.value},
+        )
+
+    def _append_index(self, rel: str, page: WikiPage) -> None:
+        index = self._wiki_dir / INDEX_FILE
+        line = f"- [{page.title}]({rel}) — draft compiled from memkernel ({page.extra.get('memory_type', '')})."
+        if not index.exists():
+            index.write_text(
+                "---\npage_type: index\nslug: index\ntitle: Wiki Index\nstatus: draft\n"
+                f"last_reviewed: {_today()}\n---\n\n# Wiki Index\n\n## Compiled from memkernel\n\n",
+                encoding="utf-8",
+            )
+        text = index.read_text(encoding="utf-8")
+        if f"]({rel})" in text:
+            return
+        if not text.endswith("\n"):
+            text += "\n"
+        index.write_text(text + line + "\n", encoding="utf-8")
+
+    def _append_log(self, page: WikiPage, verb: str) -> None:
+        log = self._wiki_dir / LOG_FILE
+        if not log.exists():
+            log.write_text(
+                "---\npage_type: log\nslug: log\ntitle: Wiki Edit Log\nstatus: draft\n"
+                f"last_reviewed: {_today()}\n---\n\n# Wiki Edit Log\n\n## Entries\n\n",
+                encoding="utf-8",
+            )
+        ref = page.extra.get("memkernel_ref", "")
+        entry = f"- {_today()} — {verb} `{page.slug}` ({page.page_type}): compiled from memkernel {ref}\n"
+        with log.open("a", encoding="utf-8") as fh:
+            fh.write(entry)
+
+
+class WikiImporter:
+    """Index curated wiki pages as long-term memories."""
+
+    def __init__(self, service: MemoryService, wiki_dir: Path) -> None:
+        self._service = service
+        self._wiki_dir = Path(wiki_dir)
+
+    def import_pages(self, session_id: str | None = None) -> ImportReport:
+        """Retain every page with a mapped page_type; return what was imported/skipped."""
+        report = ImportReport()
+        existing = {
+            self._page_tag(m): m
+            for m in self._service.list_memories(tier=MemoryTier.LONG_TERM)
+            if WIKI_TAG in m.tags and self._page_tag(m)
+        }
+        for path in sorted(self._wiki_dir.rglob("*.md")):
+            rel = path.relative_to(self._wiki_dir).as_posix()
+            if rel.startswith(("schema/", "scripts/", ".")):
+                continue
+            page = parse_page(path.read_text(encoding="utf-8"))
+            memory_type = IMPORT_MAP.get(page.page_type) if page else None
+            if page is None or memory_type is None or page.status == "deprecated":
+                report.skipped.append(rel)
+                continue
+            content = f"{page.title}\n\n{page.body}".strip()
+            tags = [WIKI_TAG, f"page:{page.slug}", f"page_type:{page.page_type}"]
+            previous = existing.get(f"page:{page.slug}")
+            supersedes = previous.ref_id if previous is not None and previous.content != content else None
+            self._service.retain(
+                content=content,
+                type=memory_type,
+                session_id=session_id,
+                source=rel,
+                importance=0.9 if page.status == "stable" else 0.7,
+                tags=tags,
+                supersedes=supersedes,
+            )
+            report.imported.append(rel)
+        return report
+
+    @staticmethod
+    def _page_tag(memory: Memory) -> str:
+        for tag in memory.tags:
+            if tag.startswith("page:"):
+                return tag
+        return ""