rdstudio 0.2.0__py3-none-any.whl

rdstudio/learner.py

@@ -0,0 +1,187 @@
+"""The private learner record: what a person has done to learn a project.
+
+Kept apart from the project (``~/.local/share/rdstudio/learners/<project>/`` by
+default, or under ``[learner] path`` in the user config), one JSON event per
+line in ``record.jsonl``, only ever appended. Off until ``[learner] enabled``.
+
+Every event has an ``id``: a ULID (a millisecond timestamp, then randomness, in
+26 sortable characters), and the ``device`` that wrote it. Records from several
+devices merge as a union by id, in any order and as often as needed; sorting by
+id puts them in time order. Corrections are new events that name the id they
+correct, so nothing is ever rewritten.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import secrets
+import subprocess
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+from .config import Config, _read_toml, user_config_path
+from .okf import iso, now, to_datetime
+
+KINDS = ("autodidactic", "interactive", "ai")
+MAX_EVENT_BYTES = 64 * 1024
+_CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+ID_RE = re.compile(r"^[0-9A-HJKMNP-TV-Z]{26}$")
+DEVICE_RE = re.compile(r"^[a-z0-9]{4,16}$")
+
+
+class LearnerError(ValueError):
+    pass
+
+
+def settings() -> dict[str, Any]:
+    return _read_toml(user_config_path()).get("learner", {})
+
+
+def enabled(cfg: Config) -> bool:
+    return bool(settings().get("enabled", False)) and cfg.is_project
+
+
+def project_id(root: Path) -> str:
+    """The repository's first root commit, the same in every clone; outside git,
+    a hash of the path."""
+    try:
+        out = subprocess.run(["git", "rev-list", "--max-parents=0", "HEAD"], cwd=root,
+                             capture_output=True, text=True, timeout=10)
+        roots = sorted(out.stdout.split())
+        if out.returncode == 0 and roots:
+            return roots[0][:16]
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return "path-" + hashlib.sha256(str(root.resolve()).encode()).hexdigest()[:12]
+
+
+def learners_root() -> Path:
+    base = settings().get("path")
+    if base:
+        return Path(base).expanduser()
+    data = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share")
+    return Path(data) / "rdstudio" / "learners"
+
+
+def record_dir(cfg: Config) -> Path:
+    return learners_root() / project_id(cfg.root)
+
+
+def device_id() -> str:
+    """This machine's id in learner records, made once and kept beside them."""
+    path = learners_root() / "device"
+    try:
+        dev = path.read_text(encoding="utf-8").strip()
+        if DEVICE_RE.match(dev):
+            return dev
+    except OSError:
+        pass
+    dev = secrets.token_hex(4)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(dev + "\n", encoding="utf-8")
+    return dev
+
+
+def _encode(n: int, length: int) -> str:
+    return "".join(_CROCKFORD[(n >> (5 * i)) & 31] for i in reversed(range(length)))
+
+
+_last = [0, 0]  # the last id's time and random part, for ids made in the same millisecond
+_lock = threading.Lock()
+
+
+def new_id(ms: int | None = None) -> str:
+    """A ULID. Within one millisecond the random part counts up, so ids from one
+    device sort in the order they were made."""
+    ms = time.time_ns() // 1_000_000 if ms is None else ms
+    with _lock:
+        rand = _last[1] + 1 if ms == _last[0] else secrets.randbits(80)
+        _last[:] = [ms, rand]
+    return _encode(ms, 10) + _encode(rand & ((1 << 80) - 1), 16)
+
+
+def legacy_id(line: str, at: object) -> str:
+    """A fixed id for an event written before events had ids: its time, and a
+    hash of the line in place of randomness, so every device derives the same one."""
+    when = to_datetime(at)
+    ms = int(when.timestamp() * 1000) if when else 0
+    return _encode(ms, 10) + _encode(int(hashlib.sha256(line.encode()).hexdigest()[:20], 16), 16)
+
+
+def content_hash(body: str) -> str:
+    """The version of a note an event refers to."""
+    return hashlib.sha256(body.strip().encode("utf-8")).hexdigest()[:12]
+
+
+def _clean(event: dict[str, Any]) -> dict[str, Any]:
+    if not isinstance(event, dict):
+        raise LearnerError("an event is a JSON object")
+    name = event.get("event")
+    if not isinstance(name, str) or not name or len(name) > 40:
+        raise LearnerError("an event needs an 'event' name")
+    kind = event.get("kind")
+    if kind is not None and kind not in KINDS:
+        raise LearnerError(f"'kind' is one of {', '.join(KINDS)}")
+    eid = event.get("id")
+    if eid is not None and not (isinstance(eid, str) and ID_RE.match(eid)):
+        raise LearnerError("'id' is a ULID (26 characters)")  # given when made offline
+    dev = event.get("device")
+    if dev is not None and not (isinstance(dev, str) and DEVICE_RE.match(dev)):
+        raise LearnerError("'device' is 4 to 16 lowercase letters and digits")
+    rest = {k: v for k, v in event.items() if k not in ("at", "id", "device")}
+    out = {"id": eid or new_id(), "at": iso(now()), "device": dev or device_id(), **rest}
+    if len(json.dumps(out, ensure_ascii=False, separators=(",", ":")).encode()) > MAX_EVENT_BYTES:
+        raise LearnerError("event too large")
+    return out
+
+
+def append(cfg: Config, event: dict[str, Any]) -> dict[str, Any]:
+    """Add one event to the record and return it as stored."""
+    out = _clean(event)
+    path = record_dir(cfg) / "record.jsonl"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(out, ensure_ascii=False, separators=(",", ":")) + "\n")
+    return out
+
+
+def read(path: Path) -> list[dict[str, Any]]:
+    """The events in one record file, in the order written, each once (a retried
+    write can repeat one); lines that do not parse are skipped."""
+    if not path.is_file():
+        return []
+    out, seen = [], set()
+    for line in path.read_text(encoding="utf-8").splitlines():
+        try:
+            e = json.loads(line)
+        except ValueError:
+            continue
+        if not isinstance(e, dict):
+            continue
+        if not isinstance(e.get("id"), str):
+            e = {"id": legacy_id(line, e.get("at")), **e}
+        if e["id"] not in seen:
+            seen.add(e["id"])
+            out.append(e)
+    return out
+
+
+def merge(*records: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """The union of several devices' records, by id, in time order. The same in
+    any order and however often it is repeated."""
+    by_id: dict[str, dict[str, Any]] = {}
+    for record in records:
+        for e in record:
+            by_id.setdefault(e["id"], e)
+    return [by_id[k] for k in sorted(by_id)]
+
+
+def events(cfg: Config, *, concept: str | None = None) -> list[dict[str, Any]]:
+    """Every event, oldest first."""
+    out = read(record_dir(cfg) / "record.jsonl")
+    return [e for e in out if concept is None or e.get("concept") == concept]

rdstudio/mcp_server.py

@@ -0,0 +1,313 @@
+"""Local stdio MCP server over the project knowledge bundle.
+
+Every tool returns compact text so an agent spends as little context as
+possible: search returns ids and one-line snippets, outline returns headings,
+and read can return a single section.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from mcp.server.mcpserver import MCPServer
+
+from . import __version__, classify, procedures, references, scopes
+from .brief import brief as project_brief
+from .config import Config
+from .okf import Bundle, dump_frontmatter, headings, jsonable, section
+from .search import Index
+from .store import StoreError, record as store_record
+
+INSTRUCTIONS = """\
+Project knowledge base (OKF markdown bundle). Retrieve progressively:
+search -> outline -> read(section). Prefer reading one section over a whole
+concept. Before recording, search for an existing concept to update rather
+than creating a duplicate. Record decisions, answered questions, findings and
+procedures as they happen. Mark an edit significant=false only for trivial or
+dictated changes; significant edits flag human-reviewed concepts for re-review.
+Never claim human verification; only humans verify (rdstudio verify)."""
+
+
+def _fmt(obj: Any) -> str:
+    return json.dumps(jsonable(obj), ensure_ascii=False, indent=1)
+
+
+def create_server(cfg: Config) -> MCPServer:
+    server = MCPServer("rdstudio", instructions=INSTRUCTIONS, version=__version__)
+    classifier = classify.from_config(cfg.raw)
+
+    def bundle() -> Bundle:
+        return Bundle.load(cfg.knowledge_dir)
+
+    def locate(ref: str) -> tuple[str, Config, Bundle, str | None]:
+        """Resolve "id" (project) or "global:id" (global knowledge base)."""
+        if ref.startswith("global:"):
+            g = scopes.global_config(cfg)
+            if g is None:
+                raise scopes.ScopeError("no global knowledge base")
+            b = Bundle.load(g.knowledge_dir)
+            return "global", g, b, b.resolve_id(ref[len("global:"):])
+        b = bundle()
+        return "project", cfg, b, b.resolve_id(ref)
+
+    @server.tool()
+    def brief() -> str:
+        """A short orientation to the project knowledge base: what exists, active
+        tasks, what awaits the developer and recent commits. Call it at the start
+        of a session unless one was already provided."""
+        return project_brief(cfg)
+
+    @server.tool()
+    def search(query: str, type: str | None = None, tags: list[str] | None = None,
+               under: str | None = None, limit: int = 8, scope: str = "project") -> str:
+        """Keyword (BM25) search over the knowledge base. Returns concept ids, titles,
+        descriptions and a one-line snippet. Filter by concept type (e.g. "Decision"),
+        tags (all must match) or a directory prefix (e.g. "design"). scope: "project"
+        (default), "global" (the developer's cross-project knowledge base) or "all";
+        global ids are prefixed "global:"."""
+        try:
+            targets = scopes.scoped(cfg, scope)
+        except scopes.ScopeError as exc:
+            return str(exc)
+        limit = max(1, min(limit, 25))
+        results = []
+        for name, c in targets:
+            for h in Index(Bundle.load(c.knowledge_dir)).search(query, limit=limit, type=type, tags=tags, under=under):
+                d = h.as_dict()
+                if name == "global":
+                    d["id"] = "global:" + d["id"]
+                results.append(d)
+        results.sort(key=lambda d: -d["score"])
+        if not results:
+            return "No matches. Try other words, drop filters, or list a directory."
+        return _fmt(results[:limit])
+
+    @server.tool()
+    def outline(id: str) -> str:
+        """Metadata and heading outline of one concept, without its body. Use it to
+        decide which section to read. Prefix the id with "global:" for the global base."""
+        try:
+            _, _, b, cid = locate(id)
+        except scopes.ScopeError as exc:
+            return str(exc)
+        if cid is None:
+            return f"No concept {id!r}."
+        c = b.concepts[cid]
+        return _fmt({
+            "id": c.id, "title": c.title, "type": c.type, "description": c.description,
+            "tags": c.tags, "status": c.status, "trust": c.trust,
+            "verification_stale": c.verification_stale,
+            "generated": c.meta.get("generated"),
+            "chars": len(c.body),
+            "headings": ["#" * h.level + " " + h.text for h in headings(c.body)],
+            "links_to": sorted({l.target for l in c.links if not l.broken}),
+            "linked_from": b.backlinks(cid),
+        })
+
+    @server.tool()
+    def read(id: str, section_heading: str | None = None, frontmatter: bool = False) -> str:
+        """Read a concept's body, or only the section under one heading (matched by
+        text, case-insensitive). Set frontmatter=true to include the YAML metadata.
+        Prefix the id with "global:" for the global base."""
+        try:
+            _, _, b, cid = locate(id)
+        except scopes.ScopeError as exc:
+            return str(exc)
+        if cid is None:
+            return f"No concept {id!r}."
+        c = b.concepts[cid]
+        text = c.body
+        if section_heading:
+            text = section(c.body, section_heading)
+            if text is None:
+                options = ", ".join(h.text for h in headings(c.body)) or "none"
+                return f"No section {section_heading!r} in {cid}. Headings: {options}."
+        if frontmatter:
+            return f"---\n{dump_frontmatter(c.meta)}---\n\n{text}"
+        return f"# {c.title} ({cid})\n\n{text}" if not section_heading else text
+
+    @server.tool()
+    def list_concepts(directory: str = "", scope: str = "project") -> str:
+        """List a directory of the knowledge base: its concepts (id, type, title,
+        description) and subdirectories. Use "" for the root. scope: "project" or
+        "global"."""
+        try:
+            b = Bundle.load(scopes.scoped(cfg, scope)[-1][1].knowledge_dir) if scope == "global" else bundle()
+        except scopes.ScopeError as exc:
+            return str(exc)
+        d = b.directories.get(directory.strip("/"))
+        if d is None:
+            return f"No directory {directory!r}. Top level: {sorted(b.directories[''].children)}"
+        return _fmt({
+            "directory": d.id,
+            "subdirectories": sorted(d.children),
+            "concepts": [
+                {"id": cid, "type": b.concepts[cid].type, "title": b.concepts[cid].title,
+                 "description": b.concepts[cid].description}
+                for cid in sorted(d.concepts)
+            ],
+        })
+
+    @server.tool()
+    def record(id: str, type: str | None = None, title: str | None = None,
+               description: str | None = None, tags: list[str] | None = None,
+               body: str | None = None, section_heading: str | None = None,
+               append: str | None = None, significant: bool | None = None,
+               meta: dict[str, Any] | None = None, actor: str | None = None,
+               scope: str = "project") -> str:
+        """Create or update a concept at `id` (path without .md, e.g.
+        "decisions/activation-function"). New concepts need `type`, and should have a
+        title and one-sentence description. `body` replaces the whole body, or only the
+        section under `section_heading`; `append` adds to the end. Existing frontmatter
+        is preserved; `meta` sets extra keys (null deletes). Links between concepts use
+        bundle-absolute paths like [text](/design/model.md). `significant`: true for a
+        meaningful change, false for trivial or dictated edits (keeps provenance and
+        review state), or omit it to let rdstudio judge from the change. scope="global"
+        writes to the developer's global knowledge base: only for knowledge that is
+        not specific to this project, and only when the developer asked for it."""
+        updates: dict[str, Any] = dict(meta or {})
+        for key, value in (("type", type), ("title", title), ("description", description), ("tags", tags)):
+            if value is not None:
+                updates[key] = value
+        try:
+            target = cfg if scope == "project" else scopes.scoped(cfg, "global")[0][1]
+            result = store_record(
+                target.knowledge_dir, id.removeprefix("global:"), actor=actor or cfg.agent, body=body,
+                meta=updates, section=section_heading, append=append, significant=significant,
+                classifier=classifier,
+            )
+        except (StoreError, scopes.ScopeError) as exc:
+            return f"Not recorded: {exc}"
+        b = Bundle.load(target.knowledge_dir)
+        b.write_indexes()
+        issues = [f"{i.level}: {i.message}" for i in b.lint() if i.path == result.path]
+        out = result.as_dict()
+        if issues:
+            out["issues"] = issues
+        return _fmt(out)
+
+    @server.tool()
+    def backlinks(id: str) -> str:
+        """Concepts that link to the given concept."""
+        b = bundle()
+        cid = b.resolve_id(id)
+        if cid is None:
+            return f"No concept {id!r}."
+        return _fmt([{"id": x, "title": b.concepts[x].title} for x in b.backlinks(cid)])
+
+    @server.tool()
+    def study_path(id: str) -> str:
+        """What to read before a concept: everything it requires (links titled
+        "requires"), directly or through a chain, in reading order. ``depth`` is the
+        longest chain of prerequisites below each concept."""
+        b = bundle()
+        cid = b.resolve_id(id)
+        if cid is None:
+            return f"No concept {id!r}."
+        order = b.prerequisite_order()
+        return _fmt([{"id": x, "title": b.concepts[x].title, "depth": order[x]["depth"]}
+                     for x in [*b.prerequisites(cid), cid]])
+
+    @server.tool()
+    def review_queue(limit: int = 10) -> str:
+        """What needs human attention: concepts changed since human review, open
+        questions, unverified concepts, and format errors."""
+        b = bundle()
+        cs = list(b.concepts.values())
+        brief = lambda c: {"id": c.id, "title": c.title, "type": c.type}  # noqa: E731
+        questions = [c for c in cs if c.type.lower() == "question" and "answered" not in c.tags]
+        return _fmt({
+            "changed_since_review": [brief(c) for c in cs if c.verification_stale][:limit],
+            "open_questions": [brief(c) for c in questions][:limit],
+            "unverified_count": sum(c.trust == "unverified" for c in cs),
+            "errors": [f"{i.path}: {i.message}" for i in b.lint() if i.level == "error"][:limit],
+        })
+
+    @server.tool()
+    def procedure_next(procedure: str, step: str | None = None, hops: int = 2) -> str:
+        """Guidance for a recorded procedure (type: Procedure). Give the step you just
+        completed (node id or label); returns the steps reachable within `hops`
+        transitions, with each transition's condition, guidance and pitfalls. With no
+        step, starts at the beginning. Use it to follow a procedure step by step
+        without loading the whole graph."""
+        b = bundle()
+        cid = b.resolve_id(procedure) or b.resolve_id("procedures/" + procedure)
+        if cid is None or not procedures.is_procedure(b.concepts[cid]):
+            names = [c.id for c in b.concepts.values() if procedures.is_procedure(c)]
+            return f"No procedure {procedure!r}. Procedures: {names or 'none recorded'}."
+        c = b.concepts[cid]
+        graph = procedures.graph_of(c)
+        node = graph.match(step)
+        matched_by = "exact"
+        if node is None and step:
+            steps = {nid: str(attrs.get("label", nid)) for nid, attrs in graph.nodes.items()}
+            decision = classifier.choose("procedure_step", list(steps), {"description": step, "steps": steps})
+            node, matched_by = decision.choice, f"{decision.backend} (confidence {decision.confidence})"
+        if node is None:
+            out = procedures.describe(c, graph, None)
+            out["note"] = f"Step {step!r} is not in this procedure; showing the whole graph."
+            return _fmt(out)
+        out = procedures.describe(c, graph.neighbourhood(node, max(1, min(hops, 4))), node)
+        if matched_by != "exact":
+            out["matched_by"] = matched_by
+        return _fmt(out)
+
+    @server.tool()
+    def procedure_propose(procedure: str, edits: list[dict[str, Any]], rationale: str,
+                          actor: str | None = None) -> str:
+        """Propose changes to a procedure's graph for the developer to accept or reject.
+        Each edit: {"op": add_node|update_node|delete_node|add_edge|update_edge|delete_edge,
+        ...}. Nodes take id and label; edges take from, to, relation (LEADS_TO, TRIGGERS,
+        PROVIDES_INPUT_FOR, CONVERGES_TO) and condition, guidance, pitfalls. Explain
+        in `rationale` what went wrong or right that motivates the change. Check the
+        procedure's rejected proposals first (read it with frontmatter=true) and do not
+        repeat them."""
+        b = bundle()
+        cid = b.resolve_id(procedure) or b.resolve_id("procedures/" + procedure)
+        if cid is None:
+            return f"No procedure {procedure!r}."
+        try:
+            return _fmt(procedures.propose(cfg.knowledge_dir, cid, edits=edits, rationale=rationale,
+                                           actor=actor or cfg.agent))
+        except procedures.ProcedureError as exc:
+            return f"Not proposed: {exc}"
+
+    @server.tool()
+    def promote(id: str, as_id: str | None = None, keep: bool = False) -> str:
+        """Move a project concept into the developer's global knowledge base (for
+        knowledge that applies beyond this project). Only do this when the developer
+        has agreed. keep=true copies instead of moving. Refuses to move a concept other
+        project concepts link to unless keep=true."""
+        try:
+            return _fmt(scopes.promote(cfg, id, as_id=as_id, keep=keep))
+        except (scopes.ScopeError, StoreError) as exc:
+            return f"Not promoted: {exc}"
+
+    if references.enabled(cfg):
+        @server.tool()
+        def ref_search(query: str, limit: int = 8) -> str:
+            """Search the bibliography (papis library) by title, authors, tags and
+            abstract. Returns citekeys (`ref`), metadata, whether a PDF is attached, and
+            the id of the Reference concept for notes."""
+            try:
+                hits = references.search(cfg, query, limit=max(1, min(limit, 25)))
+            except references.ReferenceError as exc:
+                return str(exc)
+            return _fmt(hits) if hits else "No matching references."
+
+        @server.tool()
+        def ref_text(ref: str, pages: str | None = None, query: str | None = None) -> str:
+            """Read part of a reference's PDF as plain text: `pages` like "1" or "3-4,7",
+            or `query` to get the two pages that best match. Read the pages you need
+            rather than the whole paper."""
+            try:
+                return references.text(cfg, ref, pages=pages, query=query)
+            except (references.ReferenceError, ValueError) as exc:
+                return str(exc)
+
+    return server
+
+
+def run(cfg: Config) -> None:
+    create_server(cfg).run()