rdstudio 0.2.0__py3-none-any.whl

rdstudio/procedures.py

@@ -0,0 +1,233 @@
+"""Procedural graphs (after Lu et al., arXiv:2609.09153) stored in OKF concepts.
+
+A ``type: Procedure`` concept carries, in frontmatter::
+
+    start: <node id>                       # optional; default: first node
+    nodes: [{id, label, description?}]
+    edges: [{from, to, relation, condition?, guidance?, pitfalls?}]
+    proposals: [{id, by, at, rationale, edits: [...], state}]   # pending/applied/rejected
+
+Agents retrieve only the neighbourhood of their current step; edits to a graph
+are proposed by agents and applied or rejected by the developer. Rejected
+proposals are kept as negative evidence for future proposals.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .okf import Bundle, Concept, now, render_concept, split_frontmatter
+from .store import concept_path
+
+RELATIONS = ("LEADS_TO", "TRIGGERS", "PROVIDES_INPUT_FOR", "CONVERGES_TO")
+ATTRIBUTES = ("condition", "guidance", "pitfalls")
+EDIT_OPS = ("add_node", "update_node", "delete_node", "add_edge", "update_edge", "delete_edge")
+
+
+class ProcedureError(ValueError):
+    pass
+
+
+def is_procedure(c: Concept) -> bool:
+    return c.type.lower() == "procedure"
+
+
+@dataclass
+class Graph:
+    nodes: dict[str, dict[str, Any]]
+    edges: list[dict[str, Any]]
+    start: str | None
+    issues: list[str] = field(default_factory=list)
+
+    @classmethod
+    def from_meta(cls, meta: dict[str, Any]) -> "Graph":
+        issues: list[str] = []
+        nodes: dict[str, dict[str, Any]] = {}
+        for raw in meta.get("nodes") or []:
+            if isinstance(raw, str):
+                raw = {"id": raw}
+            if not isinstance(raw, dict) or not raw.get("id"):
+                issues.append(f"node without id: {raw!r}")
+                continue
+            nid = str(raw["id"])
+            if nid in nodes:
+                issues.append(f"duplicate node {nid!r}")
+            nodes[nid] = {"label": str(raw.get("label") or nid), **{k: v for k, v in raw.items() if k not in ("id", "label")}}
+        edges: list[dict[str, Any]] = []
+        for raw in meta.get("edges") or []:
+            if not isinstance(raw, dict):
+                issues.append(f"edge is not a mapping: {raw!r}")
+                continue
+            src, dst = str(raw.get("from", "")), str(raw.get("to", ""))
+            relation = str(raw.get("relation") or "LEADS_TO")
+            for end in (src, dst):
+                if end not in nodes:
+                    issues.append(f"edge {src!r} -> {dst!r} references unknown node {end!r}")
+            if relation not in RELATIONS:
+                issues.append(f"edge {src!r} -> {dst!r} has unknown relation {relation!r}")
+            edges.append({**raw, "from": src, "to": dst, "relation": relation})
+        start = meta.get("start")
+        if start is not None and str(start) not in nodes:
+            issues.append(f"start node {start!r} does not exist")
+        start = str(start) if start is not None else (next(iter(nodes)) if nodes else None)
+        return cls(nodes, edges, start, issues)
+
+    def to_meta(self) -> dict[str, Any]:
+        return {
+            "nodes": [{"id": nid, **attrs} for nid, attrs in self.nodes.items()],
+            "edges": self.edges,
+        }
+
+    def match(self, step: str | None) -> str | None:
+        """Localise the current step: exact id, then case-insensitive label."""
+        if not step:
+            return self.start
+        if step in self.nodes:
+            return step
+        wanted = step.strip().lower()
+        for nid, attrs in self.nodes.items():
+            if nid.lower() == wanted or str(attrs.get("label", "")).lower() == wanted:
+                return nid
+        return None
+
+    def neighbourhood(self, node: str, hops: int = 2) -> "Graph":
+        """``node`` and everything reachable within ``hops`` outgoing transitions."""
+        seen = {node: 0}
+        queue = deque([node])
+        while queue:
+            current = queue.popleft()
+            if seen[current] >= hops:
+                continue
+            for e in self.edges:
+                if e["from"] == current and e["to"] not in seen:
+                    seen[e["to"]] = seen[current] + 1
+                    queue.append(e["to"])
+        nodes = {nid: self.nodes[nid] for nid in seen if nid in self.nodes}
+        edges = [e for e in self.edges if e["from"] in seen and e["to"] in seen and seen[e["from"]] < hops]
+        return Graph(nodes, edges, node)
+
+
+def graph_of(c: Concept) -> Graph:
+    return Graph.from_meta(c.meta)
+
+
+def lint(bundle: Bundle) -> list[tuple[str, str]]:
+    out = []
+    for c in bundle.concepts.values():
+        if is_procedure(c):
+            out.extend((c.path, msg) for msg in graph_of(c).issues)
+    return out
+
+
+def describe(proc: Concept, graph: Graph, current: str | None) -> dict[str, Any]:
+    """Compact, agent-facing description of a (sub)graph."""
+    def edge(e: dict[str, Any]) -> dict[str, Any]:
+        out = {"from": e["from"], "to": e["to"], "relation": e["relation"]}
+        out.update({k: e[k] for k in ATTRIBUTES if e.get(k)})
+        return out
+
+    return {
+        "procedure": proc.id,
+        "title": proc.title,
+        "current": current,
+        "steps": {nid: attrs.get("label", nid) for nid, attrs in graph.nodes.items()},
+        "transitions": [edge(e) for e in graph.edges],
+    }
+
+
+# --------------------------------------------------------------------------- #
+# Proposals
+# --------------------------------------------------------------------------- #
+
+
+def _load(root: Path, cid: str) -> tuple[Path, dict[str, Any], str]:
+    path = concept_path(root, cid)
+    if not path.exists():
+        raise ProcedureError(f"no such procedure: {cid}")
+    meta, body = split_frontmatter(path.read_text(encoding="utf-8"))
+    meta = meta or {}
+    if str(meta.get("type", "")).lower() != "procedure":
+        raise ProcedureError(f"{cid} is not a Procedure concept")
+    return path, meta, body
+
+
+def _validate_edits(edits: list[dict[str, Any]]) -> None:
+    if not edits:
+        raise ProcedureError("a proposal needs at least one edit")
+    for e in edits:
+        op = e.get("op")
+        if op not in EDIT_OPS:
+            raise ProcedureError(f"unknown edit op {op!r}; use one of {', '.join(EDIT_OPS)}")
+        if op.endswith("_node") and not e.get("id"):
+            raise ProcedureError(f"{op} needs an 'id'")
+        if op.endswith("_edge") and not (e.get("from") and e.get("to")):
+            raise ProcedureError(f"{op} needs 'from' and 'to'")
+
+
+def propose(root: Path, cid: str, *, edits: list[dict[str, Any]], rationale: str, actor: str) -> dict[str, Any]:
+    _validate_edits(edits)
+    path, meta, body = _load(root, cid)
+    proposals = list(meta.get("proposals") or [])
+    pid = max((int(p.get("id", 0)) for p in proposals), default=0) + 1
+    proposal = {"id": pid, "by": actor, "at": now(), "state": "pending", "rationale": rationale, "edits": edits}
+    # Dry-run so structurally invalid proposals are refused up front.
+    trial = apply_edits(Graph.from_meta(meta), edits)
+    if trial.issues:
+        raise ProcedureError("proposal would leave the graph invalid: " + "; ".join(trial.issues))
+    proposals.append(proposal)
+    meta["proposals"] = proposals
+    path.write_text(render_concept(meta, body), encoding="utf-8")
+    return {"procedure": cid, "proposal": pid, "state": "pending"}
+
+
+def apply_edits(graph: Graph, edits: list[dict[str, Any]]) -> Graph:
+    nodes = {k: dict(v) for k, v in graph.nodes.items()}
+    edges = [dict(e) for e in graph.edges]
+    for e in edits:
+        op = e["op"]
+        attrs = {k: v for k, v in e.items() if k not in ("op",)}
+        if op == "add_node":
+            nodes[str(e["id"])] = {"label": str(e.get("label") or e["id"]), **{k: v for k, v in attrs.items() if k not in ("id", "label")}}
+        elif op == "update_node":
+            nodes.setdefault(str(e["id"]), {}).update({k: v for k, v in attrs.items() if k != "id"})
+        elif op == "delete_node":
+            nodes.pop(str(e["id"]), None)
+            edges = [x for x in edges if str(e["id"]) not in (x["from"], x["to"])]
+        elif op == "add_edge":
+            edges.append({"from": e["from"], "to": e["to"], "relation": e.get("relation", "LEADS_TO"),
+                          **{k: e[k] for k in ATTRIBUTES if e.get(k)}})
+        elif op == "update_edge":
+            for x in edges:
+                if (x["from"], x["to"]) == (e["from"], e["to"]) and (not e.get("relation") or x["relation"] == e["relation"]):
+                    x.update({k: e[k] for k in (*ATTRIBUTES, "relation") if k in e})
+        elif op == "delete_edge":
+            edges = [x for x in edges if not ((x["from"], x["to"]) == (e["from"], e["to"])
+                                              and (not e.get("relation") or x["relation"] == e["relation"]))]
+    meta = {"nodes": [{"id": k, **v} for k, v in nodes.items()], "edges": edges}
+    if graph.start in nodes:
+        meta["start"] = graph.start
+    return Graph.from_meta(meta)
+
+
+def resolve(root: Path, cid: str, pid: int, *, accept: bool, actor: str) -> dict[str, Any]:
+    path, meta, body = _load(root, cid)
+    proposals = list(meta.get("proposals") or [])
+    match = next((p for p in proposals if int(p.get("id", 0)) == pid), None)
+    if match is None:
+        raise ProcedureError(f"no proposal {pid} on {cid}")
+    if match.get("state") != "pending":
+        raise ProcedureError(f"proposal {pid} is already {match.get('state')}")
+    if accept:
+        graph = apply_edits(Graph.from_meta(meta), match["edits"])
+        if graph.issues:
+            raise ProcedureError("cannot apply: " + "; ".join(graph.issues))
+        meta.update(graph.to_meta())
+        meta["generated"] = {"by": actor, "at": now()}
+    match["state"] = "applied" if accept else "rejected"
+    match["resolved"] = {"by": actor, "at": now()}
+    meta["proposals"] = proposals
+    path.write_text(render_concept(meta, body), encoding="utf-8")
+    return {"procedure": cid, "proposal": pid, "state": match["state"]}

rdstudio/references.py

@@ -0,0 +1,228 @@
+"""Optional papis backend: reference stubs in the bundle, and cheap access to
+bibliographic metadata and PDF text for agents.
+
+papis stores one directory per document with an ``info.yaml``; this module
+reads those files directly, so papis itself is not a dependency. Configure in
+``rdstudio.toml``::
+
+    [references]
+    backend = "papis"
+    library = "thesis"        # a papis library name (from ~/.config/papis/config)
+    # path = "~/papis/thesis" # or the library directory itself
+    directory = "references"  # where stubs live in the knowledge bundle
+"""
+
+from __future__ import annotations
+
+import configparser
+import os
+import re
+import shutil
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .config import Config
+from .okf import _Loader as YAML_LOADER, now, render_concept
+from .search import tokenize
+
+_SAFE = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+class ReferenceError(RuntimeError):
+    pass
+
+
+@dataclass
+class Entry:
+    ref: str
+    folder: Path
+    info: dict[str, Any] = field(repr=False)
+
+    @property
+    def title(self) -> str:
+        return " ".join(str(self.info.get("title") or self.ref).split())
+
+    @property
+    def authors(self) -> list[str]:
+        out = []
+        for a in self.info.get("author_list") or []:
+            if isinstance(a, dict):
+                name = " ".join(str(x) for x in (a.get("given"), a.get("family")) if x)
+                if name:
+                    out.append(name)
+        if not out and self.info.get("author"):
+            out = [s.strip() for s in str(self.info["author"]).split(" and ")]
+        return out
+
+    @property
+    def year(self) -> str:
+        return str(self.info.get("year") or "")
+
+    @property
+    def venue(self) -> str:
+        return str(self.info.get("journal") or self.info.get("booktitle") or self.info.get("publisher") or "")
+
+    @property
+    def url(self) -> str:
+        doi = self.info.get("doi")
+        return f"https://doi.org/{doi}" if doi else str(self.info.get("url") or "")
+
+    @property
+    def pdfs(self) -> list[Path]:
+        files = self.info.get("files") or []
+        return [self.folder / f for f in files if str(f).lower().endswith(".pdf") and (self.folder / f).exists()]
+
+    def citation(self) -> str:
+        names = self.authors
+        who = names[0].split()[-1] + (" et al." if len(names) > 2 else f" and {names[1].split()[-1]}" if len(names) == 2 else "") if names else "Anon."
+        return f"{who} ({self.year or 'n.d.'}). {self.title}." + (f" {self.venue}." if self.venue else "")
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "ref": self.ref, "title": self.title, "authors": self.authors, "year": self.year,
+            "venue": self.venue, "url": self.url, "tags": _tags(self.info), "has_pdf": bool(self.pdfs),
+        }
+
+
+def _tags(info: dict[str, Any]) -> list[str]:
+    tags = info.get("tags") or []
+    if isinstance(tags, str):
+        tags = re.split(r"[,\s]+", tags)
+    return [str(t) for t in tags if t]
+
+
+def library_path(cfg: Config) -> Path:
+    ref_cfg = cfg.raw.get("references", {})
+    if ref_cfg.get("path"):
+        return Path(ref_cfg["path"]).expanduser()
+    papis_cfg = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") / "papis" / "config"
+    parser = configparser.ConfigParser()
+    parser.read(papis_cfg)
+    name = ref_cfg.get("library") or parser.get("settings", "default-library", fallback=None)
+    if name and parser.has_section(name) and parser.has_option(name, "dir"):
+        return Path(parser.get(name, "dir")).expanduser()
+    raise ReferenceError("no papis library configured: set [references] library or path in rdstudio.toml")
+
+
+def enabled(cfg: Config) -> bool:
+    return cfg.raw.get("references", {}).get("backend") == "papis"
+
+
+def stub_dir(cfg: Config) -> str:
+    return str(cfg.raw.get("references", {}).get("directory", "references")).strip("/")
+
+
+def load(cfg: Config) -> dict[str, Entry]:
+    root = library_path(cfg)
+    if not root.is_dir():
+        raise ReferenceError(f"papis library not found at {root}")
+    entries: dict[str, Entry] = {}
+    for info_path in sorted(root.glob("*/info.yaml")):
+        try:
+            info = yaml.load(info_path.read_text(encoding="utf-8"), Loader=YAML_LOADER) or {}  # YAML 1.2, as for notes
+        except yaml.YAMLError:
+            continue
+        ref = str(info.get("ref") or info_path.parent.name)
+        entries[ref] = Entry(ref, info_path.parent, info)
+    return entries
+
+
+def stub_id(cfg: Config, ref: str) -> str:
+    return f"{stub_dir(cfg)}/{_SAFE.sub('-', ref)}"
+
+
+def sync_stubs(cfg: Config) -> list[str]:
+    """Create a ``type: Reference`` concept for every papis entry that lacks one.
+    Existing concepts are never modified."""
+    created = []
+    for ref, e in load(cfg).items():
+        cid = stub_id(cfg, ref)
+        path = cfg.knowledge_dir / f"{cid}.md"
+        if path.exists():
+            continue
+        meta: dict[str, Any] = {
+            "type": "Reference",
+            "title": e.title,
+            "description": e.citation(),
+        }
+        if e.url:
+            meta["resource"] = e.url
+        tags = _tags(e.info)
+        if tags:
+            meta["tags"] = tags
+        meta["papis"] = {"ref": ref}
+        meta["generated"] = {"by": "process:rdstudio-refs", "at": now()}
+        body = (
+            "# Summary\n\nNot yet summarised.\n\n"
+            "# Relevance\n\nHow this bears on the project: which decisions, questions or designs it supports or challenges.\n"
+        )
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(render_concept(meta, body), encoding="utf-8")
+        created.append(cid)
+    return created
+
+
+def search(cfg: Config, query: str, limit: int = 8) -> list[dict[str, Any]]:
+    terms = set(tokenize(query))
+    scored = []
+    for e in load(cfg).values():
+        fields = [e.title] * 3 + [" ".join(e.authors)] * 2 + [" ".join(_tags(e.info)), str(e.info.get("abstract") or ""), e.venue]
+        tokens = tokenize(" ".join(fields))
+        score = sum(tokens.count(t) for t in terms)
+        if e.ref.lower() in query.lower():
+            score += 10
+        if score:
+            scored.append((score, e))
+    scored.sort(key=lambda x: (-x[0], x[1].ref))
+    return [{**e.summary(), "concept": stub_id(cfg, e.ref)} for _, e in scored[:limit]]
+
+
+def text_pages(cfg: Config, ref: str) -> list[str]:
+    """Plain text of an entry's first PDF, one string per page, cached under the
+    build output directory."""
+    entries = load(cfg)
+    if ref not in entries:
+        raise ReferenceError(f"no papis entry {ref!r}")
+    e = entries[ref]
+    if not e.pdfs:
+        raise ReferenceError(f"{ref} has no PDF attached")
+    pdf = e.pdfs[0]
+    cache = cfg.output_dir / "cache" / "text" / f"{_SAFE.sub('-', ref)}.txt"
+    if not cache.exists() or cache.stat().st_mtime < pdf.stat().st_mtime:
+        if not shutil.which("pdftotext"):
+            raise ReferenceError("pdftotext (poppler) is not installed")
+        cache.parent.mkdir(parents=True, exist_ok=True)
+        subprocess.run(["pdftotext", "-layout", str(pdf), str(cache)], check=True, capture_output=True, timeout=120)
+    pages = cache.read_text(encoding="utf-8", errors="replace").split("\f")
+    if pages and not pages[-1].strip():
+        pages.pop()  # pdftotext ends every page, including the last, with a form feed
+    return pages
+
+
+def text(cfg: Config, ref: str, *, pages: str | None = None, query: str | None = None, max_chars: int = 6000) -> str:
+    all_pages = text_pages(cfg, ref)
+    n = len(all_pages)
+    chosen: list[int] = []
+    if pages:
+        for part in pages.split(","):
+            if "-" in part:
+                a, b = part.split("-", 1)
+                chosen.extend(range(int(a), int(b) + 1))
+            elif part.strip():
+                chosen.append(int(part))
+    elif query:
+        terms = set(tokenize(query))
+        ranked = sorted(range(1, n + 1), key=lambda p: -sum(tokenize(all_pages[p - 1]).count(t) for t in terms))
+        chosen = sorted(p for p in ranked[:2] if any(t in tokenize(all_pages[p - 1]) for t in terms))
+    else:
+        chosen = [1]
+    out = []
+    for p in chosen:
+        if 1 <= p <= n:
+            out.append(f"--- page {p} of {n} ---\n{all_pages[p - 1].strip()}")
+    result = "\n\n".join(out) or f"No matching pages (the PDF has {n} pages)."
+    return result if len(result) <= max_chars else result[:max_chars] + f"\n[truncated; ask for fewer pages]"

rdstudio/reports.py

@@ -0,0 +1,75 @@
+"""HTML reports: metadata from <meta name="rdstudio:*"> tags and links into knowledge."""
+
+from __future__ import annotations
+
+import html
+import posixpath
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+_TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
+_META = re.compile(r"<meta\s+[^>]*>", re.I)
+_ATTR = re.compile(r'(\w[\w:-]*)\s*=\s*("([^"]*)"|\'([^\']*)\')')
+_HREF = re.compile(r'href\s*=\s*("([^"]*)"|\'([^\']*)\')', re.I)
+_H1 = re.compile(r"<h1[^>]*>(.*?)</h1>", re.I | re.S)
+_TAGS = re.compile(r"<[^>]+>")
+
+
+def _meta(text: str) -> dict[str, str]:
+    out: dict[str, str] = {}
+    for tag in _META.findall(text):
+        attrs = {m.group(1).lower(): html.unescape(m.group(3) if m.group(3) is not None else m.group(4) or "")
+                 for m in _ATTR.finditer(tag)}
+        name = attrs.get("name", "")
+        if name.startswith("rdstudio:") or name == "description":
+            out[name.removeprefix("rdstudio:")] = attrs.get("content", "")
+    return out
+
+
+def knowledge_links(text: str, report_rel: str, knowledge: str, reports: str) -> list[str]:
+    """Concept ids linked from a report (``/knowledge/x.md`` or relative paths into it)."""
+    ids: list[str] = []
+    here = posixpath.dirname(posixpath.join(reports, report_rel))
+    for m in _HREF.finditer(text):
+        href = html.unescape(m.group(2) if m.group(2) is not None else m.group(3) or "")
+        if href.startswith("#/k/"):  # a dashboard route
+            ids.append(href[4:])
+            continue
+        href = href.split("#", 1)[0].split("?", 1)[0]
+        if not href or re.match(r"^[a-z][a-z0-9+.-]*:", href, re.I):
+            continue
+        path = href.lstrip("/") if href.startswith("/") else posixpath.normpath(posixpath.join(here, href))
+        prefix = knowledge.strip("/") + "/"
+        if path.startswith(prefix) and path.endswith(".md"):
+            ids.append(path[len(prefix) : -3])
+    return sorted(set(ids))
+
+
+def scan(reports_dir: Path, knowledge: str, reports: str) -> list[dict[str, Any]]:
+    if not reports_dir.is_dir():
+        return []
+    items = []
+    for path in sorted(reports_dir.rglob("*.html")):
+        rel = path.relative_to(reports_dir).as_posix()
+        if any(part.startswith((".", "_")) for part in rel.split("/")):
+            continue
+        text = path.read_text(encoding="utf-8", errors="replace")
+        meta = _meta(text)
+        title = meta.get("title")
+        if not title:
+            m = _TITLE.search(text) or _H1.search(text)
+            title = html.unescape(_TAGS.sub("", m.group(1))).strip() if m else path.stem
+        mtime = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
+        items.append({
+            "path": rel,
+            "title": " ".join(title.split()),
+            "date": meta.get("date") or mtime.strftime("%Y-%m-%d"),
+            "author": meta.get("author", ""),
+            "description": meta.get("description", ""),
+            "activity": meta.get("activity", ""),
+            "links": knowledge_links(text, rel, knowledge, reports),
+        })
+    items.sort(key=lambda r: (r["date"], r["path"]), reverse=True)
+    return items