mindslice-forge 0.1.0__py3-none-any.whl

mindslice_forge/__init__.py

@@ -0,0 +1,12 @@
+"""mindslice-forge: turn documents into a MindSlice workspace by driving an agent harness.
+
+The forge itself only orchestrates: it inventories the sources, runs agent sessions, keeps
+resumable state and checks every result with the ``mindslice`` command. Everything that needs
+understanding of the text is done by the agent, following the templates in ``prompts/``.
+"""
+
+__version__ = "0.1.0"
+
+
+class ForgeError(Exception):
+    """A failure the user can act on; the CLI prints it without a traceback."""

mindslice_forge/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

mindslice_forge/checkpoint.py

@@ -0,0 +1,106 @@
+"""Git checkpoints of the output workspace: one commit per finished step.
+
+They give a clean rollback when a session fails, and ``git log -p`` in the workspace shows what
+each chunk added or changed.
+
+The checkpoints cover what the forge builds: the slices and the mount table. Once the workspace
+is connected to an agent it also holds what that agent learns (``LIVE``), which is never
+committed and never rolled back. The engine changes the slices and the mount table as well, when
+the user bakes or mounts; a rollback that would remove such a change is refused.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from . import ForgeError
+
+# The workspace needs no git configuration of its own, and paths are reported unescaped.
+OPTIONS = ["-c", "user.name=mindslice-forge", "-c", "user.email=forge@localhost", "-c", "core.quotepath=off"]
+WORKSPACE_DIRS = ("overlay", "staging", "slices")  # empty ones are not tracked by git
+LIVE = ("overlay", "staging", ".history")  # written by the engine while the workspace is in use as memory
+PENDING = "forge-pending"  # in .git, from the start of an agent session until the next checkpoint
+
+
+def git(out: Path, *args: str) -> str:
+    try:
+        done = subprocess.run(["git", "-C", str(out), *OPTIONS, *args], capture_output=True, text=True,
+                              encoding="utf-8")
+    except FileNotFoundError:
+        raise ForgeError("the `git` command is not on PATH; the forge uses it for checkpoints") from None
+    if done.returncode != 0:
+        raise ForgeError(f"git {' '.join(args)} failed in {out}:\n{done.stderr.strip()}")
+    return done.stdout
+
+
+def init(out: Path) -> None:
+    git(out, "init", "-q")
+    _exclude_live(out)
+
+
+def begin(out: Path) -> None:
+    """An agent session is about to work in ``out``: until the next checkpoint, what differs from
+    the last one is the forge's own unfinished work, and a rollback may discard it."""
+    (out / ".git" / PENDING).touch()
+
+
+def commit(out: Path, message: str, tag: str | None = None) -> None:
+    git(out, "add", "-A")
+    git(out, "commit", "-q", "--allow-empty", "-m", message)
+    if tag:
+        git(out, "tag", "-f", tag)
+    (out / ".git" / PENDING).unlink(missing_ok=True)
+
+
+def rollback(out: Path, ref: str = "HEAD") -> None:
+    """Discard the forge's work after ``ref``: an unfinished step, and later commits when ``ref`` is a tag.
+
+    Live memory stays as it is. Changes found while no step is unfinished were made by something
+    else (a bake, a mount, an edit by hand), so they are reported and nothing is discarded."""
+    if not (out / ".git").is_dir():  # git would act on a repository that contains ``out``
+        raise ForgeError(f"{out} has no forge checkpoints (its .git is missing); choose another --output")
+    _exclude_live(out)  # also for a workspace that an earlier forge version created
+    pending = out / ".git" / PENDING
+    foreign = [] if pending.exists() else git(out, "status", "--porcelain").splitlines()
+    if foreign:
+        raise ForgeError(f"{out} has changes that the forge did not make:\n"
+                         + "\n".join(f"  {line}" for line in foreign)
+                         + "\nContinuing this build would discard them. Undo them, or build into another "
+                           "--output and mount its slices where they are needed.")
+    git(out, "reset", "-q", "--hard", ref)
+    git(out, "clean", "-fdq")
+    pending.unlink(missing_ok=True)
+    for name in WORKSPACE_DIRS:  # `git clean` removes them when they are empty
+        (out / name).mkdir(exist_ok=True)
+
+
+def changed(out: Path) -> set[Path]:
+    """Files created or modified since the last commit."""
+    return _diff(out) | _untracked(out)
+
+
+def created(out: Path) -> set[Path]:
+    """Files created since the last commit."""
+    return _diff(out, "--diff-filter=A") | _untracked(out)
+
+
+def _diff(out: Path, *args: str) -> set[Path]:
+    # Compare both the index and working tree to HEAD; staged additions also count as new.
+    # Treat renames as delete/add so their destination cannot evade the lookup check.
+    return _paths(out, git(out, "diff", "--name-only", "--no-renames", "-z", *args, "HEAD", "--"))
+
+
+def _untracked(out: Path) -> set[Path]:
+    return _paths(out, git(out, "ls-files", "--others", "--exclude-standard", "-z"))
+
+
+def _exclude_live(out: Path) -> None:
+    """Keep live memory out of every commit, comparison and rollback."""
+    info = out / ".git" / "info"
+    info.mkdir(exist_ok=True)
+    (info / "exclude").write_text("".join(f"/{name}/\n" for name in LIVE), encoding="utf-8")
+
+
+def _paths(out: Path, names: str) -> set[Path]:
+    return {path for name in names.split("\0") if name and (path := (out / name).resolve()).is_file()}

mindslice_forge/cli.py

@@ -0,0 +1,85 @@
+"""Command line: ``mindslice-forge SOURCE... --harness-cmd COMMAND [--dict PATH] [--prompt TEXT_OR_FILE] [-o DIR]``."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import ForgeError, __version__, harness, pipeline
+from .state import STAGES
+
+
+def parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="mindslice-forge",
+        description="Turn documents into a MindSlice workspace by driving an agent harness.\n"
+                    "Running the same command again continues an unfinished build.",
+        epilog=harness.CONTRACT, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("sources", nargs="+", type=Path, metavar="SOURCE", help="document or folder to convert")
+    p.add_argument("--dict", dest="dicts", action="append", default=[], type=Path, metavar="PATH",
+                   help="reference frame (outline, glossary, wiki page): shapes names and structure, "
+                        "is never a source of facts; repeatable")
+    p.add_argument("--prompt", default="", metavar="TEXT_OR_FILE",
+                   help="conversion instructions, or a file that holds them")
+    p.add_argument("-o", "--output", type=Path, metavar="DIR",
+                   help="workspace to create (default: ./<first source name>.workspace)")
+    p.add_argument("--sample", type=int, metavar="N", help="process only the first N chunks")
+    p.add_argument("--chunk-size", type=int, default=100_000, metavar="CHARS",
+                   help="target characters per chunk (default: %(default)s)")
+    p.add_argument("--harness-cmd", metavar="COMMAND",
+                   help="command line that runs one agent session, described below; "
+                        "needed for everything except --dry-run")
+    p.add_argument("--timeout", type=int, default=3600, metavar="SECONDS",
+                   help="time limit of one agent session (default: %(default)s)")
+    p.add_argument("--max-repairs", type=int, default=2, metavar="N",
+                   help="repair sessions allowed per step (default: %(default)s)")
+    p.add_argument("--from", dest="from_stage", choices=STAGES, metavar="STAGE",
+                   help=f"redo the build from this stage on: {', '.join(STAGES)}")
+    p.add_argument("--dry-run", action="store_true",
+                   help="show the source inventory, the directories an agent session must reach and "
+                        "what would run; call no agent, write nothing")
+    p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return p
+
+
+def read_prompt(value: str) -> str:
+    """``--prompt`` is the instructions themselves, or the path of a file that holds them."""
+    try:
+        if value and "\n" not in value and Path(value).is_file():
+            return Path(value).read_text(encoding="utf-8").strip()
+    except OSError:  # too long to be a file name
+        pass
+    return value.strip()
+
+
+def job_from(args: argparse.Namespace) -> pipeline.Job:
+    if args.sample is not None and args.sample < 1:
+        raise ForgeError("--sample must be at least 1")
+    if args.chunk_size < 1:
+        raise ForgeError("--chunk-size must be at least 1")
+    sources = [p.resolve() for p in args.sources]
+    dicts = [p.resolve() for p in args.dicts]
+    first = sources[0]
+    out = (args.output or Path(f"{first.name if first.is_dir() else first.stem}.workspace")).resolve()
+    if not args.harness_cmd and not args.dry_run:
+        raise ForgeError("--harness-cmd is needed: the command line that runs one agent session of your harness. "
+                         "`mindslice-forge --help` says what it must do; --dry-run works without it.")
+    return pipeline.Job(
+        sources=sources, dicts=dicts, prompt=read_prompt(args.prompt), out=out,
+        harness=harness.template(args.harness_cmd) if args.harness_cmd else [],
+        sample=args.sample, chunk_size=args.chunk_size, timeout=args.timeout, max_repairs=args.max_repairs,
+        from_stage=args.from_stage, dry_run=args.dry_run)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parser().parse_args(argv)
+    try:
+        pipeline.run(job_from(args))
+    except ForgeError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        print("\ninterrupted; run the same command again to continue", file=sys.stderr)
+        return 130
+    return 0

mindslice_forge/engine.py

@@ -0,0 +1,91 @@
+"""The MindSlice engine, used only through its ``mindslice`` command."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import tomllib
+from pathlib import Path
+
+from . import ForgeError
+
+# Warnings an agent can fix in the card they are reported on. Others are left for the final report.
+FIXABLE = {"card_too_long", "missing_summary", "long_summary", "missing_title", "missing_frontmatter",
+           "broken_wikilink", "frontmatter_schema"}
+MINIMUM = (0, 1, 0)  # the oldest engine with the commands, JSON output and card format this forge relies on
+
+
+def _engine(*args: str) -> subprocess.CompletedProcess:
+    try:
+        return subprocess.run(["mindslice", *args], capture_output=True, text=True, encoding="utf-8")
+    except FileNotFoundError:
+        raise ForgeError("the `mindslice` command is not on PATH; install the MindSlice engine first") from None
+
+
+def _run(*args: str) -> subprocess.CompletedProcess:
+    return _engine(*args, "--json")
+
+
+def version() -> str:
+    """What ``mindslice --version`` prints. An engine older than ``MINIMUM`` is an error."""
+    done = _engine("--version")
+    line = done.stdout.strip()
+    found = re.search(r"(\d+)\.(\d+)\.(\d+)", line)
+    if done.returncode != 0 or not found:
+        raise ForgeError(f"`mindslice --version` did not give a version:\n{line or done.stderr.strip()}")
+    if tuple(map(int, found.groups())) < MINIMUM:
+        raise ForgeError(f"this forge needs MindSlice {'.'.join(map(str, MINIMUM))} or newer, and `mindslice` on PATH "
+                         f"is {found.group()}; upgrade the engine, for example with `pipx upgrade mindslice`")
+    return line
+
+
+def _call(*args: str) -> dict:
+    done = _run(*args)
+    if done.returncode != 0:
+        raise ForgeError(f"mindslice {' '.join(args)} failed:\n{done.stderr.strip() or done.stdout.strip()}")
+    return json.loads(done.stdout)
+
+
+def init(out: Path) -> None:
+    _call("init", str(out))
+
+
+def mount_slices(out: Path) -> list[str]:
+    """Mount every ``slices/*.slice`` directory that is not mounted yet; returns the new aliases."""
+    mounted = {(out / m["path"]).resolve() for m in _call("-w", str(out), "status")["mounts"]}
+    new = [d for d in sorted((out / "slices").glob("*.slice")) if d.resolve() not in mounted]
+    return [_call("-w", str(out), "mount", str(d), "--precedence", "50", "--role", "curated")["alias"] for d in new]
+
+
+def validate(out: Path, slice_dir: Path | None = None) -> dict:
+    """The engine's lint report for the workspace, or for one slice directory. Errors do not raise."""
+    done = _run("-w", str(out), "validate", *([str(slice_dir)] if slice_dir else []))
+    try:
+        return json.loads(done.stdout)
+    except json.JSONDecodeError:
+        raise ForgeError(f"mindslice validate failed:\n{done.stderr.strip() or done.stdout.strip()}") from None
+
+
+def card_names(card: Path) -> list[str]:
+    """A card's title and aliases, or no names when its frontmatter is unreadable."""
+    text = card.read_text(encoding="utf-8", errors="replace")
+    try:
+        lines = text.splitlines()
+        meta = tomllib.loads("\n".join(lines[1:lines.index("+++", 1)])) if lines and lines[0] == "+++" else {}
+    except (ValueError, tomllib.TOMLDecodeError):
+        meta = {}
+    aliases = meta.get("aliases") if isinstance(meta.get("aliases"), list) else []
+    names = [name for name in [meta.get("title"), *aliases] if isinstance(name, str) and name.strip()]
+    return names
+
+
+def issues(report: dict, files: set[Path] | None = None) -> list[dict]:
+    """What a repair session should fix: every error, and the fixable warnings on ``files``
+    (on any file when ``files`` is None)."""
+    out = list(report["errors"])
+    for warning in report["warnings"]:
+        file = warning.get("file")
+        if warning["code"] in FIXABLE and file and (files is None or Path(file).resolve() in files):
+            out.append(warning)
+    return out

mindslice_forge/harness.py

@@ -0,0 +1,81 @@
+"""Run one non-interactive agent session.
+
+A harness is a command line, given by the caller: the forge knows no agent product, only what
+it needs from a session (``CONTRACT``). The prompt is written to a file and the command only
+carries a one-line message pointing at it, so prompt length and quoting never matter and any
+agent with file tools can be used.
+"""
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+from pathlib import Path
+
+from . import ForgeError
+
+MESSAGE = "Read the file {prompt_file} and carry out the task it describes, completely and exactly."
+
+# Shown by ``mindslice-forge --help``: an agent that composes the command reads it there.
+CONTRACT = """\
+the harness command:
+  --harness-cmd is the command line that runs one agent session of the harness you choose, with
+  its model and options. The forge runs it once per step, from the forge directory, with no
+  terminal attached, after filling in these placeholders (do not quote them):
+    {message}      one line telling the agent to read the prompt file and do what it says
+    {prompt_file}  the path of the prompt file
+    {workdir}      the forge directory
+  The command must:
+    - contain {message} or {prompt_file}, so that the agent receives its task;
+    - name files and directories by absolute paths, as it does not run from your directory;
+    - end when the task is done, with exit status 0 on success;
+    - never wait for an answer: no questions, and no permission prompts for reading and writing
+      files or running shell commands;
+    - let the agent write in the workspace and the forge directory and read the sources and
+      every --dict (--dry-run lists these directories);
+    - give the agent's shell a PATH with the `mindslice` command.
+  Before the first step with a new command, one short session checks all of this.
+"""
+
+BACKOFF = (30, 120, 300)  # seconds to wait before each retry of a failed session
+
+
+def template(command_line: str) -> list[str]:
+    """The ``--harness-cmd`` command line as a list of arguments, placeholders still in place."""
+    try:
+        cmd = shlex.split(command_line)
+    except ValueError as error:
+        raise ForgeError(f"--harness-cmd cannot be read as a command line: {error}") from None
+    if not any("{message}" in part or "{prompt_file}" in part for part in cmd[1:]):
+        raise ForgeError("--harness-cmd must be a command with {message} or {prompt_file} among its arguments; "
+                         "without one the agent never receives its task. See `mindslice-forge --help`.")
+    return cmd
+
+
+def command(template: list[str], prompt_file: Path, workdir: Path) -> list[str]:
+    values = {"{message}": MESSAGE.format(prompt_file=prompt_file), "{prompt_file}": str(prompt_file),
+              "{workdir}": str(workdir)}
+    cmd = []
+    for part in template:
+        for placeholder, value in values.items():
+            part = part.replace(placeholder, value)
+        cmd.append(part)
+    return cmd
+
+
+def run(cmd: list[str], cwd: Path, log: Path, timeout: int) -> bool:
+    """Run the session, appending its output to ``log``. True when it exited with status 0 in time."""
+    log.parent.mkdir(parents=True, exist_ok=True)
+    with open(log, "ab") as fh:
+        fh.write(f"$ {shlex.join(cmd)}\n".encode())
+        fh.flush()
+        try:
+            done = subprocess.run(cmd, cwd=cwd, stdin=subprocess.DEVNULL, stdout=fh, stderr=subprocess.STDOUT,
+                                  timeout=timeout)
+        except subprocess.TimeoutExpired:
+            fh.write(f"\n[forge] timed out after {timeout}s\n".encode())
+            return False
+        except FileNotFoundError:
+            raise ForgeError(f"harness command not found: {cmd[0]}") from None
+        fh.write(f"\n[forge] exit status {done.returncode}\n".encode())
+        return done.returncode == 0

mindslice_forge/lookup.py

@@ -0,0 +1,91 @@
+"""Search the workspace for a name, and record that the search was made.
+
+    python lookup.py WORKSPACE NAME...
+
+Agents use this, not ``mindslice search`` directly, to find out whether something already has a
+card. The record is what lets the forge enforce "look before you write": a new card whose name
+was never looked up is sent back. This file is run as a script by its path, so it imports
+nothing from the package.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import time
+import unicodedata
+from pathlib import Path
+
+
+def forge_dir(workspace: Path) -> Path:
+    """The forge's own directory for a workspace: next to it, so it never ends up in a workspace backup."""
+    return workspace.with_name(workspace.name + ".forge")
+
+
+def log_file(workspace: Path) -> Path:
+    return forge_dir(workspace) / "lookups.jsonl"
+
+
+def queries(workspace: Path) -> list[str]:
+    """Every lookup made so far, oldest first."""
+    log = log_file(workspace)
+    if not log.exists():
+        return []
+    return [json.loads(line)["query"] for line in log.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def covered(names: list[str], queries: list[str]) -> bool:
+    """Match a whole lookup name within a title/alias, or a whole title/alias in a lookup.
+
+    Substrings allow a base name to cover its period cards. Latin letters and numbers must
+    match at word boundaries, so searching for "API" cannot cover "Capital".
+    """
+    def normalize(value: str) -> str:
+        return " ".join(unicodedata.normalize("NFKC", value).casefold().split())
+
+    def word_char(char: str) -> bool:
+        return char.isdigit() or "LATIN" in unicodedata.name(char, "") or char == "_"
+
+    def contains(text: str, name: str) -> bool:
+        start = text.find(name)
+        while start >= 0:
+            end = start + len(name)
+            if (not word_char(name[0]) or start == 0 or not word_char(text[start - 1])) and (
+                    not word_char(name[-1]) or end == len(text) or not word_char(text[end])):
+                return True
+            start = text.find(name, start + 1)
+        return False
+
+    names = [normalized for name in names if (normalized := normalize(name))]
+    for query in queries:
+        query = normalize(query)
+        if query and any(contains(name, query) or contains(query, name) for name in names):
+            return True
+    return False
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) < 2:
+        print(__doc__, file=sys.stderr)
+        return 2
+    workspace, query = Path(argv[0]).resolve(), " ".join(argv[1:]).strip()
+    if not query:
+        print("error: lookup needs a non-empty name", file=sys.stderr)
+        return 2
+    try:
+        # JSON also avoids the engine's text renderer failing on unresolved lineage references.
+        done = subprocess.run(["mindslice", "-w", str(workspace), "search", query, "--json"])
+        if done.returncode == 0:
+            log = log_file(workspace)
+            log.parent.mkdir(parents=True, exist_ok=True)
+            with open(log, "a", encoding="utf-8") as fh:
+                fh.write(json.dumps({"time": time.time(), "query": query}, ensure_ascii=False) + "\n")
+    except OSError as error:
+        print(f"error: lookup failed: {error}", file=sys.stderr)
+        return 1
+    return done.returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))