roundtable-cli 0.4.0__py3-none-any.whl

roundtable/scan.py

@@ -0,0 +1,187 @@
+"""Codebase scanning: turn an existing project into a compact text "digest".
+
+The digest is the provider-agnostic input to the ``map`` command's analyst: a
+pruned file tree plus the truncated contents of the files that explain a project
+fastest (README, package manifests, entry points). It deliberately stays within a
+byte budget so it fits in one LLM call, and is stdlib-only (no extra deps).
+
+With ``provider: cli`` the analyst CLI agent additionally has live file access
+(it runs with ``cwd`` set to the scanned project), so the digest is a reliable
+floor, not a ceiling.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+# Directories we never descend into: VCS, caches, deps, build output, IDE noise.
+IGNORE_DIRS = frozenset({
+    ".git", ".hg", ".svn", ".roundtable",
+    "node_modules", ".venv", "venv", "env", "__pycache__",
+    ".mypy_cache", ".pytest_cache", ".ruff_cache", ".tox", ".cache",
+    "dist", "build", "target", ".next", "out", "coverage", ".nyc_output",
+    ".idea", ".vscode", "vendor", ".gradle", ".dart_tool",
+})
+
+# Files we list in the tree but never read (binary/asset/lock noise).
+SKIP_READ_EXTS = frozenset({
+    # images / media
+    ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico", ".bmp",
+    ".mp4", ".mov", ".webm", ".mp3", ".wav", ".ogg",
+    # fonts
+    ".ttf", ".otf", ".woff", ".woff2", ".eot",
+    # archives / binaries
+    ".zip", ".tar", ".gz", ".tgz", ".bz2", ".7z", ".rar",
+    ".pdf", ".so", ".dylib", ".dll", ".o", ".a", ".class", ".pyc",
+    ".bin", ".exe", ".wasm", ".db", ".sqlite",
+})
+SKIP_READ_NAMES = frozenset({
+    "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock",
+    "uv.lock", "Cargo.lock", "Gemfile.lock", "composer.lock", "go.sum",
+})
+
+# Manifest -> stack label, also the manifests we always inline.
+_STACK_MANIFESTS: list[tuple[str, str]] = [
+    ("pyproject.toml", "Python"),
+    ("setup.py", "Python"),
+    ("requirements.txt", "Python"),
+    ("package.json", "Node"),
+    ("Cargo.toml", "Rust"),
+    ("go.mod", "Go"),
+    ("pom.xml", "JVM"),
+    ("build.gradle", "JVM"),
+    ("build.gradle.kts", "JVM"),
+    ("Gemfile", "Ruby"),
+    ("composer.json", "PHP"),
+    ("pubspec.yaml", "Dart"),
+]
+
+# Entry-point filenames worth inlining when present anywhere shallow.
+_ENTRYPOINT_NAMES = frozenset({
+    "main.py", "__main__.py", "app.py", "cli.py", "manage.py",
+    "index.js", "index.ts", "main.js", "main.ts", "main.go", "main.rs",
+})
+
+
+def detect_stack(root: Path) -> list[str]:
+    """Best-effort language/stack labels from manifest files at the root."""
+    found: list[str] = []
+    for name, label in _STACK_MANIFESTS:
+        if (root / name).exists() and label not in found:
+            found.append(label)
+    return found
+
+
+def _should_read(path: Path) -> bool:
+    return path.suffix.lower() not in SKIP_READ_EXTS and path.name not in SKIP_READ_NAMES
+
+
+def _iter_files(root: Path, *, max_files: int) -> list[Path]:
+    """Deterministically walk ``root``, pruning ignored dirs; cap at ``max_files``."""
+    out: list[Path] = []
+    stack: list[Path] = [root]
+    while stack:
+        d = stack.pop()
+        try:
+            entries = sorted(d.iterdir(), key=lambda p: (p.is_file(), p.name.lower()))
+        except (PermissionError, OSError):
+            continue
+        dirs: list[Path] = []
+        for e in entries:
+            if e.is_symlink():
+                continue
+            if e.is_dir():
+                if e.name not in IGNORE_DIRS:
+                    dirs.append(e)
+            elif e.is_file():
+                out.append(e)
+                if len(out) >= max_files:
+                    return out
+        # Push dirs reversed so the sorted order is preserved on pop (DFS).
+        stack.extend(reversed(dirs))
+    return out
+
+
+def _key_files(root: Path, files: list[Path]) -> list[Path]:
+    """Order the files most worth inlining first: README, manifests, entrypoints."""
+    manifest_names = {name for name, _ in _STACK_MANIFESTS}
+    readmes, manifests, entrypoints = [], [], []
+    for f in files:
+        if not _should_read(f):
+            continue
+        name = f.name
+        if name.lower().startswith("readme"):
+            readmes.append(f)
+        elif name in manifest_names:
+            manifests.append(f)
+        elif name in _ENTRYPOINT_NAMES:
+            entrypoints.append(f)
+    # README first, then manifests, then a handful of entrypoints.
+    return readmes + manifests + entrypoints
+
+
+def _read_text(path: Path, limit: int) -> str:
+    try:
+        data = path.read_text(encoding="utf-8", errors="replace")
+    except (OSError, ValueError):
+        return ""
+    if len(data) > limit:
+        return data[:limit] + "\n...[truncated]..."
+    return data
+
+
+def build_digest(
+    root: Path,
+    *,
+    max_files: int = 400,
+    max_bytes_per_file: int = 8000,
+    max_total_bytes: int = 60000,
+) -> str:
+    """Build a compact Markdown digest of ``root`` for the analyst.
+
+    Sections: a stack line, a pruned file tree (capped at ``max_files``), and the
+    inlined contents of key files (README/manifests/entrypoints), each truncated
+    to ``max_bytes_per_file`` and the whole inline section bounded by
+    ``max_total_bytes``.
+    """
+    root = Path(root).resolve()
+    files = _iter_files(root, max_files=max_files)
+    stack = detect_stack(root) or ["unknown"]
+
+    lines: list[str] = [
+        f"# Codebase digest: {root.name}",
+        "",
+        f"## Stack: {', '.join(stack)}",
+        f"_{len(files)} files scanned (cap {max_files})._",
+        "",
+        "## File tree",
+        "",
+    ]
+    for f in files:
+        try:
+            rel = f.relative_to(root).as_posix()
+            size = f.stat().st_size
+        except (OSError, ValueError):
+            continue
+        lines.append(f"- {rel} ({size}B)")
+    if len(files) >= max_files:
+        lines.append(f"- …(truncated at {max_files} files)")
+
+    lines += ["", "## Key files", ""]
+    budget = max_total_bytes
+    inlined = 0
+    for f in _key_files(root, files):
+        if budget <= 0:
+            lines.append("_…(content budget exhausted; remaining key files omitted)_")
+            break
+        text = _read_text(f, min(max_bytes_per_file, budget))
+        if not text.strip():
+            continue
+        rel = f.relative_to(root).as_posix()
+        lines += [f"### {rel}", "", "```", text, "```", ""]
+        budget -= len(text)
+        inlined += 1
+    if inlined == 0:
+        lines.append("_(no README, manifest, or entry point found to inline)_")
+
+    return "\n".join(lines)

roundtable/store.py

@@ -0,0 +1,339 @@
+"""Filesystem persistence and project layout.
+
+Layout::
+
+    <root>/
+      roundtable.config.yaml
+      plan/{BRIEF.md, PLAN.md, plan.json}
+      phases/phase-NN-<slug>/
+        PHASE.md
+        phase-summary.md            # Phase Orchestrator -> Main report
+        tasks/task-NN-<slug>/
+          TASK.md                   # the task's work definition
+          result.md                 # the agent's completed work
+          output/                   # artifacts produced by the agent
+      docs/                         # maintained by the Main Orchestrator
+      runs/run.log                  # append-only event log
+      archive/<YYYY-MM-DDTHH-MM-SS>/
+        plan/  phases/  runs/  docs/  # snapshot of the previous run
+
+The ``plan.json`` manifest is the source of truth; the ``*.md`` files are
+human-readable renderings/definitions.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import json
+import shutil
+import threading
+from pathlib import Path
+
+from .models import Phase, Plan, Task
+
+PLAN_DIR = "plan"
+PHASES_DIR = "phases"
+DOCS_DIR = "docs"
+RUNS_DIR = "runs"
+HITL_DIR = "hitl"
+MANIFEST = "plan.json"
+
+
+def _atomic_write(path: Path, text: str) -> None:
+    """Write text to path atomically: write a sibling .tmp then rename."""
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(text, encoding="utf-8")
+    tmp.replace(path)
+
+
+def _utcnow() -> str:
+    return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
+
+
+class Store:
+    """All disk reads/writes for one roundtable project.
+
+    ``root`` is the target project directory (where agents do their work).
+    All roundtable artifacts live under ``root/workdir`` (default ``.roundtable``) so
+    they never clutter an existing repo. Only ``roundtable.config.yaml`` sits at the
+    project root.
+    """
+
+    def __init__(self, root: Path | str, workdir: str = ".roundtable"):
+        self.root = Path(root).resolve()
+        self.base = self.root / workdir
+        self._log_lock = threading.Lock()  # serialize run.log appends across threads
+
+    # ---- paths -----------------------------------------------------------
+    @property
+    def plan_dir(self) -> Path:
+        return self.base / PLAN_DIR
+
+    @property
+    def phases_dir(self) -> Path:
+        return self.base / PHASES_DIR
+
+    @property
+    def docs_dir(self) -> Path:
+        return self.base / DOCS_DIR
+
+    @property
+    def runs_dir(self) -> Path:
+        return self.base / RUNS_DIR
+
+    @property
+    def hitl_dir(self) -> Path:
+        return self.base / HITL_DIR
+
+    def hitl_path(self, task_id: str) -> Path:
+        return self.hitl_dir / f"{task_id}.json"
+
+    @property
+    def run_pid_path(self) -> Path:
+        return self.runs_dir / "run.pid"
+
+    def write_run_pid(self, pid: int) -> None:
+        self.runs_dir.mkdir(parents=True, exist_ok=True)
+        _atomic_write(self.run_pid_path, str(pid))
+
+    def read_run_pid(self) -> int | None:
+        try:
+            return int(self.run_pid_path.read_text().strip())
+        except (OSError, ValueError):
+            return None
+
+    def clear_run_pid(self) -> None:
+        try:
+            self.run_pid_path.unlink()
+        except OSError:
+            pass
+
+    def list_waiting_checkpoints(self) -> list[dict]:
+        """Return all HITL checkpoints with status 'waiting'."""
+        results: list[dict] = []
+        if not self.hitl_dir.exists():
+            return results
+        for p in self.hitl_dir.glob("*.json"):
+            try:
+                data = json.loads(p.read_text())
+                if data.get("status") == "waiting":
+                    results.append(data)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return results
+
+    @property
+    def manifest_path(self) -> Path:
+        return self.plan_dir / MANIFEST
+
+    def phase_dir(self, phase: Phase) -> Path:
+        return self.phases_dir / phase.dir_name
+
+    def _task_index(self, phase: Phase, task: Task) -> int:
+        for i, t in enumerate(phase.tasks, start=1):
+            if t.id == task.id:
+                return i
+        raise KeyError(f"task {task.id!r} not in phase {phase.id!r}")
+
+    def task_dir(self, phase: Phase, task: Task) -> Path:
+        idx = self._task_index(phase, task)
+        return self.phase_dir(phase) / "tasks" / phase.task_dir_name(task, idx)
+
+    def task_output_dir(self, phase: Phase, task: Task) -> Path:
+        return self.task_dir(phase, task) / "output"
+
+    # ---- scaffolding -----------------------------------------------------
+    def scaffold(self) -> None:
+        for d in (self.plan_dir, self.phases_dir, self.docs_dir, self.runs_dir):
+            d.mkdir(parents=True, exist_ok=True)
+
+    def scaffold_plan_tree(self, plan: Plan) -> None:
+        """Create every phase/task directory for an approved plan."""
+        self.scaffold()
+        for phase in plan.phases:
+            self.phase_dir(phase).mkdir(parents=True, exist_ok=True)
+            for task in phase.tasks:
+                self.task_output_dir(phase, task).mkdir(parents=True, exist_ok=True)
+
+    def archive_current_run(self) -> Path | None:
+        """Move the current run's artifacts to .roundtable/archive/<created_at>/.
+
+        Moves plan/, phases/, runs/ entirely and the run-specific docs
+        (OVERVIEW.md, PROGRESS.md, FINAL.md).  Project-level docs
+        (ARCHITECTURE.md, PRD.md) stay in place.
+
+        Returns the archive directory path, or None if there was no plan to archive.
+        """
+        if not self.has_plan():
+            return None
+
+        try:
+            ts = Plan.model_validate_json(self.manifest_path.read_text()).created_at
+            # Normalise to a filesystem-safe name: 2025-06-30T14-22-00
+            ts = ts[:19].replace(":", "-")
+        except Exception:
+            ts = _utcnow()[:19].replace(":", "-")
+
+        archive_dir = self.base / "archive" / ts
+        # Avoid clobbering an existing archive slot (e.g. two plans in one second)
+        suffix, attempt = "", 1
+        while (archive_dir.parent / (ts + suffix)).exists():
+            attempt += 1
+            suffix = f"-{attempt}"
+        archive_dir = archive_dir.parent / (ts + suffix)
+        archive_dir.mkdir(parents=True, exist_ok=True)
+
+        for name in ("plan", "phases", "runs"):
+            src = self.base / name
+            if src.exists():
+                shutil.move(str(src), str(archive_dir / name))
+
+        run_docs = ("OVERVIEW.md", "PROGRESS.md", "FINAL.md")
+        for doc_name in run_docs:
+            src = self.docs_dir / doc_name
+            if src.exists():
+                (archive_dir / "docs").mkdir(exist_ok=True)
+                shutil.move(str(src), str(archive_dir / "docs" / doc_name))
+
+        return archive_dir
+
+    # ---- manifest --------------------------------------------------------
+    def save_plan(self, plan: Plan) -> None:
+        self.plan_dir.mkdir(parents=True, exist_ok=True)
+        _atomic_write(self.manifest_path, plan.model_dump_json(indent=2))
+
+    def load_plan(self) -> Plan:
+        if not self.manifest_path.exists():
+            raise FileNotFoundError(
+                f"no plan manifest at {self.manifest_path}; run `roundtable plan` first"
+            )
+        return Plan.model_validate_json(self.manifest_path.read_text())
+
+    def has_plan(self) -> bool:
+        return self.manifest_path.exists()
+
+    # ---- documents -------------------------------------------------------
+    def write_brief(self, goal: str) -> Path:
+        self.plan_dir.mkdir(parents=True, exist_ok=True)
+        p = self.plan_dir / "BRIEF.md"
+        _atomic_write(p, f"# Brief\n\n{goal.strip()}\n")
+        return p
+
+    def write_plan_md(self, plan: Plan) -> Path:
+        self.plan_dir.mkdir(parents=True, exist_ok=True)
+        p = self.plan_dir / "PLAN.md"
+        _atomic_write(p, render_plan_md(plan))
+        return p
+
+    def write_phase_md(self, phase: Phase, content: str) -> Path:
+        d = self.phase_dir(phase)
+        d.mkdir(parents=True, exist_ok=True)
+        p = d / "PHASE.md"
+        _atomic_write(p, content)
+        return p
+
+    def write_task_def(self, phase: Phase, task: Task, content: str) -> Path:
+        d = self.task_dir(phase, task)
+        d.mkdir(parents=True, exist_ok=True)
+        p = d / "TASK.md"
+        _atomic_write(p, content)
+        return p
+
+    def write_result(self, phase: Phase, task: Task, content: str) -> Path:
+        d = self.task_dir(phase, task)
+        d.mkdir(parents=True, exist_ok=True)
+        p = d / "result.md"
+        _atomic_write(p, content)
+        return p
+
+    def write_artifact(self, phase: Phase, task: Task, name: str, content: str) -> Path:
+        d = self.task_output_dir(phase, task)
+        d.mkdir(parents=True, exist_ok=True)
+        p = d / name
+        p.parent.mkdir(parents=True, exist_ok=True)
+        _atomic_write(p, content)
+        return p
+
+    def write_phase_summary(self, phase: Phase, content: str) -> Path:
+        d = self.phase_dir(phase)
+        d.mkdir(parents=True, exist_ok=True)
+        p = d / "phase-summary.md"
+        _atomic_write(p, content)
+        return p
+
+    def write_doc(self, name: str, content: str) -> Path:
+        self.docs_dir.mkdir(parents=True, exist_ok=True)
+        p = self.docs_dir / name
+        p.parent.mkdir(parents=True, exist_ok=True)
+        _atomic_write(p, content)
+        return p
+
+    def read_doc(self, name: str) -> str:
+        p = self.docs_dir / name
+        return p.read_text() if p.exists() else ""
+
+    # ---- event log -------------------------------------------------------
+    def record_event(self, event_type: str, message: str = "", **fields: object) -> None:
+        """Append one structured JSONL event to ``runs/run.log``.
+
+        ``type`` + ``fields`` (e.g. ``task_id``, ``agent``, ``model``) let the
+        dashboard reconstruct live activity and timings; ``msg`` stays
+        human-readable for tailing the log.
+
+        Thread safety: guarded by ``self._log_lock``. Most callers run in the
+        asyncio event loop (single-threaded, cooperative), but streamed
+        ``task_output`` events arrive from the PTY drain worker thread, so the
+        lock is needed to keep appends from interleaving.
+        """
+        self.runs_dir.mkdir(parents=True, exist_ok=True)
+        rec: dict[str, object] = {"ts": _utcnow(), "type": event_type}
+        if message:
+            rec["msg"] = message
+        rec.update(fields)
+        line = json.dumps(rec) + "\n"
+        with self._log_lock, (self.runs_dir / "run.log").open("a") as f:
+            f.write(line)
+
+    def read_events(self) -> list[dict]:
+        """All recorded events in order; malformed/partial lines are skipped."""
+        p = self.runs_dir / "run.log"
+        if not p.exists():
+            return []
+        events: list[dict] = []
+        for line in p.read_text().splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                events.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue  # a concurrent writer's partial last line
+        return events
+
+
+def render_plan_md(plan: Plan) -> str:
+    lines = [
+        "# Plan",
+        "",
+        f"**Goal:** {plan.goal}",
+        "",
+        f"- Created: {plan.created_at or 'n/a'}",
+        f"- Main orchestrator: `{plan.main_runner or 'n/a'}`",
+        f"- Approved: {plan.approved}",
+        "",
+        "## Phases",
+        "",
+    ]
+    for phase in plan.phases:
+        lines.append(f"### Phase {phase.index}: {phase.title}  `[{phase.id}]`")
+        if phase.objective:
+            lines.append(f"\n{phase.objective}\n")
+        lines.append(f"- Phase orchestrator: `{phase.runner or 'n/a'}`")
+        lines.append("")
+        for i, task in enumerate(phase.tasks, start=1):
+            dep = f" (depends on: {', '.join(task.depends_on)})" if task.depends_on else ""
+            lines.append(f"{i}. **{task.title}** `[{task.id}]` — `{task.runner or 'n/a'}`{dep}")
+            if task.description:
+                lines.append(f"   - {task.description}")
+        lines.append("")
+    return "\n".join(lines)