si-deckhand 0.1.0__py3-none-any.whl

deckhand/__init__.py

@@ -0,0 +1,23 @@
+"""
+deckhand — Lightweight local agent that indexes, retrieves, and wikis the workspace.
+
+The deckhand is the crew member who knows where everything is.
+File-first (exo-filed). Human-readable. No hidden vector DB.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+__all__ = [
+    "Indexer",
+    "Retriever",
+    "WikiGenerator",
+    "TaskBatcher",
+    "Deckhand",
+]
+
+from deckhand.agent import Deckhand
+from deckhand.indexer import Indexer
+from deckhand.retriever import Retriever
+from deckhand.wiki import WikiGenerator
+from deckhand.task_batcher import TaskBatcher

deckhand/agent.py

@@ -0,0 +1,143 @@
+"""Agent — the main deckhand loop.
+
+Index → Study → Batch → Sleep → Repeat.
+When no tasks are queued, study the workspace and update the wiki.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any
+
+from deckhand.indexer import Indexer
+from deckhand.retriever import Retriever
+from deckhand.wiki import WikiGenerator
+from deckhand.task_batcher import TaskBatcher
+
+
+class Deckhand:
+    """The main agent loop."""
+
+    def __init__(
+        self,
+        workspace_root: str,
+        wiki_dir: str | None = None,
+        sleep_seconds: float = 300.0,
+    ) -> None:
+        self.root = workspace_root
+        self.wiki_dir = wiki_dir or str(Path(workspace_root) / "deckhand-wiki")
+        self.sleep_seconds = sleep_seconds
+        self.indexer = Indexer(workspace_root)
+        self.retriever: Retriever | None = None
+        self.wiki_gen: WikiGenerator | None = None
+        self.batcher = TaskBatcher(workspace_root)
+        self._running = False
+
+    def index(self) -> dict[str, Any]:
+        """Run a full index pass."""
+        stats = self.indexer.index()
+        self.retriever = Retriever(self.indexer.files)
+        self.wiki_gen = WikiGenerator(self.indexer)
+        return stats
+
+    def search(self, query: str, top_k: int = 10) -> list[dict[str, Any]]:
+        """Search the workspace."""
+        if not self.retriever:
+            self.index()
+        assert self.retriever is not None
+        results = self.retriever.search(query, top_k=top_k)
+        return [r.to_dict() for r in results]
+
+    def generate_wiki(self) -> list[str]:
+        """Generate all wiki pages."""
+        if not self.wiki_gen:
+            self.index()
+        assert self.wiki_gen is not None
+        return self.wiki_gen.generate(self.wiki_dir)
+
+    def batch_tasks(self) -> str:
+        """Collect and batch tasks."""
+        batch_path = str(Path(self.wiki_dir) / "BATCH.md")
+        return self.batcher.write_batch(batch_path)
+
+    def study(self) -> str:
+        """Study a random repo and write observations."""
+        if not self.indexer.repos:
+            self.index()
+
+        # Pick a repo that hasn't been studied recently
+        repos_by_age = sorted(
+            self.indexer.repos.values(),
+            key=lambda r: r.last_modified,
+        )
+        if not repos_by_age:
+            return "nothing to study"
+
+        target = repos_by_age[0]  # oldest modified
+
+        log_dir = Path(self.wiki_dir) / "logs"
+        log_dir.mkdir(parents=True, exist_ok=True)
+        log_path = log_dir / f"{time.strftime('%Y-%m-%d')}.md"
+
+        lines = [
+            f"# Deckhand Study Log — {time.strftime('%Y-%m-%d')}",
+            f"",
+            f"## Studied: `{target.name}`",
+            f"",
+            f"- Files: {target.file_count}",
+            f"- Health: {target.health}",
+            f"- Has tests: {target.has_tests}",
+            f"- Has README: {target.has_readme}",
+            f"- Languages: {target.languages}",
+            f"",
+        ]
+
+        if target.health == "untested":
+            lines.append("⚠️ This repo has code but no tests. Needs a test suite.")
+        elif target.health == "no-readme":
+            lines.append("❌ This repo has no README. Needs documentation.")
+        elif target.health == "stub":
+            lines.append("🔩 This repo is a stub (< 3 files). May be abandoned.")
+        else:
+            lines.append("✅ This repo looks healthy.")
+
+        lines.append("")
+
+        with open(log_path, "a") as fh:
+            fh.write("\n".join(lines))
+
+        return str(log_path)
+
+    def run_once(self) -> dict[str, Any]:
+        """Run one full cycle: index → wiki → batch → study."""
+        stats = self.index()
+        wiki_files = self.generate_wiki()
+        batch_path = self.batch_tasks()
+        study_log = self.study()
+
+        return {
+            "index": stats,
+            "wiki_files": wiki_files,
+            "batch": batch_path,
+            "study_log": study_log,
+        }
+
+    def run_forever(self) -> None:
+        """Run the main loop forever."""
+        self._running = True
+        while self._running:
+            try:
+                result = self.run_once()
+                print(
+                    f"[deckhand] indexed {result['index']['total_files']} files, "
+                    f"wrote {len(result['wiki_files'])} wiki pages, "
+                    f"batch at {result['batch']}"
+                )
+            except Exception as e:
+                print(f"[deckhand] error: {e}")
+
+            time.sleep(self.sleep_seconds)
+
+    def stop(self) -> None:
+        self._running = False

deckhand/cli.py

@@ -0,0 +1,98 @@
+"""CLI for deckhand."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from deckhand.agent import Deckhand
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="deckhand",
+        description="🐑 The deckhand — indexes, retrieves, and wikis the workspace.",
+    )
+
+    sub = parser.add_subparsers(dest="command")
+
+    # index
+    p_index = sub.add_parser("index", help="Index the workspace")
+    p_index.add_argument("workspace", help="Workspace root directory")
+    p_index.add_argument("--wiki", default=None, help="Wiki output directory")
+
+    # search
+    p_search = sub.add_parser("search", help="Search the workspace")
+    p_search.add_argument("workspace", help="Workspace root directory")
+    p_search.add_argument("query", help="Search query")
+    p_search.add_argument("--top", type=int, default=10, help="Number of results")
+
+    # wiki
+    p_wiki = sub.add_parser("wiki", help="Generate wiki pages")
+    p_wiki.add_argument("workspace", help="Workspace root directory")
+    p_wiki.add_argument("--output", default=None, help="Wiki output directory")
+
+    # batch
+    p_batch = sub.add_parser("batch", help="Collect and batch tasks")
+    p_batch.add_argument("workspace", help="Workspace root directory")
+
+    # study
+    p_study = sub.add_parser("study", help="Study a repo and write observations")
+    p_study.add_argument("workspace", help="Workspace root directory")
+
+    # run
+    p_run = sub.add_parser("run", help="Run the deckhand loop forever")
+    p_run.add_argument("workspace", help="Workspace root directory")
+    p_run.add_argument("--sleep", type=float, default=300, help="Sleep seconds between cycles")
+
+    args = parser.parse_args(argv)
+
+    if not args.command:
+        parser.print_help()
+        return 0
+
+    dh = Deckhand(
+        workspace_root=args.workspace,
+        wiki_dir=getattr(args, "wiki", None) or getattr(args, "output", None),
+        sleep_seconds=getattr(args, "sleep", 300),
+    )
+
+    if args.command == "index":
+        stats = dh.index()
+        print(f"Indexed {stats['total_files']} files across {stats['total_repos']} repos in {stats['elapsed_seconds']}s")
+        dh.generate_wiki()
+        print(f"Wiki written to {dh.wiki_dir}")
+
+    elif args.command == "search":
+        results = dh.search(args.query, top_k=args.top)
+        if not results:
+            print("No results found.")
+        for i, r in enumerate(results, 1):
+            print(f"{i}. [{r['score']:.2f}] {r['rel_path']}")
+            if r["snippet"]:
+                print(f"   {r['snippet'][:150]}...")
+
+    elif args.command == "wiki":
+        dh.index()
+        files = dh.generate_wiki()
+        print(f"Wiki pages written:")
+        for f in files:
+            print(f"  {f}")
+
+    elif args.command == "batch":
+        path = dh.batch_tasks()
+        print(f"Batch written to {path}")
+
+    elif args.command == "study":
+        log = dh.study()
+        print(f"Study log written to {log}")
+
+    elif args.command == "run":
+        print(f"Deckhand running on {args.workspace} (sleep={args.sleep}s)")
+        dh.run_forever()
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

deckhand/indexer.py

@@ -0,0 +1,309 @@
+"""Indexer — walks the workspace and builds a file index.
+
+The index is exo-filed: stored as JSON + markdown that a human can read.
+No hidden database. No opaque binary format.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class FileEntry:
+    """One file in the index."""
+    path: str
+    rel_path: str
+    size: int
+    lines: int
+    suffix: str
+    sha256: str
+    modified: float
+    words: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "rel_path": self.rel_path,
+            "size": self.size,
+            "lines": self.lines,
+            "suffix": self.suffix,
+            "sha256": self.sha256[:12],
+            "modified": self.modified,
+            "words": self.words,
+        }
+
+
+@dataclass
+class RepoEntry:
+    """One repo (or directory) in the index."""
+    name: str
+    path: str
+    has_git: bool
+    has_tests: bool
+    has_readme: bool
+    has_pyproject: bool
+    has_package_json: bool
+    has_cargo_toml: bool
+    file_count: int
+    total_size: int
+    languages: dict[str, int] = field(default_factory=dict)
+    last_modified: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "path": self.path,
+            "has_git": self.has_git,
+            "has_tests": self.has_tests,
+            "has_readme": self.has_readme,
+            "has_pyproject": self.has_pyproject,
+            "has_package_json": self.has_package_json,
+            "has_cargo_toml": self.has_cargo_toml,
+            "file_count": self.file_count,
+            "total_size": self.total_size,
+            "languages": self.languages,
+            "last_modified": self.last_modified,
+        }
+
+    @property
+    def health(self) -> str:
+        """Quick health indicator."""
+        if not self.has_readme:
+            return "no-readme"
+        if self.has_pyproject and not self.has_tests:
+            return "untested"
+        if self.file_count < 3:
+            return "stub"
+        return "ok"
+
+
+class Indexer:
+    """Walks a workspace directory and builds an exo-filed index."""
+
+    SKIP_DIRS = frozenset({
+        ".git", "__pycache__", "node_modules", ".pytest_cache",
+        "dist", "build", ".egg-info", ".venv", "venv", ".tox",
+        ".wrangler", ".crush", ".swarm",
+    })
+
+    SKIP_FILES = frozenset({
+        ".DS_Store", "Thumbs.db", ".gitkeep",
+    })
+
+    CODE_SUFFIXES = frozenset({
+        ".py", ".rs", ".js", ".ts", ".go", ".rb", ".java",
+        ".c", ".cpp", ".h", ".hpp", ".ex", ".exs", ".zig", ".lua",
+    })
+
+    DOC_SUFFIXES = frozenset({
+        ".md", ".rst", ".txt", ".org",
+    })
+
+    CONFIG_SUFFIXES = frozenset({
+        ".toml", ".yaml", ".yml", ".json", ".ini", ".cfg",
+    })
+
+    def __init__(self, root: str, max_depth: int = 4) -> None:
+        self.root = Path(root).resolve()
+        self.max_depth = max_depth
+        self.files: list[FileEntry] = []
+        self.repos: dict[str, RepoEntry] = {}
+        self.indexed_at: float = 0.0
+
+    def index(self) -> dict[str, Any]:
+        """Walk the workspace and build the index."""
+        self.files.clear()
+        self.repos.clear()
+        start = time.time()
+
+        for entry in self._walk():
+            self.files.append(entry)
+
+        self._build_repo_index()
+        self.indexed_at = time.time()
+        elapsed = self.indexed_at - start
+
+        return {
+            "root": str(self.root),
+            "indexed_at": self.indexed_at,
+            "elapsed_seconds": round(elapsed, 2),
+            "total_files": len(self.files),
+            "total_repos": len(self.repos),
+        }
+
+    def _walk(self) -> list[FileEntry]:
+        """Walk directory tree, yield FileEntry objects."""
+        entries: list[FileEntry] = []
+
+        for dirpath, dirnames, filenames in os.walk(self.root):
+            # Prune skip dirs
+            dirnames[:] = [d for d in dirnames if d not in self.SKIP_DIRS]
+
+            # Check depth
+            rel = os.path.relpath(dirpath, self.root)
+            depth = 0 if rel == "." else rel.count(os.sep) + 1
+            if depth > self.max_depth:
+                dirnames.clear()
+                continue
+
+            for fname in filenames:
+                if fname in self.SKIP_FILES:
+                    continue
+
+                fpath = os.path.join(dirpath, fname)
+                if not os.path.isfile(fpath):
+                    continue
+
+                try:
+                    stat = os.stat(fpath)
+                except OSError:
+                    continue
+
+                suffix = Path(fname).suffix.lower()
+                lines = 0
+                words = 0
+
+                if suffix in self.CODE_SUFFIXES or suffix in self.DOC_SUFFIXES:
+                    try:
+                        with open(fpath, "r", errors="ignore") as fh:
+                            content = fh.read()
+                            lines = content.count("\n") + 1
+                            words = len(content.split())
+                    except Exception:
+                        pass
+
+                sha = self._hash_file(fpath)
+
+                entries.append(FileEntry(
+                    path=fpath,
+                    rel_path=os.path.relpath(fpath, self.root),
+                    size=stat.st_size,
+                    lines=lines,
+                    suffix=suffix,
+                    sha256=sha,
+                    modified=stat.st_mtime,
+                    words=words,
+                ))
+
+        return entries
+
+    def _hash_file(self, path: str, max_bytes: int = 65536) -> str:
+        """Hash first 64KB of a file for identity."""
+        try:
+            h = hashlib.sha256()
+            with open(path, "rb") as fh:
+                h.update(fh.read(max_bytes))
+            return h.hexdigest()
+        except Exception:
+            return ""
+
+    def _build_repo_index(self) -> None:
+        """Group files into repos (top-level directories with .git or source code)."""
+        repo_files: dict[str, list[FileEntry]] = {}
+
+        for f in self.files:
+            parts = f.rel_path.split(os.sep)
+            if len(parts) < 2:
+                continue  # root-level file, skip
+            repo_name = parts[0]
+            repo_files.setdefault(repo_name, []).append(f)
+
+        for repo_name, files in repo_files.items():
+            repo_path = str(self.root / repo_name)
+            has_git = os.path.isdir(os.path.join(repo_path, ".git"))
+            has_tests = any(
+                "test" in f.rel_path.lower() or f.rel_path.endswith("__test__.py")
+                for f in files
+            )
+            has_readme = any(
+                os.path.basename(f.rel_path).upper().startswith("README")
+                for f in files
+            )
+            has_pyproject = any(
+                f.rel_path == f"{repo_name}/pyproject.toml"
+                for f in files
+            )
+            has_package_json = any(
+                f.rel_path == f"{repo_name}/package.json"
+                for f in files
+            )
+            has_cargo = any(
+                f.rel_path == f"{repo_name}/Cargo.toml"
+                for f in files
+            )
+
+            languages: dict[str, int] = {}
+            total_size = 0
+            last_mod = 0.0
+
+            for f in files:
+                if f.suffix in self.CODE_SUFFIXES:
+                    lang = f.suffix.lstrip(".")
+                    languages[lang] = languages.get(lang, 0) + f.lines
+                total_size += f.size
+                last_mod = max(last_mod, f.modified)
+
+            self.repos[repo_name] = RepoEntry(
+                name=repo_name,
+                path=repo_path,
+                has_git=has_git,
+                has_tests=has_tests,
+                has_readme=has_readme,
+                has_pyproject=has_pyproject,
+                has_package_json=has_package_json,
+                has_cargo_toml=has_cargo,
+                file_count=len(files),
+                total_size=total_size,
+                languages=languages,
+                last_modified=last_mod,
+            )
+
+    def save(self, output_dir: str) -> None:
+        """Save index as exo-filed JSON + markdown."""
+        out = Path(output_dir)
+        out.mkdir(parents=True, exist_ok=True)
+
+        # JSON index (machine-readable)
+        index_data = {
+            "root": str(self.root),
+            "indexed_at": self.indexed_at,
+            "total_files": len(self.files),
+            "total_repos": len(self.repos),
+            "files": [f.to_dict() for f in self.files],
+            "repos": {n: r.to_dict() for n, r in self.repos.items()},
+        }
+        with open(out / "index.json", "w") as fh:
+            json.dump(index_data, fh, indent=2)
+
+        # Markdown summary (human-readable)
+        lines = [
+            f"# Workspace Index",
+            f"",
+            f"**Root:** `{self.root}`",
+            f"**Indexed:** {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(self.indexed_at))}",
+            f"**Files:** {len(self.files):,}  |  **Repos:** {len(self.repos)}",
+            f"",
+            f"## Repos",
+            f"",
+            f"| Repo | Files | Health | Git | Tests | README | Languages |",
+            f"|------|-------|--------|-----|-------|--------|-----------|",
+        ]
+        for name in sorted(self.repos):
+            r = self.repos[name]
+            langs = ", ".join(sorted(r.languages, key=lambda x: -r.languages[x])[:3])
+            lines.append(
+                f"| `{name}` | {r.file_count} | {r.health} | "
+                f"{'✓' if r.has_git else '—'} | "
+                f"{'✓' if r.has_tests else '—'} | "
+                f"{'✓' if r.has_readme else '—'} | "
+                f"{langs} |"
+            )
+        with open(out / "INDEX.md", "w") as fh:
+            fh.write("\n".join(lines))

deckhand/retriever.py

@@ -0,0 +1,146 @@
+"""Retriever — BM25 search over the exo-filed index.
+
+Pure Python. No dependencies. No database. Just ranking.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass
+from typing import Any
+
+from deckhand.indexer import FileEntry
+
+
+@dataclass
+class SearchResult:
+    """A single search result."""
+    path: str
+    rel_path: str
+    score: float
+    snippet: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "rel_path": self.rel_path,
+            "score": round(self.score, 4),
+            "snippet": self.snippet[:200],
+        }
+
+
+class Retriever:
+    """BM25 search over indexed files."""
+
+    def __init__(self, files: list[FileEntry], k1: float = 1.5, b: float = 0.75) -> None:
+        self.k1 = k1
+        self.b = b
+        self.files = [f for f in files if f.words > 0]
+        self._build_index()
+
+    def _tokenize(self, text: str) -> list[str]:
+        return re.findall(r"\w+", text.lower())
+
+    def _build_index(self) -> None:
+        """Build BM25 statistics."""
+        self.doc_tokens: list[list[str]] = []
+        self.doc_freq: Counter[str] = Counter()
+        self.doc_len: list[int] = []
+        self.total_docs = len(self.files)
+        self.avg_len: float = 0.0
+        self.term_freqs: list[Counter[str]] = []
+
+        for f in self.files:
+            try:
+                with open(f.path, "r", errors="ignore") as fh:
+                    content = fh.read()
+            except Exception:
+                content = ""
+
+            tokens = self._tokenize(content)
+            self.doc_tokens.append(tokens)
+            tf = Counter(tokens)
+            self.term_freqs.append(tf)
+
+            for term in tf:
+                self.doc_freq[term] += 1
+
+            self.doc_len.append(len(tokens))
+
+        self.avg_len = sum(self.doc_len) / max(1, self.total_docs)
+
+    def _idf(self, term: str) -> float:
+        df = self.doc_freq.get(term, 0)
+        if df == 0:
+            return 0.0
+        return math.log(1 + (self.total_docs - df + 0.5) / (df + 0.5))
+
+    def _bm25_score(self, query_terms: list[str], doc_idx: int) -> float:
+        score = 0.0
+        tf = self.term_freqs[doc_idx]
+        dl = self.doc_len[doc_idx]
+
+        for term in query_terms:
+            idf = self._idf(term)
+            if idf == 0:
+                continue
+            f = tf.get(term, 0)
+            if f == 0:
+                continue
+            numerator = f * (self.k1 + 1)
+            denominator = f + self.k1 * (1 - self.b + self.b * dl / max(1, self.avg_len))
+            score += idf * numerator / denominator
+
+        return score
+
+    def search(self, query: str, top_k: int = 10) -> list[SearchResult]:
+        """Search the corpus. Returns ranked results."""
+        query_terms = self._tokenize(query)
+        if not query_terms:
+            return []
+
+        scored: list[tuple[int, float]] = []
+        for i in range(self.total_docs):
+            s = self._bm25_score(query_terms, i)
+            if s > 0:
+                scored.append((i, s))
+
+        scored.sort(key=lambda x: -x[1])
+
+        results: list[SearchResult] = []
+        for idx, score in scored[:top_k]:
+            f = self.files[idx]
+            snippet = self._snippet(idx, query_terms)
+            results.append(SearchResult(
+                path=f.path,
+                rel_path=f.rel_path,
+                score=score,
+                snippet=snippet,
+            ))
+
+        return results
+
+    def _snippet(self, doc_idx: int, query_terms: list[str], context: int = 50) -> str:
+        """Extract a snippet around the first query term match."""
+        tokens = self.doc_tokens[doc_idx]
+        query_set = set(query_terms)
+
+        for i, tok in enumerate(tokens):
+            if tok in query_set:
+                start = max(0, i - context)
+                end = min(len(tokens), i + context)
+                snippet = " ".join(tokens[start:end])
+                return f"...{snippet}..."
+
+        return ""
+
+    def stats(self) -> dict[str, Any]:
+        """Corpus statistics."""
+        return {
+            "total_docs": self.total_docs,
+            "avg_doc_length": round(self.avg_len, 1),
+            "total_terms": sum(self.doc_len),
+            "unique_terms": len(self.doc_freq),
+        }

deckhand/task_batcher.py

@@ -0,0 +1,129 @@
+"""TaskBatcher — collects tasks from multiple sources into one batch.
+
+Reduces director API calls by batching small tasks into a single list.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+
+class TaskBatcher:
+    """Collects tasks from workspace files into a single batch."""
+
+    def __init__(self, workspace_root: str) -> None:
+        self.root = Path(workspace_root)
+
+    def collect(self) -> dict[str, Any]:
+        """Collect tasks from all sources."""
+        tasks: list[dict[str, str]] = []
+        sources: dict[str, int] = {}
+
+        # From PLANNER_QUEUE.md (checkboxes)
+        planner_tasks = self._read_checkboxes("PLANNER_QUEUE.md")
+        for t in planner_tasks:
+            tasks.append({"source": "PLANNER_QUEUE", "task": t})
+        sources["PLANNER_QUEUE"] = len(planner_tasks)
+
+        # From OPEN_QUESTIONS.md (question bullets)
+        questions = self._read_questions("OPEN_QUESTIONS.md")
+        for q in questions:
+            tasks.append({"source": "OPEN_QUESTION", "task": q})
+        sources["OPEN_QUESTIONS"] = len(questions)
+
+        # From recent git commits (what just changed)
+        recent = self._recent_changes()
+        for c in recent:
+            tasks.append({"source": "RECENT_CHANGE", "task": c})
+        sources["RECENT_CHANGES"] = len(recent)
+
+        return {
+            "collected_at": time.time(),
+            "total_tasks": len(tasks),
+            "sources": sources,
+            "tasks": tasks,
+        }
+
+    def _read_checkboxes(self, filename: str) -> list[str]:
+        """Read unchecked items from a markdown file."""
+        path = self.root / filename
+        if not path.exists():
+            return []
+
+        items: list[str] = []
+        with open(path) as fh:
+            for line in fh:
+                line = line.strip()
+                if line.startswith("- [ ]"):
+                    items.append(line[5:].strip())
+        return items
+
+    def _read_questions(self, filename: str) -> list[str]:
+        """Read question bullets from OPEN_QUESTIONS.md."""
+        path = self.root / filename
+        if not path.exists():
+            return []
+
+        items: list[str] = []
+        with open(path) as fh:
+            for line in fh:
+                line = line.strip()
+                if line.startswith("- [") and "UTC]" in line:
+                    items.append(line)
+        return items
+
+    def _recent_changes(self, hours: float = 6.0) -> list[str]:
+        """Find files modified in the last N hours across repos."""
+        cutoff = time.time() - hours * 3600
+        changes: list[str] = []
+
+        for entry in self.root.iterdir():
+            if not entry.is_dir() or entry.name.startswith("."):
+                continue
+            git_dir = entry / ".git"
+            if not git_dir.exists():
+                continue
+
+            for dirpath, dirnames, filenames in os.walk(entry):
+                dirnames[:] = [d for d in dirnames if d not in {".git", "__pycache__", "node_modules", "dist", "build"}]
+                for fname in filenames:
+                    fpath = os.path.join(dirpath, fname)
+                    try:
+                        if os.path.getmtime(fpath) > cutoff:
+                            rel = os.path.relpath(fpath, self.root)
+                            changes.append(f"modified: {rel}")
+                    except OSError:
+                        pass
+
+        return changes[:50]  # cap at 50
+
+    def write_batch(self, output_path: str) -> str:
+        """Write collected tasks as a single BATCH.md file."""
+        data = self.collect()
+
+        lines = [
+            "# Task Batch",
+            "",
+            f"*Collected by deckhand at {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(data['collected_at']))}*",
+            f"",
+            f"**Total tasks:** {data['total_tasks']}",
+            f"**Sources:** {', '.join(f'{k}({v})' for k,v in data['sources'].items())}",
+            f"",
+        ]
+
+        current_source = None
+        for task in data["tasks"]:
+            if task["source"] != current_source:
+                current_source = task["source"]
+                lines.append(f"\n## {current_source}\n")
+            lines.append(f"- {task['task']}")
+
+        lines.append("")
+
+        with open(output_path, "w") as fh:
+            fh.write("\n".join(lines))
+
+        return output_path

deckhand/wiki.py

@@ -0,0 +1,209 @@
+"""WikiGenerator — builds human-readable wiki pages from the index.
+
+Exo-filed: the wiki IS the interface. A human can cat it.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+from deckhand.indexer import Indexer, RepoEntry
+
+
+class WikiGenerator:
+    """Generates wiki pages from the index."""
+
+    def __init__(self, indexer: Indexer) -> None:
+        self.indexer = indexer
+
+    def generate(self, output_dir: str) -> list[str]:
+        """Generate all wiki pages. Returns list of files written."""
+        out = Path(output_dir)
+        out.mkdir(parents=True, exist_ok=True)
+
+        written: list[str] = []
+        written.append(self._write_repos(out))
+        written.append(self._write_connections(out))
+        written.append(self._write_glossary(out))
+        written.append(self._write_status(out))
+
+        return written
+
+    def _write_repos(self, out: Path) -> str:
+        """REPOS.md — one-line summary of every repo."""
+        lines = [
+            "# Fleet Repos",
+            "",
+            f"*Generated by deckhand at {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime())}*",
+            "",
+            f"**Total repos:** {len(self.indexer.repos)}",
+            f"**Total files:** {len(self.indexer.files):,}",
+            "",
+            "| Repo | Files | Health | Git | Tests | README | Py | JS | Rust | Top Languages |",
+            "|------|-------|--------|-----|-------|--------|----|----|------|---------------|",
+        ]
+
+        for name in sorted(self.indexer.repos):
+            r = self.indexer.repos[name]
+            langs = sorted(r.languages, key=lambda x: -r.languages[x])[:3]
+            lang_str = ", ".join(f"{l}({r.languages[l]}L)" for l in langs) if langs else "—"
+            health_emoji = {"ok": "✅", "untested": "⚠️", "stub": "🔩", "no-readme": "❌"}.get(r.health, "?")
+            lines.append(
+                f"| `{name}` | {r.file_count} | {health_emoji} {r.health} | "
+                f"{'✓' if r.has_git else '—'} | "
+                f"{'✓' if r.has_tests else '—'} | "
+                f"{'✓' if r.has_readme else '—'} | "
+                f"{'✓' if r.has_pyproject else '—'} | "
+                f"{'✓' if r.has_package_json else '—'} | "
+                f"{'✓' if r.has_cargo_toml else '—'} | "
+                f"{lang_str} |"
+            )
+
+        path = str(out / "REPOS.md")
+        with open(path, "w") as fh:
+            fh.write("\n".join(lines))
+        return path
+
+    def _write_connections(self, out: Path) -> str:
+        """CONNECTIONS.md — cross-reference graph."""
+        lines = [
+            "# Repo Connections",
+            "",
+            "*Which repos reference which. Generated by deckhand.*",
+            "",
+        ]
+
+        # Find cross-references by searching for repo names in other repos' files
+        repo_names = sorted(self.indexer.repos.keys())
+        connections: dict[str, list[str]] = {n: [] for n in repo_names}
+
+        for f in self.indexer.files:
+            parts = f.rel_path.split(os.sep)
+            if len(parts) < 2:
+                continue
+            source_repo = parts[0]
+
+            try:
+                with open(f.path, "r", errors="ignore") as fh:
+                    content = fh.read().lower()
+            except Exception:
+                continue
+
+            for target_repo in repo_names:
+                if target_repo == source_repo:
+                    continue
+                if target_repo.lower() in content:
+                    if target_repo not in connections[source_repo]:
+                        connections[source_repo].append(target_repo)
+
+        for repo in sorted(connections):
+            targets = connections[repo]
+            if targets:
+                links = ", ".join(f"`{t}`" for t in sorted(targets))
+                lines.append(f"- **`{repo}`** → {links}")
+            else:
+                lines.append(f"- **`{repo}`** → (isolated)")
+
+        path = str(out / "CONNECTIONS.md")
+        with open(path, "w") as fh:
+            fh.write("\n".join(lines))
+        return path
+
+    def _write_glossary(self, out: Path) -> str:
+        """GLOSSARY.md — top terms across the workspace."""
+        from collections import Counter
+
+        # Simple word frequency (not TF-IDF — that's the Retriever's job)
+        word_freq: Counter[str] = Counter()
+        stop = frozenset({
+            "the", "a", "an", "is", "are", "was", "were", "be", "been",
+            "to", "of", "in", "for", "on", "at", "by", "with", "from",
+            "and", "or", "not", "but", "if", "then", "else", "when",
+            "this", "that", "these", "those", "it", "its", "they", "them",
+            "their", "there", "here", "which", "who", "what", "where",
+            "import", "from", "def", "class", "return", "self", "none",
+            "true", "false", "raise", "try", "except", "with", "as",
+        })
+
+        import re
+        for f in self.indexer.files:
+            if f.words == 0:
+                continue
+            try:
+                with open(f.path, "r", errors="ignore") as fh:
+                    content = fh.read()
+            except Exception:
+                continue
+
+            words = re.findall(r"\b[a-z_]{4,30}\b", content.lower())
+            for w in words:
+                if w not in stop:
+                    word_freq[w] += 1
+
+        lines = [
+            "# Workspace Glossary",
+            "",
+            "*Top 100 terms across the workspace. Generated by deckhand.*",
+            "",
+            "| Term | Frequency |",
+            "|------|-----------|",
+        ]
+
+        for term, freq in word_freq.most_common(100):
+            lines.append(f"| `{term}` | {freq:,} |")
+
+        path = str(out / "GLOSSARY.md")
+        with open(path, "w") as fh:
+            fh.write("\n".join(lines))
+        return path
+
+    def _write_status(self, out: Path) -> str:
+        """STATUS.md — health of each repo."""
+        lines = [
+            "# Fleet Status",
+            "",
+            f"*Generated by deckhand at {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime())}*",
+            "",
+        ]
+
+        healthy = [r for r in self.indexer.repos.values() if r.health == "ok"]
+        untested = [r for r in self.indexer.repos.values() if r.health == "untested"]
+        stubs = [r for r in self.indexer.repos.values() if r.health == "stub"]
+        no_readme = [r for r in self.indexer.repos.values() if r.health == "no-readme"]
+
+        lines.append(f"## Summary")
+        lines.append(f"")
+        lines.append(f"- ✅ **Healthy:** {len(healthy)}")
+        lines.append(f"- ⚠️ **Untested (has code, no tests):** {len(untested)}")
+        lines.append(f"- 🔩 **Stub (< 3 files):** {len(stubs)}")
+        lines.append(f"- ❌ **No README:** {len(no_readme)}")
+        lines.append(f"")
+
+        if untested:
+            lines.append(f"## Needs Tests")
+            lines.append(f"")
+            for r in sorted(untested, key=lambda x: -x.file_count):
+                lines.append(f"- `{r.name}` ({r.file_count} files, {sum(r.languages.values())} LOC)")
+            lines.append("")
+
+        if no_readme:
+            lines.append(f"## Needs README")
+            lines.append(f"")
+            for r in sorted(no_readme, key=lambda x: -x.file_count):
+                lines.append(f"- `{r.name}` ({r.file_count} files)")
+            lines.append("")
+
+        if stubs:
+            lines.append(f"## Stubs (may be abandoned)")
+            lines.append(f"")
+            for r in sorted(stubs, key=lambda x: -x.file_count):
+                lines.append(f"- `{r.name}` ({r.file_count} files)")
+            lines.append("")
+
+        path = str(out / "STATUS.md")
+        with open(path, "w") as fh:
+            fh.write("\n".join(lines))
+        return path

si_deckhand-0.1.0.dist-info/METADATA

@@ -0,0 +1,106 @@
+Metadata-Version: 2.4
+Name: si-deckhand
+Version: 0.1.0
+Summary: Lightweight local agent that indexes, retrieves, and wikis the workspace. The crew member who knows where everything is.
+Author: SuperInstance
+License-Expression: MIT
+Keywords: agent,retriever,wiki,index,exo-filed,local
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Systems Administration
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: vector
+Requires-Dist: numpy>=1.20; extra == "vector"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+
+# deckhand
+
+*The crew member who knows where everything is.*
+
+---
+
+**deckhand** is a lightweight local agent that indexes, retrieves, and wikis your workspace. File-first (exo-filed). Human-readable. No hidden database.
+
+## What it does
+
+- **Indexes** every file in your workspace (code, docs, configs)
+- **Builds wiki pages** that a human can `cat` — REPOS.md, CONNECTIONS.md, GLOSSARY.md, STATUS.md
+- **Searches** via BM25 (pure Python, no dependencies)
+- **Batches tasks** from PLANNER_QUEUE.md, OPEN_QUESTIONS.md, and recent file changes
+- **Studies** repos off-watch — writes daily observation logs
+- **Runs forever** in a loop: index → wiki → batch → study → sleep → repeat
+
+## Install
+
+```bash
+pip install deckhand
+```
+
+## Use
+
+```bash
+# Index the workspace and generate wiki
+deckhand index /path/to/workspace
+
+# Search for something
+deckhand search /path/to/workspace "conservation law"
+
+# Generate just the wiki
+deckhand wiki /path/to/workspace
+
+# Collect tasks into a batch
+deckhand batch /path/to/workspace
+
+# Run the deckhand loop forever (index → wiki → batch → study → sleep)
+deckhand run /path/to/workspace --sleep 300
+```
+
+## The wiki output
+
+After indexing, deckhand writes to `deckhand-wiki/`:
+
+- **REPOS.md** — table of every repo, one line each. Health indicators (✅ ok, ⚠️ untested, 🔩 stub, ❌ no-readme)
+- **CONNECTIONS.md** — cross-reference graph. Which repos reference which.
+- **GLOSSARY.md** — top 100 terms across the workspace
+- **STATUS.md** — health summary. Lists repos that need tests, need READMEs, or may be abandoned
+- **INDEX.md** — machine-readable JSON index
+- **BATCH.md** — collected tasks from PLANNER_QUEUE + OPEN_QUESTIONS + recent changes
+- **logs/YYYY-MM-DD.md** — daily study observations
+
+## Why exo-filed?
+
+A vector database is opaque. You can't `cat` it. You can't grep it. You can't diff it in git.
+
+deckhand's index is markdown + JSON. A human can open REPOS.md and understand the fleet in 30 seconds. An agent can read CONNECTIONS.md instead of re-reading 4,000 repos.
+
+When the corpus gets too large for file-based search (>10K files), build the optional vector twin:
+
+```python
+from deckhand.vector_twin import VectorTwin  # coming in v0.2
+```
+
+The vector twin is ALWAYS accompanied by its file-twin. You can always fall back to the human-readable version.
+
+## The maritime metaphor
+
+On a boat, the deckhand is the crew member who:
+- Knows where every line, every shackle, every tool is stored
+- Prepares tasks for the captain (batches)
+- Maintains the logbook (study logs)
+- Studies charts off-watch (studies repos when idle)
+- Doesn't need to be told what to do — knows the direction of the voyage
+
+The deckhand doesn't replace the captain (the director model). The deckhand makes the captain more efficient by pre-processing the workspace into a form the captain can act on in fewer calls.
+
+## License
+
+MIT
+
+---
+
+*Built by SuperInstance. The deckhand knows where everything is.*

si_deckhand-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+deckhand/__init__.py,sha256=0NLBQR2dHquc_dGhZ7KDCYtGLnyqIvW7Kg0AAHp7og4,582
+deckhand/agent.py,sha256=f0KekZ-DJRDKm9artXDD8HZESSv6nzs-LPFxJxAaL1s,4638
+deckhand/cli.py,sha256=uQPmasGuIUKeI0vMu0yAbohqjF_sp0H-46p70Npj--0,3192
+deckhand/indexer.py,sha256=0UOBklFIgB9aVHKToHOa3uWHGidWuQedIV5ATMXzzSU,9936
+deckhand/retriever.py,sha256=MhfRXayt5Y5wQSLGWCD5ICf-nvug6HGQbRKPQtAD4Q0,4344
+deckhand/task_batcher.py,sha256=A7gAJmIXeSqnbJmHxStvtA2mdFI4h5MLPqZs_O-ZPKc,4396
+deckhand/wiki.py,sha256=NibnYfE1t2PkFVggw_3EUAAzGT3SDxPMSODN3zjO2qs,7831
+si_deckhand-0.1.0.dist-info/METADATA,sha256=GGCCdNBPKEnzqBfEtb6VDuKC5KZToNRkiSGyKsaaFhk,3747
+si_deckhand-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+si_deckhand-0.1.0.dist-info/entry_points.txt,sha256=ceZa8_rmihTBe1XOdkxMaiYU42FAdotPdH-f1Nbcozg,47
+si_deckhand-0.1.0.dist-info/top_level.txt,sha256=G4gYaKu5Ipp7G_G-YiBbvZwXB95Jmm42BaXzffzh620,9
+si_deckhand-0.1.0.dist-info/RECORD,,

si_deckhand-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

si_deckhand-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+deckhand = deckhand.cli:main

si_deckhand-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+deckhand