code2okf 0.1.0__py3-none-any.whl

code2okf/__init__.py

@@ -0,0 +1,8 @@
+"""code2okf: compile Markdown into an OKF wiki with the Pi coding agent."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("code2okf")
+except PackageNotFoundError:
+    __version__ = "0.0.0+local"

code2okf/cli.py

@@ -0,0 +1,234 @@
+"""The code2okf command.
+
+Compiles Markdown into an OKF wiki with the Pi coding agent, via a sandboxed
+sbx runtime. See .claude/plans/interface-plan.md.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from code2okf import __version__, resources, sandbox, workbench
+from code2okf import compile as compile_mod
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="code2okf",
+        description="Compile Markdown into an OKF wiki with the Pi coding agent.",
+    )
+    parser.add_argument(
+        "paths", nargs="*", metavar="FILE|DIR", help="Markdown files or folders; '-' or none means stdin"
+    )
+    parser.add_argument("-o", "--output", default="okf", metavar="DIR", help="wiki output directory (default: ./okf)")
+    parser.add_argument("--spec", metavar="FILE", help="OKF spec file (default: the bundled SPEC.md)")
+    parser.add_argument(
+        "-n",
+        type=int,
+        default=compile_mod.DEFAULT_MAX_ITERATIONS,
+        metavar="N",
+        help="max Ralph loop iterations per document (default: 10)",
+    )
+    parser.add_argument("--fresh", action="store_true", help="recreate the sandbox even if it could be reused")
+    parser.add_argument("--dry-run", action="store_true", help="resolve and print what would run; do nothing paid")
+    verbosity = parser.add_mutually_exclusive_group()
+    verbosity.add_argument(
+        "-q", "--quiet", action="store_true", help="suppress progress and TSV rows (fatal errors still print)"
+    )
+    verbosity.add_argument("-v", "--verbose", action="store_true", help="also show Pi's tool calls and prose")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return parser
+
+
+def _resolve_inputs(
+    args: argparse.Namespace,
+) -> tuple[list[compile_mod.Document], Path, Path, workbench.Workbench] | int:
+    """Everything decided before any work starts. Returns exit code 2 on failure."""
+    try:
+        if args.n < 1:
+            raise compile_mod.UsageError(f"-n must be at least 1 (got {args.n})")
+        documents = compile_mod.resolve_documents(args.paths)
+        spec_path = Path(args.spec) if args.spec else resources.spec_md()
+        workbench.reject_if_unsafe(spec_path, what="--spec")
+        if not spec_path.is_file():
+            raise compile_mod.UsageError(f"--spec is not a file: {spec_path}")
+
+        output_dir = Path(args.output)
+        if not workbench.is_adoptable_output(output_dir):
+            raise compile_mod.UsageError(
+                f"-o {output_dir} is not empty and is not a recognised OKF bundle root; refusing to adopt it"
+            )
+
+        wb = workbench.Workbench.default()
+        overlap_paths = [Path(raw) for raw in args.paths if raw != "-"]
+        overlap_paths += [spec_path, output_dir, wb.root]
+        workbench.check_no_overlap(overlap_paths)
+    except (compile_mod.UsageError, workbench.WorkbenchError, resources.ResourcesError) as exc:
+        print(f"code2okf: {exc}", file=sys.stderr)
+        return 2
+    return documents, spec_path, output_dir, wb
+
+
+def _print_dry_run(
+    documents: list[compile_mod.Document], spec_path: Path, output_dir: Path, wb: workbench.Workbench
+) -> None:
+    print("code2okf --dry-run: resolving only -- no sandbox will be created, nothing paid will run.")
+    print(f"  spec:   {spec_path}")
+    print(f"  output: {output_dir}")
+    print("  documents:")
+    for doc in documents:
+        print(f"    {doc.display} -> work/md/{doc.basename}")
+    print("  mounts:")
+    for mount in wb.mounts():
+        print(f"    {mount.as_arg()}")
+    print("  commands:")
+    print(f"    {_format_sbx_run(wb)}")
+    for doc in documents:
+        print(f"    {_format_sbx_exec_pi(wb, doc)}")
+
+
+def _format_sbx_run(wb: workbench.Workbench) -> str:
+    """The `sbx run` line that would create the sandbox, if one is needed.
+
+    Whether it actually runs depends on sandbox reuse -- a live decision
+    --dry-run must not make (that would mean querying `sbx`). This shows
+    what *would* run if a (re)creation turns out to be necessary.
+    """
+    mount_args = " ".join(mount.as_arg() for mount in wb.mounts())
+    return (
+        f"sbx run --detached --name {workbench.SANDBOX_NAME} "
+        f"-e CODE2OKF_STATE_DIR={wb.root} {resources.kit_dir()} {mount_args}"
+    )
+
+
+def _format_sbx_exec_pi(wb: workbench.Workbench, doc: compile_mod.Document) -> str:
+    """The first-iteration `sbx exec ... pi` line for one document.
+
+    Later Ralph loop iterations append the continuation prompt; --dry-run
+    shows only the first, since how many would actually run is exactly
+    what compiling determines.
+    """
+    document_path = wb.work_md / doc.basename
+    prompt = compile_mod.COMPILE_PROMPT.format(document=document_path)
+    return f"sbx exec {workbench.SANDBOX_NAME} -- pi --mode json {prompt!r}"
+
+
+def _ensure_sandbox(wb: workbench.Workbench, args: argparse.Namespace) -> int | None:
+    """Reuse or (re)create the sandbox. Returns an exit code on failure, else None."""
+    try:
+        workbench.ensure_sandbox(wb, fresh=args.fresh)
+    except (
+        workbench.UnownedSandboxError,
+        workbench.KeyNotProxyManagedError,
+        sandbox.SandboxError,
+        resources.ResourcesError,
+    ) as exc:
+        print(f"code2okf: {exc}", file=sys.stderr)
+        return 2
+    return None
+
+
+def _run(
+    args: argparse.Namespace,
+    documents: list[compile_mod.Document],
+    spec_path: Path,
+    output_dir: Path,
+    wb: workbench.Workbench,
+) -> int:
+    wb.ensure_roots()
+
+    failure = _ensure_sandbox(wb, args)
+    if failure is not None:
+        return failure
+
+    try:
+        clis_dir = resources.clis_dir()
+    except resources.ResourcesError as exc:
+        print(f"code2okf: {exc}", file=sys.stderr)
+        return 2
+
+    try:
+        workbench.restage(
+            wb,
+            inputs=compile_mod.stage_items(documents),
+            clis_dir=clis_dir,
+            spec_source=spec_path,
+            output_dir=output_dir,
+        )
+    except workbench.WorkbenchError as exc:
+        print(f"code2okf: {exc}", file=sys.stderr)
+        return 1
+
+    def on_progress(line: str) -> None:
+        if not args.quiet:
+            print(line, file=sys.stderr)
+
+    def on_event(line: str) -> None:
+        if args.verbose:
+            print(line, file=sys.stderr)
+
+    for doc in documents:
+        try:
+            row = compile_mod.compile_document(
+                workbench.SANDBOX_NAME, doc, wb, output_dir, args.n, on_progress=on_progress, on_event=on_event
+            )
+        except compile_mod.CompileError as exc:
+            print(f"code2okf: {exc}", file=sys.stderr)
+            return 1
+        if not args.quiet:
+            print(row.as_tsv())
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Parse argv, run, and return a process exit code."""
+    args = _build_parser().parse_args(argv)
+
+    resolved = _resolve_inputs(args)
+    if isinstance(resolved, int):
+        return resolved
+    documents, spec_path, output_dir, wb = resolved
+
+    if args.dry_run:
+        try:
+            _print_dry_run(documents, spec_path, output_dir, wb)
+        except resources.ResourcesError as exc:
+            print(f"code2okf: {exc}", file=sys.stderr)
+            return 2
+        return 0
+
+    try:
+        sandbox.preflight()
+    except sandbox.SandboxError as exc:
+        print(f"code2okf: {exc}", file=sys.stderr)
+        return 2
+
+    try:
+        with workbench.lock():
+            return _run(args, documents, spec_path, output_dir, wb)
+    except workbench.LockHeld:
+        print("code2okf: another code2okf run is using the sandbox; try again later", file=sys.stderr)
+        return 2
+    except workbench.UnsafeLockFile as exc:
+        # Caught here rather than with the setup errors above because lock()
+        # is entered after them; it is still an environment problem decided
+        # before any work starts, so exit 2. Kept narrow on purpose: a broad
+        # WorkbenchError clause here would also swallow run-phase failures
+        # that owe the caller exit 1.
+        print(f"code2okf: {exc}", file=sys.stderr)
+        return 2
+    except KeyboardInterrupt:
+        # Ctrl-C is a failed run (exit 1), not a crash: a bare traceback tells
+        # the user nothing about what survived. The lock is already released
+        # by lock()'s own finally, and mirror_out() only ever runs after a
+        # completed iteration, so -o DIR cannot hold a half-written pass --
+        # though the copy itself is not atomic, hence "may be partial" for
+        # the workbench side.
+        print(
+            f"\ncode2okf: interrupted. {output_dir} holds the last completed pass; "
+            f"the workbench copy at {wb.work_okf} may be a partial one.",
+            file=sys.stderr,
+        )
+        return 1

code2okf/clis/inspectmd/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "inspectmd"
+version = "0.1.0"
+description = "Print a Markdown heading map with line ranges and word counts"
+requires-python = ">=3.12"
+dependencies = []
+
+[project.scripts]
+inspectmd = "inspectmd.cli:entrypoint"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+test = ["pytest>=8.4"]
+
+# Paths are relative to this file, so they resolve wherever the project is
+# invoked from. pythonpath keeps `import inspectmd` working without an install.
+[tool.pytest.ini_options]
+minversion = "8.0"
+addopts = ["-ra", "--strict-markers", "--strict-config"]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 120
+
+# Same rule set as web2md: owned here under the zero-overlap rule, not shared.
+[tool.ruff.lint]
+select = ["E", "W", "F", "I", "UP", "B", "SIM", "C4", "RET", "PT", "D"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+# A test's name is its documentation; a docstring on each of them would be noise.
+# Fixtures and helpers are still expected to explain themselves.
+"tests/*" = ["D100", "D103"]

code2okf/clis/inspectmd/src/inspectmd/__init__.py

@@ -0,0 +1,8 @@
+"""Inspect Markdown heading structure (stdlib only)."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("inspectmd")
+except PackageNotFoundError:
+    __version__ = "0.0.0+local"

code2okf/clis/inspectmd/src/inspectmd/__main__.py

@@ -0,0 +1,5 @@
+"""Allow `python -m inspectmd`."""
+
+from inspectmd.cli import main
+
+raise SystemExit(main())

code2okf/clis/inspectmd/src/inspectmd/cli.py

@@ -0,0 +1,159 @@
+"""Command-line interface for inspectmd."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from inspectmd import __version__
+from inspectmd.parse import Section, inspect_markdown
+
+
+def format_table(sections: Sequence[Section], *, max_level: int | None = None) -> str:
+    """Render sections as a fixed-width table.
+
+    When ``max_level`` is set, only the preamble (level 0) and headings with
+    ``level <= max_level`` are shown.
+    """
+    visible = [
+        s
+        for s in sections
+        if max_level is None or s.level == 0 or s.level <= max_level
+    ]
+    if not visible:
+        return "(no sections at this depth)\n"
+
+    headers = ("Index", "Level", "Lines", "Words", "Slug", "Title")
+    rows: list[tuple[str, str, str, str, str, str]] = []
+    for s in visible:
+        rows.append(
+            (
+                str(s.index),
+                str(s.level),
+                f"{s.start}-{s.end}",
+                str(s.words),
+                s.slug,
+                s.title,
+            )
+        )
+
+    widths = [len(h) for h in headers]
+    for row in rows:
+        for i, cell in enumerate(row):
+            widths[i] = max(widths[i], len(cell))
+
+    def fmt(row: tuple[str, ...]) -> str:
+        return "  ".join(cell.ljust(widths[i]) for i, cell in enumerate(row))
+
+    lines = [fmt(headers), fmt(tuple("-" * w for w in widths))]
+    lines.extend(fmt(row) for row in rows)
+    return "\n".join(lines) + "\n"
+
+
+def format_section_range(section: Section) -> str:
+    """One section as ``start:end`` plus word count, for a ranged read."""
+    return f"{section.start}:{section.end}  {section.words} words\n"
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="inspectmd",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="Print a Markdown heading map with line ranges and word counts.",
+        epilog="""\
+Output columns (default table):
+  Index       Section number in document order (0 = preamble when present).
+              Pass this value to --section.
+  Level       Heading depth: 0 for the preamble, 1 for #, 2 for ##, …, 6 for ######.
+  Lines       1-based inclusive line range of the section (start-end).
+  Words       Whitespace-split word count of that range.
+  Slug        Kebab-case slug derived from the heading title (OKF file-name style).
+  Title       Heading text as written (or "(preamble)" / "(empty)").
+
+--section N prints only "start:end  N words" for ranged reads.
+""",
+    )
+    parser.add_argument(
+        "file",
+        type=Path,
+        help="Markdown file to inspect",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+    parser.add_argument(
+        "--section",
+        type=int,
+        metavar="N",
+        help="print only section N as start:end and word count",
+    )
+    parser.add_argument(
+        "-L",
+        "--level",
+        type=int,
+        metavar="N",
+        help="show only headings at this level or above (1=H1, …)",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Parse argv and print the heading map. Returns a process exit code."""
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    # Checked before the path so the message is the same wherever it is run
+    # from, and so every `-L` in this repo rejects the same values.
+    level: int | None = args.level
+    if level is not None and level < 1:
+        print(f"inspectmd: --level must be 1 or greater (got {level})", file=sys.stderr)
+        return 2
+
+    path: Path = args.file
+    if not path.is_file():
+        print(f"inspectmd: not a file: {path}", file=sys.stderr)
+        return 2
+
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        print(f"inspectmd: {exc}", file=sys.stderr)
+        return 2
+
+    line_count = text.count("\n") + (0 if text.endswith("\n") or text == "" else 1)
+    if text == "":
+        line_count = 0
+    sections = inspect_markdown(text)
+
+    if args.section is not None:
+        match = next((s for s in sections if s.index == args.section), None)
+        if match is None:
+            print(
+                f"inspectmd: section {args.section} out of range "
+                f"(0..{sections[-1].index if sections else 'none'})",
+                file=sys.stderr,
+            )
+            return 2
+        sys.stdout.write(format_section_range(match))
+        return 0
+
+    summary = (
+        f"{path.name}: {line_count} lines, {len(text.split())} words, "
+        f"{len(sections)} sections"
+    )
+    sys.stdout.write(summary + "\n")
+    if not sections:
+        sys.stdout.write("(no headings)\n")
+        return 0
+
+    sys.stdout.write(format_table(sections, max_level=args.level))
+    return 0
+
+
+def entrypoint() -> None:
+    """Console-script entry: exit with ``main``'s return code."""
+    raise SystemExit(main())

code2okf/clis/inspectmd/src/inspectmd/parse.py

@@ -0,0 +1,212 @@
+"""Parse ATX Markdown headings into a section map.
+
+Only ATX headings (``#`` … ``######``) count. Setext underlines are ignored on
+purpose: both producers in this repo emit ATX (``web2md`` sets
+``heading_style = ATX``, and Marker output under ``md/`` is ATX throughout).
+Headings inside fenced code blocks (backticks or tildes) are ignored. A leading
+YAML frontmatter block is skipped so its ``---`` lines are not mistaken for
+content. Text between the frontmatter and the first heading is section 0, the
+preamble.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass
+
+_ATX_RE = re.compile(r"^( {0,3})(#{1,6})(?:[ \t]+(.*))?$")
+_CLOSING_HASHES_RE = re.compile(r"[ \t]+#*[ \t]*$")
+_FENCE_RE = re.compile(r"^( {0,3})(`{3,}|~{3,})(.*)$")
+_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+")
+
+
+@dataclass(frozen=True)
+class Section:
+    """One preamble or ATX-headed region of a Markdown document."""
+
+    index: int
+    """0 is the preamble, if any; otherwise the first section is 0."""
+
+    level: int
+    """1–6 for ATX headings; 0 for the preamble."""
+
+    title: str
+    slug: str
+    """Kebab-case slug matching the AGENTS.md file-name convention."""
+
+    start: int
+    """1-based inclusive line number (the heading line itself, or first body)."""
+
+    end: int
+    """1-based inclusive line number."""
+
+    words: int
+    """Whitespace-split word count of the section's lines."""
+
+
+def count_words(text: str) -> int:
+    """Count whitespace-separated tokens in ``text``."""
+    return len(text.split())
+
+
+def slugify(title: str) -> str:
+    """Reduce a heading title to a kebab-case slug.
+
+    Lowercases ASCII letters, strips combining marks from non-ASCII characters,
+    replaces any remaining non ``[a-z0-9]`` run with a single hyphen, and trims
+    leading/trailing hyphens. Empty input yields an empty string.
+    """
+    normalized = unicodedata.normalize("NFKD", title)
+    ascii_only = "".join(c for c in normalized if not unicodedata.combining(c))
+    return _NON_ALNUM_RE.sub("-", ascii_only.casefold()).strip("-")
+
+
+def split_frontmatter(text: str) -> tuple[int, str]:
+    """Strip a leading YAML frontmatter block.
+
+    Returns ``(last_frontmatter_line, remainder)``. Line numbers are 1-based:
+    ``last_frontmatter_line`` is the closing ``---`` line, or ``0`` when there
+    is no frontmatter. The remainder starts at the first content line.
+    """
+    if text.startswith("\ufeff"):
+        text = text[1:]
+
+    if not text.startswith("---"):
+        return 0, text
+
+    lines = text.splitlines(keepends=True)
+    if not lines or lines[0].strip() != "---":
+        return 0, text
+
+    for i in range(1, len(lines)):
+        if lines[i].strip() == "---":
+            return i + 1, "".join(lines[i + 1 :])
+    return 0, text
+
+
+def _fence_opener(line: str) -> tuple[str, int] | None:
+    """Return ``(fence_char, length)`` if ``line`` opens a fenced code block."""
+    match = _FENCE_RE.match(line.rstrip("\n"))
+    if match is None:
+        return None
+    marker = match.group(2)
+    return marker[0], len(marker)
+
+
+def _fence_closer(line: str, char: str, length: int) -> bool:
+    """True if ``line`` closes a fence opened with ``char`` × ``length``."""
+    match = _FENCE_RE.match(line.rstrip("\n"))
+    if match is None:
+        return False
+    marker = match.group(2)
+    info = match.group(3)
+    if marker[0] != char or len(marker) < length:
+        return False
+    # Closing fences may only carry trailing whitespace as "info".
+    return info.strip() == ""
+
+
+def _parse_atx(line: str) -> tuple[int, str] | None:
+    """Return ``(level, title)`` for an ATX heading line, else ``None``."""
+    match = _ATX_RE.match(line.rstrip("\n"))
+    if match is None:
+        return None
+    hashes = match.group(2)
+    rest = match.group(3)
+    if rest is None:
+        return len(hashes), ""
+    title = _CLOSING_HASHES_RE.sub("", rest).strip()
+    return len(hashes), title
+
+
+def parse_sections(text: str, *, line_offset: int = 0) -> list[Section]:
+    """Parse ``text`` into sections.
+
+    ``line_offset`` is the number of lines already consumed before ``text``
+    (typically the frontmatter). Section line numbers are 1-based in the full
+    file: the first line of ``text`` is ``line_offset + 1``.
+    """
+    lines = text.splitlines(keepends=True)
+    if not lines:
+        return []
+
+    headings: list[tuple[int, int, str]] = []
+    fence: tuple[str, int] | None = None
+
+    for i, line in enumerate(lines):
+        if fence is not None:
+            char, length = fence
+            if _fence_closer(line, char, length):
+                fence = None
+            continue
+        opener = _fence_opener(line)
+        if opener is not None:
+            fence = opener
+            continue
+        atx = _parse_atx(line)
+        if atx is not None:
+            headings.append((i, atx[0], atx[1]))
+
+    sections: list[Section] = []
+
+    def text_between(start_i: int, end_i: int) -> str:
+        return "".join(lines[j] for j in range(start_i, end_i + 1))
+
+    first_heading_i = headings[0][0] if headings else None
+
+    if first_heading_i is None:
+        body = "".join(lines)
+        if body.strip():
+            sections.append(
+                Section(
+                    index=0,
+                    level=0,
+                    title="(preamble)",
+                    slug="preamble",
+                    start=line_offset + 1,
+                    end=line_offset + len(lines),
+                    words=count_words(body),
+                )
+            )
+        return sections
+
+    if first_heading_i > 0:
+        preamble_text = "".join(lines[:first_heading_i])
+        if preamble_text.strip():
+            sections.append(
+                Section(
+                    index=0,
+                    level=0,
+                    title="(preamble)",
+                    slug="preamble",
+                    start=line_offset + 1,
+                    end=line_offset + first_heading_i,
+                    words=count_words(preamble_text),
+                )
+            )
+
+    for idx, (start_i, level, title) in enumerate(headings):
+        end_i = headings[idx + 1][0] - 1 if idx + 1 < len(headings) else len(lines) - 1
+        next_index = sections[-1].index + 1 if sections else 0
+        display_title = title if title else "(empty)"
+        display_slug = slugify(title) if title else "empty"
+        sections.append(
+            Section(
+                index=next_index,
+                level=level,
+                title=display_title,
+                slug=display_slug,
+                start=line_offset + start_i + 1,
+                end=line_offset + end_i + 1,
+                words=count_words(text_between(start_i, end_i)),
+            )
+        )
+
+    return sections
+
+
+def inspect_markdown(text: str) -> list[Section]:
+    """Split frontmatter and parse the remainder into sections."""
+    offset, body = split_frontmatter(text)
+    return parse_sections(body, line_offset=offset)