code-jig 0.12.0__py3-none-any.whl

codejig/migrate.py

@@ -0,0 +1,515 @@
+from __future__ import annotations
+
+import contextlib
+import errno
+import functools
+import hashlib
+import importlib.util
+import json
+import os
+import re
+import shutil
+import sqlite3
+import subprocess
+import sys
+from pathlib import Path
+
+from codejig import collect, install, paths
+from codejig.atomicwrite import atomic_write
+from codejig.overlay import Overlay, load
+
+NOTICE = "code-jig migrate is deprecated and will be removed in the next release"
+KEYS = ("email", "allowProse")
+OLD_MARKER = "generated by codex-harness sha256:"
+OLD_STATUSES = ("codex-harness collect", "codex-harness watermark")
+OLD_SIDECAR = ".codex-harness"
+OLD_LAUNCHER_MARK = "codex-harness launcher "
+SHARED_DEV_SKILLS = {"grilling", "web-consensus", "remove-ai-attribution", "skill-validate", "bug-hunt"}
+OLD_LINK = re.compile(r"(?:^|[/\\])(codex-harness|dev-harness)[/\\]skills[/\\]([^/\\]+)[/\\]?$")
+OLD_NAMES = ("CODEX_HARNESS_", "codex-harness")
+SKIPPED = "skipped the remaining steps because of the refused or failed line above; fix it and rerun"
+GATE_OFF = "failed: the git gate is off until code-jig install succeeds"
+
+
+def old_state(home: Path) -> Path:
+    return home / "codex-harness"
+
+
+def _has_rows(path: Path) -> bool:
+    try:
+        with contextlib.closing(sqlite3.connect(path)) as con:
+            tables = [r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type = 'table'")]
+            return any(con.execute(f'SELECT count(*) FROM "{t}"').fetchone()[0] for t in tables)
+    except sqlite3.DatabaseError:
+        return True
+
+
+def _move(src: Path, dst: Path) -> str | None:
+    in_use = f"refused: {src} is in use; close Codex and Claude Code and rerun"
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        os.replace(src, dst)
+        return None
+    except PermissionError:
+        return in_use
+    except OSError as e:
+        if e.errno != errno.EXDEV:
+            raise
+    tmp = dst.with_name(dst.name + ".migrate-tmp")
+    try:
+        shutil.copy2(src, tmp)
+        os.replace(tmp, dst)
+    finally:
+        tmp.unlink(missing_ok=True)
+    try:
+        src.unlink()
+    except OSError:
+        dst.unlink()
+        return in_use
+    return None
+
+
+def move_db(home: Path, dry_run: bool = False) -> list[str]:
+    old, new = old_state(home) / "usage.sqlite3", collect.db_path()
+    if not old.is_file():
+        return []
+    if old.with_name(old.name + "-journal").exists():
+        return [f"refused: {old} has an unfinished write; open and close Codex once and rerun"]
+    if new.exists() and _has_rows(new):
+        return [f"refused: {new} already has data; collect rebuilds it from the rollouts, so move {new} away and rerun"]
+    if not dry_run:
+        refused = _move(old, new)
+        if refused:
+            return [refused]
+    return [f"{'would ' if dry_run else ''}move {old} to {new}"]
+
+
+def _git(*args: str) -> subprocess.CompletedProcess | None:
+    try:
+        return subprocess.run(["git", *args], capture_output=True, text=True, encoding="utf-8", check=False)
+    except OSError:
+        return None
+
+
+def _config(repo: Path | None, *args: str) -> subprocess.CompletedProcess | None:
+    scope = ["-C", str(repo), "config", "--local"] if repo else ["config", "--global"]
+    return _git(*scope, *args)
+
+
+def _values(repo: Path | None, key: str) -> list[str]:
+    r = _config(repo, "--get-all", "-z", key)
+    return r.stdout.split("\0")[:-1] if r and r.returncode == 0 else []
+
+
+def _write(repo: Path | None, *args: str) -> str | None:
+    r = _config(repo, *args)
+    if r and r.returncode == 0:
+        return None
+    why = r.stderr.strip() if r else "git not found"
+    return f"failed: git config {' '.join(args)} ({'global' if repo is None else repo}): {why}"
+
+
+def _is_work_tree(path: Path) -> bool:
+    r = _git("-C", str(path), "rev-parse", "--is-inside-work-tree")
+    return bool(r) and r.stdout.strip() == "true"
+
+
+def default_repos() -> list[Path]:
+    return [Path.cwd()] if _is_work_tree(Path.cwd()) else []
+
+
+def move_git_config(repos: list[Path], dry_run: bool = False) -> list[str]:
+    lines = []
+    p = "would " if dry_run else ""
+    for repo in [None, *repos]:
+        if repo is not None and not _is_work_tree(repo):
+            lines.append(f"skip {repo}: not a git repo")
+            continue
+        where = "global" if repo is None else str(repo)
+        for key in KEYS:
+            old, new = f"codexharness.{key}", f"codejig.{key}"
+            olds = [v or "true" for v in _values(repo, old)] if key == "allowProse" else _values(repo, old)
+            if not olds:
+                continue
+            have = _values(repo, new)
+            for value in olds:
+                if value in have:
+                    continue
+                if dry_run:
+                    have.append(value)
+                    lines.append(f"{p}copy {old} = {value} to {new} ({where})")
+                    continue
+                failed = _write(repo, "--add", new, value)
+                lines.append(failed or f"copy {old} = {value} to {new} ({where})")
+            if not dry_run:
+                if not set(olds) <= set(_values(repo, new)):
+                    lines.append(f"failed: kept {old} ({where}) because {new} does not hold every value")
+                    continue
+                failed = _write(repo, "--unset-all", old)
+                if failed:
+                    lines.append(failed)
+                    continue
+            lines.append(f"{p}unset {old} ({where})")
+    return lines
+
+
+def _old_hooks_path(home: Path) -> str | None:
+    current = install._git_config("--get", "core.hooksPath")
+    return current if current and install._same_path(current, old_state(home) / "git-hooks") else None
+
+
+def release_hooks_path(home: Path, dry_run: bool = False) -> list[str]:
+    current = _old_hooks_path(home)
+    if not current:
+        return []
+    failed = None if dry_run else _write(None, "--unset", "core.hooksPath")
+    if failed:
+        return [failed]
+    return [f"{'would ' if dry_run else ''}unset core.hooksPath {current}"]
+
+
+def claude_settings() -> Path:
+    return Path.home() / ".claude" / "settings.json"
+
+
+def claude_skills() -> Path:
+    return Path.home() / ".claude" / "skills"
+
+
+def _old_generated(path: Path) -> bool | None:
+    try:
+        data = path.read_bytes().removeprefix(install.BOM)
+    except OSError:
+        return None
+    first, _, body = data.partition(b"\n")
+    for head, tail in install.STYLES.values():
+        prefix, suffix = (head + OLD_MARKER).encode(), (install.MARKER_NOTE + tail).encode()
+        if first.startswith(prefix):
+            digest = first[len(prefix): len(first) - len(suffix)].decode("ascii", "replace")
+            return first.endswith(suffix) and hashlib.sha256(body).hexdigest() == digest
+    return None
+
+
+OLD_FILE_GLOBS = ("AGENTS.md", "*.config.toml", "agents/*.toml", "rules/codex-harness-*.rules")
+
+
+def _old_files(home: Path) -> list[Path]:
+    return [f for g in OLD_FILE_GLOBS for f in sorted(home.glob(g))]
+
+
+def _old_backups(home: Path) -> list[Path]:
+    return [f for g in OLD_FILE_GLOBS for f in sorted(home.glob(g + ".bak-*")) if _old_generated(f)]
+
+
+def _old_copy(d: Path) -> bool:
+    side = d / OLD_SIDECAR
+    try:
+        want = side.read_text(encoding="utf-8").strip()
+        files = sorted((f for f in d.rglob("*") if f.is_file() and f != side and install.installed_file(d, f)),
+                       key=lambda f: f.relative_to(d).parts)
+        h = hashlib.sha256()
+        for f in files:
+            h.update(f.relative_to(d).as_posix().encode() + b"\0" + f.read_bytes() + b"\0")
+    except OSError:
+        return False
+    return h.hexdigest() == want
+
+
+def _old_skill_backups() -> list[Path]:
+    roots = {paths.agents_skills().parent / "skills-backup", claude_skills().parent / "skills-backup"}
+    return [d for r in sorted(roots) if r.is_dir() for d in sorted(r.glob("*.bak-*"))
+            if d.is_dir() and not d.is_symlink() and _old_copy(d)]
+
+
+def _hook_files(home: Path) -> list[Path]:
+    return [p for p in (home / "hooks.json", claude_settings()) if p.is_file()]
+
+
+def _without_old_hooks(path: Path) -> tuple[dict, bool]:
+    data = json.loads(path.read_text(encoding="utf-8-sig"))
+    hooks = data.get("hooks") if isinstance(data, dict) else None
+    changed = False
+    for event, groups in (hooks.items() if isinstance(hooks, dict) else []):
+        if not install.valid_hook_groups(groups):
+            continue
+        kept = groups
+        for status in OLD_STATUSES:
+            kept = install._without_status(kept, status)
+        if len(kept) != len(groups):
+            hooks[event], changed = kept, True
+    return data, changed
+
+
+def remove_old_generated(home: Path, dry_run: bool = False) -> list[str]:
+    p = "would " if dry_run else ""
+    lines = []
+    for f in _old_files(home):
+        verdict = _old_generated(f)
+        if verdict is False:
+            lines.append(f"left {f}: edited by hand")
+        elif verdict:
+            if not dry_run:
+                f.unlink()
+            lines.append(f"{p}delete {f}")
+    for f in _old_backups(home):
+        if not dry_run:
+            f.unlink()
+        lines.append(f"{p}delete {f}: a backup of a file the old version wrote")
+    for d in _old_skill_backups():
+        if not dry_run:
+            shutil.rmtree(d)
+        lines.append(f"{p}delete {d}: a backup of a skill the old version copied")
+    for f in _hook_files(home):
+        try:
+            data, changed = _without_old_hooks(f)
+        except (OSError, ValueError) as e:
+            lines.append(f"failed: {f}: {e}")
+            continue
+        if changed:
+            if not dry_run:
+                atomic_write(f, (json.dumps(data, indent=2, ensure_ascii=False) + "\n").encode("utf-8"))
+            lines.append(f"{p}remove old hook entries from {f}")
+    old = old_state(home)
+    keep = not dry_run and _old_hooks_path(home)
+    for d in (old / "lib", old / "git-hooks"):
+        if d.is_dir() and keep:
+            lines.append(f"left {d}: global core.hooksPath still points at it")
+        elif d.is_dir():
+            if not dry_run:
+                shutil.rmtree(d)
+            lines.append(f"{p}delete {d}")
+    gone = {"lib", "git-hooks", "usage.sqlite3"} if dry_run else set()
+    if old.is_dir() and all(e.name in gone for e in old.iterdir()):
+        if not dry_run:
+            old.rmdir()
+        lines.append(f"{p}delete {old}")
+    return lines
+
+
+def _sidecars(dirs: list[Path]) -> list[Path]:
+    return [f for d in dirs if d.is_dir() for f in sorted(d.glob(f"*/{OLD_SIDECAR}"))]
+
+
+def move_skill_sidecars(dirs: list[Path], dry_run: bool = False) -> list[str]:
+    p = "would " if dry_run else ""
+    lines = []
+    for old in _sidecars(dirs):
+        new = old.with_name(install.SKILL_SIDECAR)
+        if new.exists():
+            if not dry_run:
+                old.unlink()
+            lines.append(f"{p}delete {old}: {new} exists")
+        else:
+            if not dry_run:
+                old.rename(new)
+            lines.append(f"{p}rename {old} to {new}")
+    return lines
+
+
+def _old_link(path: Path) -> str | None:
+    if not path.is_symlink():
+        return None
+    m = OLD_LINK.search(os.readlink(path))
+    if not m or (m[1] == "dev-harness" and m[2] not in SHARED_DEV_SKILLS):
+        return None
+    return m[2]
+
+
+def replace_skill_links(core: Path, ov: Overlay, claude_skills: Path, dry_run: bool = False) -> list[str]:
+    p = "would " if dry_run else ""
+    lines = []
+    sources = install.skill_sources(core, ov.root)
+    for d in (claude_skills, paths.agents_skills()):
+        replaced = {}
+        for link in sorted(d.iterdir()) if d.is_dir() else []:
+            name = _old_link(link)
+            if name is None:
+                continue
+            if name not in sources:
+                if not dry_run:
+                    link.unlink()
+                lines.append(f"{p}remove {link}: {name} is no longer in the core")
+                continue
+            lines.append(f"{p}replace the link {link} with a copy")
+            replaced[link] = sources[name]
+        if not replaced or dry_run:
+            continue
+        owned = {n: src for n, src in sources.items()
+                 if not (d / n).is_symlink() and (d / n / install.SKILL_SIDECAR).is_file()}
+        for link in replaced:
+            link.unlink()
+        lines += install.install_skills({**owned, **{link.name: src for link, src in replaced.items()}}, d)
+    return lines
+
+
+@functools.cache
+def _bootstrap(core: Path):
+    spec = importlib.util.spec_from_file_location("codejig_bootstrap", core / "bootstrap.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _tag(core: Path) -> str:
+    r = _git("-C", str(core), "describe", "--tags", "--always")
+    if r and r.returncode == 0 and r.stdout.strip():
+        return r.stdout.strip()
+    try:
+        return (core / "VERSION").read_text(encoding="utf-8").split()[0]
+    except (OSError, IndexError):
+        return "checkout"
+
+
+def _hand_written(text: str) -> bool:
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    return ("CODEX_HARNESS_OVERLAY" in text and "-m harness" in text
+            and all(line.startswith(("#!", "export ", "PYTHONPATH", "exec ")) for line in lines))
+
+
+def _old_launchers(bs) -> list[Path]:
+    bins = {bs.bin_dir(None, bs.data_dir(None)), bs.bin_dir(None, bs.data_dir(None).parent / "codex-harness")}
+    return [b / name for b in sorted(bins) for name in ("harness", "harness.cmd") if (b / name).is_file()]
+
+
+def _known_launcher(f: Path) -> bool | None:
+    try:
+        text = f.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return None
+    return OLD_LAUNCHER_MARK in text or _hand_written(text)
+
+
+def write_launchers(core: Path, overlay: Path, dry_run: bool = False) -> list[str]:
+    p = "would " if dry_run else ""
+    bs = _bootstrap(core)
+    lines = []
+    for f in _old_launchers(bs):
+        known = _known_launcher(f)
+        if known is None:
+            lines.append(f"delete {f} yourself: cannot read it")
+        elif known:
+            if not dry_run:
+                f.unlink()
+            lines.append(f"{p}delete {f}")
+        else:
+            lines.append(f"left {f}: not a code-jig launcher")
+    bins = bs.bin_dir(None, bs.data_dir(None))
+    try:
+        if dry_run:
+            for f in bs.launcher_paths(bins):
+                bs.check_launcher(f)
+            return [*lines, *(f"would write {f}" for f in bs.launcher_paths(bins))]
+        written = bs.write_launcher(bins, install.interpreter(), core, overlay, _tag(core))
+    except bs.Refused as e:
+        return [*lines, f"refused: {e}"]
+    return [*lines, *(f"wrote {f}" for f in written)]
+
+
+def _scan_files(home: Path) -> list[Path]:
+    config = home / ".config"
+    files = [home / ".bashrc", home / ".zshrc", home / ".profile", *sorted(config.glob("environment.d/*"))]
+    for ps in ("PowerShell", "WindowsPowerShell"):
+        files += sorted((home / "Documents" / ps).glob("*profile.ps1"))
+    for depth in ("*", "*/*", "*/*/*"):
+        for ext in ("json", "toml"):
+            files += sorted(config.glob(f"{depth}.{ext}"))
+    return list(dict.fromkeys(files))
+
+
+def leftovers(home: Path) -> list[str]:
+    lines = []
+    for f in _scan_files(home):
+        try:
+            if not f.is_file() or f.stat().st_size > 1_000_000:
+                continue
+            text = f.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        lines += [f"still names old paths: {f}:{n}" for n, line in enumerate(text.splitlines(), 1)
+                  if any(name in line for name in OLD_NAMES)]
+    return lines
+
+
+def _old_keys(includes: bool) -> list[tuple[str, str]]:
+    r = _git("config", "--global", *(["--includes"] if includes else []), "--show-origin", "-z",
+             "--get-regexp", r"^codexharness\.")
+    parts = r.stdout.split("\0")[:-1] if r and r.returncode == 0 else []
+    return [(origin, entry.split("\n")[0]) for origin, entry in zip(parts[::2], parts[1::2])]
+
+
+def _new_gate_active() -> bool:
+    return install._same_path(install._git_config("--get", "core.hooksPath") or "", paths.state_dir() / "git-hooks")
+
+
+def pending(home: Path) -> list[str]:
+    lines = []
+    old = old_state(home)
+    if old.exists():
+        lines.append(f"{old} still exists")
+    current = _old_hooks_path(home)
+    if current:
+        lines.append(f"global core.hooksPath is {current}")
+    own = {origin for origin, _ in _old_keys(False)}
+    for origin, key in _old_keys(True):
+        lines.append(f"global git key {key} is set" if origin in own
+                     else f"{key} in {origin.removeprefix('file:')}: edit by hand")
+    lines += [f"{f} was written by the old version" for f in [*_old_files(home), *_old_backups(home)] if _old_generated(f)]
+    lines += [f"{d} is a backup of an old skill copy" for d in _old_skill_backups()]
+    for f in _hook_files(home):
+        try:
+            if _without_old_hooks(f)[1]:
+                lines.append(f"{f} has old hook entries")
+        except (OSError, ValueError):
+            pass
+    lines += [f"{f} is an old skill sidecar" for f in _sidecars([paths.agents_skills(), claude_skills()])]
+    return lines
+
+
+def _stopped(lines: list[str]) -> bool:
+    return any(line.startswith(("refused:", "failed:")) for line in lines)
+
+
+def run(core: Path, ov: Overlay, home: Path, repos: list[Path], dry_run: bool) -> list[str]:
+    settings = claude_settings()
+    try:
+        install.plan(core, ov, home, settings if settings.is_file() else None)
+        install.skill_sources(core, ov.root)
+    except (ValueError, OSError) as e:
+        return [f"failed: code-jig install would fail: {e}", "nothing was changed; fix it and rerun"]
+    lines = [*move_db(home, dry_run), *move_git_config(repos, dry_run)]
+    if _stopped(lines):
+        return [*lines, SKIPPED]
+    lines += release_hooks_path(home, dry_run)
+    if _stopped(lines):
+        return [*lines, SKIPPED]
+    lines += [*remove_old_generated(home, dry_run),
+              *move_skill_sidecars([paths.agents_skills(), claude_skills()], dry_run),
+              *replace_skill_links(core, ov, claude_skills(), dry_run)]
+    bs = _bootstrap(core)
+    done = (all(f.is_file() for f in bs.launcher_paths(bs.bin_dir(None, bs.data_dir(None))))
+            and not any(_known_launcher(f) for f in _old_launchers(bs))
+            and (not ov.git_gate or _new_gate_active()))
+    if not lines and done:
+        return ["nothing to do", *leftovers(Path.home())]
+    if dry_run:
+        lines.append("would run code-jig install")
+    else:
+        try:
+            lines += install.run(core, ov, home)
+        except (ValueError, OSError) as e:
+            return [*lines, f"failed: code-jig install: {e}", *([GATE_OFF] if ov.git_gate and not _new_gate_active() else [])]
+    return [*lines, *write_launchers(core, ov.root, dry_run), *leftovers(Path.home())]
+
+
+def cli(args) -> int:
+    print(NOTICE)
+    try:
+        ov = load(paths.overlay_dir(args.overlay))
+    except ValueError as e:
+        print(f"code-jig: {e}", file=sys.stderr)
+        return 2
+    repos = [Path(r) for r in args.repos] if args.repos else default_repos()
+    lines = run(paths.CORE, ov, paths.codex_home(args.codex_home), repos, args.dry_run)
+    for line in lines:
+        print(line)
+    return 1 if any(line.startswith(("refused:", "failed:")) for line in lines) else 0

codejig/overlay.py

@@ -0,0 +1,43 @@
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+REQUIRED = ("core_version", "core_writable")
+
+
+@dataclass(frozen=True)
+class Overlay:
+    root: Path
+    core_version: str
+    core_writable: bool
+    auto_compact_token_limit: int | None = None
+    git_gate: bool = False
+    prose_gate: bool = False
+    push_cleanup: bool | None = None
+
+
+def _flags(path: Path, data: dict, compact) -> dict:
+    if compact is not None and (isinstance(compact, bool) or not isinstance(compact, int) or compact <= 0):
+        raise ValueError(f"{path}: auto_compact_token_limit must be a positive integer")
+    flags = {k: data.get(k, False) for k in ("git_gate", "prose_gate")}
+    flags["push_cleanup"] = data.get("push_cleanup")
+    bad = [k for k, v in flags.items() if not isinstance(v, bool) and not (k == "push_cleanup" and v is None)]
+    if bad:
+        raise ValueError(f"{path}: {', '.join(bad)} must be true or false")
+    return flags
+
+
+def load(root: Path) -> Overlay:
+    path = root / "overlay.toml"
+    try:
+        data = tomllib.loads(path.read_text(encoding="utf-8-sig"))
+    except (tomllib.TOMLDecodeError, UnicodeDecodeError) as e:
+        raise ValueError(f"{path}: {e}") from e
+    missing = [k for k in REQUIRED if k not in data]
+    if missing:
+        raise ValueError(f"{path}: missing {', '.join(missing)}")
+    if not isinstance(data["core_writable"], bool):
+        raise ValueError(f"{path}: core_writable must be true or false")
+    compact = data.get("auto_compact_token_limit")
+    flags = _flags(path, data, compact)
+    return Overlay(root, str(data["core_version"]), data["core_writable"], compact, **flags)

codejig/paths.py

@@ -0,0 +1,40 @@
+import os
+from pathlib import Path
+
+CODE = Path(__file__).resolve().parent
+
+
+def locate_core(code: Path) -> Path:
+    return code / "_core" if (code / "_core").is_dir() else code.parent
+
+
+CORE = locate_core(CODE)
+
+
+def codex_home(arg: str | None) -> Path:
+    return Path(arg or os.environ.get("CODEX_HOME") or Path.home() / ".codex").expanduser()
+
+
+def state_dir() -> Path:
+    if os.name == "nt":
+        return Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData" / "Local") / "code-jig"
+    return Path(os.environ.get("XDG_DATA_HOME") or Path.home() / ".local" / "share") / "code-jig"
+
+
+def overlay_dir(arg: str | None) -> Path:
+    value = arg or os.environ.get("CODE_JIG_OVERLAY")
+    if not value:
+        raise SystemExit("code-jig: no overlay; pass --overlay PATH or set CODE_JIG_OVERLAY")
+    root = Path(value).expanduser().absolute()
+    if not (root / "overlay.toml").is_file():
+        raise SystemExit(f"code-jig: {root / 'overlay.toml'} not found")
+    return root
+
+
+def agents_skills() -> Path:
+    return Path(os.environ.get("CODE_JIG_SKILLS_DIR") or Path.home() / ".agents" / "skills").expanduser()
+
+
+def claude_code_settings() -> Path | None:
+    path = Path.home() / ".claude" / "settings.json"
+    return path if path.is_file() else None