tandem-cli 0.1.0__py3-none-any.whl

tandem/memory_sync.py

@@ -0,0 +1,220 @@
+"""Instruction/memory file sync: CLAUDE.md <-> AGENTS.md, plus opt-in MCP
+server copying.
+
+v1 mapping: shared content lives in a marker-delimited common block; text
+outside the markers is tool-specific and never touched.
+
+    <!-- tandem:shared:begin -->
+    ...shared instructions...
+    <!-- tandem:shared:end -->
+
+Rules (documented in README):
+- A file without markers contributes its ENTIRE content as the shared block
+  but is itself never modified (we don't rewrite user files to add markers).
+- The counterpart file is created/refreshed with the shared block wrapped in
+  markers; anything it keeps outside the markers survives.
+- When both files carry markers, the newer mtime wins the shared block.
+- When both exist and neither has markers we refuse to guess and report it.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from . import paths
+from .util import write_file_atomic
+
+BEGIN = "<!-- tandem:shared:begin -->"
+END = "<!-- tandem:shared:end -->"
+
+_BLOCK_RE = re.compile(
+    re.escape(BEGIN) + r"\n?(.*?)\n?" + re.escape(END), re.S
+)
+
+FILES = {"claude": "CLAUDE.md", "codex": "AGENTS.md"}
+
+
+@dataclass
+class MemorySyncReport:
+    actions: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+
+def _shared_block(text: str) -> str | None:
+    m = _BLOCK_RE.search(text)
+    return m.group(1).strip() if m else None
+
+
+def _replace_block(text: str, shared: str) -> str:
+    new_block = f"{BEGIN}\n{shared}\n{END}"
+    if _BLOCK_RE.search(text):
+        # replacement via callable: content must be literal, never treated
+        # as backreference syntax
+        return _BLOCK_RE.sub(lambda _m: new_block, text, count=1)
+    prefix = text.rstrip() + "\n\n" if text.strip() else ""
+    return prefix + new_block + "\n"
+
+
+def sync_memory_files(cwd: str) -> MemorySyncReport:
+    """Reflect shared instructions between CLAUDE.md and AGENTS.md in cwd."""
+    report = MemorySyncReport()
+    root = Path(cwd)
+    paths_ = {k: root / v for k, v in FILES.items()}
+    texts = {}
+    for k, p in paths_.items():
+        try:
+            texts[k] = p.read_text() if p.exists() else None
+        except OSError as exc:
+            report.warnings.append(f"cannot read {p.name}: {exc}")
+            return report
+
+    existing = [k for k, t in texts.items() if t is not None]
+    if not existing:
+        return report
+
+    if len(existing) == 1:
+        src = existing[0]
+        dst = "codex" if src == "claude" else "claude"
+        shared = _shared_block(texts[src])
+        if shared is None:
+            shared = texts[src].strip()
+        if not shared:
+            return report
+        write_file_atomic(paths_[dst], _replace_block("", shared))
+        report.actions.append(
+            f"created {FILES[dst]} from {FILES[src]} (shared block)"
+        )
+        return report
+
+    blocks = {k: _shared_block(texts[k]) for k in ("claude", "codex")}
+    if blocks["claude"] is None and blocks["codex"] is None:
+        report.warnings.append(
+            f"both {FILES['claude']} and {FILES['codex']} exist without tandem "
+            f"markers; not merging. Wrap the shared part of one of them in "
+            f"'{BEGIN}' / '{END}' to enable sync."
+        )
+        return report
+
+    # Effective shared content per side: the marker block, or (for a
+    # marker-less file) its whole content.
+    effective = {
+        k: blocks[k] if blocks[k] is not None else texts[k].strip()
+        for k in ("claude", "codex")
+    }
+    if effective["claude"] == effective["codex"]:
+        return report
+
+    newer = max(("claude", "codex"), key=lambda k: paths_[k].stat().st_mtime)
+    older = "codex" if newer == "claude" else "claude"
+    if blocks[older] is None:
+        # never rewrite a user file that has no markers
+        report.warnings.append(
+            f"{FILES[older]} has no tandem markers; not updating it from "
+            f"{FILES[newer]}."
+        )
+        return report
+    write_file_atomic(
+        paths_[older], _replace_block(texts[older], effective[newer])
+    )
+    report.actions.append(
+        f"updated shared block in {FILES[older]} from {FILES[newer]}"
+    )
+    return report
+
+
+# -- MCP config copy (opt-in via `tandem sync-mcp`) --------------------------
+
+
+def claude_mcp_servers() -> dict[str, dict]:
+    """Global stdio MCP servers from ~/.claude.json."""
+    cfg = Path.home() / ".claude.json"
+    try:
+        data = json.loads(cfg.read_text())
+    except (OSError, json.JSONDecodeError):
+        return {}
+    servers = data.get("mcpServers") or {}
+    return {
+        name: s
+        for name, s in servers.items()
+        if isinstance(s, dict) and s.get("command")
+    }
+
+
+def codex_mcp_servers() -> dict[str, dict]:
+    """[mcp_servers.*] from ~/.codex/config.toml."""
+    import tomllib
+
+    cfg = paths.codex_home() / "config.toml"
+    try:
+        with open(cfg, "rb") as f:
+            data = tomllib.load(f)
+    except (OSError, ValueError):
+        return {}
+    servers = data.get("mcp_servers") or {}
+    return {
+        name: s
+        for name, s in servers.items()
+        if isinstance(s, dict) and s.get("command")
+    }
+
+
+def _toml_str(value: str) -> str:
+    return json.dumps(value)  # JSON string escaping is valid TOML
+
+
+def copy_mcp(report: MemorySyncReport | None = None) -> MemorySyncReport:
+    """Bidirectional additive copy of stdio MCP servers: each side gains the
+    servers it is missing; existing definitions are never modified."""
+    report = report or MemorySyncReport()
+    claude_servers = claude_mcp_servers()
+    codex_servers = codex_mcp_servers()
+
+    # claude -> codex: append TOML sections for missing servers
+    missing_in_codex = {
+        n: s for n, s in claude_servers.items() if n not in codex_servers
+    }
+    if missing_in_codex:
+        cfg = paths.codex_home() / "config.toml"
+        text = cfg.read_text() if cfg.exists() else ""
+        chunks = [text.rstrip() + "\n" if text.strip() else ""]
+        for name, s in missing_in_codex.items():
+            chunks.append(f"\n# added by tandem sync-mcp\n[mcp_servers.{name}]")
+            chunks.append(f"command = {_toml_str(s['command'])}")
+            args = s.get("args") or []
+            chunks.append(
+                "args = [" + ", ".join(_toml_str(a) for a in args) + "]"
+            )
+            env = s.get("env") or {}
+            if env:
+                chunks.append(f"\n[mcp_servers.{name}.env]")
+                for k, v in env.items():
+                    chunks.append(f"{k} = {_toml_str(str(v))}")
+            report.actions.append(f"added MCP server {name!r} to codex config.toml")
+        write_file_atomic(cfg, "\n".join(chunks) + "\n")
+
+    # codex -> claude: merge into ~/.claude.json mcpServers
+    missing_in_claude = {
+        n: s for n, s in codex_servers.items() if n not in claude_servers
+    }
+    if missing_in_claude:
+        cfg = Path.home() / ".claude.json"
+        try:
+            data = json.loads(cfg.read_text()) if cfg.exists() else {}
+        except json.JSONDecodeError:
+            report.warnings.append("~/.claude.json is not valid JSON; skipping")
+            return report
+        servers = data.setdefault("mcpServers", {})
+        for name, s in missing_in_claude.items():
+            entry: dict = {"command": s["command"], "args": list(s.get("args") or [])}
+            if s.get("env"):
+                entry["env"] = {k: str(v) for k, v in s["env"].items()}
+            servers[name] = entry
+            report.actions.append(f"added MCP server {name!r} to ~/.claude.json")
+        write_file_atomic(cfg, json.dumps(data, indent=2) + "\n")
+
+    if not report.actions:
+        report.actions.append("MCP configs already in sync (nothing copied)")
+    return report

tandem/ops.py

@@ -0,0 +1,207 @@
+"""Session operations shared by the CLI: catch-up drain, cursor
+fast-forward, switch, and one-off routing.
+
+Echo suppression: every entry tandem appends to a shadow is, by
+construction, already represented in the other file. So whenever a harness
+changes role (switch, or a one-off turn routed to it), we first drain the
+outgoing source's unsynced tail, then fast-forward the incoming source's
+cursor to end-of-file — from that point only genuinely new turns flow, and
+nothing synced ever bounces back.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from pathlib import Path
+
+from . import paths
+from .harness import get_adapter, other
+from .runner import TailLoop, await_codex_rollout
+from .state import PairedSession, StateStore
+from .sync import SyncEngine, SyncSetupError
+
+# seam for tests (patching subprocess.run itself would also intercept the
+# CLI version probes)
+_run = subprocess.run
+
+
+def source_transcript(session: PairedSession, source: str) -> Path | None:
+    sid = getattr(session, f"{source}_session_id")
+    if not sid:
+        return None
+    adapter = get_adapter(source)
+    return adapter.transcript_path(session.cwd, sid)
+
+
+def drain_source(
+    store: StateStore, session: PairedSession, source: str,
+    *, flush_dangling: bool = False,
+) -> int:
+    """Translate any unsynced tail of `source`'s file into the other file.
+    Pure local file I/O. Returns lines consumed. With flush_dangling=True,
+    close any still-unpaired tool calls with placeholder results afterwards
+    (required when the source is being handed off: both replay APIs reject
+    a dangling call)."""
+    transcript = source_transcript(session, source)
+    if transcript is None:
+        return 0
+    engine = SyncEngine(store, session, source)
+    loop = TailLoop(store, session, source, transcript, engine)
+    total = 0
+    while True:
+        n = loop.drain()
+        total += n
+        if n == 0:
+            break
+    if loop.errors:
+        raise SyncSetupError("; ".join(loop.errors))
+    if flush_dangling:
+        engine.flush_dangling(loop.ctx, loop.cursor)
+    return total
+
+
+def fast_forward(store: StateStore, session: PairedSession, source: str) -> None:
+    """Mark everything currently in `source`'s file as already-synced."""
+    transcript = source_transcript(session, source)
+    cursor = store.get_cursor(session.tandem_id, source)
+    if transcript is None:
+        cursor.byte_offset = 0
+        cursor.line_index = 0
+    else:
+        data = transcript.read_bytes()
+        cursor.byte_offset = len(data)
+        cursor.line_index = data.count(b"\n")
+    cursor.pending.pop("intent", None)
+    store.save_cursor(cursor)
+
+
+def unsynced_lines(session: PairedSession, store: StateStore, source: str) -> int:
+    transcript = source_transcript(session, source)
+    if transcript is None:
+        return 0
+    cursor = store.get_cursor(session.tandem_id, source)
+    try:
+        data = transcript.read_bytes()
+    except OSError:
+        return 0
+    if len(data) <= cursor.byte_offset:
+        return 0
+    return data[cursor.byte_offset :].count(b"\n")
+
+
+def switch_session(store: StateStore, session: PairedSession):
+    """Flip active/shadow. Returns (new_active, problems-with-new-active-file,
+    memory-sync report). Instant: catch-up drain of the old source + cursor
+    fast-forward, no bulk re-conversion."""
+    from .doctor import validate_transcript
+
+    old_active, new_active = session.active, session.shadow
+
+    # If codex never ran (id pending), its shadow file does not exist yet;
+    # create it now so the flip has something to resume.
+    if new_active == "codex" and not session.codex_session_id:
+        _create_codex_shadow_late(store, session)
+        session = store.get_session(session.tandem_id) or session
+
+    drain_source(store, session, old_active, flush_dangling=True)
+    fast_forward(store, session, new_active)
+    store.set_active(session.tandem_id, new_active)
+
+    from .memory_sync import sync_memory_files
+
+    memory_report = sync_memory_files(session.cwd)
+
+    problems: list[str] = []
+    transcript = source_transcript(session, new_active)
+    if transcript is None:
+        if new_active == "claude" and session.claude_session_id:
+            # claude never launched; it will be created fresh on next run
+            problems = []
+        else:
+            problems = ["transcript for newly active harness does not exist yet"]
+    else:
+        problems = validate_transcript(new_active, transcript,
+                                       getattr(session, f"{new_active}_session_id"))
+    return new_active, problems, memory_report
+
+
+def _create_codex_shadow_late(store: StateStore, session: PairedSession) -> None:
+    from .constants import SEED_NOTE
+    from .runner import ctx_from_cursor
+
+    adapter = get_adapter("codex")
+    sid = adapter.mint_session_id()
+    cursor = store.get_cursor(session.tandem_id, session.active)
+    ctx = ctx_from_cursor(session, session.active, cursor)
+    ctx.codex_session_id = sid
+    note = SEED_NOTE.format(
+        tandem_id=session.tandem_id, other=get_adapter(session.active).display_name
+    )
+    adapter.create_shadow_transcript(session.cwd, sid, ctx, note)
+    store.set_native_session_id(session.tandem_id, "codex", sid)
+
+
+def run_oneoff(
+    store: StateStore, session: PairedSession, target: str, prompt: str
+) -> int:
+    """Route one prompt to `target` as a single native non-interactive turn,
+    then sync that turn into the other file. Exactly one model (target's) is
+    invoked."""
+    adapter = get_adapter(target)
+    sid = getattr(session, f"{target}_session_id")
+
+    # Catch up the active side first, then mark the target's whole file as
+    # known so only the new turn flows back afterwards. (When target IS the
+    # active side there is nothing to fast-forward — its cursor is live.)
+    drain_source(store, session, session.active, flush_dangling=True)
+    if target != session.active and sid and source_transcript(session, target) is not None:
+        fast_forward(store, session, target)
+
+    started = time.time()
+    if target == "codex" and not sid:
+        # codex never ran: no session to resume; a fresh exec creates one.
+        argv = [adapter.binary, "exec", "--skip-git-repo-check", prompt]
+    else:
+        argv = adapter.oneoff_argv(sid, prompt)
+    code = _run(argv, cwd=session.cwd).returncode
+
+    if target == "codex" and not sid:
+        rollout = await_codex_rollout(session.cwd, started, timeout=10)
+        if rollout:
+            new_sid = paths.codex_rollout_session_id(rollout)
+            if new_sid:
+                store.set_native_session_id(session.tandem_id, "codex", new_sid)
+                session = store.get_session(session.tandem_id) or session
+                fast_forward_to_zero = store.get_cursor(session.tandem_id, "codex")
+                fast_forward_to_zero.byte_offset = 0
+                fast_forward_to_zero.line_index = 0
+                store.save_cursor(fast_forward_to_zero)
+
+    # Echo suppression (see the module docstring): this drain appends
+    # tandem's translation of the target's turn to the OTHER harness's file.
+    # Those appends are by construction already represented in the target's
+    # file, so the echo side's cursor has to move past them — otherwise its
+    # next drain translates them straight back, duplicating call ids and text.
+    echo_side = other(target)
+    echo_pre_size = _file_size(source_transcript(session, echo_side))
+    echo_pre_offset = store.get_cursor(session.tandem_id, echo_side).byte_offset
+
+    drain_source(store, session, target, flush_dangling=True)
+
+    # Only when the echo side was fully synced before the drain is everything
+    # now in its file known to be ours. If it had an unsynced tail (a
+    # concurrent writer), fast-forwarding would swallow a live turn: leave the
+    # cursor alone and let the normal drain pick both up.
+    if echo_pre_size is not None and echo_pre_offset == echo_pre_size:
+        fast_forward(store, session, echo_side)
+    return code
+
+
+def _file_size(path: Path | None) -> int | None:
+    if path is None:
+        return None
+    try:
+        return path.stat().st_size
+    except OSError:
+        return None

tandem/paths.py

@@ -0,0 +1,92 @@
+"""Filesystem locations for tandem and both wrapped harnesses.
+
+Everything here was derived by inspecting the files written by the installed
+CLIs (claude 2.1.220, codex-cli 0.145.0) — see docs/formats.md.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+
+def tandem_home() -> Path:
+    return Path(os.environ.get("TANDEM_HOME", Path.home() / ".tandem"))
+
+
+def state_db_path() -> Path:
+    return tandem_home() / "state.db"
+
+
+def quarantine_dir(tandem_id: str) -> Path:
+    return tandem_home() / "quarantine" / tandem_id
+
+
+def log_dir() -> Path:
+    return tandem_home() / "logs"
+
+
+# --- Claude Code ------------------------------------------------------------
+
+def claude_home() -> Path:
+    return Path(os.environ.get("CLAUDE_CONFIG_DIR", Path.home() / ".claude"))
+
+
+def claude_munge_cwd(cwd: str | Path) -> str:
+    """Claude Code names each project dir after the cwd with every character
+    outside [A-Za-z0-9] replaced by '-'. Observed: /private/tmp/claude-501/x
+    -> -private-tmp-claude-501-x (leading '/' becomes a leading '-',
+    pre-existing '-' survive unchanged)."""
+    return re.sub(r"[^A-Za-z0-9]", "-", str(cwd))
+
+
+def claude_project_dir(cwd: str | Path) -> Path:
+    return claude_home() / "projects" / claude_munge_cwd(cwd)
+
+
+def claude_transcript_path(cwd: str | Path, session_id: str) -> Path:
+    return claude_project_dir(cwd) / f"{session_id}.jsonl"
+
+
+# --- Codex ------------------------------------------------------------------
+
+def codex_home() -> Path:
+    return Path(os.environ.get("CODEX_HOME", Path.home() / ".codex"))
+
+
+def codex_sessions_dir() -> Path:
+    return codex_home() / "sessions"
+
+
+_ROLLOUT_RE = re.compile(
+    r"rollout-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-([0-9a-f-]{36})\.jsonl$"
+)
+
+
+def codex_rollout_session_id(path: Path) -> str | None:
+    """The session UUID is embedded in the rollout filename."""
+    m = _ROLLOUT_RE.search(path.name)
+    return m.group(1) if m else None
+
+
+def find_codex_rollout(session_id: str) -> Path | None:
+    """Locate a rollout file by session id. Filenames embed the id, so this
+    never needs to open files."""
+    root = codex_sessions_dir()
+    if not root.is_dir():
+        return None
+    for p in root.glob(f"**/rollout-*-{session_id}.jsonl"):
+        return p
+    return None
+
+
+def iter_codex_rollouts_newest_first() -> list[Path]:
+    root = codex_sessions_dir()
+    if not root.is_dir():
+        return []
+    return sorted(
+        root.glob("**/rollout-*.jsonl"),
+        key=lambda p: p.stat().st_mtime,
+        reverse=True,
+    )

tandem/ptyrun.py

@@ -0,0 +1,99 @@
+"""PTY passthrough: run the real harness CLI with its untouched native UX.
+
+The child gets a real pty (raw-mode stdin, terminal resize via SIGWINCH
+forwarding, control bytes delivered through the pty line discipline). Tandem
+never reads meaning from the terminal stream — transcript files are the
+source of truth — so this module just pumps bytes.
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import select
+import signal
+import struct
+import subprocess
+import sys
+import termios
+import tty
+
+from ptyprocess import PtyProcess
+
+
+def _winsize(fd: int) -> tuple[int, int]:
+    try:
+        import fcntl
+
+        packed = fcntl.ioctl(fd, termios.TIOCGWINSZ, struct.pack("HHHH", 0, 0, 0, 0))
+        rows, cols = struct.unpack("HHHH", packed)[:2]
+        return (rows or 24, cols or 80)
+    except OSError:
+        return (24, 80)
+
+
+def run_in_pty(argv: list[str], cwd: str | None = None, env: dict | None = None) -> int:
+    """Run argv on a pty, mirroring the controlling terminal. Returns the
+    child's exit status. Falls back to a plain subprocess when stdin is not a
+    tty (tests, pipes)."""
+    try:
+        stdin_fd = sys.stdin.fileno()
+        is_tty = os.isatty(stdin_fd)
+    except (ValueError, OSError, io.UnsupportedOperation):
+        is_tty = False
+    if not is_tty:
+        return subprocess.run(argv, cwd=cwd, env=env).returncode
+
+    child = PtyProcess.spawn(
+        argv, cwd=cwd, env=env or dict(os.environ), dimensions=_winsize(stdin_fd)
+    )
+
+    def on_winch(signum, frame):
+        rows, cols = _winsize(stdin_fd)
+        try:
+            child.setwinsize(rows, cols)
+        except Exception:
+            pass
+
+    def on_term(signum, frame):
+        try:
+            child.kill(signal.SIGTERM)
+        except Exception:
+            pass
+
+    old_winch = signal.signal(signal.SIGWINCH, on_winch)
+    old_term = signal.signal(signal.SIGTERM, on_term)
+    old_attrs = termios.tcgetattr(stdin_fd)
+    try:
+        tty.setraw(stdin_fd)
+        stdin_open = True
+        while child.isalive():
+            rlist = [child.fd] + ([stdin_fd] if stdin_open else [])
+            try:
+                ready, _, _ = select.select(rlist, [], [], 0.2)
+            except InterruptedError:
+                continue  # signal (e.g. SIGWINCH) — loop again
+            if child.fd in ready:
+                try:
+                    data = child.read(65536)
+                except EOFError:
+                    break
+                if not data:
+                    break
+                os.write(sys.stdout.fileno(), data)
+            if stdin_open and stdin_fd in ready:
+                data = os.read(stdin_fd, 65536)
+                if data:
+                    child.write(data)
+                else:
+                    child.sendeof()
+                    stdin_open = False
+    finally:
+        termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_attrs)
+        signal.signal(signal.SIGWINCH, old_winch)
+        signal.signal(signal.SIGTERM, old_term)
+    try:
+        child.wait()
+    except Exception:
+        pass
+    return child.exitstatus if child.exitstatus is not None else 1