syncmill 0.2.0__py3-none-any.whl

syncmill/telemetry_hook.py

@@ -0,0 +1,231 @@
+"""Body-free workspace hook collector for Kimi Code and Antigravity CLI."""
+
+from __future__ import annotations
+
+import contextlib
+import fcntl
+import json
+import os
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+
+from syncmill.agents import normalize_tool_key
+from syncmill.models import ToolCallRecord, ToolOutcome, ToolTelemetry
+
+
+@contextmanager
+def _locked(path: Path) -> Iterator[None]:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        os.chmod(path, 0o600)
+        fcntl.flock(handle, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(handle, fcntl.LOCK_UN)
+
+
+def _mark_incomplete(spool: Path, code: str) -> None:
+    marker = Path(f"{spool}.incomplete")
+    marker.write_text(code, encoding="utf-8")
+    os.chmod(marker, 0o600)
+
+
+def _read_records(spool: Path) -> tuple[list[dict], int]:
+    """Parsed records plus the count of unreadable (torn / non-dict) lines —
+    the reader must surface a torn write as incompleteness, not silence."""
+    if not spool.exists():
+        return [], 0
+    records: list[dict] = []
+    unreadable = 0
+    for line in spool.read_text(encoding="utf-8").splitlines():
+        try:
+            item = json.loads(line)
+        except json.JSONDecodeError:
+            unreadable += 1
+            continue
+        if isinstance(item, dict):
+            records.append(item)
+        else:
+            unreadable += 1
+    return records, unreadable
+
+
+def _next_sequence(spool: Path) -> int:
+    """The next sequence number, tracked in a ``{spool}.seq`` sidecar.
+
+    Deriving it from a record COUNT was the bug: a torn line is skipped by the
+    reader, so the count under-reports and every later record reuses a live
+    sequence number (which the reader then sorts by). The sidecar is one int
+    read per event (also killing the O(n²) full-spool rescan); a missing or
+    unreadable sidecar (first event, or a legacy spool) seeds from the highest
+    sequence actually on disk — max, not count, so a torn line can only leave
+    a harmless gap, never a collision with a readable record.
+    """
+    seq_path = Path(f"{spool}.seq")
+    try:
+        value = int(seq_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        records, _ = _read_records(spool)
+        value = max(
+            (r["sequence"] for r in records if isinstance(r.get("sequence"), int)),
+            default=0,
+        )
+    return value + 1
+
+
+def _append(spool: Path, *, provider: str, tool_key: str, outcome: str) -> None:
+    sequence = _next_sequence(spool)
+    record = {
+        "source": provider,
+        "sequence": sequence,
+        "tool_key": tool_key,
+        "outcome": outcome,
+    }
+    spool.parent.mkdir(parents=True, exist_ok=True)
+    # Reserve the sequence BEFORE appending the record: a crash in between
+    # skips a number (harmless gap); the reverse order could reuse one. The
+    # write must be ATOMIC (unique temp + fsync + replace): a plain write_text
+    # truncates first and a crash mid-write can leave a valid PREFIX of the
+    # old value ("12" → "1"), which _next_sequence would trust — re-minting
+    # already-used numbers, the exact bug the sidecar exists to prevent.
+    seq_path = Path(f"{spool}.seq")
+    fd, tmp = tempfile.mkstemp(dir=str(spool.parent), prefix=f".{seq_path.name}.")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(str(sequence))
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.chmod(tmp, 0o600)
+        os.replace(tmp, seq_path)
+        # fsync the DIRECTORY so the rename is durable before the record write
+        # below begins: without it a filesystem crash could keep the appended
+        # record while reverting the sidecar to an older value — whose next
+        # trusted read would re-mint an existing sequence number.
+        dir_fd = os.open(str(spool.parent), os.O_RDONLY)
+        try:
+            os.fsync(dir_fd)
+        finally:
+            os.close(dir_fd)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            os.unlink(tmp)
+        raise
+    with spool.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(record, separators=(",", ":")) + "\n")
+        handle.flush()
+        # Durable record after a durable reserve: with both ordered this way, a
+        # record that survives a crash implies its reservation did too, so the
+        # sidecar can never lag behind the spool's highest sequence.
+        os.fsync(handle.fileno())
+    os.chmod(spool, 0o600)
+
+
+def collect_hook_event(
+    *,
+    provider: str,
+    event: str,
+    spool: Path,
+    payload: dict,
+    tool_map: dict[str, str] | None = None,
+) -> None:
+    """Project a provider hook payload without ever persisting its body fields."""
+    lock = Path(f"{spool}.lock")
+    state_path = Path(f"{spool}.state")
+    with _locked(lock):
+        if provider == "kimi-code":
+            raw_name = payload.get("tool_name")
+            if not isinstance(raw_name, str):
+                _mark_incomplete(spool, "missing_tool_name")
+                return
+            tool_key = normalize_tool_key(raw_name, tool_map)
+            if tool_key is None:
+                _mark_incomplete(spool, "unqualified_tool_name")
+                return
+            outcome = "failed" if event == "failure" else "succeeded"
+            _append(spool, provider=provider, tool_key=tool_key, outcome=outcome)
+            return
+
+        if provider != "antigravity":
+            _mark_incomplete(spool, "unknown_provider")
+            return
+
+        conversation = payload.get("conversationId")
+        step = payload.get("stepIdx")
+        if not isinstance(conversation, str) or not isinstance(step, int):
+            _mark_incomplete(spool, "missing_correlation")
+            return
+        correlation = f"{conversation}:{step}"
+        state = {}
+        if state_path.exists():
+            try:
+                state = json.loads(state_path.read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError):
+                _mark_incomplete(spool, "invalid_state")
+                return
+        if event == "pre":
+            tool_call = payload.get("toolCall")
+            raw_name = tool_call.get("name") if isinstance(tool_call, dict) else None
+            tool_key = normalize_tool_key(raw_name, tool_map) if isinstance(raw_name, str) else None
+            if tool_key is None:
+                _mark_incomplete(spool, "unqualified_tool_name")
+                return
+            state[correlation] = tool_key
+            state_path.write_text(json.dumps(state, separators=(",", ":")), encoding="utf-8")
+            os.chmod(state_path, 0o600)
+            return
+        tool_key = state.pop(correlation, None)
+        if not isinstance(tool_key, str):
+            _mark_incomplete(spool, "missing_pre_event")
+            return
+        state_path.write_text(json.dumps(state, separators=(",", ":")), encoding="utf-8")
+        os.chmod(state_path, 0o600)
+        outcome = "failed" if event == "failure" or bool(payload.get("error")) else "succeeded"
+        _append(spool, provider=provider, tool_key=tool_key, outcome=outcome)
+
+
+def load_hook_telemetry(spool: Path, *, source: str) -> ToolTelemetry:
+    calls: list[ToolCallRecord] = []
+    records, malformed = _read_records(spool)
+    for item in records:
+        try:
+            outcome = ToolOutcome(item["outcome"])
+            calls.append(
+                ToolCallRecord(
+                    sequence=int(item["sequence"]),
+                    tool_key=str(item["tool_key"]),
+                    outcome=outcome,
+                )
+            )
+        except (KeyError, TypeError, ValueError):
+            malformed += 1
+    incomplete = Path(f"{spool}.incomplete").exists()
+    # A crash BETWEEN the sequence reserve and the spool append loses an event
+    # without a torn line or an .incomplete marker — the only evidence is the
+    # numbering itself: an internal gap in the recorded sequences, or a sidecar
+    # reservation newer than any record. Either must flip `complete`, or the
+    # consumer would trust a spool that silently dropped an event.
+    # Sequences are minted from 1, so completeness means EXACTLY 1..max — a
+    # min-to-max contiguity check would miss a lost PREFIX (event 1 reserved
+    # and lost, event 2 recorded → singleton [2] looked contiguous).
+    sequences = sorted({call.sequence for call in calls})
+    lost = bool(sequences) and (
+        sequences[0] != 1 or len(sequences) != sequences[-1]
+    )
+    try:
+        reserved = int(Path(f"{spool}.seq").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        reserved = None
+    if reserved is not None:
+        if sequences:
+            lost = lost or reserved > sequences[-1]
+        else:
+            lost = True  # a reservation exists but no record survived
+    return ToolTelemetry(
+        source=source,
+        complete=not incomplete and malformed == 0 and not lost,
+        calls=tuple(sorted(calls, key=lambda call: call.sequence)),
+        malformed_count=malformed,
+    )

syncmill/terminal.py

@@ -0,0 +1,245 @@
+"""The one place syncmill writes to the terminal.
+
+Everything the CLI prints may quote a stored record, an agent's output, a
+provider's error, a configured path or an exception, and any of them can carry
+characters a terminal interprets (C1 CSI, OSC sequences, bidi overrides, CR).
+``echo``/``secho`` pass every message through ``printable_text`` before click
+styles and writes it, so colour still works and line breaks survive while
+nothing else non-printable reaches the terminal. ``echo_patch`` is the one
+writer that can be byte-exact, and only off a terminal; a test pins where it may
+be used and that nothing else in the package writes to the terminal directly.
+
+``install_console_guards`` covers the interpreter's report of an uncaught
+exception, in any thread, and syncmill's own log records; ``EscapingArgumentParser``
+covers argparse's own output.
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import logging
+import os
+import shlex
+import sys
+import threading
+import traceback
+from collections.abc import Sequence
+from typing import Any
+
+import click
+
+from syncmill.artifacts import printable_name, printable_text
+
+
+def _escaped(message: Any) -> Any:
+    """The display form of anything echo may be handed. ``None`` stays (an
+    empty line); bytes are decoded the way a stored patch is, so an invalid
+    byte shows as ``\\xNN``; any other object is shown as ``str()`` -- click
+    would stringify it anyway, after this point, unescaped."""
+    if message is None:
+        return None
+    if isinstance(message, (bytes, bytearray)):
+        message = bytes(message).decode("utf-8", errors="surrogateescape")
+    return printable_text(str(message))
+
+
+def echo(message: Any = None, **kwargs: Any) -> None:
+    click.echo(_escaped(message), **kwargs)
+
+
+def secho(message: Any = None, **kwargs: Any) -> None:
+    click.secho(_escaped(message), **kwargs)
+
+
+NO_COMMAND = "a value has characters that cannot be shown, so no command is printed"
+
+
+def shell_arg(value: str) -> str | None:
+    """``value`` quoted for a command the user copies, or ``None`` when the
+    terminal cannot show it unchanged: an escaped argument in a printed command
+    names a different file, run or session, so such a command is not printed."""
+    return shlex.quote(value) if printable_name(value) == value else None
+
+
+def printable_command(
+    *words: object, args: Sequence[object] = (), options: Sequence[object] = ()
+) -> str:
+    """One copy-paste command line: the fixed ``words``, the positional
+    ``args``, then ``options``, each quoted; or a short notice when any of them
+    cannot be shown unchanged (see ``shell_arg``).
+
+    A positional value that starts with ``-`` would be parsed as an option
+    (``--discard=x`` selects a different, destructive action), so then the
+    options come first and ``--`` ends option parsing before the values."""
+    positional = [str(a) for a in args]
+    if any(a.startswith("-") for a in positional):
+        argv = [*map(str, words), *map(str, options), "--", *positional]
+    else:
+        argv = [*map(str, words), *positional, *map(str, options)]
+    quoted = [shell_arg(word) for word in argv]
+    if any(word is None for word in quoted):
+        return f"({NO_COMMAND})"
+    return " ".join(quoted)
+
+
+def _stdout_is_terminal() -> bool:
+    return sys.stdout is not None and sys.stdout.isatty()
+
+
+def echo_patch(data: bytes) -> None:
+    """Print a stored patch: escaped at a terminal, as ``echo`` shows any
+    bytes; otherwise the stored bytes unchanged, because redirected output is
+    data (``> fix.patch``, ``| git apply``). Bytes reach click's binary
+    writer, which neither decodes them nor strips ANSI sequences off a
+    terminal, as it does for text."""
+    if not isinstance(data, (bytes, bytearray)):
+        raise TypeError(f"a patch is bytes, not {type(data).__name__}")
+    if _stdout_is_terminal():
+        echo(data, nl=False)
+    else:
+        click.echo(bytes(data), nl=False)
+
+
+class EscapedStderr(io.TextIOBase):
+    """A text stream that writes to whatever ``sys.stderr`` is, escaped, for a
+    library that prints through a stream of its own (the TUI's error console)."""
+
+    def writable(self) -> bool:
+        return True
+
+    def write(self, text: str) -> int:
+        stream = sys.stderr
+        if stream is not None:
+            stream.write(printable_text(text))
+        return len(text)
+
+    def flush(self) -> None:
+        stream = sys.stderr
+        if stream is not None:
+            stream.flush()
+
+
+class EscapingArgumentParser(argparse.ArgumentParser):
+    """argparse writes usage, help and errors itself (then exits), outside
+    click; subparsers inherit the class, so they are covered too."""
+
+    def _print_message(self, message: str, file=None) -> None:
+        super()._print_message(printable_text(message) if message else message, file)
+
+
+def _escaping_excepthook(exc_type, exc, tb) -> None:
+    text = "".join(traceback.format_exception(exc_type, exc, tb))
+    sys.stderr.write(printable_text(text))
+
+
+def _escaping_thread_excepthook(args: threading.ExceptHookArgs) -> None:
+    """``threading.excepthook``'s report, escaped, thread name included."""
+    if args.exc_type is SystemExit:
+        return
+    stream = sys.stderr
+    if stream is None:  # interpreter shutdown, as the default hook allows
+        return
+    name = args.thread.name if args.thread is not None else threading.get_ident()
+    text = "".join(
+        traceback.format_exception(args.exc_type, args.exc_value, args.exc_traceback)
+    )
+    stream.write(printable_text(f"Exception in thread {name}:\n{text}"))
+
+
+class _EscapingFormatter(logging.Formatter):
+    """lastResort's format (the message, then any traceback or stack), escaped.
+
+    The message is one line: ``printable_name`` shows an LF in an id, a path or
+    an exception string as ``\\x0a``, so an argument cannot forge a log line of
+    its own (#198). The traceback and stack keep their lines: the finished
+    string is escaped with ``printable_text``, which also covers a traceback
+    another formatter cached on the record first. ``msg`` and ``args`` are left
+    as they are."""
+
+    def formatMessage(self, record: logging.LogRecord) -> str:
+        return printable_name(super().formatMessage(record))
+
+    def format(self, record: logging.LogRecord) -> str:
+        return printable_text(super().format(record))
+
+
+class _ConsoleLogHandler(logging.StreamHandler):
+    """Where syncmill's own log records are printed: like ``logging.lastResort``
+    it writes to whatever ``sys.stderr`` is when the record arrives (CliRunner,
+    pytest and Textual swap it), but everything it writes is escaped, including
+    the report of a record that failed to format."""
+
+    def __init__(self) -> None:
+        logging.Handler.__init__(self)
+        self.setFormatter(_EscapingFormatter())
+
+    @property
+    def stream(self):
+        return sys.stderr
+
+    def handleError(self, record: logging.LogRecord) -> None:
+        """The stdlib report (``logging.Handler.handleError``), built in a
+        buffer and escaped before it is written: the stdlib version writes the
+        failing exception and the record's ``msg``/``args`` to ``sys.stderr``
+        unescaped."""
+        stream = sys.stderr
+        if not (logging.raiseExceptions and stream):
+            return
+        t, v, tb = sys.exc_info()
+        try:
+            buf = io.StringIO()
+            buf.write("--- Logging error ---\n")
+            traceback.print_exception(t, v, tb, None, buf)
+            buf.write("Call stack:\n")
+            # The first frame outside the logging package is the caller.
+            frame = tb.tb_frame if tb is not None else None
+            while frame and os.path.dirname(frame.f_code.co_filename) == _LOGGING_DIR:
+                frame = frame.f_back
+            if frame:
+                traceback.print_stack(frame, file=buf)
+            else:
+                buf.write(f"Logged from file {record.filename}, line {record.lineno}\n")
+            try:
+                buf.write("Message: %r\nArguments: %s\n" % (record.msg, record.args))
+            except RecursionError:
+                raise
+            except Exception:
+                buf.write(
+                    "Unable to print the message and arguments - possible formatting "
+                    "error.\nUse the traceback above to help find the error.\n"
+                )
+            stream.write(printable_text(buf.getvalue()))
+        except OSError:
+            pass
+        finally:
+            del t, v, tb
+
+
+_LOGGING_DIR = os.path.dirname(logging.__file__)
+
+
+def _install_log_handler() -> None:
+    """Print syncmill's records through ``_ConsoleLogHandler`` only. With
+    ``propagate`` off they never reach a handler someone else installed on the
+    root (``serve``'s RichHandler), so no other handler, formatter or filter
+    ever sees, or has to be guessed about, a syncmill record, and nothing
+    outside the ``syncmill`` hierarchy is touched. The handler has no level of
+    its own: a record prints at its logger's effective level, as it did through
+    the root's handlers, rather than at lastResort's fixed WARNING."""
+    logger = logging.getLogger("syncmill")
+    if not any(isinstance(h, _ConsoleLogHandler) for h in logger.handlers):
+        logger.addHandler(_ConsoleLogHandler())
+    logger.propagate = False
+
+
+def install_console_guards() -> None:
+    """Escape the interpreter's report of an uncaught exception, in the main
+    thread or any other, and every record logged under ``syncmill``. Idempotent.
+    Other libraries' log records (memtomem, mcp, uvicorn, asyncio) are not
+    covered: escaping inside stdlib logging means guessing what each handler,
+    formatter and filter reads, and attempts broke third-party formatters
+    (#202). Click's own help and usage text is not covered either."""
+    sys.excepthook = _escaping_excepthook
+    threading.excepthook = _escaping_thread_excepthook
+    _install_log_handler()