shikiru-cli 0.2.0__py3-none-any.whl

shikiru/plugins/agents/codex/model_catalog.py

@@ -0,0 +1,128 @@
+"""Codex CLI model catalog parsing."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from typing import Any, cast
+
+from shikiru.agents.contracts import ModelChoice
+
+_MODELS_FIELD = "models"
+_MODEL_SLUG_FIELD = "slug"
+_MODEL_CONTEXT_WINDOW_FIELD = "context_window"
+_MODEL_VISIBILITY_FIELD = "visibility"
+_MODEL_SUPPORTED_IN_API_FIELD = "supported_in_api"
+_MODEL_REASONING_LEVELS_FIELD = "supported_reasoning_levels"
+_MODEL_EFFORT_FIELD = "effort"
+_LIST_VISIBILITY = "list"
+_HIDDEN_VISIBILITY = "hide"
+_INVALID_CATALOG_MESSAGE = "Codex CLI model catalog has an invalid structure"
+_INVALID_JSON_MESSAGE = "Codex CLI model catalog is not valid JSON"
+
+
+def parse_bundled_context_windows(output: str) -> dict[str, int]:
+    """Parse positive model context windows from Codex CLI's bundled model catalog."""
+    windows: dict[str, int] = {}
+    for raw_model in _bundled_models(output):
+        if not isinstance(raw_model, Mapping):
+            continue
+        model = cast("Mapping[str, Any]", raw_model)
+        slug = model.get(_MODEL_SLUG_FIELD)
+        window = model.get(_MODEL_CONTEXT_WINDOW_FIELD)
+        if (
+            isinstance(slug, str)
+            and isinstance(window, int)
+            and not isinstance(window, bool)
+            and window > 0
+        ):
+            windows[slug] = window
+    return windows
+
+
+def parse_bundled_model_choices(output: str) -> tuple[ModelChoice, ...]:
+    """Parse API-supported models and only their reported effort choices."""
+    return parse_catalog(output)[0]
+
+
+def parse_catalog(output: str) -> tuple[tuple[ModelChoice, ...], dict[str, int]]:
+    """Parse model choices and context windows from one Codex catalog payload."""
+    choices: list[ModelChoice] = []
+    seen: set[str] = set()
+    context_windows: dict[str, int] = {}
+    for raw_model in _bundled_models(output):
+        if not isinstance(raw_model, Mapping):
+            raise TypeError(_INVALID_CATALOG_MESSAGE)
+        model = cast("Mapping[str, Any]", raw_model)
+        slug = model.get(_MODEL_SLUG_FIELD)
+        window = model.get(_MODEL_CONTEXT_WINDOW_FIELD)
+        if (
+            isinstance(slug, str)
+            and isinstance(window, int)
+            and not isinstance(window, bool)
+            and window > 0
+        ):
+            context_windows[slug] = window
+        choice = _model_choice(model)
+        if choice is None:
+            continue
+        if choice.model in seen:
+            raise TypeError(_INVALID_CATALOG_MESSAGE)
+        seen.add(choice.model)
+        choices.append(choice)
+    return tuple(choices), context_windows
+
+
+def _bundled_models(output: str) -> list[object]:
+    try:
+        payload = json.loads(output)
+    except json.JSONDecodeError as error:
+        raise TypeError(_INVALID_JSON_MESSAGE) from error
+    if not isinstance(payload, Mapping):
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    typed_payload = cast("Mapping[str, Any]", payload)
+    raw_models = typed_payload.get(_MODELS_FIELD)
+    if not isinstance(raw_models, list):
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    return cast("list[object]", raw_models)
+
+
+def _model_choice(raw_model: object) -> ModelChoice | None:
+    if not isinstance(raw_model, Mapping):
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    model = cast("Mapping[str, Any]", raw_model)
+    visibility = model.get(_MODEL_VISIBILITY_FIELD)
+    if not isinstance(visibility, str):
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    if visibility not in {_LIST_VISIBILITY, _HIDDEN_VISIBILITY}:
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    supported_in_api = model.get(_MODEL_SUPPORTED_IN_API_FIELD)
+    if not isinstance(supported_in_api, bool):
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    if not supported_in_api:
+        return None
+    slug = model.get(_MODEL_SLUG_FIELD)
+    if not isinstance(slug, str) or not slug:
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    return ModelChoice(
+        model=slug,
+        efforts=_effort_choices(model),
+        offered=visibility == _LIST_VISIBILITY,
+    )
+
+
+def _effort_choices(model: Mapping[str, Any]) -> tuple[str, ...] | None:
+    raw_levels = model.get(_MODEL_REASONING_LEVELS_FIELD)
+    if raw_levels is None:
+        return None
+    if not isinstance(raw_levels, list):
+        raise TypeError(_INVALID_CATALOG_MESSAGE)
+    effort_values: list[str] = []
+    for raw_level in cast("list[object]", raw_levels):
+        if not isinstance(raw_level, Mapping):
+            raise TypeError(_INVALID_CATALOG_MESSAGE)
+        effort = cast("Mapping[str, Any]", raw_level).get(_MODEL_EFFORT_FIELD)
+        if not isinstance(effort, str) or not effort or effort in effort_values:
+            raise TypeError(_INVALID_CATALOG_MESSAGE)
+        effort_values.append(effort)
+    return tuple(effort_values)

shikiru/plugins/agents/codex/options.py

@@ -0,0 +1,94 @@
+"""Codex plugin options and protocol constants."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from shikiru.agents.contracts import PERCENT, BillingMode
+from shikiru.plugins.agents.shared_settings import (
+    billing_declaration_setting,
+    cli_args_setting,
+    compact_threshold_setting,
+)
+
+if TYPE_CHECKING:
+    from shikiru.agents.settings import AgentSetting
+
+CODEX_HOME_ENV_VAR = "CODEX_HOME"
+CODEX_COMMAND = "codex"
+DEBUG_SUBCOMMAND = "debug"
+MODELS_SUBCOMMAND = "models"
+MODELS_BUNDLED_FLAG = "--bundled"
+MODELS_CACHE_FILENAME = "models_cache.json"
+CODEX_MODEL_CATALOG_ARGV = (CODEX_COMMAND, DEBUG_SUBCOMMAND, MODELS_SUBCOMMAND)
+PROFILE_FLAG = "--profile"
+MODEL_LONG_FLAG = "--model"
+EFFORT_LONG_FLAG = "--effort"
+CONFIG_LONG_FLAG = "--config"
+MODEL_CONFIG_OVERRIDE = "model_provider"
+MODEL_OVERRIDE = "model"
+REASONING_EFFORT_CONFIG = "model_reasoning_effort"
+MODEL_CATALOG_ARGS_DETAIL = (
+    "Codex agent args select a model, effort, or profile; its effective model catalog "
+    "cannot be determined safely"
+)
+CODEX_BUNDLED_MODELS_ARGV = (
+    CODEX_COMMAND,
+    DEBUG_SUBCOMMAND,
+    MODELS_SUBCOMMAND,
+    MODELS_BUNDLED_FLAG,
+)
+RESUME_SUBCOMMAND = "resume"
+NO_DAEMON_FLAG = "--no-daemon"
+MODEL_FLAG = "-m"
+CONFIG_FLAG = "-c"
+EXEC_SUBCOMMAND = "exec"
+EPHEMERAL_FLAG = "--ephemeral"
+JSON_FLAG = "--json"
+OUTPUT_SCHEMA_FLAG = "--output-schema"
+SANDBOX_FLAG = "-s"
+SANDBOX_MODE_CONFIG = "sandbox_mode"
+WRITABLE_ROOTS_CONFIG = "sandbox_workspace_write.writable_roots"
+READ_ONLY_SANDBOX = "read-only"
+WORKSPACE_WRITE_SANDBOX = "workspace-write"
+SKIP_GIT_REPO_CHECK_FLAG = "--skip-git-repo-check"
+AUTO_COMPACT_LIMIT_CONFIG = "model_auto_compact_token_limit"
+VERSION_ARGV = (CODEX_COMMAND, "--version")
+LOGIN_STATUS_ARGV = (CODEX_COMMAND, "login", "status")
+DEFAULT_COMPACT_AT_PCT = 80.0
+USAGE_EVENT_TYPE = "token_count"
+GIT_DIRS_TIMEOUT_SECONDS = 10.0
+
+
+class CodexOptions(BaseModel):
+    """Options accepted by the Codex agent factory."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    args: list[str] = Field(default_factory=list)
+    compact_at_pct: float = Field(default=DEFAULT_COMPACT_AT_PCT, ge=0, le=PERCENT)
+    billing_mode: BillingMode | None = None
+
+
+TESTED_CODEX_VERSION = "0.159.2"
+SANDBOX_PROBE_TIMEOUT_SECONDS = 60.0
+SANDBOX_PROBE_PARENT = (".cache", "shikiru")
+SANDBOX_PROBE_PREFIX = "codex-sandbox-"
+GIT_COMMAND = "git"
+END_OF_OPTIONS_FLAG = "--"
+GIT_DIRS_ARGV = (
+    GIT_COMMAND,
+    "rev-parse",
+    "--path-format=absolute",
+    "--absolute-git-dir",
+    "--git-common-dir",
+)
+
+
+CODEX_SETTINGS: tuple[AgentSetting, ...] = (
+    compact_threshold_setting(DEFAULT_COMPACT_AT_PCT),
+    billing_declaration_setting(),
+    cli_args_setting(()),
+)

shikiru/plugins/agents/codex/quota.py

@@ -0,0 +1,195 @@
+"""Codex rollout quota parsing."""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any, cast
+
+from shikiru.agents.contracts import (
+    QUOTA_WINDOW_FIVE_HOUR,
+    QUOTA_WINDOW_SEVEN_DAY,
+    QuotaReading,
+    QuotaWindow,
+)
+from shikiru.plugins.agents.codex.options import USAGE_EVENT_TYPE
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from shikiru.agents.sdk.reset_time import ResetTimeParser
+    from shikiru.clock import Clock
+
+_EVENT_MESSAGE_TYPE = "event_msg"
+_RATE_LIMITS_FIELD = "rate_limits"
+_PRIMARY_WINDOW = "primary"
+_SECONDARY_WINDOW = "secondary"
+_DEFAULT_WINDOW_NAMES = {
+    _PRIMARY_WINDOW: QUOTA_WINDOW_FIVE_HOUR,
+    _SECONDARY_WINDOW: QUOTA_WINDOW_SEVEN_DAY,
+}
+_WINDOW_MINUTES_FIELD = "window_minutes"
+_TIMESTAMP_FIELD = "timestamp"
+_MINUTES_PER_HOUR = 60
+_MINUTES_PER_DAY = 24 * _MINUTES_PER_HOUR
+_USED_PERCENT_FIELD = "used_percent"
+_RESETS_AT_FIELD = "resets_at"
+_ROLLOUT_GLOB = "rollout-*.jsonl"
+_LIMIT_ID_FIELD = "limit_id"
+CODEX_LIMIT_ID = "codex"
+_ROLLOUT_TAIL_BYTES = 1 << 20
+_NEWLINE = b"\n"
+_FILE_START = 0
+_AFTER_SEPARATOR = -1
+
+
+@dataclass(frozen=True)
+class QuotaObservation:
+    """A Codex quota sample and its provider timestamp, when present."""
+
+    windows: tuple[QuotaWindow, ...]
+    timestamp: datetime | None
+
+
+class CodexQuotaProbe:
+    """Reads the Codex-bucket quota windows from Codex rollouts."""
+
+    def __init__(self, home: Path, clock: Clock, reset_time: ResetTimeParser) -> None:
+        """Use `home`, `clock` and `reset_time` to inspect Codex session files."""
+        self._home = home
+        self._clock = clock
+        self._reset_time = reset_time
+
+    def get(self, rollout_path: Path | None = None) -> list[QuotaWindow]:
+        """Return the latest primary and secondary quota windows, or no windows."""
+        path = rollout_path or self._newest_rollout()
+        if path is None or not path.is_file():
+            return []
+        return self.parse(path.read_text(encoding="utf-8"), self._clock.now())
+
+    def parse(self, text: str, observed_at: datetime) -> list[QuotaWindow]:
+        """Parse quota windows from Codex rollout JSONL text."""
+        observation = self.parse_observation(text, observed_at)
+        return list(observation.windows) if observation is not None else []
+
+    def parse_observation(self, text: str, observed_at: datetime) -> QuotaObservation | None:
+        """Parse the latest quota sample and preserve its provider timestamp."""
+        latest: tuple[Mapping[str, Any], datetime | None] | None = None
+        for line in text.splitlines():
+            try:
+                raw = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(raw, Mapping):
+                continue
+            payload = _payload(cast("Mapping[str, Any]", raw))
+            if payload.get("type") != USAGE_EVENT_TYPE:
+                continue
+            limits = payload.get(_RATE_LIMITS_FIELD)
+            if _is_codex_bucket(limits) and _has_windows(cast("Mapping[str, Any]", limits)):
+                latest = (
+                    cast("Mapping[str, Any]", limits),
+                    _event_time(cast("Mapping[str, Any]", raw)),
+                )
+        if latest is None:
+            return None
+        rate_limits, event_time = latest
+        window_time = event_time or observed_at
+        windows = tuple(
+            self._window(key, cast("Mapping[str, Any]", rate_limits[key]), window_time)
+            for key in _DEFAULT_WINDOW_NAMES
+            if isinstance(rate_limits.get(key), Mapping)
+        )
+        return QuotaObservation(windows, event_time)
+
+    def latest_reading(self, since: datetime) -> QuotaReading | None:
+        """Return the newest Codex-bucket sample stamped after `since` in any recent rollout.
+
+        Only rollouts written after `since` are read, from their tail; a sample without a
+        provider timestamp cannot show it is fresh and is ignored.
+        """
+        sessions = self._home / "sessions"
+        if not sessions.is_dir():
+            return None
+        readings = [
+            QuotaReading(observation.windows, observation.timestamp)
+            for path in sessions.rglob(_ROLLOUT_GLOB)
+            if datetime.fromtimestamp(path.stat().st_mtime, tz=UTC) > since
+            for observation in [self.parse_observation(_rollout_tail(path), self._clock.now())]
+            if observation is not None
+            and observation.timestamp is not None
+            and observation.timestamp > since
+        ]
+        return max(readings, key=lambda reading: reading.observed_at, default=None)
+
+    def _window(self, key: str, raw: Mapping[str, Any], observed_at: datetime) -> QuotaWindow:
+        value = raw.get(_USED_PERCENT_FIELD)
+        if not isinstance(value, int | float):
+            message = f"Codex quota window {key!r} has no numeric used_percent"
+            raise TypeError(message)
+        reset_value = raw.get(_RESETS_AT_FIELD)
+        resets_at = self._reset_time.parse(str(reset_value)) if reset_value is not None else None
+        return QuotaWindow(
+            name=_window_name(key, raw.get(_WINDOW_MINUTES_FIELD)),
+            used_pct=float(value),
+            resets_at=resets_at,
+            observed_at=observed_at,
+        )
+
+    def _newest_rollout(self) -> Path | None:
+        sessions = self._home / "sessions"
+        paths = list(sessions.rglob(_ROLLOUT_GLOB)) if sessions.is_dir() else []
+        return max(paths, key=lambda path: path.stat().st_mtime_ns) if paths else None
+
+
+def _is_codex_bucket(limits: object) -> bool:
+    if not isinstance(limits, Mapping):
+        return False
+    limit_id = cast("Mapping[str, Any]", limits).get(_LIMIT_ID_FIELD)
+    return limit_id is None or limit_id == CODEX_LIMIT_ID
+
+
+def _has_windows(limits: Mapping[str, Any]) -> bool:
+    return any(isinstance(limits.get(key), Mapping) for key in _DEFAULT_WINDOW_NAMES)
+
+
+def _rollout_tail(path: Path) -> str:
+    with path.open("rb") as handle:
+        size = handle.seek(_FILE_START, os.SEEK_END)
+        handle.seek(max(size - _ROLLOUT_TAIL_BYTES, _FILE_START))
+        data = handle.read()
+    if size > _ROLLOUT_TAIL_BYTES:
+        data = data.partition(_NEWLINE)[_AFTER_SEPARATOR]
+    return data.decode("utf-8", errors="ignore")
+
+
+def _payload(entry: Mapping[str, Any]) -> Mapping[str, Any]:
+    if entry.get("type") == _EVENT_MESSAGE_TYPE:
+        payload = entry.get("payload")
+        if isinstance(payload, Mapping):
+            return cast("Mapping[str, Any]", payload)
+    return entry
+
+
+def _event_time(entry: Mapping[str, Any]) -> datetime | None:
+    value = entry.get(_TIMESTAMP_FIELD)
+    if not isinstance(value, str):
+        return None
+    try:
+        parsed = datetime.fromisoformat(value)
+    except ValueError:
+        return None
+    return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
+
+
+def _window_name(key: str, minutes: object) -> str:
+    if not isinstance(minutes, int) or isinstance(minutes, bool) or minutes <= 0:
+        return _DEFAULT_WINDOW_NAMES[key]
+    if minutes % _MINUTES_PER_DAY == 0:
+        return f"{minutes // _MINUTES_PER_DAY}d"
+    if minutes % _MINUTES_PER_HOUR == 0:
+        return f"{minutes // _MINUTES_PER_HOUR}h"
+    return f"{minutes}m"

shikiru/plugins/agents/codex/sandbox.py

@@ -0,0 +1,31 @@
+"""Codex sandbox configuration shared by turns and the doctor probe."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from shikiru.plugins.agents.codex.options import (
+    CONFIG_FLAG,
+    SANDBOX_MODE_CONFIG,
+    WORKSPACE_WRITE_SANDBOX,
+    WRITABLE_ROOTS_CONFIG,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from pathlib import Path
+
+
+def sandbox_config_args(roots: Sequence[Path]) -> list[str]:
+    """Return the `-c` arguments that make `roots` writable under `workspace-write`.
+
+    Configuration overrides are the one mechanism `codex exec` and `codex exec resume` share;
+    `exec resume` has no `--sandbox` or `--add-dir`.
+    """
+    return [
+        CONFIG_FLAG,
+        f'{SANDBOX_MODE_CONFIG}="{WORKSPACE_WRITE_SANDBOX}"',
+        CONFIG_FLAG,
+        f"{WRITABLE_ROOTS_CONFIG}={json.dumps([str(root) for root in roots])}",
+    ]

shikiru/plugins/agents/codex/sandbox_probe.py

@@ -0,0 +1,97 @@
+"""Free Codex sandbox probe: a real `codex sandbox` run in a scratch linked worktree."""
+
+from __future__ import annotations
+
+import asyncio
+import shutil
+import tempfile
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from shikiru.agents.contracts import CheckResult
+from shikiru.errors import ShikiruError
+from shikiru.plugins.agents.codex.options import (
+    CODEX_COMMAND,
+    END_OF_OPTIONS_FLAG,
+    GIT_COMMAND,
+    GIT_DIRS_ARGV,
+    SANDBOX_PROBE_PARENT,
+    SANDBOX_PROBE_PREFIX,
+    SANDBOX_PROBE_TIMEOUT_SECONDS,
+)
+from shikiru.plugins.agents.codex.sandbox import sandbox_config_args
+
+if TYPE_CHECKING:
+    from shikiru.process import SubprocessRunner
+
+SANDBOX_CHECK = "sandbox"
+_SANDBOX_SUBCOMMAND = "sandbox"
+_WORKTREE_DIR = "worktree"
+_RUN_DIR = "run"
+_MAIN_BRANCH = "main"
+_PROBE_BRANCH = "probe"
+_PROBE_FILE = ".shikiru-probe"
+_GIT_IDENTITY = ("-c", "user.name=shikiru", "-c", "user.email=shikiru@localhost")
+_TOUCH_SCRIPT = (
+    'for directory; do touch "$directory/'
+    + _PROBE_FILE
+    + '" 2>/dev/null || echo "$directory"; done'
+)
+_SHELL_ARGV = ("sh", "-c", _TOUCH_SCRIPT, "sh")
+
+
+class CodexSandboxProbe:
+    """Check that a linked worktree's git directory and the run directory are writable."""
+
+    def __init__(self, runner: SubprocessRunner) -> None:
+        self._runner = runner
+
+    async def check(self) -> CheckResult:
+        """Run the probe outside `$TMPDIR` and report every root the sandbox kept read-only."""
+        parent = Path.home().joinpath(*SANDBOX_PROBE_PARENT)
+        await asyncio.to_thread(parent.mkdir, parents=True, exist_ok=True)
+        base = Path(
+            await asyncio.to_thread(tempfile.mkdtemp, prefix=SANDBOX_PROBE_PREFIX, dir=parent)
+        )
+        try:
+            return await self._probe(base)
+        except (OSError, ShikiruError) as error:
+            return CheckResult(SANDBOX_CHECK, False, str(error))
+        finally:
+            await asyncio.to_thread(shutil.rmtree, base, ignore_errors=True)
+
+    async def _probe(self, base: Path) -> CheckResult:
+        repository, worktree, run_dir = base / "repo", base / _WORKTREE_DIR, base / _RUN_DIR
+        run_dir.mkdir()
+        await self._git(base, "init", "-q", "-b", _MAIN_BRANCH, str(repository))
+        await self._git(repository, *_GIT_IDENTITY, "commit", "-q", "--allow-empty", "-m", "probe")
+        await self._git(repository, "worktree", "add", "-q", "-b", _PROBE_BRANCH, str(worktree))
+        git_dirs = (await self._git(worktree, *GIT_DIRS_ARGV[1:])).split()
+        targets = (*git_dirs, str(run_dir))
+        result = await self._runner.run(
+            (
+                CODEX_COMMAND,
+                _SANDBOX_SUBCOMMAND,
+                *sandbox_config_args((run_dir, *map(Path, git_dirs))),
+                END_OF_OPTIONS_FLAG,
+                *_SHELL_ARGV,
+                *targets,
+            ),
+            timeout=SANDBOX_PROBE_TIMEOUT_SECONDS,
+            cwd=worktree,
+            check=False,
+        )
+        if result.returncode != 0:
+            return CheckResult(SANDBOX_CHECK, False, result.stderr.strip())
+        denied = result.stdout.split()
+        if denied:
+            return CheckResult(
+                SANDBOX_CHECK, False, f"not writable under the sandbox: {', '.join(denied)}"
+            )
+        return CheckResult(SANDBOX_CHECK, True, "worktree git directory and run directory writable")
+
+    async def _git(self, cwd: Path, *args: str) -> str:
+        result = await self._runner.run(
+            (GIT_COMMAND, *args), timeout=SANDBOX_PROBE_TIMEOUT_SECONDS, cwd=cwd, check=True
+        )
+        return result.stdout