comodor 0.1.0__py3-none-any.whl

comodor/agent/prompts.py

@@ -0,0 +1,209 @@
+"""Assembling the system prompt.
+
+The prompt is built in fixed order — identity, environment, mode, playbook —
+because that order is also a caching order: the stable parts come first, so a
+provider's prefix cache keeps hitting even as the recalled playbook changes from
+turn to turn.
+
+Everything the model is told about *how to behave* lives here. Everything it is
+told about *what it has learned* is injected as a separate playbook block, kept
+visibly distinct so a bad lesson can be traced and deleted rather than quietly
+shaping every answer.
+"""
+
+from __future__ import annotations
+
+import os
+import platform
+import sys
+from datetime import date
+from pathlib import Path
+
+from ..config import Config
+
+IDENTITY = """\
+You are Comodor, a terminal coding agent. You work inside the user's project, \
+with real tools, on a real filesystem.
+
+How you work:
+- Act on what was asked. Do not widen the scope, and do not narrow it either.
+- Investigate before you change anything. Read the file before you edit it; \
+search for existing helpers before you write new ones.
+- Prefer small, verifiable steps. After a change that can be checked — a test, \
+a build, a script — run it.
+- When something fails, read the actual error and fix the cause. Do not guess \
+twice at the same thing.
+- Say what you did, plainly. If part of a task is unfinished or blocked, say \
+which part and why, rather than implying it is done.
+- Never invent file contents, command output, or API behaviour. If you have not \
+looked, say you have not looked.
+
+How you write:
+- Answer in English. Be concise; the user is reading in a terminal panel.
+- Use Markdown sparingly — code blocks for code, short bullet lists for \
+several parallel points, plain sentences otherwise.
+- Reference files as `path/to/file.py:42` so they can be found quickly.
+- Do not narrate routine tool use ("Now I will read the file"). Just use the \
+tool; the interface already shows the user what is running."""
+
+TOOL_GUIDANCE = """\
+Using tools:
+- Call tools in parallel when the calls are independent — several reads or \
+searches at once is normal and much faster.
+- `todo_write` is for any task with more than about three steps. Write the plan \
+once, then keep it current: one item `active`, finished items `done`.
+- `edit_file` needs an `old_string` that appears exactly once. Include \
+surrounding lines to make it unique rather than guessing.
+- Use `run_shell` for builds, tests and version control; use the file tools for \
+reading and editing — they are safer and produce better diffs.
+- A tool that returns an error is information, not a dead end. Read it, then \
+adjust."""
+
+MODE_ACT = """\
+Mode: ACT. You have the full tool set and may modify the project. Work until \
+the task is genuinely done."""
+
+MODE_PLAN = """\
+Mode: PLAN. You have read-only tools only — you cannot modify anything, and \
+write tools are not available to you. Investigate the codebase, then produce a \
+concrete plan: which files change, what changes in each, and how to verify the \
+result. Name real files you have actually looked at."""
+
+MODE_CHAT = """\
+Mode: CHAT. Tools are switched off. Answer from the conversation and your own \
+knowledge. If answering properly needs to look at files, say so and suggest \
+switching to Act mode."""
+
+_MODES = {"act": MODE_ACT, "plan": MODE_PLAN, "chat": MODE_CHAT}
+
+
+def environment_block(config: Config) -> str:
+    """Facts about the machine — cheap to include, and wrong guesses are costly."""
+    root = config.paths.project
+    shell = "PowerShell / cmd" if os.name == "nt" else os.environ.get("SHELL", "sh")
+    lines = [
+        "Environment:",
+        f"- Operating system: {platform.system()} {platform.release()}",
+        f"- Shell: {shell}",
+        f"- Python: {sys.version.split()[0]}",
+        f"- Workspace: {root}",
+        f"- Today: {date.today().isoformat()}",
+    ]
+    markers = _project_markers(root)
+    if markers:
+        lines.append(f"- Project markers: {', '.join(markers)}")
+    return "\n".join(lines)
+
+
+def _project_markers(root: Path) -> list[str]:
+    candidates = ("pyproject.toml", "package.json", "Cargo.toml", "go.mod",
+                  "pom.xml", "build.gradle", "Makefile", "docker-compose.yml",
+                  ".git", "requirements.txt")
+    found = []
+    for name in candidates:
+        try:
+            if (root / name).exists():
+                found.append(name)
+        except OSError:
+            continue
+    return found
+
+
+def project_instructions(config: Config) -> str:
+    """Per-project rules from ``COMODOR.md`` — the project's own house style.
+
+    Reading a checked-in instructions file is how a team makes the agent follow
+    their conventions without every member configuring it by hand.
+    """
+    for name in ("COMODOR.md", ".comodor/COMODOR.md", "AGENTS.md"):
+        path = config.paths.project / name
+        try:
+            if path.is_file():
+                text = path.read_text(encoding="utf-8", errors="replace").strip()
+                if text:
+                    return (f"Project instructions (from {name}) — these take "
+                            f"precedence over your defaults:\n\n{text[:8000]}")
+        except OSError:
+            continue
+    return ""
+
+
+def build_system_prompt(config: Config, playbook: str = "") -> str:
+    """The complete system prompt for one turn."""
+    mode = (config.agent.mode or "act").lower()
+    sections = [
+        IDENTITY,
+        environment_block(config),
+        _MODES.get(mode, MODE_ACT),
+    ]
+    if mode != "chat":
+        sections.append(TOOL_GUIDANCE)
+
+    instructions = project_instructions(config)
+    if instructions:
+        sections.append(instructions)
+
+    if playbook:
+        sections.append(playbook)
+
+    extra = (config.agent.system_prompt_extra or "").strip()
+    if extra:
+        sections.append(extra)
+
+    return "\n\n".join(section for section in sections if section)
+
+
+# --------------------------------------------------------------------------- #
+# prompts for the agent's own bookkeeping
+# --------------------------------------------------------------------------- #
+
+TITLE_PROMPT = """\
+Summarise this conversation as a title of at most six words. Reply with the \
+title only — no quotes, no punctuation at the end."""
+
+COMPACT_PROMPT = """\
+You are compacting a long agent session so that work can continue in a smaller \
+context window.
+
+Write a factual brief covering:
+1. What the user asked for, including any constraints they stated.
+2. What has been done so far — files created or changed, commands run, results.
+3. What is known about the codebase that was expensive to discover.
+4. What remains to be done, and any decision still open.
+
+Be specific: real file paths, real function names, real error messages. Omit \
+pleasantries and anything already superseded. This brief replaces the messages \
+it summarises, so anything you leave out is lost."""
+
+REFLECT_PROMPT = """\
+You are reviewing a finished agent task in order to learn from it.
+
+Produce JSON only, matching this shape:
+
+{
+  "lessons": [
+    {
+      "kind": "preference | fact | heuristic | pitfall | env",
+      "trigger": "when this applies — the situation, in a few words",
+      "guidance": "what to do about it, imperative and specific",
+      "confidence": 0.0-1.0
+    }
+  ],
+  "skill": {
+    "name": "short_snake_case_name",
+    "description": "what this procedure accomplishes",
+    "steps": ["ordered, reusable steps"],
+    "tags": ["keyword", "keyword"]
+  } | null
+}
+
+Rules:
+- Record only what would change behaviour *next time*. Facts about this \
+codebase, a user preference they stated, a tool that failed in a predictable \
+way, an environment quirk.
+- Never record the specific content of this task ("fixed the typo in line 12"). \
+Record the transferable part.
+- If the task was routine and taught you nothing, return {"lessons": [], \
+"skill": null}. An empty answer is a good answer.
+- Propose a skill only for a multi-step procedure that clearly recurs.
+- At most four lessons."""

comodor/agent/tokens.py

@@ -0,0 +1,131 @@
+"""Counting tokens without a tokenizer.
+
+Shipping ``tiktoken`` would add a compiled dependency and still be wrong for
+every non-OpenAI model, so Comodor estimates instead — and then *calibrates*.
+Every provider reply carries a real ``usage.input_tokens``; comparing that to
+what we predicted for the same messages yields a correction factor, so after a
+couple of turns the ``Context: 1M`` gauge tracks reality closely for whatever
+model is actually in use.
+
+The base heuristic accounts for the fact that code tokenises worse than prose
+and that CJK text tokenises worse still.
+"""
+
+from __future__ import annotations
+
+import re
+import threading
+from dataclasses import dataclass
+
+from ..providers.base import Message, ToolSpec
+
+# Rough characters-per-token for Latin prose. Code, with its punctuation and
+# indentation, lands nearer 3.
+_PROSE_CHARS_PER_TOKEN = 4.0
+_CODE_CHARS_PER_TOKEN = 3.0
+# CJK, Thai and similar scripts are close to one token per character.
+_WIDE = re.compile(r"[ᄀ-ᇿ⺀-꓏ꥠ-꥿가-퟿"
+                   r"豈-﫿︰-﹏＀-￯]")
+_CODEY = re.compile(r"[{}()\[\];=<>+\-*/\\|&^%$#@~`_]")
+# Per-message framing the API adds (role markers, separators).
+_MESSAGE_OVERHEAD = 4
+_TOOL_OVERHEAD = 12
+
+
+def estimate_text(text: str) -> int:
+    """Estimated token count for one string."""
+    if not text:
+        return 0
+    wide = len(_WIDE.findall(text))
+    rest = len(text) - wide
+    if rest <= 0:
+        return wide
+
+    density = len(_CODEY.findall(text)) / max(1, rest)
+    chars_per_token = (_CODE_CHARS_PER_TOKEN if density > 0.06
+                       else _PROSE_CHARS_PER_TOKEN)
+    return int(wide + rest / chars_per_token) + 1
+
+
+def estimate_message(message: Message) -> int:
+    total = _MESSAGE_OVERHEAD + estimate_text(message.content)
+    for call in message.tool_calls:
+        total += _TOOL_OVERHEAD + estimate_text(call.name) + estimate_text(call.arguments_json())
+    # An image costs far more than its base64 length suggests; this is the
+    # commonly used ~1.1k tokens for a typical screenshot.
+    total += 1100 * len(message.images)
+    return total
+
+
+def estimate_messages(messages: list[Message]) -> int:
+    return sum(estimate_message(message) for message in messages)
+
+
+def estimate_tools(tools: list[ToolSpec]) -> int:
+    total = 0
+    for tool in tools:
+        total += _TOOL_OVERHEAD + estimate_text(tool.name) + estimate_text(tool.description)
+        total += estimate_text(str(tool.parameters))
+    return total
+
+
+@dataclass
+class Calibration:
+    """Learns the ratio between our estimate and the provider's real count."""
+
+    factor: float = 1.0
+    samples: int = 0
+    _lock: threading.Lock = None  # type: ignore[assignment]
+
+    def __post_init__(self) -> None:
+        self._lock = threading.Lock()
+
+    def observe(self, estimated: int, actual: int) -> None:
+        """Fold one real measurement into the correction factor."""
+        if estimated <= 0 or actual <= 0:
+            return
+        ratio = actual / estimated
+        # Ignore wild outliers: a cached-prefix turn reports far fewer input
+        # tokens than were sent and would otherwise poison the estimate.
+        if not 0.3 <= ratio <= 3.0:
+            return
+        with self._lock:
+            self.samples += 1
+            weight = min(0.3, 1.0 / self.samples)
+            self.factor = (1 - weight) * self.factor + weight * ratio
+
+    def apply(self, estimated: int) -> int:
+        with self._lock:
+            return int(estimated * self.factor)
+
+    @property
+    def confident(self) -> bool:
+        return self.samples >= 2
+
+
+class TokenCounter:
+    """Estimation plus calibration, for one session."""
+
+    def __init__(self) -> None:
+        self.calibration = Calibration()
+
+    def count(self, messages: list[Message], tools: list[ToolSpec] | None = None) -> int:
+        raw = estimate_messages(messages) + estimate_tools(tools or [])
+        return self.calibration.apply(raw)
+
+    def count_text(self, text: str) -> int:
+        return self.calibration.apply(estimate_text(text))
+
+    def observe_usage(self, messages: list[Message], tools: list[ToolSpec] | None,
+                      actual_input_tokens: int) -> None:
+        raw = estimate_messages(messages) + estimate_tools(tools or [])
+        self.calibration.observe(raw, actual_input_tokens)
+
+
+def humanise(count: int) -> str:
+    """Compact display: 1.2M, 340K, 812."""
+    if count >= 1_000_000:
+        return f"{count / 1_000_000:.1f}M".replace(".0M", "M")
+    if count >= 1_000:
+        return f"{count / 1_000:.0f}K"
+    return str(count)

comodor/cli.py

@@ -0,0 +1,295 @@
+"""Command line entry point.
+
+Three ways in:
+
+* ``comodor`` — the full interface.
+* ``comodor run "<task>"`` — one task, no interface, output to stdout or JSON.
+  This is the mode CI and scripts use; it needs explicit ``--yes`` before it
+  will change anything, because there is nobody there to approve a prompt.
+* ``comodor doctor`` — what is configured, what is reachable, what the terminal
+  can do. The first thing to run when something is not working.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Any
+
+from . import __version__
+from .config import Config, load as load_config
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="comodor",
+        description="Comodor — a terminal coding agent that learns as it works.",
+    )
+    parser.add_argument("--version", action="version", version=f"comodor {__version__}")
+
+    parser.add_argument("--provider", help="provider to use (openrouter, anthropic, …)")
+    parser.add_argument("--model", help="model id")
+    parser.add_argument("--mode", choices=("act", "plan", "chat"), help="starting mode")
+    parser.add_argument("--no-loop", action="store_true",
+                        help="answer once instead of iterating autonomously")
+    parser.add_argument("--theme", help="colour theme (ember, midnight, matrix, mono)")
+    parser.add_argument("--ascii", action="store_true",
+                        help="ASCII borders, for terminals without box-drawing glyphs")
+    parser.add_argument("--no-mouse", action="store_true", help="disable mouse support")
+    parser.add_argument("--cwd", help="workspace directory (default: the project root)")
+    parser.add_argument("--demo", action="store_true",
+                        help="run the interface against a scripted offline provider")
+    parser.add_argument("--resume", nargs="?", const="__pick__",
+                        help="reopen the most recent session, or one by id")
+
+    sub = parser.add_subparsers(dest="command")
+
+    run = sub.add_parser("run", help="run one task without the interface")
+    run.add_argument("task", help="what to do")
+    run.add_argument("--json", action="store_true", help="emit a JSON result")
+    run.add_argument("--yes", action="store_true",
+                     help="approve file writes and commands automatically")
+    run.add_argument("--max-steps", type=int, help="override the step limit")
+
+    sub.add_parser("doctor", help="show configuration and environment diagnostics")
+
+    preview = sub.add_parser("preview",
+                             help="render the interface at a given size and exit")
+    preview.add_argument("size", nargs="?", default="120x34", help="WIDTHxHEIGHT")
+    preview.add_argument("--svg", help="also write an SVG to this path")
+
+    return parser
+
+
+def apply_overrides(config: Config, args: argparse.Namespace) -> Config:
+    if args.provider:
+        config.provider = args.provider
+        entry = config.providers.get(args.provider)
+        if entry and not args.model:
+            config.model = entry.model
+    if args.model:
+        config.model = args.model
+    if args.mode:
+        config.agent.mode = args.mode
+    if args.no_loop:
+        config.agent.loop = False
+    if args.theme:
+        config.ui.theme = args.theme
+    if args.ascii:
+        config.ui.ascii_borders = True
+    if args.no_mouse:
+        config.ui.mouse = False
+    return config
+
+
+# --------------------------------------------------------------------------- #
+# subcommands
+# --------------------------------------------------------------------------- #
+
+
+def run_headless(config: Config, args: argparse.Namespace) -> int:
+    """One task, no TUI. Used by scripts, hooks and CI."""
+    from .agent import AgentLoop, Conversation
+    from .events import EventBus, Kind
+    from .learning import LearningEngine
+    from .providers.gateway import Gateway
+    from .safety import CheckpointStore, PermissionEngine, Redactor
+    from .tools import ToolRegistry
+
+    if args.yes:
+        config.safety.auto_approve_writes = True
+        config.safety.auto_approve_shell = True
+    if args.max_steps:
+        config.agent.max_steps = args.max_steps
+
+    bus = EventBus()
+    gateway = Gateway(config)
+    # Headless runs learn from corrections too: a scripted run whose output the
+    # user later fixes by hand should teach the same lesson an interactive one
+    # would, or the brain would depend on how the agent happened to be invoked.
+    memory = LearningEngine(
+        config, bus, gateway,
+        checkpoints=CheckpointStore(config.paths.checkpoints),
+        redact=Redactor([entry.api_key for entry in config.providers.values()
+                         if entry.api_key]),
+    )
+    permissions = PermissionEngine(config, bus)
+    permissions.on_denied = memory.on_denied
+    agent = AgentLoop(config, gateway, ToolRegistry(), bus,
+                      permissions, Conversation(), memory)
+
+    if not args.json:
+        # Progress on stderr keeps stdout clean for the answer itself.
+        def echo(event: Any) -> None:
+            if event.kind is Kind.TOOL_START:
+                print(f"· {event.get('summary', event.get('name'))}", file=sys.stderr)
+            elif event.kind is Kind.ERROR:
+                print(f"! {event.text}", file=sys.stderr)
+
+        bus.subscribe(echo)
+
+    result = agent.run(args.task)
+    memory.wait_for_reflection(timeout=20.0)
+
+    if args.json:
+        print(json.dumps({
+            "text": result.text,
+            "ok": result.ok,
+            "stopped": result.stopped,
+            "steps": result.steps,
+            "tool_calls": result.tool_calls,
+            "error": result.error,
+            "usage": {
+                "input_tokens": result.usage.input_tokens,
+                "output_tokens": result.usage.output_tokens,
+                "cost_usd": round(result.usage.cost_usd, 6),
+            },
+            "elapsed": round(result.elapsed, 2),
+        }, ensure_ascii=False, indent=2))
+    else:
+        print(result.text)
+        if result.error:
+            print(f"\nerror: {result.error}", file=sys.stderr)
+
+    memory.close()
+    gateway.close()
+    return 0 if result.ok else 1
+
+
+def run_doctor(config: Config) -> int:
+    from rich.table import Table
+
+    from .learning import BrainStore
+    from .ui import console as console_module
+
+    theme = console_module.prepare_theme(config.ui.theme, config.ui.ascii_borders,
+                                         no_color=False)
+    console = console_module.build(theme)
+
+    console.print(f"[title]Comodor {__version__}[/title]\n")
+
+    providers = Table(title="Providers", title_justify="left", box=theme.box,
+                      border_style=theme.style("border.dim"), expand=False)
+    providers.add_column("name", style=theme.style("value"))
+    providers.add_column("model")
+    providers.add_column("endpoint", style=theme.style("dim"))
+    providers.add_column("key")
+    for entry in config.providers.values():
+        status = ("[good]ready[/good]" if entry.ready else
+                  "[dim]local[/dim]" if entry.local else "[bad]missing[/bad]")
+        marker = " ←" if entry.name == config.provider else ""
+        providers.add_row(entry.name + marker, entry.model, entry.base_url, status)
+    console.print(providers)
+
+    brain = BrainStore(config.paths.brain_db)
+    stats = brain.stats()
+    console.print("\n[title]Brain[/title]")
+    console.print(f"  reflex:  {stats['rules_active']} active rules of "
+                  f"{stats['rules']} · {stats['signals']} signals observed")
+    console.print(f"  memory:  {stats['lessons']} lessons · {stats['skills']} skills · "
+                  f"{stats['episodes']} tasks "
+                  f"({stats['success_rate']:.0%} succeeded)")
+    console.print(f"  index:   {'FTS5' if stats['fts'] else 'python BM25 fallback'}, "
+                  f"{stats['indexed']} records in memory")
+    console.print(f"  file:    [dim]{stats['path']}[/dim]")
+    brain.close()
+
+    console.print("\n[title]Terminal[/title]")
+    console.print(f"  size: {console.size.width}×{console.size.height}")
+    console.print(f"  colour: {console.color_system or 'none'}")
+    console.print(f"  unicode: {console_module.supports_unicode()}")
+    console.print(f"  legacy windows console: {console.legacy_windows}")
+
+    console.print("\n[title]Paths[/title]")
+    console.print(f"  workspace: [dim]{config.paths.project}[/dim]")
+    console.print(f"  config:    [dim]{config.paths.config_file}[/dim]")
+
+    if not config.available():
+        console.print("\n[bad]No provider has an API key.[/bad] Add one to a .env "
+                      "file, or run [accent]comodor --demo[/accent] to try the "
+                      "interface offline.")
+        return 1
+    return 0
+
+
+def run_preview(config: Config, args: argparse.Namespace) -> int:
+    """Render one frame at a fixed size — for screenshots and layout checks."""
+    from .ui import console as console_module
+    from .ui import layout as layout_module
+    from .ui.screen import Screen, ScreenState
+    from .ui.widgets.statusbar import StatusModel
+
+    try:
+        width, _, height = args.size.lower().partition("x")
+        size = (int(width), int(height))
+    except ValueError:
+        print(f"bad size {args.size!r}; expected WIDTHxHEIGHT", file=sys.stderr)
+        return 2
+
+    theme = console_module.prepare_theme(config.ui.theme, config.ui.ascii_borders,
+                                         no_color=False)
+    console = console_module.build(theme, width=size[0], height=size[1],
+                                   record=bool(args.svg))
+    state = ScreenState()
+    state.status = StatusModel(
+        provider=config.active().display if config.active() else "none",
+        model=config.active_model(), connected=bool(config.available()),
+        mode=config.agent.mode, loop=config.agent.loop,
+        gateway="Disable" if not config.gateway.enabled else config.gateway.policy,
+        context_limit=config.agent.context_limit,
+    )
+    geometry = layout_module.compute(size[0], size[1])
+    console.print(Screen(console, theme).render(state, geometry))
+    if args.svg:
+        console.save_svg(args.svg, title="Comodor")
+        print(f"wrote {args.svg}", file=sys.stderr)
+    return 0
+
+
+# --------------------------------------------------------------------------- #
+# entry point
+# --------------------------------------------------------------------------- #
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    config = apply_overrides(load_config(args.cwd), args)
+
+    if args.command == "doctor":
+        return run_doctor(config)
+    if args.command == "preview":
+        return run_preview(config, args)
+    if args.command == "run":
+        return run_headless(config, args)
+
+    if args.demo:
+        from .config import ProviderConfig
+
+        config.providers["fake"] = ProviderConfig(
+            name="fake", kind="fake", base_url="offline", api_key="demo",
+            model="comodor-demo", label="Demo (offline)")
+        config.provider = "fake"
+        config.model = "comodor-demo"
+    elif not config.available():
+        print("No provider is configured.\n"
+              "  · add an API key to a .env file (see .env.example), or\n"
+              "  · run `comodor --demo` to explore the interface offline, or\n"
+              "  · run `comodor doctor` to see what is missing.", file=sys.stderr)
+        return 1
+
+    from .ui.app import App
+
+    resume = None
+    if args.resume and args.resume != "__pick__":
+        resume = args.resume
+
+    app = App(config, demo=args.demo, resume=resume)
+    try:
+        return app.run()
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())