omnipilot-agent 0.1.0__py3-none-any.whl

omnipilot/__init__.py

@@ -0,0 +1,19 @@
+"""
+OmniPilot — your computer, piloted by any LLM.
+
+A cross-platform, provider-agnostic computer-use agent: it lets supported
+large language models (OpenAI, Anthropic, Google Gemini, NVIDIA NIM, Groq,
+OpenRouter, DeepSeek, Mistral, xAI, Together, Ollama, LM Studio and any
+OpenAI-compatible endpoint) operate your machine through auditable tools —
+shell commands, a Python interpreter, the filesystem, applications,
+mouse/keyboard GUI control, screenshots, the clipboard, processes, web
+requests and more — behind a human-in-the-loop permission system.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+__author__ = "xrchris7"
+__license__ = "MIT"
+
+__all__ = ["__version__", "__author__", "__license__"]

omnipilot/__main__.py

@@ -0,0 +1,6 @@
+"""``python -m omnipilot`` entry point."""
+
+from .cli import main_entry
+
+if __name__ == "__main__":
+    main_entry()

omnipilot/agent/__init__.py

@@ -0,0 +1,6 @@
+"""The agentic control loop."""
+
+from .loop import Agent
+from .sessions import SessionStore
+
+__all__ = ["Agent", "SessionStore"]

omnipilot/agent/demo.py

@@ -0,0 +1,182 @@
+"""
+Self-contained, offline demo.
+
+A scripted provider drives the *real* agent loop against the *real* tools in a
+temporary directory, so `omnipilot demo` (and the documentation asset generator)
+show the full experience — plan, approval prompts, tool panels and results —
+without an API key or network access.
+"""
+
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from ..config.settings import PermissionConfig, Settings
+from ..providers.base import BaseProvider, ModelResponse, ToolCall
+from ..tools.base import ToolContext
+from ..tools.registry import ToolRegistry
+from .loop import Agent
+from .prompts import build_system_prompt  # noqa: F401  (re-exported for scripts)
+
+TASK = (
+    "Scaffold a tiny Python CLI project called quizzer, write the code and a "
+    "README, run it to prove it works, then summarize what you built."
+)
+
+QUIZ_PY = '''#!/usr/bin/env python3
+"""Quizzer — a tiny terminal multiple-choice quiz."""
+
+import argparse
+
+QUESTIONS = [
+    {"q": "Which language is OmniPilot written in?",
+     "options": ["Rust", "Python", "Go"], "answer": 1},
+    {"q": "What does MCP stand for?",
+     "options": ["Model Context Protocol", "Mail Control Program",
+                 "Minimal C Parser"], "answer": 0},
+]
+
+
+def run(verbose: bool = False) -> int:
+    score = 0
+    for i, item in enumerate(QUESTIONS, start=1):
+        print(f"Q{i}: {item['q']}")
+        for j, option in enumerate(item["options"], start=1):
+            print(f"  {j}. {option}")
+        if verbose:
+            print(f"  -> answer: {item['options'][item['answer']]}")
+        score += 1
+    print(f"Score: {score}/{len(QUESTIONS)}")
+    return score
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Tiny terminal quiz demo")
+    parser.add_argument("--verbose", action="store_true", help="reveal answers")
+    run(verbose=parser.parse_args().verbose)
+
+
+if __name__ == "__main__":
+    main()
+'''
+
+README_MD = """# quizzer
+
+A tiny terminal quiz generated as an OmniPilot demo.
+
+## Run
+
+```bash
+python quiz.py          # take the quiz
+python quiz.py --verbose  # reveal answers
+```
+"""
+
+
+class ScriptedDemoProvider(BaseProvider):
+    """Deterministic three-turn provider used by the offline demo."""
+
+    name = "demo"
+    supports_vision = False
+
+    def __init__(self) -> None:
+        super().__init__(model="scripted-demo-1", api_key="demo")
+        self._turn = 0
+
+    def complete(self, messages, tools=None, *, temperature=0.2, max_tokens=4096):
+        tool_results = [m for m in messages if m.get("role") == "tool"]
+        turn = len(tool_results)
+
+        if turn == 0:
+            return ModelResponse(
+                text=(
+                    "I'll scaffold a small **quizzer** CLI project: create the "
+                    "folder, write `quiz.py` and a `README.md`, then execute it to "
+                    "verify everything works."
+                ),
+                tool_calls=[
+                    ToolCall(id="d1", name="fs.make_directory", arguments={"path": "quizzer"}),
+                    ToolCall(
+                        id="d2",
+                        name="fs.write_file",
+                        arguments={"path": "quizzer/quiz.py", "content": QUIZ_PY},
+                    ),
+                    ToolCall(
+                        id="d3",
+                        name="fs.write_file",
+                        arguments={"path": "quizzer/README.md", "content": README_MD},
+                    ),
+                ],
+            )
+        if turn == 3:
+            return ModelResponse(
+                text=("Let's run it to verify the CLI works, including `--help`."),
+                tool_calls=[
+                    ToolCall(
+                        id="d4",
+                        name="shell.run_command",
+                        arguments={"command": "python quizzer/quiz.py --verbose"},
+                    ),
+                ],
+            )
+        if turn == 4:
+            return ModelResponse(
+                text=(
+                    "Done — I built **quizzer** in `./quizzer`:\n\n"
+                    "- `quiz.py` — a 2-question terminal quiz with argparse "
+                    "(`--verbose` reveals answers)\n"
+                    "- `README.md` — run instructions\n\n"
+                    "I executed `python quizzer/quiz.py --verbose` and it printed "
+                    "both questions and scored 2/2, so the project works end to end."
+                ),
+                tool_calls=[],
+            )
+        # Defensive: no more tools.
+        return ModelResponse(text="(demo complete)", tool_calls=[])
+
+
+def build_demo_agent(ui: Any, workdir: str | None = None) -> tuple[Agent, Any]:
+    """Build an Agent wired to the scripted demo provider in a temp workspace."""
+    import os
+
+    workspace = Path(workdir) if workdir else Path(tempfile.mkdtemp(prefix="omnipilot-demo-"))
+    workspace.mkdir(parents=True, exist_ok=True)
+    os.chdir(workspace)
+
+    settings = Settings()
+    settings.default_provider = "demo"
+    settings.permission = PermissionConfig(mode="ask", auto_approve_low=True)
+    settings.tools.workdir = str(workspace)
+
+    registry = ToolRegistry.build_default(settings)
+    provider = ScriptedDemoProvider()
+
+    from ..permission.policy import PermissionPolicy
+
+    # Approvals in the demo are auto-accepted but still rendered.
+    def _render_and_approve(*, tool, arguments, risk, allow_tool_wide=True):
+        from ..ui.approvals import render_approval
+
+        render_approval(ui.console, tool, arguments, risk, "y")
+        return "y"
+
+    policy = PermissionPolicy(settings, console=ui.console, prompt=_render_and_approve)
+
+    agent = Agent(
+        settings=settings,
+        provider=provider,
+        provider_name="demo",
+        registry=registry,
+        policy=policy,
+        model="offline-demo (no API key)",
+        ui=ui,
+    )
+    return agent, workspace
+
+
+def demo_tool_context(settings: Settings, cwd: str) -> ToolContext:
+    import logging
+
+    return ToolContext(settings=settings, logger=logging.getLogger("demo"), console=None, cwd=cwd)

omnipilot/agent/loop.py

@@ -0,0 +1,299 @@
+"""
+The agent loop: messages go in, the model replies or calls tools, calls are
+permission-checked and executed, results feed back, until the model produces
+a final answer (or the iteration/abort budget is exhausted).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ..config.settings import Settings
+from ..constants import APP_NAME, MODEL_PRICES
+from ..permission.policy import PermissionPolicy
+from ..providers.base import (
+    BaseProvider,
+    ModelResponse,
+    ProviderError,
+    ToolCall,
+    Usage,
+)
+from ..tools.base import ToolContext, ToolResult
+from ..tools.registry import ToolRegistry
+from ..utils.images import EncodedImage, encode_file
+from .prompts import build_system_prompt
+from .sessions import SessionStore
+
+
+class NullUI:
+    """No-op event sink (library usage / tests)."""
+
+    def assistant_message(self, text: str) -> None: ...
+    def tool_started(self, index: int, total: int, call: ToolCall) -> None: ...
+    def tool_finished(self, name: str, result: ToolResult) -> None: ...
+    def tool_denied(self, name: str, reason: str) -> None: ...
+    def info(self, message: str) -> None: ...
+    def error(self, message: str) -> None: ...
+    def system_event(self, message: str) -> None: ...
+
+
+@dataclass
+class Agent:
+    settings: Settings
+    provider: BaseProvider
+    provider_name: str
+    registry: ToolRegistry
+    policy: PermissionPolicy
+    model: str
+    ui: Any = field(default_factory=NullUI)
+    logger: logging.Logger | None = None
+    session_id: str | None = None
+    extra_system_prompt: str | None = None
+    messages: list[dict] = field(default_factory=list)
+    usage: Usage = field(default_factory=Usage)
+    cost_estimate: float = 0.0
+    tool_runs: int = 0
+    mcp_manager: Any = None
+
+    def __post_init__(self) -> None:
+        self.session_id = self.session_id or SessionStore().new_id()
+        if self.logger is None:
+            self.logger = logging.getLogger(APP_NAME.lower())
+        self.vision_enabled = self.settings.vision_enabled_for(self.provider_name)
+        self.context = ToolContext(
+            settings=self.settings,
+            logger=self.logger,
+            console=getattr(self.ui, "console", None),
+            cwd=self.settings.tools.workdir or os.getcwd(),
+        )
+
+    # -- conversation management ----------------------------------------------
+
+    def reset(self) -> None:
+        self.messages.clear()
+        self.usage = Usage()
+        self.cost_estimate = 0.0
+        self.tool_runs = 0
+        self.session_id = SessionStore().new_id()
+
+    def add_user_message(self, text: str, image_paths: list[str] | None = None) -> dict:
+        if not image_paths:
+            message = {"role": "user", "content": text}
+        else:
+            parts: list[dict] = [{"type": "text", "text": text or "Look at this image:"}]
+            for raw_path in image_paths:
+                path = Path(raw_path).expanduser()
+                encoded = _re_encode(path)
+                parts.append(
+                    {
+                        "type": "image",
+                        "image": {"media_type": encoded.media_type, "data": encoded.data},
+                    }
+                )
+            message = {"role": "user", "content": parts}
+        self.messages.append(message)
+        return message
+
+    def restore(self, messages: list[dict]) -> None:
+        """Load history from a saved session."""
+        self.messages = messages
+
+    # -- the loop --------------------------------------------------------------
+
+    @property
+    def system_prompt(self) -> str:
+        return build_system_prompt(
+            self.registry,
+            vision=self.vision_enabled,
+            permission_mode=self.settings.permission.mode,
+            extra=self.extra_system_prompt,
+        )
+
+    def run(
+        self,
+        user_text: str | None = None,
+        image_paths: list[str] | None = None,
+        should_cancel: Callable[[], bool] | None = None,
+    ) -> str:
+        if user_text:
+            self.add_user_message(user_text, image_paths if self.vision_enabled else None)
+            if image_paths and not self.vision_enabled:
+                self.ui.info(
+                    "Vision is off for this provider/model; images were saved but "
+                    "not sent to the model."
+                )
+
+        final_text = ""
+        for _step in range(1, self.settings.agent.max_iterations + 1):
+            if should_cancel and should_cancel():
+                return "(task cancelled)"
+            self._compact_if_needed()
+            request_messages = [{"role": "system", "content": self.system_prompt}, *self.messages]
+            try:
+                response: ModelResponse = self.provider.complete(
+                    request_messages,
+                    tools=self.registry.schemas(self.provider_name),
+                    temperature=self.settings.agent.temperature,
+                    max_tokens=self.settings.agent.max_tokens,
+                )
+            except ProviderError as exc:
+                self.ui.error(str(exc))
+                self.logger.error("provider error: %s", exc)
+                note = f"(provider error: {exc})"
+                self.messages.append({"role": "assistant", "content": note})
+                return note
+            except Exception as exc:  # noqa: BLE001 - network oddities
+                self.ui.error(f"Unexpected error calling the model: {exc}")
+                self.logger.exception("unexpected provider failure")
+                return f"(unexpected error: {exc})"
+
+            self._account_usage(response)
+
+            if response.text:
+                self.ui.assistant_message(response.text)
+
+            if not response.tool_calls:
+                self.messages.append(
+                    {
+                        "role": "assistant",
+                        "content": response.text or "(done)",
+                    }
+                )
+                final_text = response.text or "(task complete)"
+                break
+
+            # Assistant turn requesting tools.
+            self.messages.append(
+                {
+                    "role": "assistant",
+                    "content": response.text or "",
+                    "tool_calls": response.tool_calls,
+                }
+            )
+            if response.text:
+                self.ui.info(f"Planning {len(response.tool_calls)} action(s)...")
+
+            total = len(response.tool_calls)
+            for index, call in enumerate(response.tool_calls, 1):
+                if should_cancel and should_cancel():
+                    return "(task cancelled)"
+                result = self._execute_tool(index, total, call)
+                tool_message: dict[str, Any] = {
+                    "role": "tool",
+                    "tool_call_id": call.id,
+                    "name": call.name,
+                    "content": result.message_for_model(),
+                    "success": result.success,
+                }
+                if result.image and self.vision_enabled:
+                    tool_message["image"] = result.image
+                self.messages.append(tool_message)
+                if not result.success and self.settings.agent.abort_on_error and not result.denied:
+                    return f"(aborted after tool failure: {result.error})"
+        else:
+            final_text = (
+                f"(reached the maximum of {self.settings.agent.max_iterations} tool-calling "
+                "iterations; you can ask me to continue)"
+            )
+            self.ui.error(final_text)
+            self.messages.append({"role": "assistant", "content": final_text})
+
+        self._autosave()
+        return final_text
+
+    # -- tool execution --------------------------------------------------------
+
+    def _execute_tool(self, index: int, total: int, call: ToolCall) -> ToolResult:
+        self.ui.tool_started(index, total, call)
+        tool = self.registry.get(call.name)
+        if tool is None:
+            result = ToolResult.fail(
+                f"unknown tool {call.name!r}. Available tools: {', '.join(self.registry.names())}"
+            )
+            self.ui.tool_finished(call.name, result)
+            return result
+
+        arguments = call.arguments if isinstance(call.arguments, dict) else {}
+        try:
+            decision = self.policy.check(call.name, arguments, base_risk=tool.risk)
+        except KeyboardInterrupt:  # pragma: no cover - answering the prompt
+            return ToolResult.denied_result("approval interrupted (Ctrl+C)")
+
+        if not decision.allowed:
+            result = ToolResult.denied_result(decision.reason)
+            self.ui.tool_denied(call.name, decision.reason)
+            return result
+
+        result = tool.execute(self.context, arguments)
+        self.ui.tool_finished(call.name, result)
+        self.tool_runs += 1
+        return result
+
+    # -- accounting / context --------------------------------------------------
+
+    def _account_usage(self, response: ModelResponse) -> None:
+        self.usage.add(response.usage)
+        for needle, (in_price, out_price) in MODEL_PRICES.items():
+            if needle in self.model.lower():
+                self.cost_estimate += (
+                    response.usage.input_tokens / 1000 * in_price
+                    + response.usage.output_tokens / 1000 * out_price
+                )
+                break
+
+    def _compact_if_needed(self) -> None:
+        """Elide verbose old tool output once the history budget is exceeded."""
+        budget_chars = self.settings.agent.history_token_budget * 3
+        total = sum(len(json.dumps(m, default=str)) for m in self.messages)
+        if total <= budget_chars:
+            return
+        keep_recent = 14
+        for msg in self.messages[:-keep_recent]:
+            if msg.get("role") == "tool" and len(str(msg.get("content", ""))) > 600:
+                msg["content"] = (
+                    "[older tool output elided to save context; re-run the tool if needed]\n"
+                    + str(msg["content"])[:300]
+                )
+                msg.pop("image", None)
+
+    def _autosave(self) -> None:
+        try:
+            SessionStore().save(self.session_id, self.session_payload())
+        except Exception as exc:  # noqa: BLE001  # pragma: no cover
+            self.logger.debug("autosave failed: %s", exc)
+
+    def session_payload(self) -> dict[str, Any]:
+        return {
+            "id": self.session_id,
+            "provider": self.provider_name,
+            "model": self.model,
+            "messages": self.messages,
+            "usage": {
+                "input_tokens": self.usage.input_tokens,
+                "output_tokens": self.usage.output_tokens,
+            },
+            "cost_estimate": round(self.cost_estimate, 5),
+        }
+
+    def save(self, name: str | None = None) -> Path:
+        session_id = name or self.session_id
+        path = SessionStore().save(session_id, self.session_payload())
+        if name:
+            self.session_id = name
+        return path
+
+
+def _re_encode(path: Path) -> EncodedImage:
+    image = encode_file(path)
+    from ..utils.images import downscale as _downscale
+
+    return EncodedImage(
+        media_type=image.media_type,
+        data=__import__("base64").b64encode(_downscale(image.raw_bytes())).decode("ascii"),
+    )

omnipilot/agent/prompts.py

@@ -0,0 +1,108 @@
+"""System prompt construction."""
+
+from __future__ import annotations
+
+import datetime
+import os
+
+from ..tools.registry import ToolRegistry
+from ..utils.platform_tools import IS_WINDOWS, default_shell, platform_summary
+
+SAFETY_RULES = """\
+## Safety and ethics (non-negotiable)
+- You act on a REAL computer belonging to the user. Destructive actions (deleting files,
+  force-killing processes, formatting, registry changes, force pushes, privilege escalation)
+  trigger explicit approval. Never try to bypass, obfuscate or automate-around a denial;
+  the permission system is the user's, not an obstacle to defeat.
+- Never read, print, upload or transmit secrets (API keys, passwords, tokens, browser
+  credential stores, SSH keys) unless the user explicitly asks for that exact action.
+- Never run fork bombs, crypto miners, self-replicating code, ransomware-like encryption,
+  or anything that hides itself. Refuse and explain.
+- When a command fails TWICE, stop and reason about a different approach instead of
+  repeating it.
+- Prefer dedicated tools (fs.*, apps.*, gui.*) over shell commands when one exists;
+  they are safer and give structured results.
+- If the user's request is ambiguous in a way that could cause damage, ask a clarifying
+  question (just reply with text; do not run tools speculatively).
+"""
+
+GUI_RULES = """\
+## Controlling the graphical interface (computer use)
+- Before ANY mouse/keyboard action, call screen.screenshot and LOOK at the returned image
+  to read exact coordinates and window state. Coordinates are pixels from the top-left.
+- Act in small steps: screenshot -> one action -> screenshot to verify -> continue.
+- Use gui.hotkey for keyboard shortcuts (ctrl+c, alt+tab, command+space, ...) and
+  gui.press_key for single keys. Use gui.type_text for text fields (click the field first!).
+- If your click missed, re-screenshot and correct coordinates; do not spam clicks.
+- Moving the mouse to the screen corner (0,0) triggers the pyautogui fail-safe abort.
+"""
+
+
+def build_system_prompt(
+    registry: ToolRegistry, *, vision: bool, permission_mode: str, extra: str | None = None
+) -> str:
+    info = platform_summary()
+    shell_exe, shell_args = default_shell()
+    now = datetime.datetime.now().astimezone()
+    shell_kind = "cmd.exe (interpreter='powershell' available)" if IS_WINDOWS else shell_exe
+
+    tool_names = registry.names()
+    tool_catalog = "\n".join(f"- {n}" for n in tool_names)
+
+    prompt = f"""\
+You are OmniPilot, an expert AI computer-use agent operating the user's real machine.
+You can observe the screen, move the mouse, type, run shell commands, execute Python,
+manage files, open and close applications, browse the web and automate almost anything —
+through the tool functions provided to you.
+
+## Current environment
+- Date/time: {now.strftime("%A %Y-%m-%d %H:%M %Z")}
+- Operating system: {info["os_release"]}
+- Host: {info["hostname"]} | logged-in user: {info["user"]}
+- Current working directory: {os.getcwd()}
+- Default shell: {shell_kind}
+- Python: {info["python"]} (the python.run_python tool uses a persistent in-process namespace)
+- Permission mode: {permission_mode} (ask = the user approves actions; auto = safe read-only
+  actions run unattended; yolo = no prompts)
+- Vision: {
+        "ENABLED — you receive screenshots as images"
+        if vision
+        else "DISABLED — screenshots are only saved to disk and described to you; avoid GUI actions that require seeing the screen"
+    }
+
+## Available tools ({len(tool_names)})
+{tool_catalog}
+
+## How you operate
+1. Understand the goal, then form a short plan. For multi-step work, briefly tell the user
+   the plan in one or two sentences before calling tools.
+2. Observe first: run system/list/screenshot tools to gather state; act second.
+3. Use ONE tool call per logical step when the steps depend on each other. You may batch
+   independent calls together.
+4. After completing the task, give a concise summary: what you did, the result, and
+   anything the user should check. Then STOP calling tools.
+5. If an action is denied, accept it, do not attempt workarounds, and either pick another
+   approach or tell the user what permission/rule is needed.
+6. Shell commands run in a fresh interpreter each time (working directory persists, but
+   shell variables do not). For stateful logic use python.run_python instead.
+7. Use absolute paths or paths relative to the current working directory shown above.
+8. Keep responses tight and practical; show command output only when it matters.
+
+{
+        GUI_RULES
+        if vision
+        else GUI_RULES.replace(
+            "call screen.screenshot and LOOK at the returned image to read exact coordinates",
+            "vision is disabled; prefer CLI/app/API approaches and only use GUI tools when the user "
+            "confirms they can watch the screen",
+        )
+    }
+
+{SAFETY_RULES}
+
+Remember: you are granted real control. Be competent, be careful, and confirm anything
+that is hard to reverse.
+"""
+    if extra:
+        prompt += f"\n## Additional user instructions\n{extra}\n"
+    return prompt