gothos 0.2.0__py3-none-any.whl

gothos/__init__.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2026 GouthamRajam. MIT License.
+
+"""GothOS — Android AI agent with safety, memory, planning and provider-agnostic grounding."""
+
+__version__ = "0.2.0"
+
+from gothos import (
+    benchmark,
+    console,
+    core,
+    device,
+    extract,
+    grounding,
+    llm,
+    mcp,
+    memory,
+    planner,
+    safety,
+    screen,
+)
+
+__all__ = [
+    "benchmark",
+    "console",
+    "core",
+    "device",
+    "extract",
+    "grounding",
+    "llm",
+    "mcp",
+    "memory",
+    "planner",
+    "safety",
+    "screen",
+]

gothos/__main__.py

@@ -0,0 +1,5 @@
+"""Allow ``python -m gothos`` to invoke the CLI."""
+
+from gothos.cli import main
+
+main()

gothos/benchmark/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2026 GouthamRajam. MIT License.
+
+"""Benchmark harness — run GothOS against standardised task suites."""
+
+from gothos.benchmark.harness import BenchmarkHarness, BenchmarkResult, TaskResult
+from gothos.benchmark.suites import TaskSuite, load_suite
+
+__all__ = [
+    "BenchmarkHarness",
+    "BenchmarkResult",
+    "TaskResult",
+    "TaskSuite",
+    "load_suite",
+]

gothos/benchmark/harness.py

@@ -0,0 +1,237 @@
+# Copyright (c) 2026 GouthamRajam. MIT License.
+
+"""Benchmark harness — executes task suites and collects results."""
+
+from __future__ import annotations
+
+import logging
+import time
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from gothos.benchmark.suites import TaskCase, TaskSuite
+
+logger = logging.getLogger(__name__)
+
+
+class TaskResult(BaseModel):
+    """Outcome of running one benchmark task."""
+
+    task_id: str
+    goal: str
+    category: str = ""
+    difficulty: str = ""
+    success: bool = False
+    steps_taken: int = 0
+    max_steps: int = 0
+    duration_seconds: float = 0.0
+    execution_mode: str = ""
+    error: str | None = None
+
+
+class BenchmarkResult(BaseModel):
+    """Aggregate results from a full benchmark run."""
+
+    suite_name: str
+    suite_version: str = ""
+    provider: str = ""
+    model: str = ""
+    execution_mode: str = ""
+    total_tasks: int = 0
+    passed: int = 0
+    failed: int = 0
+    skipped: int = 0
+    pass_rate: float = Field(default=0.0, ge=0.0, le=100.0)
+    total_duration_seconds: float = 0.0
+    avg_steps: float = 0.0
+    results: list[TaskResult] = Field(default_factory=list)
+
+    def by_category(self) -> dict[str, dict[str, Any]]:
+        """Break down pass rates by category."""
+        cats: dict[str, list[TaskResult]] = {}
+        for r in self.results:
+            cats.setdefault(r.category, []).append(r)
+        return {
+            cat: {
+                "total": len(rs),
+                "passed": sum(1 for r in rs if r.success),
+                "pass_rate": round(
+                    sum(1 for r in rs if r.success) / len(rs) * 100, 1
+                ),
+            }
+            for cat, rs in cats.items()
+        }
+
+    def by_difficulty(self) -> dict[str, dict[str, Any]]:
+        """Break down pass rates by difficulty."""
+        diffs: dict[str, list[TaskResult]] = {}
+        for r in self.results:
+            diffs.setdefault(r.difficulty, []).append(r)
+        return {
+            diff: {
+                "total": len(rs),
+                "passed": sum(1 for r in rs if r.success),
+                "pass_rate": round(
+                    sum(1 for r in rs if r.success) / len(rs) * 100, 1
+                ),
+            }
+            for diff, rs in diffs.items()
+        }
+
+    def summary(self) -> str:
+        """Human-readable summary."""
+        lines = [
+            f"Benchmark: {self.suite_name} v{self.suite_version}",
+            f"Provider:  {self.provider} / {self.model}",
+            f"Mode:      {self.execution_mode}",
+            f"Results:   {self.passed}/{self.total_tasks} passed "
+            f"({self.pass_rate:.1f}%)",
+            f"Duration:  {self.total_duration_seconds:.1f}s total, "
+            f"{self.avg_steps:.1f} avg steps",
+        ]
+
+        cats = self.by_category()
+        if cats:
+            lines.append("\nBy category:")
+            for cat, stats in sorted(cats.items()):
+                lines.append(
+                    f"  {cat}: {stats['passed']}/{stats['total']} "
+                    f"({stats['pass_rate']}%)"
+                )
+
+        return "\n".join(lines)
+
+    def to_json(self, path: str | Path) -> None:
+        """Write results to a JSON file."""
+        Path(path).write_text(self.model_dump_json(indent=2))
+
+
+class BenchmarkHarness:
+    """Runs a :class:`TaskSuite` against a GothOS agent and collects results.
+
+    Usage::
+
+        harness = BenchmarkHarness(
+            provider="anthropic",
+            model="claude-sonnet-4-20250514",
+            execution_mode="auto",
+        )
+        result = await harness.run(suite)
+        print(result.summary())
+        result.to_json("benchmark_results.json")
+    """
+
+    def __init__(
+        self,
+        provider: str = "anthropic",
+        model: str = "claude-sonnet-4-20250514",
+        execution_mode: str = "auto",
+        device_serial: str | None = None,
+        **agent_kwargs: Any,
+    ) -> None:
+        self._provider = provider
+        self._model = model
+        self._execution_mode = execution_mode
+        self._device_serial = device_serial
+        self._agent_kwargs = agent_kwargs
+
+    async def run(
+        self,
+        suite: TaskSuite,
+        skip_categories: set[str] | None = None,
+    ) -> BenchmarkResult:
+        """Execute every task in *suite* and return aggregate results."""
+
+        results: list[TaskResult] = []
+        skipped = 0
+        start = time.monotonic()
+
+        for task in suite.tasks:
+            if skip_categories and task.category in skip_categories:
+                skipped += 1
+                logger.info("Skipping task %s (category: %s)", task.id, task.category)
+                continue
+
+            logger.info("Running task %s: %s", task.id, task.goal)
+            task_result = await self._run_task(task)
+            results.append(task_result)
+
+            status = "PASS" if task_result.success else "FAIL"
+            logger.info(
+                "  %s — %s (%.1fs, %d steps)",
+                task.id, status, task_result.duration_seconds, task_result.steps_taken,
+            )
+
+        total_duration = time.monotonic() - start
+        passed = sum(1 for r in results if r.success)
+        total = len(results)
+
+        return BenchmarkResult(
+            suite_name=suite.name,
+            suite_version=suite.version,
+            provider=self._provider,
+            model=self._model,
+            execution_mode=self._execution_mode,
+            total_tasks=total + skipped,
+            passed=passed,
+            failed=total - passed,
+            skipped=skipped,
+            pass_rate=round(passed / total * 100, 1) if total else 0.0,
+            total_duration_seconds=round(total_duration, 2),
+            avg_steps=round(
+                sum(r.steps_taken for r in results) / total, 1
+            ) if total else 0.0,
+            results=results,
+        )
+
+    async def _run_task(self, task: TaskCase) -> TaskResult:
+        """Run a single task and return its result."""
+        from gothos.core.agent import GothOSAgent
+        from gothos.core.config import AgentConfig
+
+        config = AgentConfig(
+            llm_provider=self._provider,
+            model=self._model,
+            execution_mode=self._execution_mode,
+            device_serial=self._device_serial,
+            max_steps=task.max_steps,
+            **self._agent_kwargs,
+        )
+
+        agent = GothOSAgent(config)
+
+        try:
+            await agent.initialize()
+            run_result = await agent.run(
+                goal=task.goal,
+                app_package=task.app_package,
+            )
+            return TaskResult(
+                task_id=task.id,
+                goal=task.goal,
+                category=task.category,
+                difficulty=task.difficulty,
+                success=run_result.success,
+                steps_taken=run_result.steps_taken,
+                max_steps=task.max_steps,
+                duration_seconds=run_result.duration_seconds,
+                execution_mode=run_result.execution_mode,
+                error=run_result.error_message,
+            )
+        except Exception as exc:
+            logger.exception("Task %s crashed: %s", task.id, exc)
+            return TaskResult(
+                task_id=task.id,
+                goal=task.goal,
+                category=task.category,
+                difficulty=task.difficulty,
+                success=False,
+                error=str(exc),
+            )
+        finally:
+            import contextlib
+
+            with contextlib.suppress(Exception):
+                await agent.shutdown()

gothos/benchmark/suites.py

@@ -0,0 +1,205 @@
+# Copyright (c) 2026 GouthamRajam. MIT License.
+
+"""Task suite definitions for benchmarking."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+
+class TaskCase(BaseModel):
+    """A single benchmark task."""
+
+    id: str
+    goal: str
+    app_package: str
+    category: str = "general"
+    difficulty: str = "medium"
+    max_steps: int = 50
+    expected_outcome: str = ""
+    verification_steps: list[str] = Field(default_factory=list)
+
+
+class TaskSuite(BaseModel):
+    """A collection of benchmark tasks."""
+
+    name: str
+    version: str = "1.0"
+    description: str = ""
+    tasks: list[TaskCase] = Field(default_factory=list)
+
+    @property
+    def task_count(self) -> int:
+        return len(self.tasks)
+
+    def by_category(self) -> dict[str, list[TaskCase]]:
+        groups: dict[str, list[TaskCase]] = {}
+        for task in self.tasks:
+            groups.setdefault(task.category, []).append(task)
+        return groups
+
+    def by_difficulty(self) -> dict[str, list[TaskCase]]:
+        groups: dict[str, list[TaskCase]] = {}
+        for task in self.tasks:
+            groups.setdefault(task.difficulty, []).append(task)
+        return groups
+
+
+def load_suite(path: str | Path) -> TaskSuite:
+    """Load a task suite from a JSON file."""
+    data = json.loads(Path(path).read_text())
+    return TaskSuite.model_validate(data)
+
+
+# ---------------------------------------------------------------------------
+# Built-in suites
+# ---------------------------------------------------------------------------
+
+CORE_SUITE = TaskSuite(
+    name="gothos-core",
+    version="1.0",
+    description="Core functionality benchmark — basic device operations",
+    tasks=[
+        TaskCase(
+            id="core-001",
+            goal="Open the Settings app",
+            app_package="com.android.settings",
+            category="navigation",
+            difficulty="easy",
+            max_steps=5,
+            expected_outcome="Settings app is visible",
+        ),
+        TaskCase(
+            id="core-002",
+            goal="Enable Wi-Fi in Settings",
+            app_package="com.android.settings",
+            category="settings",
+            difficulty="easy",
+            max_steps=10,
+            expected_outcome="Wi-Fi toggle is enabled",
+        ),
+        TaskCase(
+            id="core-003",
+            goal="Open Chrome and navigate to google.com",
+            app_package="com.android.chrome",
+            category="navigation",
+            difficulty="medium",
+            max_steps=15,
+            expected_outcome="Google search page is displayed",
+        ),
+        TaskCase(
+            id="core-004",
+            goal="Take a screenshot and save it",
+            app_package="",
+            category="device",
+            difficulty="easy",
+            max_steps=5,
+            expected_outcome="Screenshot captured",
+        ),
+        TaskCase(
+            id="core-005",
+            goal="Open Contacts and search for a name",
+            app_package="com.android.contacts",
+            category="search",
+            difficulty="medium",
+            max_steps=15,
+            expected_outcome="Contact search results displayed",
+        ),
+        TaskCase(
+            id="core-006",
+            goal="Change display brightness to maximum",
+            app_package="com.android.settings",
+            category="settings",
+            difficulty="medium",
+            max_steps=15,
+            expected_outcome="Brightness slider at maximum",
+        ),
+        TaskCase(
+            id="core-007",
+            goal="Enable Dark Mode in display settings",
+            app_package="com.android.settings",
+            category="settings",
+            difficulty="medium",
+            max_steps=20,
+            expected_outcome="Dark mode is active",
+        ),
+        TaskCase(
+            id="core-008",
+            goal="Open the calculator and compute 42 * 7",
+            app_package="com.google.android.calculator",
+            category="app_interaction",
+            difficulty="medium",
+            max_steps=15,
+            expected_outcome="Result 294 is displayed",
+        ),
+        TaskCase(
+            id="core-009",
+            goal="Set an alarm for 7:30 AM",
+            app_package="com.google.android.deskclock",
+            category="app_interaction",
+            difficulty="hard",
+            max_steps=25,
+            expected_outcome="Alarm set for 7:30 AM",
+        ),
+        TaskCase(
+            id="core-010",
+            goal="Open Google Maps and search for restaurants nearby",
+            app_package="com.google.android.apps.maps",
+            category="search",
+            difficulty="hard",
+            max_steps=20,
+            expected_outcome="Map shows restaurant results",
+        ),
+    ],
+)
+
+CROSS_APP_SUITE = TaskSuite(
+    name="gothos-cross-app",
+    version="1.0",
+    description="Cross-app workflows — tasks spanning multiple applications",
+    tasks=[
+        TaskCase(
+            id="xapp-001",
+            goal="Copy text from Chrome and paste it into Notes",
+            app_package="com.android.chrome",
+            category="cross_app",
+            difficulty="hard",
+            max_steps=30,
+            expected_outcome="Text appears in Notes app",
+        ),
+        TaskCase(
+            id="xapp-002",
+            goal="Take a photo with Camera and share it via Messages",
+            app_package="com.android.camera",
+            category="cross_app",
+            difficulty="hard",
+            max_steps=35,
+            expected_outcome="Share dialog shows Messages option",
+        ),
+        TaskCase(
+            id="xapp-003",
+            goal="Look up a contact in Contacts and call them via Phone",
+            app_package="com.android.contacts",
+            category="cross_app",
+            difficulty="hard",
+            max_steps=25,
+            expected_outcome="Phone dialer shows contact number",
+        ),
+    ],
+)
+
+BUILT_IN_SUITES: dict[str, TaskSuite] = {
+    "core": CORE_SUITE,
+    "cross-app": CROSS_APP_SUITE,
+}
+
+
+def get_suite(name: str) -> TaskSuite:
+    """Get a built-in suite by name."""
+    if name not in BUILT_IN_SUITES:
+        available = ", ".join(BUILT_IN_SUITES.keys())
+        raise ValueError(f"Unknown suite '{name}'. Available: {available}")
+    return BUILT_IN_SUITES[name]