gemcode 0.3.23__tar.gz → 0.3.24__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. {gemcode-0.3.23/src/gemcode.egg-info → gemcode-0.3.24}/PKG-INFO +1 -1
  2. {gemcode-0.3.23 → gemcode-0.3.24}/pyproject.toml +1 -1
  3. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/agent.py +141 -1
  4. gemcode-0.3.24/src/gemcode/computer_use/browser_computer.py +361 -0
  5. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/repl_commands.py +3 -0
  6. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/repl_slash.py +90 -1
  7. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/session_runtime.py +16 -1
  8. gemcode-0.3.24/src/gemcode/tools/browser.py +202 -0
  9. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tui/input_handler.py +1 -0
  10. {gemcode-0.3.23 → gemcode-0.3.24/src/gemcode.egg-info}/PKG-INFO +1 -1
  11. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode.egg-info/SOURCES.txt +1 -0
  12. gemcode-0.3.23/src/gemcode/computer_use/browser_computer.py +0 -275
  13. {gemcode-0.3.23 → gemcode-0.3.24}/LICENSE +0 -0
  14. {gemcode-0.3.23 → gemcode-0.3.24}/MANIFEST.in +0 -0
  15. {gemcode-0.3.23 → gemcode-0.3.24}/README.md +0 -0
  16. {gemcode-0.3.23 → gemcode-0.3.24}/setup.cfg +0 -0
  17. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/__init__.py +0 -0
  18. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/__main__.py +0 -0
  19. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/audit.py +0 -0
  20. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/autocompact.py +0 -0
  21. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/callbacks.py +0 -0
  22. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/capability_routing.py +0 -0
  23. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/cli.py +0 -0
  24. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/compaction.py +0 -0
  25. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/computer_use/__init__.py +0 -0
  26. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/config.py +0 -0
  27. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/context_budget.py +0 -0
  28. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/context_warning.py +0 -0
  29. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/credentials.py +0 -0
  30. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/hitl_session.py +0 -0
  31. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/interactions.py +0 -0
  32. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/invoke.py +0 -0
  33. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/kairos_daemon.py +0 -0
  34. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/limits.py +0 -0
  35. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/live_audio_engine.py +0 -0
  36. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/logging_config.py +0 -0
  37. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/mcp_loader.py +0 -0
  38. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/memory/__init__.py +0 -0
  39. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/memory/embedding_memory_service.py +0 -0
  40. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/memory/file_memory_service.py +0 -0
  41. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/modality_tools.py +0 -0
  42. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/model_errors.py +0 -0
  43. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/model_routing.py +0 -0
  44. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/paths.py +0 -0
  45. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/permissions.py +0 -0
  46. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/plugins/__init__.py +0 -0
  47. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/plugins/terminal_hooks_plugin.py +0 -0
  48. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/plugins/tool_recovery_plugin.py +0 -0
  49. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/prompt_suggestions.py +0 -0
  50. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/query/__init__.py +0 -0
  51. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/query/config.py +0 -0
  52. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/query/deps.py +0 -0
  53. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/query/engine.py +0 -0
  54. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/query/stop_hooks.py +0 -0
  55. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/query/token_budget.py +0 -0
  56. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/query/transitions.py +0 -0
  57. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/slash_commands.py +0 -0
  58. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/thinking.py +0 -0
  59. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tool_prompt_manifest.py +0 -0
  60. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tool_registry.py +0 -0
  61. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/__init__.py +0 -0
  62. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/bash.py +0 -0
  63. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/edit.py +0 -0
  64. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/filesystem.py +0 -0
  65. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/search.py +0 -0
  66. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/shell.py +0 -0
  67. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/shell_gate.py +0 -0
  68. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/subtask.py +0 -0
  69. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/think.py +0 -0
  70. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/todo.py +0 -0
  71. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools/web.py +0 -0
  72. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tools_inspector.py +0 -0
  73. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/trust.py +0 -0
  74. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tui/scrollback.py +0 -0
  75. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tui/spinner.py +0 -0
  76. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tui/welcome_banner.py +0 -0
  77. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/tui/welcome_rich.py +0 -0
  78. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/version.py +0 -0
  79. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/vertex.py +0 -0
  80. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/web/__init__.py +0 -0
  81. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/web/claude_sse_adapter.py +0 -0
  82. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/web/terminal_repl.py +0 -0
  83. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode/workspace_hints.py +0 -0
  84. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode.egg-info/dependency_links.txt +0 -0
  85. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode.egg-info/entry_points.txt +0 -0
  86. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode.egg-info/requires.txt +0 -0
  87. {gemcode-0.3.23 → gemcode-0.3.24}/src/gemcode.egg-info/top_level.txt +0 -0
  88. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_agent_instruction.py +0 -0
  89. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_autocompact.py +0 -0
  90. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_capability_routing.py +0 -0
  91. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_claude_web_adapter_sse.py +0 -0
  92. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_cli_init.py +0 -0
  93. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_computer_use_permissions.py +0 -0
  94. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_context_budget.py +0 -0
  95. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_context_warning.py +0 -0
  96. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_credentials.py +0 -0
  97. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_interactive_permission_ask.py +0 -0
  98. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_kairos_scheduler.py +0 -0
  99. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_modality_tools.py +0 -0
  100. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_model_error_retry.py +0 -0
  101. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_model_errors.py +0 -0
  102. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_model_routing.py +0 -0
  103. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_paths.py +0 -0
  104. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_permissions.py +0 -0
  105. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_prompt_suggestions.py +0 -0
  106. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_repl_commands.py +0 -0
  107. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_repl_slash.py +0 -0
  108. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_slash_commands.py +0 -0
  109. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_thinking_config.py +0 -0
  110. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_token_budget.py +0 -0
  111. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_tool_context_circulation.py +0 -0
  112. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_tools.py +0 -0
  113. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_tools_inspector.py +0 -0
  114. {gemcode-0.3.23 → gemcode-0.3.24}/tests/test_workspace_hints.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gemcode
3
- Version: 0.3.23
3
+ Version: 0.3.24
4
4
  Summary: Local-first coding agent on Google Gemini + ADK
5
5
  Author: GemCode Contributors
6
6
  License: Apache License
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "gemcode"
7
- version = "0.3.23"
7
+ version = "0.3.24"
8
8
  description = "Local-first coding agent on Google Gemini + ADK"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -71,6 +71,143 @@ def _build_runtime_facts(cfg: GemCodeConfig) -> str:
71
71
  - **Working in subfolders** — use tools: e.g. `list_directory("Desktop")`, `glob_files("**/query.ts")`, `read_file("testing/ai-edtech-app/src/app/page.tsx")`, or `run_command` with `cwd_subdir`. Never claim the sandbox cannot reach a subpath unless a tool returned an explicit error."""
72
72
 
73
73
 
74
+ def _build_computer_use_section(cfg: GemCodeConfig) -> str:
75
+ """Rich computer use guidance, only injected when enable_computer_use=True."""
76
+ w = getattr(cfg, "_cfg", None)
77
+ viewport_w = 1280
78
+ viewport_h = 720
79
+ try:
80
+ import os
81
+ viewport_w = int(os.environ.get("GEMCODE_BROWSER_WIDTH", "1280"))
82
+ viewport_h = int(os.environ.get("GEMCODE_BROWSER_HEIGHT", "720"))
83
+ except Exception:
84
+ pass
85
+ return f"""
86
+ ## Browser Computer Use
87
+ You have full browser automation capabilities via a real Chromium instance ({viewport_w}×{viewport_h} px).
88
+
89
+ ### Available tools
90
+
91
+ **Navigation:**
92
+ - `navigate(url)` — Load a URL, wait for DOM, return screenshot + URL
93
+ - `go_back()` / `go_forward()` — Browser history
94
+ - `search()` — Open Google homepage
95
+
96
+ **Mouse:**
97
+ - `click_at(x, y)` — Left-click at pixel coordinates (0,0 = top-left corner)
98
+ - `double_click_at(x, y)` — Double-click
99
+ - `right_click_at(x, y)` — Right-click (opens context menus)
100
+ - `hover_at(x, y)` — Hover to reveal tooltips / dropdown menus
101
+ - `drag_and_drop(x, y, dest_x, dest_y)` — Click-drag
102
+
103
+ **Keyboard:**
104
+ - `type_text_at(x, y, text, press_enter=True, clear_before_typing=True)` — Click field then type
105
+ - `key_combination(keys)` — Press combos: `["control+a"]`, `["control+c"]`, `["control+v"]`, `["escape"]`, `["tab"]`
106
+
107
+ **Scroll:**
108
+ - `scroll_document(direction)` — Scroll whole page: `"up"`, `"down"`, `"left"`, `"right"`
109
+ - `scroll_at(x, y, direction, magnitude)` — Scroll at a specific coordinate (for panels)
110
+
111
+ **Wait:**
112
+ - `wait(seconds)` — Pause for dynamic content (SPAs, animations, lazy-loading)
113
+ - `browser_wait_for_navigation(timeout_seconds)` — Wait for a page transition to complete
114
+
115
+ **Read-only inspection (NO side effects — always safe to call):**
116
+ - `browser_screenshot()` — Take screenshot, save to file, return path + URL + title
117
+ - `browser_get_text(max_chars)` — Extract ALL visible text from page (best for data extraction)
118
+ - `browser_get_url()` — Get current URL and page title
119
+ - `browser_find_element(selector_or_text, selector_type)` — Find element position by CSS or text; returns center (x, y) for clicking
120
+
121
+ ### Human-like execution strategy — ALWAYS follow this loop
122
+
123
+ Every computer use task MUST follow this exact loop:
124
+
125
+ 1. **THINK** — call `think` first. Reason through: What is the goal? What page do I need? What sequence of actions?
126
+ 2. **NAVIGATE** — go to the right URL with `navigate(url)`
127
+ 3. **LOOK** — call `browser_screenshot()` to see the current page state
128
+ 4. **ANALYZE** — study the screenshot carefully: Where are the buttons? Inputs? Text? What are their approximate pixel coordinates?
129
+ 5. **FIND** — use `browser_find_element(text_or_selector)` to get precise coordinates for important elements
130
+ 6. **ACT** — execute ONE action (`click_at`, `type_text_at`, etc.)
131
+ 7. **VERIFY** — study the new screenshot returned by the action. Did it work?
132
+ 8. **ADAPT** — if unexpected result, reconsider; try different coordinates or approach
133
+ 9. **REPEAT** — continue until the task is fully done
134
+
135
+ ### Critical rules
136
+
137
+ - **ALWAYS call `think` before a sequence of computer actions** — plan the exact steps before touching the browser
138
+ - **Coordinates are (x, y) from the top-left corner (0, 0)** — viewport is {viewport_w}×{viewport_h} px
139
+ - **Use `browser_find_element` for precision** — do not guess coordinates; find elements by their visible text
140
+ - **Click THEN verify** — every action returns a screenshot; always analyze it before the next action
141
+ - **For slow pages** — call `wait(2)` after navigation if content is still loading in the screenshot
142
+ - **For forms** — click each field individually, then type; `type_text_at` handles this automatically
143
+ - **For data extraction** — use `browser_get_text()` instead of trying to read screenshot text
144
+ - **If a click misses** — take `browser_screenshot()`, analyze coordinates more carefully, try again
145
+ - **For menus** — `hover_at` first (to reveal), then `click_at` the menu item
146
+
147
+ ### Common patterns
148
+
149
+ **Web search for information:**
150
+ ```
151
+ think("I need to search Google for X")
152
+ navigate("https://www.google.com")
153
+ browser_screenshot() # verify Google loaded
154
+ type_text_at(640, 360, "your query", press_enter=True)
155
+ browser_screenshot() # see search results
156
+ browser_get_text() # extract text of results
157
+ click_at(x, y) # click most relevant result
158
+ browser_get_text() # extract the article
159
+ ```
160
+
161
+ **Fill and submit a form:**
162
+ ```
163
+ think("I need to fill in: field1=value1, field2=value2, then submit")
164
+ navigate("https://example.com/form")
165
+ browser_screenshot() # see the form layout
166
+ browser_find_element("Email", selector_type="text") # get input coords
167
+ type_text_at(x, y, "user@email.com", press_enter=False)
168
+ browser_find_element("Password", selector_type="text")
169
+ type_text_at(x, y, "password", press_enter=False)
170
+ browser_find_element("Submit", selector_type="text")
171
+ click_at(center_x, center_y)
172
+ browser_screenshot() # verify submission
173
+ ```
174
+
175
+ **Log in to a website:**
176
+ ```
177
+ navigate("https://example.com/login")
178
+ browser_screenshot()
179
+ browser_find_element("input[type='email']", selector_type="css")
180
+ type_text_at(x, y, "user@email.com", press_enter=False)
181
+ browser_find_element("input[type='password']", selector_type="css")
182
+ type_text_at(x, y, "password", press_enter=True)
183
+ browser_screenshot() # verify login success
184
+ ```
185
+
186
+ **Extract data from a page:**
187
+ ```
188
+ navigate("https://example.com/data")
189
+ wait(2) # allow dynamic content to load
190
+ browser_get_text() # get all visible text — parse it to extract what you need
191
+ ```
192
+
193
+ **Copy text from page to use elsewhere:**
194
+ ```
195
+ browser_find_element("the text I want", selector_type="text")
196
+ # Note the coordinates, then use key_combination to select
197
+ click_at(x, y)
198
+ key_combination(["control+a"]) # select all text in the field
199
+ key_combination(["control+c"]) # copy
200
+ ```
201
+
202
+ ### Error recovery
203
+ - **Page didn't load**: try `wait(3)` then `browser_screenshot()` to check
204
+ - **Element not found by selector**: use `browser_get_text()` to find the exact text, then use `browser_find_element` with that text
205
+ - **Click had no effect**: double-check coordinates from screenshot, try `browser_find_element` to get precise position
206
+ - **Form submission failed**: `browser_get_text()` to read error messages, fix and resubmit
207
+ - **Unexpected page**: `browser_get_url()` to confirm where you are, `go_back()` if needed
208
+ """
209
+
210
+
74
211
  def build_instruction(cfg: GemCodeConfig) -> str:
75
212
  base = f"""You are GemCode, an expert software engineering agent powered by Google Gemini.
76
213
  You run locally via the GemCode CLI. You are the same agent the user launched — not a hosted portal.
@@ -241,8 +378,11 @@ For tasks where quality matters:
241
378
  ## Workspace scope
242
379
  All file tools use paths **relative to the project root** (where GemCode was started). The root may be the home folder — subfolders like `Desktop`, `Desktop/code`, `Documents` are inside the sandbox. Call `list_directory("Desktop")` or `glob_files("**/*name*.ts")` instead of assuming access is blocked. Only treat access as denied when a tool returns an explicit `error`."""
243
380
 
244
- tool_manifest = build_tool_manifest(cfg)
381
+ # Inject computer use strategy when the browser is enabled.
382
+ if getattr(cfg, "enable_computer_use", False):
383
+ base = f"{base}\n\n{_build_computer_use_section(cfg)}"
245
384
 
385
+ tool_manifest = build_tool_manifest(cfg)
246
386
  if tool_manifest:
247
387
  base = f"{base}\n\n{tool_manifest}"
248
388
  extra = _load_gemini_md(cfg.project_root)
@@ -0,0 +1,361 @@
1
+ """
2
+ Playwright-backed browser automation (ADK ComputerUse).
3
+
4
+ Implements Google ADK's BaseComputer so the LLM can call computer-use tools
5
+ like navigate, click_at, type_text_at, scroll_at, key_combination, etc.
6
+
7
+ Every action returns a ComputerState (viewport PNG screenshot + current URL)
8
+ so the model always sees the current page state after each action.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import platform
14
+ import sys
15
+ from dataclasses import dataclass
16
+ from typing import Literal
17
+ from typing import Tuple
18
+
19
+ from google.adk.tools.computer_use.base_computer import BaseComputer
20
+ from google.adk.tools.computer_use.base_computer import ComputerEnvironment
21
+ from google.adk.tools.computer_use.base_computer import ComputerState
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class BrowserComputerConfig:
26
+ headless: bool = True
27
+ viewport_width: int = 1280
28
+ viewport_height: int = 720
29
+ navigation_timeout_ms: int = 30_000
30
+ # Extra ms to wait after load for dynamic content (SPAs, lazy-loading, etc.)
31
+ post_load_wait_ms: int = 400
32
+
33
+
34
+ _IS_MACOS = platform.system() == "Darwin"
35
+
36
+ # On macOS, many "select all / copy / paste" shortcuts use Cmd (Meta) instead
37
+ # of Ctrl. In a Chromium browser Ctrl+A still works for web inputs, but some
38
+ # OS-level UI elements need Meta. We expose both and let the agent decide.
39
+ _MOD_MAP = {
40
+ "control": "Control", "ctrl": "Control",
41
+ "shift": "Shift",
42
+ "alt": "Alt", "option": "Alt",
43
+ "meta": "Meta", "command": "Meta", "cmd": "Meta", "super": "Meta",
44
+ }
45
+
46
+
47
+ class BrowserComputer(BaseComputer):
48
+ """
49
+ A single Chromium page controlled by Playwright.
50
+
51
+ Usage flow:
52
+ 1. LLM calls ``navigate(url)`` to open a page.
53
+ 2. LLM reads the screenshot returned by each action.
54
+ 3. LLM calls ``click_at``, ``type_text_at``, etc. to interact.
55
+ 4. Each action returns a new ``ComputerState`` (screenshot + URL).
56
+ """
57
+
58
+ def __init__(
59
+ self,
60
+ *,
61
+ headless: bool = True,
62
+ viewport_size: Tuple[int, int] = (1280, 720),
63
+ navigation_timeout_ms: int = 30_000,
64
+ post_load_wait_ms: int = 400,
65
+ ):
66
+ self._cfg = BrowserComputerConfig(
67
+ headless=headless,
68
+ viewport_width=viewport_size[0],
69
+ viewport_height=viewport_size[1],
70
+ navigation_timeout_ms=navigation_timeout_ms,
71
+ post_load_wait_ms=post_load_wait_ms,
72
+ )
73
+ self._playwright = None
74
+ self._browser = None
75
+ self._context = None
76
+ self._page = None
77
+
78
+ # ── Lifecycle ─────────────────────────────────────────────────────────────
79
+
80
+ async def initialize(self) -> None:
81
+ """Lazy-start Playwright Chromium. Called automatically before every action."""
82
+ if self._page is not None:
83
+ return
84
+ try:
85
+ from playwright.async_api import async_playwright
86
+ except ImportError as exc:
87
+ raise RuntimeError(
88
+ "Browser computer requires Playwright.\n"
89
+ " pip install playwright\n"
90
+ " playwright install chromium"
91
+ ) from exc
92
+
93
+ self._playwright = await async_playwright().start()
94
+ self._browser = await self._playwright.chromium.launch(headless=self._cfg.headless)
95
+ self._context = await self._browser.new_context(
96
+ viewport={
97
+ "width": self._cfg.viewport_width,
98
+ "height": self._cfg.viewport_height,
99
+ }
100
+ )
101
+ self._page = await self._context.new_page()
102
+ self._page.set_default_navigation_timeout(self._cfg.navigation_timeout_ms)
103
+
104
+ async def close(self) -> None:
105
+ """Shut down the Playwright process (called by session_runtime on runner close)."""
106
+ for attr in ("_context", "_browser", "_playwright"):
107
+ obj = getattr(self, attr, None)
108
+ if obj is not None:
109
+ try:
110
+ await obj.close() if attr != "_playwright" else await obj.stop()
111
+ except Exception:
112
+ pass
113
+ self._playwright = None
114
+ self._browser = None
115
+ self._context = None
116
+ self._page = None
117
+
118
+ # ── ADK BaseComputer interface ─────────────────────────────────────────────
119
+
120
+ async def environment(self) -> ComputerEnvironment:
121
+ return ComputerEnvironment.ENVIRONMENT_BROWSER
122
+
123
+ async def screen_size(self) -> tuple[int, int]:
124
+ await self.initialize()
125
+ return (self._cfg.viewport_width, self._cfg.viewport_height)
126
+
127
+ async def current_state(self) -> ComputerState:
128
+ """Return viewport PNG + current URL (called after every action)."""
129
+ await self.initialize()
130
+ screenshot = await self._page.screenshot(type="png", full_page=False)
131
+ url = self._page.url
132
+ return ComputerState(screenshot=screenshot, url=url)
133
+
134
+ # ── Navigation ─────────────────────────────────────────────────────────────
135
+
136
+ async def open_web_browser(self) -> ComputerState:
137
+ await self.initialize()
138
+ await self._page.goto("about:blank")
139
+ return await self.current_state()
140
+
141
+ async def navigate(self, url: str) -> ComputerState:
142
+ """Navigate to a URL. Waits for DOM content, then an extra grace period."""
143
+ await self.initialize()
144
+ try:
145
+ await self._page.goto(url, wait_until="domcontentloaded")
146
+ except Exception:
147
+ # Best-effort on timeout/error; still return current state.
148
+ pass
149
+ await self._page.wait_for_timeout(self._cfg.post_load_wait_ms)
150
+ return await self.current_state()
151
+
152
+ async def go_back(self) -> ComputerState:
153
+ await self.initialize()
154
+ try:
155
+ await self._page.go_back(wait_until="domcontentloaded")
156
+ except Exception:
157
+ pass
158
+ await self._page.wait_for_timeout(250)
159
+ return await self.current_state()
160
+
161
+ async def go_forward(self) -> ComputerState:
162
+ await self.initialize()
163
+ try:
164
+ await self._page.go_forward(wait_until="domcontentloaded")
165
+ except Exception:
166
+ pass
167
+ await self._page.wait_for_timeout(250)
168
+ return await self.current_state()
169
+
170
+ async def search(self) -> ComputerState:
171
+ return await self.navigate("https://www.google.com")
172
+
173
+ # ── Mouse actions ──────────────────────────────────────────────────────────
174
+
175
+ async def click_at(self, x: int, y: int) -> ComputerState:
176
+ """Left-click at pixel coordinates (0,0 = top-left)."""
177
+ await self.initialize()
178
+ await self._page.mouse.click(x, y)
179
+ await self._page.wait_for_timeout(150)
180
+ return await self.current_state()
181
+
182
+ async def double_click_at(self, x: int, y: int) -> ComputerState:
183
+ """Double-click at pixel coordinates."""
184
+ await self.initialize()
185
+ await self._page.mouse.dblclick(x, y)
186
+ await self._page.wait_for_timeout(150)
187
+ return await self.current_state()
188
+
189
+ async def right_click_at(self, x: int, y: int) -> ComputerState:
190
+ """Right-click (context menu) at pixel coordinates."""
191
+ await self.initialize()
192
+ await self._page.mouse.click(x, y, button="right")
193
+ await self._page.wait_for_timeout(150)
194
+ return await self.current_state()
195
+
196
+ async def hover_at(self, x: int, y: int) -> ComputerState:
197
+ """Hover mouse (reveals tooltips and dropdowns)."""
198
+ await self.initialize()
199
+ await self._page.mouse.move(x, y)
200
+ await self._page.wait_for_timeout(200)
201
+ return await self.current_state()
202
+
203
+ async def drag_and_drop(
204
+ self, x: int, y: int, destination_x: int, destination_y: int
205
+ ) -> ComputerState:
206
+ await self.initialize()
207
+ await self._page.mouse.move(x, y)
208
+ await self._page.mouse.down()
209
+ await self._page.mouse.move(destination_x, destination_y, steps=10)
210
+ await self._page.mouse.up()
211
+ await self._page.wait_for_timeout(250)
212
+ return await self.current_state()
213
+
214
+ # ── Scroll ─────────────────────────────────────────────────────────────────
215
+
216
+ async def scroll_document(
217
+ self, direction: Literal["up", "down", "left", "right"]
218
+ ) -> ComputerState:
219
+ """Scroll the whole page by a fixed amount."""
220
+ await self.initialize()
221
+ dx, dy = self._direction_delta(direction, magnitude=700)
222
+ await self._page.mouse.wheel(dx=dx, dy=dy)
223
+ await self._page.wait_for_timeout(250)
224
+ return await self.current_state()
225
+
226
+ async def scroll_at(
227
+ self,
228
+ x: int,
229
+ y: int,
230
+ direction: Literal["up", "down", "left", "right"],
231
+ magnitude: int,
232
+ ) -> ComputerState:
233
+ """Scroll at a specific (x, y) coordinate (for scrollable panels)."""
234
+ await self.initialize()
235
+ await self._page.mouse.move(x, y)
236
+ dx, dy = self._direction_delta(direction, magnitude=abs(magnitude))
237
+ await self._page.mouse.wheel(dx=dx, dy=dy)
238
+ await self._page.wait_for_timeout(250)
239
+ return await self.current_state()
240
+
241
+ @staticmethod
242
+ def _direction_delta(
243
+ direction: str, magnitude: int
244
+ ) -> tuple[int, int]:
245
+ if direction == "up":
246
+ return 0, -magnitude
247
+ if direction == "down":
248
+ return 0, magnitude
249
+ if direction == "left":
250
+ return -magnitude, 0
251
+ return magnitude, 0
252
+
253
+ # ── Keyboard ───────────────────────────────────────────────────────────────
254
+
255
+ async def type_text_at(
256
+ self,
257
+ x: int,
258
+ y: int,
259
+ text: str,
260
+ press_enter: bool = True,
261
+ clear_before_typing: bool = True,
262
+ ) -> ComputerState:
263
+ """
264
+ Click a field, optionally clear it, then type text.
265
+
266
+ On macOS browsers Ctrl+A selects all text in web inputs correctly.
267
+ For native OS dialogs use key_combination(["Meta+a"]) instead.
268
+ """
269
+ await self.initialize()
270
+ await self._page.mouse.click(x, y)
271
+ await self._page.wait_for_timeout(100)
272
+ if clear_before_typing:
273
+ select_all = "Meta+A" if _IS_MACOS else "Control+A"
274
+ await self._page.keyboard.press(select_all)
275
+ await self._page.keyboard.press("Backspace")
276
+ await self._page.keyboard.type(text, delay=30)
277
+ if press_enter:
278
+ await self._page.keyboard.press("Enter")
279
+ return await self.current_state()
280
+
281
+ async def key_combination(self, keys: list[str]) -> ComputerState:
282
+ """
283
+ Press key combinations.
284
+
285
+ Each element in ``keys`` is a combo string like ``"control+c"`` or
286
+ ``"shift+enter"``. Modifier names: control/ctrl, shift, alt/option,
287
+ meta/command/cmd.
288
+
289
+ Example: ``key_combination(["control+a", "control+c"])``
290
+ """
291
+ await self.initialize()
292
+ for combo in keys:
293
+ await self._press_combo(combo)
294
+ await self._page.wait_for_timeout(100)
295
+ return await self.current_state()
296
+
297
+ async def _press_combo(self, combo: str) -> None:
298
+ """Press a single key combination string (e.g. 'control+shift+t')."""
299
+ parts = combo.strip().lower().replace(" ", "").split("+")
300
+ if len(parts) == 1:
301
+ await self._page.keyboard.press(parts[0])
302
+ return
303
+ mods = parts[:-1]
304
+ key = parts[-1]
305
+ for m in mods:
306
+ pw = _MOD_MAP.get(m)
307
+ if pw:
308
+ await self._page.keyboard.down(pw)
309
+ await self._page.keyboard.press(key)
310
+ for m in reversed(mods):
311
+ pw = _MOD_MAP.get(m)
312
+ if pw:
313
+ await self._page.keyboard.up(pw)
314
+
315
+ # ── Waiting ────────────────────────────────────────────────────────────────
316
+
317
+ async def wait(self, seconds: int) -> ComputerState:
318
+ """Wait N seconds (use when page is loading dynamic content)."""
319
+ await self.initialize()
320
+ await self._page.wait_for_timeout(max(1, int(seconds)) * 1000)
321
+ return await self.current_state()
322
+
323
+ async def wait_for_text(self, text: str, timeout_seconds: int = 10) -> ComputerState:
324
+ """Wait until a specific text appears on the page."""
325
+ await self.initialize()
326
+ try:
327
+ await self._page.wait_for_function(
328
+ f"document.body.innerText.includes({repr(text)})",
329
+ timeout=timeout_seconds * 1000,
330
+ )
331
+ except Exception:
332
+ pass
333
+ return await self.current_state()
334
+
335
+ # ── Page inspection ────────────────────────────────────────────────────────
336
+
337
+ async def get_page_text(self) -> str:
338
+ """Extract all visible text from the current page (for data extraction)."""
339
+ await self.initialize()
340
+ try:
341
+ return await self._page.evaluate("() => document.body.innerText")
342
+ except Exception as e:
343
+ return f"(error reading page text: {e})"
344
+
345
+ async def get_page_title(self) -> str:
346
+ """Return the current page <title>."""
347
+ await self.initialize()
348
+ try:
349
+ return await self._page.title()
350
+ except Exception:
351
+ return ""
352
+
353
+ async def get_current_url(self) -> str:
354
+ """Return the current page URL."""
355
+ await self.initialize()
356
+ return self._page.url or ""
357
+
358
+ async def screenshot_bytes(self) -> bytes:
359
+ """Return the current viewport as a PNG bytes object (no action)."""
360
+ await self.initialize()
361
+ return await self._page.screenshot(type="png", full_page=False)
@@ -186,6 +186,9 @@ def slash_help_lines() -> list[str]:
186
186
  " /mode <fast|balanced|quality|auto> Set model selection strategy",
187
187
  "",
188
188
  " Capabilities (all require runner rebuild — shown automatically):",
189
+ " /computer Show browser automation status (Playwright Chromium)",
190
+ " /computer on|off Enable/disable browser computer use",
191
+ " /computer url Show current browser URL",
189
192
  " /research Show deep-research status",
190
193
  " /research on|off Enable/disable Google Search + URL Context tools",
191
194
  " /embeddings on|off Enable/disable semantic file search (Embeddings API)",
@@ -190,7 +190,10 @@ async def process_repl_slash(
190
190
  out(f" deep_research: {'on ✓' if cfg.enable_deep_research else 'off'}")
191
191
  out(f" embeddings: {'on ✓' if cfg.enable_embeddings else 'off'}")
192
192
  out(f" memory: {'on ✓' if cfg.enable_memory else 'off'}")
193
- out(f" computer_use: {'on ✓' if cfg.enable_computer_use else 'off'}")
193
+ _bc = getattr(cfg, "_browser_computer", None)
194
+ _bl = _bc is not None and getattr(_bc, "_page", None) is not None
195
+ out(f" computer_use: {'on ✓' if cfg.enable_computer_use else 'off'}"
196
+ + (f" [browser {'live' if _bl else 'ready/idle'}]" if cfg.enable_computer_use else ""))
194
197
  out(f" maps_grounding: {'on ✓' if cfg.enable_maps_grounding else 'off'}")
195
198
  out(f" auto_routing: {cfg.capability_mode}")
196
199
  out()
@@ -315,6 +318,86 @@ async def process_repl_slash(
315
318
  if name in ("exit", "quit"):
316
319
  return ReplSlashResult(exit_repl=True)
317
320
 
321
+ # ── /computer ────────────────────────────────────────────────────────────
322
+ if name in ("computer", "browser"):
323
+ args_s = (sc.args or "").strip().lower()
324
+ enabled = bool(cfg.enable_computer_use)
325
+ if not args_s or args_s in ("status", "show"):
326
+ out(f"computer_use: {'on ✓' if enabled else 'off'}")
327
+ if enabled:
328
+ bc = getattr(cfg, "_browser_computer", None)
329
+ headless_env = os.environ.get("GEMCODE_COMPUTER_HEADLESS", "1").lower()
330
+ headless = headless_env in ("1", "true", "yes", "on")
331
+ out(f" headless: {headless}")
332
+ w = int(os.environ.get("GEMCODE_BROWSER_WIDTH", "1280"))
333
+ h = int(os.environ.get("GEMCODE_BROWSER_HEIGHT", "720"))
334
+ out(f" viewport: {w}×{h}")
335
+ out(f" browser_initialized: {bc is not None and bc._page is not None}")
336
+ if bc is not None and bc._page is not None:
337
+ try:
338
+ import asyncio
339
+ url = asyncio.get_event_loop().run_until_complete(bc.get_current_url()) if not asyncio.get_event_loop().is_running() else "(running)"
340
+ except Exception:
341
+ url = "(check with /computer url)"
342
+ out(f" model_computer_use: {cfg.model_computer_use}")
343
+ out()
344
+ out("Available slash commands:")
345
+ out(" /computer url — show current browser URL")
346
+ out(" /computer off — disable computer use")
347
+ out(" /computer show — show browser window (GEMCODE_COMPUTER_HEADLESS=0 required at startup)")
348
+ else:
349
+ out()
350
+ out("Enable browser automation:")
351
+ out(" /computer on — enable (rebuilds runner with Playwright Chromium)")
352
+ out(" GEMCODE_COMPUTER_HEADLESS=0 — show browser window (set before start)")
353
+ out()
354
+ out("Requirements: pip install playwright && playwright install chromium")
355
+ out()
356
+ return ReplSlashResult(skip_model_turn=True)
357
+
358
+ if args_s == "on":
359
+ cfg.enable_computer_use = True
360
+ out("computer_use: on — Playwright Chromium browser automation enabled")
361
+ out(" Runner will rebuild on the next turn to inject browser tools.")
362
+ out()
363
+ out("Browser tools available to the agent:")
364
+ out(" navigate, click_at, type_text_at, scroll_at, key_combination,")
365
+ out(" browser_screenshot, browser_get_text, browser_find_element, ...")
366
+ out()
367
+ out("Tip: set GEMCODE_COMPUTER_HEADLESS=0 to see the browser window.")
368
+ out()
369
+ return ReplSlashResult(skip_model_turn=True, force_rebuild_runner=True)
370
+
371
+ if args_s == "off":
372
+ cfg.enable_computer_use = False
373
+ out("computer_use: off")
374
+ out()
375
+ return ReplSlashResult(skip_model_turn=True, force_rebuild_runner=True)
376
+
377
+ if args_s == "url":
378
+ bc = getattr(cfg, "_browser_computer", None)
379
+ if bc is None or bc._page is None:
380
+ out("Browser not initialized yet. Send a message to the agent to start it.")
381
+ else:
382
+ try:
383
+ import asyncio
384
+ if asyncio.get_event_loop().is_running():
385
+ out("(run '/computer' to check browser status — URL read not available while async loop is running)")
386
+ else:
387
+ url = asyncio.get_event_loop().run_until_complete(bc.get_current_url())
388
+ title = asyncio.get_event_loop().run_until_complete(bc.get_page_title())
389
+ out(f"url: {url}")
390
+ out(f"title: {title}")
391
+ except Exception as e:
392
+ out(f"Error reading URL: {e}")
393
+ out()
394
+ return ReplSlashResult(skip_model_turn=True)
395
+
396
+ out(f"Unknown /computer subcommand: '{args_s}'")
397
+ out("Usage: /computer [on|off|url|status]")
398
+ out()
399
+ return ReplSlashResult(skip_model_turn=True)
400
+
318
401
  # ── /research ────────────────────────────────────────────────────────────
319
402
  if name == "research":
320
403
  args_s = (sc.args or "").strip().lower()
@@ -475,6 +558,12 @@ async def process_repl_slash(
475
558
  out("enable_embeddings: on (runner rebuilding…)")
476
559
  out()
477
560
  return ReplSlashResult(skip_model_turn=True, force_rebuild_runner=True)
561
+ if args_s in ("computer", "browser"):
562
+ cfg.enable_computer_use = True
563
+ out("enable_computer_use: on (runner rebuilding…)")
564
+ out("Tip: set GEMCODE_COMPUTER_HEADLESS=0 before starting gemcode to see the browser window.")
565
+ out()
566
+ return ReplSlashResult(skip_model_turn=True, force_rebuild_runner=True)
478
567
  if args_s == "all":
479
568
  cfg.enable_deep_research = True
480
569
  cfg.enable_embeddings = True