gemcode 0.3.22__tar.gz → 0.3.24__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gemcode-0.3.22/src/gemcode.egg-info → gemcode-0.3.24}/PKG-INFO +1 -1
- {gemcode-0.3.22 → gemcode-0.3.24}/pyproject.toml +1 -1
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/agent.py +141 -1
- gemcode-0.3.24/src/gemcode/computer_use/browser_computer.py +361 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/config.py +7 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/repl_commands.py +33 -6
- gemcode-0.3.24/src/gemcode/repl_slash.py +713 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/session_runtime.py +16 -1
- gemcode-0.3.24/src/gemcode/tools/browser.py +202 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tui/input_handler.py +10 -3
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tui/scrollback.py +28 -1
- {gemcode-0.3.22 → gemcode-0.3.24/src/gemcode.egg-info}/PKG-INFO +1 -1
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode.egg-info/SOURCES.txt +1 -0
- gemcode-0.3.22/src/gemcode/computer_use/browser_computer.py +0 -275
- gemcode-0.3.22/src/gemcode/repl_slash.py +0 -383
- {gemcode-0.3.22 → gemcode-0.3.24}/LICENSE +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/MANIFEST.in +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/README.md +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/setup.cfg +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/__init__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/__main__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/audit.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/autocompact.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/callbacks.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/capability_routing.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/cli.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/compaction.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/computer_use/__init__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/context_budget.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/context_warning.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/credentials.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/hitl_session.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/interactions.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/invoke.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/kairos_daemon.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/limits.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/live_audio_engine.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/logging_config.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/mcp_loader.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/memory/__init__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/memory/embedding_memory_service.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/memory/file_memory_service.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/modality_tools.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/model_errors.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/model_routing.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/paths.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/permissions.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/plugins/__init__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/plugins/terminal_hooks_plugin.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/plugins/tool_recovery_plugin.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/prompt_suggestions.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/query/__init__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/query/config.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/query/deps.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/query/engine.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/query/stop_hooks.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/query/token_budget.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/query/transitions.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/slash_commands.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/thinking.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tool_prompt_manifest.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tool_registry.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/__init__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/bash.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/edit.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/filesystem.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/search.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/shell.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/shell_gate.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/subtask.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/think.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/todo.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools/web.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tools_inspector.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/trust.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tui/spinner.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tui/welcome_banner.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/tui/welcome_rich.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/version.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/vertex.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/web/__init__.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/web/claude_sse_adapter.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/web/terminal_repl.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode/workspace_hints.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode.egg-info/dependency_links.txt +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode.egg-info/entry_points.txt +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode.egg-info/requires.txt +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/src/gemcode.egg-info/top_level.txt +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_agent_instruction.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_autocompact.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_capability_routing.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_claude_web_adapter_sse.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_cli_init.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_computer_use_permissions.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_context_budget.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_context_warning.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_credentials.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_interactive_permission_ask.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_kairos_scheduler.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_modality_tools.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_model_error_retry.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_model_errors.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_model_routing.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_paths.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_permissions.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_prompt_suggestions.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_repl_commands.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_repl_slash.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_slash_commands.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_thinking_config.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_token_budget.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_tool_context_circulation.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_tools.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_tools_inspector.py +0 -0
- {gemcode-0.3.22 → gemcode-0.3.24}/tests/test_workspace_hints.py +0 -0
|
@@ -71,6 +71,143 @@ def _build_runtime_facts(cfg: GemCodeConfig) -> str:
|
|
|
71
71
|
- **Working in subfolders** — use tools: e.g. `list_directory("Desktop")`, `glob_files("**/query.ts")`, `read_file("testing/ai-edtech-app/src/app/page.tsx")`, or `run_command` with `cwd_subdir`. Never claim the sandbox cannot reach a subpath unless a tool returned an explicit error."""
|
|
72
72
|
|
|
73
73
|
|
|
74
|
+
def _build_computer_use_section(cfg: GemCodeConfig) -> str:
|
|
75
|
+
"""Rich computer use guidance, only injected when enable_computer_use=True."""
|
|
76
|
+
w = getattr(cfg, "_cfg", None)
|
|
77
|
+
viewport_w = 1280
|
|
78
|
+
viewport_h = 720
|
|
79
|
+
try:
|
|
80
|
+
import os
|
|
81
|
+
viewport_w = int(os.environ.get("GEMCODE_BROWSER_WIDTH", "1280"))
|
|
82
|
+
viewport_h = int(os.environ.get("GEMCODE_BROWSER_HEIGHT", "720"))
|
|
83
|
+
except Exception:
|
|
84
|
+
pass
|
|
85
|
+
return f"""
|
|
86
|
+
## Browser Computer Use
|
|
87
|
+
You have full browser automation capabilities via a real Chromium instance ({viewport_w}×{viewport_h} px).
|
|
88
|
+
|
|
89
|
+
### Available tools
|
|
90
|
+
|
|
91
|
+
**Navigation:**
|
|
92
|
+
- `navigate(url)` — Load a URL, wait for DOM, return screenshot + URL
|
|
93
|
+
- `go_back()` / `go_forward()` — Browser history
|
|
94
|
+
- `search()` — Open Google homepage
|
|
95
|
+
|
|
96
|
+
**Mouse:**
|
|
97
|
+
- `click_at(x, y)` — Left-click at pixel coordinates (0,0 = top-left corner)
|
|
98
|
+
- `double_click_at(x, y)` — Double-click
|
|
99
|
+
- `right_click_at(x, y)` — Right-click (opens context menus)
|
|
100
|
+
- `hover_at(x, y)` — Hover to reveal tooltips / dropdown menus
|
|
101
|
+
- `drag_and_drop(x, y, dest_x, dest_y)` — Click-drag
|
|
102
|
+
|
|
103
|
+
**Keyboard:**
|
|
104
|
+
- `type_text_at(x, y, text, press_enter=True, clear_before_typing=True)` — Click field then type
|
|
105
|
+
- `key_combination(keys)` — Press combos: `["control+a"]`, `["control+c"]`, `["control+v"]`, `["escape"]`, `["tab"]`
|
|
106
|
+
|
|
107
|
+
**Scroll:**
|
|
108
|
+
- `scroll_document(direction)` — Scroll whole page: `"up"`, `"down"`, `"left"`, `"right"`
|
|
109
|
+
- `scroll_at(x, y, direction, magnitude)` — Scroll at a specific coordinate (for panels)
|
|
110
|
+
|
|
111
|
+
**Wait:**
|
|
112
|
+
- `wait(seconds)` — Pause for dynamic content (SPAs, animations, lazy-loading)
|
|
113
|
+
- `browser_wait_for_navigation(timeout_seconds)` — Wait for a page transition to complete
|
|
114
|
+
|
|
115
|
+
**Read-only inspection (NO side effects — always safe to call):**
|
|
116
|
+
- `browser_screenshot()` — Take screenshot, save to file, return path + URL + title
|
|
117
|
+
- `browser_get_text(max_chars)` — Extract ALL visible text from page (best for data extraction)
|
|
118
|
+
- `browser_get_url()` — Get current URL and page title
|
|
119
|
+
- `browser_find_element(selector_or_text, selector_type)` — Find element position by CSS or text; returns center (x, y) for clicking
|
|
120
|
+
|
|
121
|
+
### Human-like execution strategy — ALWAYS follow this loop
|
|
122
|
+
|
|
123
|
+
Every computer use task MUST follow this exact loop:
|
|
124
|
+
|
|
125
|
+
1. **THINK** — call `think` first. Reason through: What is the goal? What page do I need? What sequence of actions?
|
|
126
|
+
2. **NAVIGATE** — go to the right URL with `navigate(url)`
|
|
127
|
+
3. **LOOK** — call `browser_screenshot()` to see the current page state
|
|
128
|
+
4. **ANALYZE** — study the screenshot carefully: Where are the buttons? Inputs? Text? What are their approximate pixel coordinates?
|
|
129
|
+
5. **FIND** — use `browser_find_element(text_or_selector)` to get precise coordinates for important elements
|
|
130
|
+
6. **ACT** — execute ONE action (`click_at`, `type_text_at`, etc.)
|
|
131
|
+
7. **VERIFY** — study the new screenshot returned by the action. Did it work?
|
|
132
|
+
8. **ADAPT** — if unexpected result, reconsider; try different coordinates or approach
|
|
133
|
+
9. **REPEAT** — continue until the task is fully done
|
|
134
|
+
|
|
135
|
+
### Critical rules
|
|
136
|
+
|
|
137
|
+
- **ALWAYS call `think` before a sequence of computer actions** — plan the exact steps before touching the browser
|
|
138
|
+
- **Coordinates are (x, y) from the top-left corner (0, 0)** — viewport is {viewport_w}×{viewport_h} px
|
|
139
|
+
- **Use `browser_find_element` for precision** — do not guess coordinates; find elements by their visible text
|
|
140
|
+
- **Click THEN verify** — every action returns a screenshot; always analyze it before the next action
|
|
141
|
+
- **For slow pages** — call `wait(2)` after navigation if content is still loading in the screenshot
|
|
142
|
+
- **For forms** — click each field individually, then type; `type_text_at` handles this automatically
|
|
143
|
+
- **For data extraction** — use `browser_get_text()` instead of trying to read screenshot text
|
|
144
|
+
- **If a click misses** — take `browser_screenshot()`, analyze coordinates more carefully, try again
|
|
145
|
+
- **For menus** — `hover_at` first (to reveal), then `click_at` the menu item
|
|
146
|
+
|
|
147
|
+
### Common patterns
|
|
148
|
+
|
|
149
|
+
**Web search for information:**
|
|
150
|
+
```
|
|
151
|
+
think("I need to search Google for X")
|
|
152
|
+
navigate("https://www.google.com")
|
|
153
|
+
browser_screenshot() # verify Google loaded
|
|
154
|
+
type_text_at(640, 360, "your query", press_enter=True)
|
|
155
|
+
browser_screenshot() # see search results
|
|
156
|
+
browser_get_text() # extract text of results
|
|
157
|
+
click_at(x, y) # click most relevant result
|
|
158
|
+
browser_get_text() # extract the article
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
**Fill and submit a form:**
|
|
162
|
+
```
|
|
163
|
+
think("I need to fill in: field1=value1, field2=value2, then submit")
|
|
164
|
+
navigate("https://example.com/form")
|
|
165
|
+
browser_screenshot() # see the form layout
|
|
166
|
+
browser_find_element("Email", selector_type="text") # get input coords
|
|
167
|
+
type_text_at(x, y, "user@email.com", press_enter=False)
|
|
168
|
+
browser_find_element("Password", selector_type="text")
|
|
169
|
+
type_text_at(x, y, "password", press_enter=False)
|
|
170
|
+
browser_find_element("Submit", selector_type="text")
|
|
171
|
+
click_at(center_x, center_y)
|
|
172
|
+
browser_screenshot() # verify submission
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
**Log in to a website:**
|
|
176
|
+
```
|
|
177
|
+
navigate("https://example.com/login")
|
|
178
|
+
browser_screenshot()
|
|
179
|
+
browser_find_element("input[type='email']", selector_type="css")
|
|
180
|
+
type_text_at(x, y, "user@email.com", press_enter=False)
|
|
181
|
+
browser_find_element("input[type='password']", selector_type="css")
|
|
182
|
+
type_text_at(x, y, "password", press_enter=True)
|
|
183
|
+
browser_screenshot() # verify login success
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
**Extract data from a page:**
|
|
187
|
+
```
|
|
188
|
+
navigate("https://example.com/data")
|
|
189
|
+
wait(2) # allow dynamic content to load
|
|
190
|
+
browser_get_text() # get all visible text — parse it to extract what you need
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
**Copy text from page to use elsewhere:**
|
|
194
|
+
```
|
|
195
|
+
browser_find_element("the text I want", selector_type="text")
|
|
196
|
+
# Note the coordinates, then use key_combination to select
|
|
197
|
+
click_at(x, y)
|
|
198
|
+
key_combination(["control+a"]) # select all text in the field
|
|
199
|
+
key_combination(["control+c"]) # copy
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Error recovery
|
|
203
|
+
- **Page didn't load**: try `wait(3)` then `browser_screenshot()` to check
|
|
204
|
+
- **Element not found by selector**: use `browser_get_text()` to find the exact text, then use `browser_find_element` with that text
|
|
205
|
+
- **Click had no effect**: double-check coordinates from screenshot, try `browser_find_element` to get precise position
|
|
206
|
+
- **Form submission failed**: `browser_get_text()` to read error messages, fix and resubmit
|
|
207
|
+
- **Unexpected page**: `browser_get_url()` to confirm where you are, `go_back()` if needed
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
|
|
74
211
|
def build_instruction(cfg: GemCodeConfig) -> str:
|
|
75
212
|
base = f"""You are GemCode, an expert software engineering agent powered by Google Gemini.
|
|
76
213
|
You run locally via the GemCode CLI. You are the same agent the user launched — not a hosted portal.
|
|
@@ -241,8 +378,11 @@ For tasks where quality matters:
|
|
|
241
378
|
## Workspace scope
|
|
242
379
|
All file tools use paths **relative to the project root** (where GemCode was started). The root may be the home folder — subfolders like `Desktop`, `Desktop/code`, `Documents` are inside the sandbox. Call `list_directory("Desktop")` or `glob_files("**/*name*.ts")` instead of assuming access is blocked. Only treat access as denied when a tool returns an explicit `error`."""
|
|
243
380
|
|
|
244
|
-
|
|
381
|
+
# Inject computer use strategy when the browser is enabled.
|
|
382
|
+
if getattr(cfg, "enable_computer_use", False):
|
|
383
|
+
base = f"{base}\n\n{_build_computer_use_section(cfg)}"
|
|
245
384
|
|
|
385
|
+
tool_manifest = build_tool_manifest(cfg)
|
|
246
386
|
if tool_manifest:
|
|
247
387
|
base = f"{base}\n\n{tool_manifest}"
|
|
248
388
|
extra = _load_gemini_md(cfg.project_root)
|
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Playwright-backed browser automation (ADK ComputerUse).
|
|
3
|
+
|
|
4
|
+
Implements Google ADK's BaseComputer so the LLM can call computer-use tools
|
|
5
|
+
like navigate, click_at, type_text_at, scroll_at, key_combination, etc.
|
|
6
|
+
|
|
7
|
+
Every action returns a ComputerState (viewport PNG screenshot + current URL)
|
|
8
|
+
so the model always sees the current page state after each action.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import platform
|
|
14
|
+
import sys
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Literal
|
|
17
|
+
from typing import Tuple
|
|
18
|
+
|
|
19
|
+
from google.adk.tools.computer_use.base_computer import BaseComputer
|
|
20
|
+
from google.adk.tools.computer_use.base_computer import ComputerEnvironment
|
|
21
|
+
from google.adk.tools.computer_use.base_computer import ComputerState
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class BrowserComputerConfig:
|
|
26
|
+
headless: bool = True
|
|
27
|
+
viewport_width: int = 1280
|
|
28
|
+
viewport_height: int = 720
|
|
29
|
+
navigation_timeout_ms: int = 30_000
|
|
30
|
+
# Extra ms to wait after load for dynamic content (SPAs, lazy-loading, etc.)
|
|
31
|
+
post_load_wait_ms: int = 400
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
_IS_MACOS = platform.system() == "Darwin"
|
|
35
|
+
|
|
36
|
+
# On macOS, many "select all / copy / paste" shortcuts use Cmd (Meta) instead
|
|
37
|
+
# of Ctrl. In a Chromium browser Ctrl+A still works for web inputs, but some
|
|
38
|
+
# OS-level UI elements need Meta. We expose both and let the agent decide.
|
|
39
|
+
_MOD_MAP = {
|
|
40
|
+
"control": "Control", "ctrl": "Control",
|
|
41
|
+
"shift": "Shift",
|
|
42
|
+
"alt": "Alt", "option": "Alt",
|
|
43
|
+
"meta": "Meta", "command": "Meta", "cmd": "Meta", "super": "Meta",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class BrowserComputer(BaseComputer):
|
|
48
|
+
"""
|
|
49
|
+
A single Chromium page controlled by Playwright.
|
|
50
|
+
|
|
51
|
+
Usage flow:
|
|
52
|
+
1. LLM calls ``navigate(url)`` to open a page.
|
|
53
|
+
2. LLM reads the screenshot returned by each action.
|
|
54
|
+
3. LLM calls ``click_at``, ``type_text_at``, etc. to interact.
|
|
55
|
+
4. Each action returns a new ``ComputerState`` (screenshot + URL).
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
*,
|
|
61
|
+
headless: bool = True,
|
|
62
|
+
viewport_size: Tuple[int, int] = (1280, 720),
|
|
63
|
+
navigation_timeout_ms: int = 30_000,
|
|
64
|
+
post_load_wait_ms: int = 400,
|
|
65
|
+
):
|
|
66
|
+
self._cfg = BrowserComputerConfig(
|
|
67
|
+
headless=headless,
|
|
68
|
+
viewport_width=viewport_size[0],
|
|
69
|
+
viewport_height=viewport_size[1],
|
|
70
|
+
navigation_timeout_ms=navigation_timeout_ms,
|
|
71
|
+
post_load_wait_ms=post_load_wait_ms,
|
|
72
|
+
)
|
|
73
|
+
self._playwright = None
|
|
74
|
+
self._browser = None
|
|
75
|
+
self._context = None
|
|
76
|
+
self._page = None
|
|
77
|
+
|
|
78
|
+
# ── Lifecycle ─────────────────────────────────────────────────────────────
|
|
79
|
+
|
|
80
|
+
async def initialize(self) -> None:
|
|
81
|
+
"""Lazy-start Playwright Chromium. Called automatically before every action."""
|
|
82
|
+
if self._page is not None:
|
|
83
|
+
return
|
|
84
|
+
try:
|
|
85
|
+
from playwright.async_api import async_playwright
|
|
86
|
+
except ImportError as exc:
|
|
87
|
+
raise RuntimeError(
|
|
88
|
+
"Browser computer requires Playwright.\n"
|
|
89
|
+
" pip install playwright\n"
|
|
90
|
+
" playwright install chromium"
|
|
91
|
+
) from exc
|
|
92
|
+
|
|
93
|
+
self._playwright = await async_playwright().start()
|
|
94
|
+
self._browser = await self._playwright.chromium.launch(headless=self._cfg.headless)
|
|
95
|
+
self._context = await self._browser.new_context(
|
|
96
|
+
viewport={
|
|
97
|
+
"width": self._cfg.viewport_width,
|
|
98
|
+
"height": self._cfg.viewport_height,
|
|
99
|
+
}
|
|
100
|
+
)
|
|
101
|
+
self._page = await self._context.new_page()
|
|
102
|
+
self._page.set_default_navigation_timeout(self._cfg.navigation_timeout_ms)
|
|
103
|
+
|
|
104
|
+
async def close(self) -> None:
|
|
105
|
+
"""Shut down the Playwright process (called by session_runtime on runner close)."""
|
|
106
|
+
for attr in ("_context", "_browser", "_playwright"):
|
|
107
|
+
obj = getattr(self, attr, None)
|
|
108
|
+
if obj is not None:
|
|
109
|
+
try:
|
|
110
|
+
await obj.close() if attr != "_playwright" else await obj.stop()
|
|
111
|
+
except Exception:
|
|
112
|
+
pass
|
|
113
|
+
self._playwright = None
|
|
114
|
+
self._browser = None
|
|
115
|
+
self._context = None
|
|
116
|
+
self._page = None
|
|
117
|
+
|
|
118
|
+
# ── ADK BaseComputer interface ─────────────────────────────────────────────
|
|
119
|
+
|
|
120
|
+
async def environment(self) -> ComputerEnvironment:
|
|
121
|
+
return ComputerEnvironment.ENVIRONMENT_BROWSER
|
|
122
|
+
|
|
123
|
+
async def screen_size(self) -> tuple[int, int]:
|
|
124
|
+
await self.initialize()
|
|
125
|
+
return (self._cfg.viewport_width, self._cfg.viewport_height)
|
|
126
|
+
|
|
127
|
+
async def current_state(self) -> ComputerState:
|
|
128
|
+
"""Return viewport PNG + current URL (called after every action)."""
|
|
129
|
+
await self.initialize()
|
|
130
|
+
screenshot = await self._page.screenshot(type="png", full_page=False)
|
|
131
|
+
url = self._page.url
|
|
132
|
+
return ComputerState(screenshot=screenshot, url=url)
|
|
133
|
+
|
|
134
|
+
# ── Navigation ─────────────────────────────────────────────────────────────
|
|
135
|
+
|
|
136
|
+
async def open_web_browser(self) -> ComputerState:
|
|
137
|
+
await self.initialize()
|
|
138
|
+
await self._page.goto("about:blank")
|
|
139
|
+
return await self.current_state()
|
|
140
|
+
|
|
141
|
+
async def navigate(self, url: str) -> ComputerState:
|
|
142
|
+
"""Navigate to a URL. Waits for DOM content, then an extra grace period."""
|
|
143
|
+
await self.initialize()
|
|
144
|
+
try:
|
|
145
|
+
await self._page.goto(url, wait_until="domcontentloaded")
|
|
146
|
+
except Exception:
|
|
147
|
+
# Best-effort on timeout/error; still return current state.
|
|
148
|
+
pass
|
|
149
|
+
await self._page.wait_for_timeout(self._cfg.post_load_wait_ms)
|
|
150
|
+
return await self.current_state()
|
|
151
|
+
|
|
152
|
+
async def go_back(self) -> ComputerState:
|
|
153
|
+
await self.initialize()
|
|
154
|
+
try:
|
|
155
|
+
await self._page.go_back(wait_until="domcontentloaded")
|
|
156
|
+
except Exception:
|
|
157
|
+
pass
|
|
158
|
+
await self._page.wait_for_timeout(250)
|
|
159
|
+
return await self.current_state()
|
|
160
|
+
|
|
161
|
+
async def go_forward(self) -> ComputerState:
|
|
162
|
+
await self.initialize()
|
|
163
|
+
try:
|
|
164
|
+
await self._page.go_forward(wait_until="domcontentloaded")
|
|
165
|
+
except Exception:
|
|
166
|
+
pass
|
|
167
|
+
await self._page.wait_for_timeout(250)
|
|
168
|
+
return await self.current_state()
|
|
169
|
+
|
|
170
|
+
async def search(self) -> ComputerState:
|
|
171
|
+
return await self.navigate("https://www.google.com")
|
|
172
|
+
|
|
173
|
+
# ── Mouse actions ──────────────────────────────────────────────────────────
|
|
174
|
+
|
|
175
|
+
async def click_at(self, x: int, y: int) -> ComputerState:
|
|
176
|
+
"""Left-click at pixel coordinates (0,0 = top-left)."""
|
|
177
|
+
await self.initialize()
|
|
178
|
+
await self._page.mouse.click(x, y)
|
|
179
|
+
await self._page.wait_for_timeout(150)
|
|
180
|
+
return await self.current_state()
|
|
181
|
+
|
|
182
|
+
async def double_click_at(self, x: int, y: int) -> ComputerState:
|
|
183
|
+
"""Double-click at pixel coordinates."""
|
|
184
|
+
await self.initialize()
|
|
185
|
+
await self._page.mouse.dblclick(x, y)
|
|
186
|
+
await self._page.wait_for_timeout(150)
|
|
187
|
+
return await self.current_state()
|
|
188
|
+
|
|
189
|
+
async def right_click_at(self, x: int, y: int) -> ComputerState:
|
|
190
|
+
"""Right-click (context menu) at pixel coordinates."""
|
|
191
|
+
await self.initialize()
|
|
192
|
+
await self._page.mouse.click(x, y, button="right")
|
|
193
|
+
await self._page.wait_for_timeout(150)
|
|
194
|
+
return await self.current_state()
|
|
195
|
+
|
|
196
|
+
async def hover_at(self, x: int, y: int) -> ComputerState:
|
|
197
|
+
"""Hover mouse (reveals tooltips and dropdowns)."""
|
|
198
|
+
await self.initialize()
|
|
199
|
+
await self._page.mouse.move(x, y)
|
|
200
|
+
await self._page.wait_for_timeout(200)
|
|
201
|
+
return await self.current_state()
|
|
202
|
+
|
|
203
|
+
async def drag_and_drop(
|
|
204
|
+
self, x: int, y: int, destination_x: int, destination_y: int
|
|
205
|
+
) -> ComputerState:
|
|
206
|
+
await self.initialize()
|
|
207
|
+
await self._page.mouse.move(x, y)
|
|
208
|
+
await self._page.mouse.down()
|
|
209
|
+
await self._page.mouse.move(destination_x, destination_y, steps=10)
|
|
210
|
+
await self._page.mouse.up()
|
|
211
|
+
await self._page.wait_for_timeout(250)
|
|
212
|
+
return await self.current_state()
|
|
213
|
+
|
|
214
|
+
# ── Scroll ─────────────────────────────────────────────────────────────────
|
|
215
|
+
|
|
216
|
+
async def scroll_document(
|
|
217
|
+
self, direction: Literal["up", "down", "left", "right"]
|
|
218
|
+
) -> ComputerState:
|
|
219
|
+
"""Scroll the whole page by a fixed amount."""
|
|
220
|
+
await self.initialize()
|
|
221
|
+
dx, dy = self._direction_delta(direction, magnitude=700)
|
|
222
|
+
await self._page.mouse.wheel(dx=dx, dy=dy)
|
|
223
|
+
await self._page.wait_for_timeout(250)
|
|
224
|
+
return await self.current_state()
|
|
225
|
+
|
|
226
|
+
async def scroll_at(
|
|
227
|
+
self,
|
|
228
|
+
x: int,
|
|
229
|
+
y: int,
|
|
230
|
+
direction: Literal["up", "down", "left", "right"],
|
|
231
|
+
magnitude: int,
|
|
232
|
+
) -> ComputerState:
|
|
233
|
+
"""Scroll at a specific (x, y) coordinate (for scrollable panels)."""
|
|
234
|
+
await self.initialize()
|
|
235
|
+
await self._page.mouse.move(x, y)
|
|
236
|
+
dx, dy = self._direction_delta(direction, magnitude=abs(magnitude))
|
|
237
|
+
await self._page.mouse.wheel(dx=dx, dy=dy)
|
|
238
|
+
await self._page.wait_for_timeout(250)
|
|
239
|
+
return await self.current_state()
|
|
240
|
+
|
|
241
|
+
@staticmethod
|
|
242
|
+
def _direction_delta(
|
|
243
|
+
direction: str, magnitude: int
|
|
244
|
+
) -> tuple[int, int]:
|
|
245
|
+
if direction == "up":
|
|
246
|
+
return 0, -magnitude
|
|
247
|
+
if direction == "down":
|
|
248
|
+
return 0, magnitude
|
|
249
|
+
if direction == "left":
|
|
250
|
+
return -magnitude, 0
|
|
251
|
+
return magnitude, 0
|
|
252
|
+
|
|
253
|
+
# ── Keyboard ───────────────────────────────────────────────────────────────
|
|
254
|
+
|
|
255
|
+
async def type_text_at(
|
|
256
|
+
self,
|
|
257
|
+
x: int,
|
|
258
|
+
y: int,
|
|
259
|
+
text: str,
|
|
260
|
+
press_enter: bool = True,
|
|
261
|
+
clear_before_typing: bool = True,
|
|
262
|
+
) -> ComputerState:
|
|
263
|
+
"""
|
|
264
|
+
Click a field, optionally clear it, then type text.
|
|
265
|
+
|
|
266
|
+
On macOS browsers Ctrl+A selects all text in web inputs correctly.
|
|
267
|
+
For native OS dialogs use key_combination(["Meta+a"]) instead.
|
|
268
|
+
"""
|
|
269
|
+
await self.initialize()
|
|
270
|
+
await self._page.mouse.click(x, y)
|
|
271
|
+
await self._page.wait_for_timeout(100)
|
|
272
|
+
if clear_before_typing:
|
|
273
|
+
select_all = "Meta+A" if _IS_MACOS else "Control+A"
|
|
274
|
+
await self._page.keyboard.press(select_all)
|
|
275
|
+
await self._page.keyboard.press("Backspace")
|
|
276
|
+
await self._page.keyboard.type(text, delay=30)
|
|
277
|
+
if press_enter:
|
|
278
|
+
await self._page.keyboard.press("Enter")
|
|
279
|
+
return await self.current_state()
|
|
280
|
+
|
|
281
|
+
async def key_combination(self, keys: list[str]) -> ComputerState:
|
|
282
|
+
"""
|
|
283
|
+
Press key combinations.
|
|
284
|
+
|
|
285
|
+
Each element in ``keys`` is a combo string like ``"control+c"`` or
|
|
286
|
+
``"shift+enter"``. Modifier names: control/ctrl, shift, alt/option,
|
|
287
|
+
meta/command/cmd.
|
|
288
|
+
|
|
289
|
+
Example: ``key_combination(["control+a", "control+c"])``
|
|
290
|
+
"""
|
|
291
|
+
await self.initialize()
|
|
292
|
+
for combo in keys:
|
|
293
|
+
await self._press_combo(combo)
|
|
294
|
+
await self._page.wait_for_timeout(100)
|
|
295
|
+
return await self.current_state()
|
|
296
|
+
|
|
297
|
+
async def _press_combo(self, combo: str) -> None:
|
|
298
|
+
"""Press a single key combination string (e.g. 'control+shift+t')."""
|
|
299
|
+
parts = combo.strip().lower().replace(" ", "").split("+")
|
|
300
|
+
if len(parts) == 1:
|
|
301
|
+
await self._page.keyboard.press(parts[0])
|
|
302
|
+
return
|
|
303
|
+
mods = parts[:-1]
|
|
304
|
+
key = parts[-1]
|
|
305
|
+
for m in mods:
|
|
306
|
+
pw = _MOD_MAP.get(m)
|
|
307
|
+
if pw:
|
|
308
|
+
await self._page.keyboard.down(pw)
|
|
309
|
+
await self._page.keyboard.press(key)
|
|
310
|
+
for m in reversed(mods):
|
|
311
|
+
pw = _MOD_MAP.get(m)
|
|
312
|
+
if pw:
|
|
313
|
+
await self._page.keyboard.up(pw)
|
|
314
|
+
|
|
315
|
+
# ── Waiting ────────────────────────────────────────────────────────────────
|
|
316
|
+
|
|
317
|
+
async def wait(self, seconds: int) -> ComputerState:
|
|
318
|
+
"""Wait N seconds (use when page is loading dynamic content)."""
|
|
319
|
+
await self.initialize()
|
|
320
|
+
await self._page.wait_for_timeout(max(1, int(seconds)) * 1000)
|
|
321
|
+
return await self.current_state()
|
|
322
|
+
|
|
323
|
+
async def wait_for_text(self, text: str, timeout_seconds: int = 10) -> ComputerState:
|
|
324
|
+
"""Wait until a specific text appears on the page."""
|
|
325
|
+
await self.initialize()
|
|
326
|
+
try:
|
|
327
|
+
await self._page.wait_for_function(
|
|
328
|
+
f"document.body.innerText.includes({repr(text)})",
|
|
329
|
+
timeout=timeout_seconds * 1000,
|
|
330
|
+
)
|
|
331
|
+
except Exception:
|
|
332
|
+
pass
|
|
333
|
+
return await self.current_state()
|
|
334
|
+
|
|
335
|
+
# ── Page inspection ────────────────────────────────────────────────────────
|
|
336
|
+
|
|
337
|
+
async def get_page_text(self) -> str:
|
|
338
|
+
"""Extract all visible text from the current page (for data extraction)."""
|
|
339
|
+
await self.initialize()
|
|
340
|
+
try:
|
|
341
|
+
return await self._page.evaluate("() => document.body.innerText")
|
|
342
|
+
except Exception as e:
|
|
343
|
+
return f"(error reading page text: {e})"
|
|
344
|
+
|
|
345
|
+
async def get_page_title(self) -> str:
|
|
346
|
+
"""Return the current page <title>."""
|
|
347
|
+
await self.initialize()
|
|
348
|
+
try:
|
|
349
|
+
return await self._page.title()
|
|
350
|
+
except Exception:
|
|
351
|
+
return ""
|
|
352
|
+
|
|
353
|
+
async def get_current_url(self) -> str:
|
|
354
|
+
"""Return the current page URL."""
|
|
355
|
+
await self.initialize()
|
|
356
|
+
return self._page.url or ""
|
|
357
|
+
|
|
358
|
+
async def screenshot_bytes(self) -> bytes:
|
|
359
|
+
"""Return the current viewport as a PNG bytes object (no action)."""
|
|
360
|
+
await self.initialize()
|
|
361
|
+
return await self._page.screenshot(type="png", full_page=False)
|
|
@@ -244,6 +244,13 @@ class GemCodeConfig:
|
|
|
244
244
|
default_factory=lambda: _opt_int("GEMCODE_THINKING_BUDGET")
|
|
245
245
|
)
|
|
246
246
|
|
|
247
|
+
# Controls how the TUI renders model thinking: True = full Rich Markdown,
|
|
248
|
+
# False = collapsed one-line excerpt (default, like OpenClaude).
|
|
249
|
+
# Toggled at runtime via /thinking verbose|brief.
|
|
250
|
+
show_full_thinking: bool = field(
|
|
251
|
+
default_factory=lambda: _truthy_env("GEMCODE_SHOW_FULL_THINKING", default=False)
|
|
252
|
+
)
|
|
253
|
+
|
|
247
254
|
def __post_init__(self) -> None:
|
|
248
255
|
self.project_root = self.project_root.resolve()
|
|
249
256
|
# Default agentic depth when env omits GEMCODE_MAX_LLM_CALLS (was: None → SDK default).
|
|
@@ -162,9 +162,11 @@ def slash_help_lines() -> list[str]:
|
|
|
162
162
|
"Slash commands:",
|
|
163
163
|
" (CLI) gemcode -C DIR Use a project folder as root (recommended vs. ~ )",
|
|
164
164
|
" (CLI) gemcode login Save or change API key (~/.gemcode/credentials.json)",
|
|
165
|
+
"",
|
|
166
|
+
" Session:",
|
|
165
167
|
" /help Show this help",
|
|
166
|
-
" /status
|
|
167
|
-
" /config
|
|
168
|
+
" /status Full status: model, capabilities, thinking, limits",
|
|
169
|
+
" /config All active config fields (model, caps, context, thinking)",
|
|
168
170
|
" /session Print current session id",
|
|
169
171
|
" /session new Start a new session id (history reset)",
|
|
170
172
|
" /clear Alias for /session new",
|
|
@@ -173,9 +175,29 @@ def slash_help_lines() -> list[str]:
|
|
|
173
175
|
" /audit [N] Tail of .gemcode/audit.log (default 40 lines)",
|
|
174
176
|
" /tools List tool inventory for this config",
|
|
175
177
|
" /doctor Environment sanity check",
|
|
178
|
+
" /version Print GemCode version hint",
|
|
179
|
+
" /exit Exit the REPL",
|
|
180
|
+
"",
|
|
181
|
+
" Model:",
|
|
176
182
|
" /model Show model routing info",
|
|
177
183
|
" /model use <id> Override model for this REPL session",
|
|
178
|
-
" /model list
|
|
184
|
+
" /model list List available Gemini model IDs",
|
|
185
|
+
" /mode Show current model_mode",
|
|
186
|
+
" /mode <fast|balanced|quality|auto> Set model selection strategy",
|
|
187
|
+
"",
|
|
188
|
+
" Capabilities (all require runner rebuild — shown automatically):",
|
|
189
|
+
" /computer Show browser automation status (Playwright Chromium)",
|
|
190
|
+
" /computer on|off Enable/disable browser computer use",
|
|
191
|
+
" /computer url Show current browser URL",
|
|
192
|
+
" /research Show deep-research status",
|
|
193
|
+
" /research on|off Enable/disable Google Search + URL Context tools",
|
|
194
|
+
" /embeddings on|off Enable/disable semantic file search (Embeddings API)",
|
|
195
|
+
" /caps View all capability flags",
|
|
196
|
+
" /caps <research|embeddings|all|reset> Bulk capability control",
|
|
197
|
+
" /memory Show persistent memory status",
|
|
198
|
+
" /memory on|off Enable/disable session memory",
|
|
199
|
+
"",
|
|
200
|
+
" Thinking:",
|
|
179
201
|
" /thinking Show current thinking config",
|
|
180
202
|
" /thinking verbose Show full thinking text each turn",
|
|
181
203
|
" /thinking brief Show collapsed one-line excerpt (default)",
|
|
@@ -183,9 +205,14 @@ def slash_help_lines() -> list[str]:
|
|
|
183
205
|
" /thinking on Re-enable thinking (auto budget/level)",
|
|
184
206
|
" /thinking budget <N> Set thinking token budget (Gemini 2.5, 0=off, -1=dynamic)",
|
|
185
207
|
" /thinking level <L> Set thinking level: minimal|low|medium|high (Gemini 3.x)",
|
|
208
|
+
"",
|
|
209
|
+
" Limits:",
|
|
210
|
+
" /limits Show all execution limits",
|
|
211
|
+
" /limits calls <N> Set max LLM calls per turn",
|
|
212
|
+
" /budget Show per-turn token budget",
|
|
213
|
+
" /budget <N>|off Set or clear per-turn token budget",
|
|
214
|
+
"",
|
|
215
|
+
" Other:",
|
|
186
216
|
" /permissions Show permission / HITL settings",
|
|
187
|
-
" /memory Show persistent memory settings",
|
|
188
217
|
" /hooks Show post-turn hook configuration",
|
|
189
|
-
" /version Print GemCode version hint",
|
|
190
|
-
" /exit Exit the REPL",
|
|
191
218
|
]
|