use-computer 0.0.45__tar.gz → 0.0.46__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- use_computer-0.0.46/PKG-INFO +108 -0
- use_computer-0.0.46/README.md +78 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/pyproject.toml +4 -8
- {use_computer-0.0.45 → use_computer-0.0.46}/scripts/smoke_wheel.py +3 -45
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/__init__.py +2 -43
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/__init__.py +0 -2
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/agent.py +1 -1
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/desktop.py +0 -52
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/recordings.py +1 -92
- use_computer-0.0.46/use_computer/agents/base/sandbox.py +7 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/screenshots.py +1 -1
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/task_setup.py +3 -70
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/debug/actions.py +1 -59
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/debug/agent.py +3 -62
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/types.py +0 -6
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/automation/osworld.py +2 -2
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/core/__init__.py +1 -18
- use_computer-0.0.46/use_computer/core/client.py +305 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/core/errors.py +4 -45
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/core/models.py +1 -1
- use_computer-0.0.46/use_computer/core/sandbox.py +474 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/uv.lock +1 -9
- use_computer-0.0.45/PKG-INFO +0 -239
- use_computer-0.0.45/README.md +0 -205
- use_computer-0.0.45/use_computer/agents/base/sandbox.py +0 -11
- use_computer-0.0.45/use_computer/agents/ios/__init__.py +0 -1
- use_computer-0.0.45/use_computer/agents/ios/agent.py +0 -390
- use_computer-0.0.45/use_computer/agents/ios/tools.py +0 -368
- use_computer-0.0.45/use_computer/agents/prompts/ios.txt +0 -49
- use_computer-0.0.45/use_computer/core/client.py +0 -703
- use_computer-0.0.45/use_computer/core/sandbox.py +0 -882
- use_computer-0.0.45/use_computer/harbor/__init__.py +0 -1
- use_computer-0.0.45/use_computer/harbor/agents.py +0 -156
- use_computer-0.0.45/use_computer/ios/__init__.py +0 -33
- use_computer-0.0.45/use_computer/ios/apps.py +0 -57
- use_computer-0.0.45/use_computer/ios/environment.py +0 -49
- use_computer-0.0.45/use_computer/ios/input.py +0 -163
- use_computer-0.0.45/use_computer/ios/simulators.py +0 -179
- use_computer-0.0.45/use_computer/tasks/__init__.py +0 -376
- use_computer-0.0.45/use_computer/tasks/templates/pre_command.ps1 +0 -1
- use_computer-0.0.45/use_computer/tasks/templates/pre_command.sh +0 -2
- use_computer-0.0.45/use_computer/tasks/templates/task.toml +0 -18
- use_computer-0.0.45/use_computer/tasks/templates/test_ios.sh +0 -20
- use_computer-0.0.45/use_computer/tasks/templates/test_ios_nograder.sh +0 -4
- use_computer-0.0.45/use_computer/tasks/templates/test_macos.sh +0 -10
- use_computer-0.0.45/use_computer/tasks/templates/test_macos_check.sh +0 -10
- use_computer-0.0.45/use_computer/tasks/templates/test_macos_nograder.sh +0 -7
- use_computer-0.0.45/use_computer/tasks/templates/test_ubuntu.sh +0 -19
- use_computer-0.0.45/use_computer/tasks/templates/test_ubuntu_check.sh +0 -10
- use_computer-0.0.45/use_computer/tasks/templates/test_ubuntu_nograder.sh +0 -6
- use_computer-0.0.45/use_computer/tasks/templates/test_windows.sh +0 -19
- use_computer-0.0.45/use_computer/tasks/templates/test_windows_check.sh +0 -10
- use_computer-0.0.45/use_computer/tasks/templates/test_windows_nograder.sh +0 -6
- use_computer-0.0.45/use_computer/windows/__init__.py +0 -3
- use_computer-0.0.45/use_computer/windows/shell.py +0 -61
- {use_computer-0.0.45 → use_computer-0.0.46}/.env.example +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/.gitignore +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/.pre-commit-config.yaml +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/__init__.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/actions.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/artifacts.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/final_state.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/prompts.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/base/trajectory.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/core.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/debug/README.md +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/debug/__init__.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/debug/models.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/prompts/anthropic.txt +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/prompts/gemini.txt +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/prompts/openai.txt +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/prompts/osworld_pyautogui.txt +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/prompts/pyautogui.txt +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/providers/__init__.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/providers/anthropic.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/providers/gemini.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/providers/generic.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/providers/openai.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/providers/osworld.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/agents/providers/tinker_backend.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/automation/__init__.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/automation/ax_transpile.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/automation/parsers.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/core/retry.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/macos/__init__.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/macos/keyboard.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/macos/mouse.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/py.typed +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/resources/__init__.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/resources/accessibility.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/resources/display.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/resources/recording.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/resources/screenshot.py +0 -0
- {use_computer-0.0.45 → use_computer-0.0.46}/use_computer/setup_files.py +0 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: use-computer
|
|
3
|
+
Version: 0.0.46
|
|
4
|
+
Summary: Python SDK for use.computer macOS sandboxes
|
|
5
|
+
Project-URL: Homepage, https://use.computer
|
|
6
|
+
Project-URL: Documentation, https://api.use.computer/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/josancamon19/use-computer-sdk
|
|
8
|
+
Author: use.computer
|
|
9
|
+
Keywords: automation,computer-use,macos,sandbox,vnc
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: httpx>=0.27
|
|
21
|
+
Provides-Extra: agents
|
|
22
|
+
Requires-Dist: anthropic>=0.86; extra == 'agents'
|
|
23
|
+
Requires-Dist: google-genai>=1; extra == 'agents'
|
|
24
|
+
Requires-Dist: litellm>=1; extra == 'agents'
|
|
25
|
+
Requires-Dist: openai>=1; extra == 'agents'
|
|
26
|
+
Requires-Dist: pillow>=10; extra == 'agents'
|
|
27
|
+
Requires-Dist: tinker-cookbook>=0.1.0; (python_full_version >= '3.11') and extra == 'agents'
|
|
28
|
+
Requires-Dist: tinker>=0.14.0; (python_full_version >= '3.11') and extra == 'agents'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# use-computer Python SDK
|
|
32
|
+
|
|
33
|
+
use.computer gives you macOS sandboxes: VMs on dedicated Apple M4 Mac minis that you reserve for 24 hours or more, up to 2 VMs at a time per Mac.
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install use-computer
|
|
37
|
+
export USE_COMPUTER_API_KEY=uc_live_...
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Optional agent integrations are installed explicitly:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install "use-computer[agents]" # computer-use agents and provider SDKs
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Base installs only the SDK client and `httpx`. The `agents` extra installs the agent runtime plus model-provider dependencies (Anthropic, OpenAI, Gemini, LiteLLM).
|
|
47
|
+
|
|
48
|
+
## Quickstart
|
|
49
|
+
|
|
50
|
+
Flow: sign up → $100 starter credit → reserve a Mac mini (dashboard, or `client.reserve(hours=24)` in the SDK) → `create()` a macOS sandbox → drive it (mouse, keyboard, screenshot, exec, files, recording, UI tree, VNC) → `delete()`. Reservations cost $1.91/hour per Mac; the starter credit pays for them.
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from use_computer import Computer
|
|
54
|
+
|
|
55
|
+
client = Computer()
|
|
56
|
+
|
|
57
|
+
# 1. Reserve one Mac Mini for 24 hours
|
|
58
|
+
reservation = client.reserve(hours=24)
|
|
59
|
+
|
|
60
|
+
# 2. Launch a macOS sandbox on the reserved Mac
|
|
61
|
+
with client.create(reservation_id=reservation.id) as mac:
|
|
62
|
+
# 3. Drive the macOS sandbox
|
|
63
|
+
mac.exec("open -a Safari")
|
|
64
|
+
mac.keyboard.type("hello from use.computer")
|
|
65
|
+
mac.mouse.click(500, 500)
|
|
66
|
+
png = mac.screenshot.take_full_screen()
|
|
67
|
+
print("VNC URL:", mac.vnc_url)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Each reserved Mac Mini supports up to 2 parallel macOS VMs. Check `Computer().platforms(reservation_id=...)["macos"]["capacity"]` for the selected reservation's `max` and `used` counts.
|
|
71
|
+
|
|
72
|
+
Sandboxes default to `ephemeral=True`, deleting automatically after the idle timeout (roughly 2 minutes without activity). Use `ephemeral=False` when a sandbox should persist until manual deletion, or call `sandbox.start_keepalive(interval=30)` during long model-think periods.
|
|
73
|
+
|
|
74
|
+
## Runtime Snapshots
|
|
75
|
+
|
|
76
|
+
macOS snapshots let you seed a VM once, snapshot it, then create new sandboxes from that snapshot version. macOS snapshots preserve disk state, so installed apps, accounts, and seeded files are available immediately:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from use_computer import Computer
|
|
80
|
+
|
|
81
|
+
client = Computer()
|
|
82
|
+
|
|
83
|
+
# Configure the macOS desktop by hand over VNC, then snapshot it.
|
|
84
|
+
with client.create(type="macos") as mac:
|
|
85
|
+
print("Set up the macOS desktop here:", mac.vnc_url)
|
|
86
|
+
input("Press Enter once the desktop is ready to snapshot...")
|
|
87
|
+
snapshot = mac.snapshot("chrome-seeded-macos")
|
|
88
|
+
|
|
89
|
+
# New sandboxes boot from that saved disk state, no setup needed.
|
|
90
|
+
with client.create(type="macos", snapshot=snapshot.version) as seeded:
|
|
91
|
+
print("Seeded sandbox ready:", seeded.vnc_url)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Use `client.snapshots()` to list saved snapshot versions.
|
|
95
|
+
|
|
96
|
+
## Examples
|
|
97
|
+
|
|
98
|
+
| File | What it shows |
|
|
99
|
+
| --- | --- |
|
|
100
|
+
| [`examples/_1_hello_macos.py`](../examples/python/_1_hello_macos.py) | reserve → create → exec → keyboard → screenshot |
|
|
101
|
+
| [`examples/_2_recording.py`](../examples/python/_2_recording.py) | start / stop / download a screen recording |
|
|
102
|
+
| [`examples/_3_file_transfer.py`](../examples/python/_3_file_transfer.py) | upload bytes, download a file back |
|
|
103
|
+
| [`examples/_4_keepalive.py`](../examples/python/_4_keepalive.py) | heartbeat for ephemeral sessions idle > 2 min |
|
|
104
|
+
| [`examples/_5_snapshots.py`](../examples/python/_5_snapshots.py) | snapshot seeded macOS state |
|
|
105
|
+
|
|
106
|
+
## HTTP API
|
|
107
|
+
|
|
108
|
+
Every SDK method wraps `https://api.use.computer/v1/...` with `Authorization: Bearer uc_live_...`. Swagger: [api.use.computer/docs](https://api.use.computer/docs). OpenAPI spec: [api.use.computer/openapi.yaml](https://api.use.computer/openapi.yaml).
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# use-computer Python SDK
|
|
2
|
+
|
|
3
|
+
use.computer gives you macOS sandboxes: VMs on dedicated Apple M4 Mac minis that you reserve for 24 hours or more, up to 2 VMs at a time per Mac.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install use-computer
|
|
7
|
+
export USE_COMPUTER_API_KEY=uc_live_...
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Optional agent integrations are installed explicitly:
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install "use-computer[agents]" # computer-use agents and provider SDKs
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Base installs only the SDK client and `httpx`. The `agents` extra installs the agent runtime plus model-provider dependencies (Anthropic, OpenAI, Gemini, LiteLLM).
|
|
17
|
+
|
|
18
|
+
## Quickstart
|
|
19
|
+
|
|
20
|
+
Flow: sign up → $100 starter credit → reserve a Mac mini (dashboard, or `client.reserve(hours=24)` in the SDK) → `create()` a macOS sandbox → drive it (mouse, keyboard, screenshot, exec, files, recording, UI tree, VNC) → `delete()`. Reservations cost $1.91/hour per Mac; the starter credit pays for them.
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from use_computer import Computer
|
|
24
|
+
|
|
25
|
+
client = Computer()
|
|
26
|
+
|
|
27
|
+
# 1. Reserve one Mac Mini for 24 hours
|
|
28
|
+
reservation = client.reserve(hours=24)
|
|
29
|
+
|
|
30
|
+
# 2. Launch a macOS sandbox on the reserved Mac
|
|
31
|
+
with client.create(reservation_id=reservation.id) as mac:
|
|
32
|
+
# 3. Drive the macOS sandbox
|
|
33
|
+
mac.exec("open -a Safari")
|
|
34
|
+
mac.keyboard.type("hello from use.computer")
|
|
35
|
+
mac.mouse.click(500, 500)
|
|
36
|
+
png = mac.screenshot.take_full_screen()
|
|
37
|
+
print("VNC URL:", mac.vnc_url)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Each reserved Mac Mini supports up to 2 parallel macOS VMs. Check `Computer().platforms(reservation_id=...)["macos"]["capacity"]` for the selected reservation's `max` and `used` counts.
|
|
41
|
+
|
|
42
|
+
Sandboxes default to `ephemeral=True`, deleting automatically after the idle timeout (roughly 2 minutes without activity). Use `ephemeral=False` when a sandbox should persist until manual deletion, or call `sandbox.start_keepalive(interval=30)` during long model-think periods.
|
|
43
|
+
|
|
44
|
+
## Runtime Snapshots
|
|
45
|
+
|
|
46
|
+
macOS snapshots let you seed a VM once, snapshot it, then create new sandboxes from that snapshot version. macOS snapshots preserve disk state, so installed apps, accounts, and seeded files are available immediately:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from use_computer import Computer
|
|
50
|
+
|
|
51
|
+
client = Computer()
|
|
52
|
+
|
|
53
|
+
# Configure the macOS desktop by hand over VNC, then snapshot it.
|
|
54
|
+
with client.create(type="macos") as mac:
|
|
55
|
+
print("Set up the macOS desktop here:", mac.vnc_url)
|
|
56
|
+
input("Press Enter once the desktop is ready to snapshot...")
|
|
57
|
+
snapshot = mac.snapshot("chrome-seeded-macos")
|
|
58
|
+
|
|
59
|
+
# New sandboxes boot from that saved disk state, no setup needed.
|
|
60
|
+
with client.create(type="macos", snapshot=snapshot.version) as seeded:
|
|
61
|
+
print("Seeded sandbox ready:", seeded.vnc_url)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Use `client.snapshots()` to list saved snapshot versions.
|
|
65
|
+
|
|
66
|
+
## Examples
|
|
67
|
+
|
|
68
|
+
| File | What it shows |
|
|
69
|
+
| --- | --- |
|
|
70
|
+
| [`examples/_1_hello_macos.py`](../examples/python/_1_hello_macos.py) | reserve → create → exec → keyboard → screenshot |
|
|
71
|
+
| [`examples/_2_recording.py`](../examples/python/_2_recording.py) | start / stop / download a screen recording |
|
|
72
|
+
| [`examples/_3_file_transfer.py`](../examples/python/_3_file_transfer.py) | upload bytes, download a file back |
|
|
73
|
+
| [`examples/_4_keepalive.py`](../examples/python/_4_keepalive.py) | heartbeat for ephemeral sessions idle > 2 min |
|
|
74
|
+
| [`examples/_5_snapshots.py`](../examples/python/_5_snapshots.py) | snapshot seeded macOS state |
|
|
75
|
+
|
|
76
|
+
## HTTP API
|
|
77
|
+
|
|
78
|
+
Every SDK method wraps `https://api.use.computer/v1/...` with `Authorization: Bearer uc_live_...`. Swagger: [api.use.computer/docs](https://api.use.computer/docs). OpenAPI spec: [api.use.computer/openapi.yaml](https://api.use.computer/openapi.yaml).
|
|
@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "use-computer"
|
|
7
|
-
version = "0.0.
|
|
8
|
-
description = "Python SDK for use.computer
|
|
7
|
+
version = "0.0.46"
|
|
8
|
+
description = "Python SDK for use.computer macOS sandboxes"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
authors = [{ name = "use.computer" }]
|
|
12
12
|
dependencies = ["httpx>=0.27"]
|
|
13
|
-
keywords = ["computer-use", "macos", "
|
|
13
|
+
keywords = ["computer-use", "macos", "automation", "sandbox", "vnc"]
|
|
14
14
|
classifiers = [
|
|
15
15
|
"Development Status :: 3 - Alpha",
|
|
16
16
|
"Intended Audience :: Developers",
|
|
@@ -33,11 +33,7 @@ agents = [
|
|
|
33
33
|
"tinker>=0.14.0 ; python_full_version >= '3.11'",
|
|
34
34
|
"tinker-cookbook>=0.1.0 ; python_full_version >= '3.11'",
|
|
35
35
|
]
|
|
36
|
-
|
|
37
|
-
"harbor>=0.4.0; python_version >= '3.12'",
|
|
38
|
-
"PyYAML>=6",
|
|
39
|
-
"tomli>=2",
|
|
40
|
-
]
|
|
36
|
+
|
|
41
37
|
|
|
42
38
|
[dependency-groups]
|
|
43
39
|
dev = [
|
|
@@ -20,7 +20,7 @@ def has(name):
|
|
|
20
20
|
return False
|
|
21
21
|
|
|
22
22
|
assert "site-packages" in (use_computer.__file__ or ""), use_computer.__file__
|
|
23
|
-
for name in ["PIL", "openai", "anthropic", "google.genai", "litellm"
|
|
23
|
+
for name in ["PIL", "openai", "anthropic", "google.genai", "litellm"]:
|
|
24
24
|
assert not has(name), name
|
|
25
25
|
"""
|
|
26
26
|
|
|
@@ -41,7 +41,7 @@ from use_computer.agents import (
|
|
|
41
41
|
AnthropicComputerAgent,
|
|
42
42
|
GeminiComputerAgent,
|
|
43
43
|
GenericComputerAgent,
|
|
44
|
-
|
|
44
|
+
|
|
45
45
|
OpenAIComputerAgent,
|
|
46
46
|
)
|
|
47
47
|
from use_computer.agents.debug import DebugComputerAgent
|
|
@@ -105,28 +105,11 @@ for agent_cls, model_name in [
|
|
|
105
105
|
(OpenAIComputerAgent, "openai/gpt-5.5"),
|
|
106
106
|
(GeminiComputerAgent, "gemini/gemini-2.5-computer-use-preview-10-2025"),
|
|
107
107
|
(GenericComputerAgent, "openai/gpt-4o"),
|
|
108
|
-
|
|
108
|
+
|
|
109
109
|
]:
|
|
110
110
|
assert agent_cls(model_name=model_name, max_steps=1).max_steps == 1
|
|
111
111
|
"""
|
|
112
112
|
|
|
113
|
-
HARBOR_SMOKE = r"""
|
|
114
|
-
import importlib
|
|
115
|
-
import importlib.util
|
|
116
|
-
|
|
117
|
-
for name in ["harbor", "yaml", "tomli"]:
|
|
118
|
-
assert importlib.util.find_spec(name) is not None, name
|
|
119
|
-
for path in ["use_computer.harbor"]:
|
|
120
|
-
importlib.import_module(path)
|
|
121
|
-
"""
|
|
122
|
-
|
|
123
|
-
HARBOR_AGENTS_SMOKE = r"""
|
|
124
|
-
import importlib
|
|
125
|
-
|
|
126
|
-
for path in ["use_computer.harbor", "use_computer.harbor.agents"]:
|
|
127
|
-
importlib.import_module(path)
|
|
128
|
-
"""
|
|
129
|
-
|
|
130
113
|
|
|
131
114
|
def main() -> None:
|
|
132
115
|
parser = argparse.ArgumentParser()
|
|
@@ -141,8 +124,6 @@ def main() -> None:
|
|
|
141
124
|
_check_metadata(wheel)
|
|
142
125
|
_smoke_install(wheel, None, BASE_SMOKE)
|
|
143
126
|
_smoke_install(wheel, "agents", AGENTS_SMOKE)
|
|
144
|
-
_smoke_install(wheel, "harbor", HARBOR_SMOKE)
|
|
145
|
-
_smoke_install(wheel, "harbor,agents", HARBOR_AGENTS_SMOKE)
|
|
146
127
|
|
|
147
128
|
|
|
148
129
|
def _check_wheel_contents(wheel: Path) -> None:
|
|
@@ -153,10 +134,8 @@ def _check_wheel_contents(wheel: Path) -> None:
|
|
|
153
134
|
"use_computer/py.typed",
|
|
154
135
|
"use_computer/agents/prompts/anthropic.txt",
|
|
155
136
|
"use_computer/agents/prompts/gemini.txt",
|
|
156
|
-
"use_computer/agents/prompts/ios.txt",
|
|
157
137
|
"use_computer/agents/prompts/openai.txt",
|
|
158
138
|
"use_computer/agents/prompts/pyautogui.txt",
|
|
159
|
-
"use_computer/harbor/agents.py",
|
|
160
139
|
}
|
|
161
140
|
missing = sorted(required - names)
|
|
162
141
|
if missing:
|
|
@@ -176,7 +155,6 @@ def _check_metadata(wheel: Path) -> None:
|
|
|
176
155
|
|
|
177
156
|
required = [
|
|
178
157
|
"Provides-Extra: agents",
|
|
179
|
-
"Provides-Extra: harbor",
|
|
180
158
|
"Requires-Dist: pillow>=10; extra == 'agents'",
|
|
181
159
|
"Requires-Dist: openai>=1; extra == 'agents'",
|
|
182
160
|
"Requires-Dist: anthropic>=0.86; extra == 'agents'",
|
|
@@ -195,26 +173,6 @@ def _check_metadata(wheel: Path) -> None:
|
|
|
195
173
|
if present:
|
|
196
174
|
raise SystemExit(f"wheel metadata still has provider-specific extras: {present}")
|
|
197
175
|
|
|
198
|
-
harbor_lines = [
|
|
199
|
-
line for line in metadata.splitlines() if line.startswith("Requires-Dist: harbor>=0.4.0;")
|
|
200
|
-
]
|
|
201
|
-
if not any(
|
|
202
|
-
"python_version >= '3.12'" in line and "extra == 'harbor'" in line for line in harbor_lines
|
|
203
|
-
):
|
|
204
|
-
raise SystemExit("wheel metadata is missing the Python 3.12+ Harbor extra")
|
|
205
|
-
|
|
206
|
-
forbidden_harbor_deps = [
|
|
207
|
-
line
|
|
208
|
-
for line in metadata.splitlines()
|
|
209
|
-
if "extra == 'harbor'" in line
|
|
210
|
-
and any(
|
|
211
|
-
f"Requires-Dist: {name}" in line
|
|
212
|
-
for name in ("pillow", "openai", "anthropic", "litellm", "google-genai")
|
|
213
|
-
)
|
|
214
|
-
]
|
|
215
|
-
if forbidden_harbor_deps:
|
|
216
|
-
raise SystemExit(f"harbor extra should not install agent deps: {forbidden_harbor_deps}")
|
|
217
|
-
|
|
218
176
|
|
|
219
177
|
def _smoke_install(wheel: Path, extra: str | None, code: str) -> None:
|
|
220
178
|
with tempfile.TemporaryDirectory(prefix="use-computer-wheel-") as tmp:
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
from use_computer.automation.osworld import AsyncOSWorldClient, OSWorldClient
|
|
2
2
|
from use_computer.automation.parsers import Action, parse_pyautogui, parse_xdotool
|
|
3
|
-
from use_computer.core.client import AsyncComputer, Computer, Reservation
|
|
3
|
+
from use_computer.core.client import AsyncComputer, Computer, Reservation
|
|
4
4
|
from use_computer.core.errors import (
|
|
5
5
|
BillingRequiredError,
|
|
6
6
|
PlatformNotSupportedError,
|
|
7
7
|
ReservationRequiredError,
|
|
8
|
-
SandboxLimitReachedError,
|
|
9
8
|
UseComputerError,
|
|
10
9
|
)
|
|
11
10
|
from use_computer.core.models import (
|
|
@@ -18,78 +17,38 @@ from use_computer.core.models import (
|
|
|
18
17
|
RecordingInfo,
|
|
19
18
|
)
|
|
20
19
|
from use_computer.core.sandbox import (
|
|
21
|
-
AsyncIOSSandbox,
|
|
22
20
|
AsyncMacOSSandbox,
|
|
23
21
|
AsyncSandbox,
|
|
24
|
-
AsyncUbuntuSandbox,
|
|
25
|
-
AsyncWindowsSandbox,
|
|
26
|
-
HostRedirect,
|
|
27
|
-
IOSSandbox,
|
|
28
22
|
MacOSSandbox,
|
|
29
23
|
Sandbox,
|
|
30
24
|
SandboxType,
|
|
31
25
|
Snapshot,
|
|
32
|
-
UbuntuSandbox,
|
|
33
|
-
WindowsSandbox,
|
|
34
26
|
)
|
|
35
|
-
from use_computer.ios.input import Button, Key, RemoteButton
|
|
36
|
-
from use_computer.ios.simulators import (
|
|
37
|
-
SimulatorChoice,
|
|
38
|
-
SimulatorFamily,
|
|
39
|
-
family_for_device,
|
|
40
|
-
is_usable_device_type,
|
|
41
|
-
required_runtime_os,
|
|
42
|
-
runtime_os,
|
|
43
|
-
select_simulator,
|
|
44
|
-
)
|
|
45
|
-
from use_computer.tasks import Task, TasksClient, TaskSummary
|
|
46
27
|
|
|
47
28
|
__all__ = [
|
|
48
29
|
"Action",
|
|
49
30
|
"AccessibilityTree",
|
|
50
31
|
"ActionResult",
|
|
51
32
|
"ActResult",
|
|
52
|
-
"AsyncIOSSandbox",
|
|
53
|
-
"AsyncMacOSSandbox",
|
|
54
33
|
"AsyncComputer",
|
|
34
|
+
"AsyncMacOSSandbox",
|
|
55
35
|
"AsyncOSWorldClient",
|
|
56
36
|
"AsyncSandbox",
|
|
57
|
-
"AsyncUbuntuSandbox",
|
|
58
|
-
"AsyncWindowsSandbox",
|
|
59
37
|
"BillingRequiredError",
|
|
60
|
-
"Button",
|
|
61
38
|
"Computer",
|
|
62
39
|
"CursorPosition",
|
|
63
40
|
"DisplayInfo",
|
|
64
41
|
"ExecResult",
|
|
65
|
-
"HostRedirect",
|
|
66
|
-
"IOSSandbox",
|
|
67
|
-
"Key",
|
|
68
42
|
"MacOSSandbox",
|
|
69
43
|
"OSWorldClient",
|
|
70
44
|
"PlatformNotSupportedError",
|
|
71
45
|
"RecordingInfo",
|
|
72
46
|
"Reservation",
|
|
73
47
|
"ReservationRequiredError",
|
|
74
|
-
"RemoteButton",
|
|
75
|
-
"RunStatus",
|
|
76
48
|
"Sandbox",
|
|
77
|
-
"SandboxLimitReachedError",
|
|
78
49
|
"SandboxType",
|
|
79
50
|
"Snapshot",
|
|
80
|
-
"SimulatorChoice",
|
|
81
|
-
"SimulatorFamily",
|
|
82
|
-
"Task",
|
|
83
|
-
"TaskSummary",
|
|
84
|
-
"TasksClient",
|
|
85
51
|
"UseComputerError",
|
|
86
|
-
"UbuntuSandbox",
|
|
87
|
-
"WindowsSandbox",
|
|
88
|
-
"family_for_device",
|
|
89
|
-
"is_usable_device_type",
|
|
90
52
|
"parse_pyautogui",
|
|
91
53
|
"parse_xdotool",
|
|
92
|
-
"required_runtime_os",
|
|
93
|
-
"runtime_os",
|
|
94
|
-
"select_simulator",
|
|
95
54
|
]
|
|
@@ -22,8 +22,6 @@ _LAZY_EXPORTS = {
|
|
|
22
22
|
"GeminiCUAAgent": ("use_computer.agents.providers.gemini", "GeminiCUAAgent"),
|
|
23
23
|
"GenericComputerAgent": ("use_computer.agents.providers.generic", "GenericComputerAgent"),
|
|
24
24
|
"GenericCUAAgent": ("use_computer.agents.providers.generic", "GenericCUAAgent"),
|
|
25
|
-
"IOSAgent": ("use_computer.agents.ios.agent", "IOSAgent"),
|
|
26
|
-
"IOSComputerAgent": ("use_computer.agents.ios.agent", "IOSComputerAgent"),
|
|
27
25
|
"OpenAIComputerAgent": ("use_computer.agents.providers.openai", "OpenAIComputerAgent"),
|
|
28
26
|
"OpenAICUAAgent": ("use_computer.agents.providers.openai", "OpenAICUAAgent"),
|
|
29
27
|
"OSWorldAgent": ("use_computer.agents.providers.osworld", "OSWorldAgent"),
|
|
@@ -255,7 +255,7 @@ class BaseCUAAgent(RecordingMixin, FinalStateMixin, ArtifactCaptureMixin):
|
|
|
255
255
|
)
|
|
256
256
|
|
|
257
257
|
async def _save_focused_app(self, sandbox: AsyncSandbox) -> None:
|
|
258
|
-
shortcut = "command+s"
|
|
258
|
+
shortcut = "command+s"
|
|
259
259
|
computer = cast(ComputerSandbox, sandbox)
|
|
260
260
|
await computer.keyboard.hotkey(shortcut)
|
|
261
261
|
await asyncio.sleep(1)
|
|
@@ -87,34 +87,9 @@ _KEY_ALIASES = {
|
|
|
87
87
|
"PAGEDOWN": "pagedown",
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
# Windows: the modifier is the Windows key, not Cmd.
|
|
91
|
-
_WINDOWS_KEY_ALIASES = {
|
|
92
|
-
**_KEY_ALIASES,
|
|
93
|
-
"Command": "win",
|
|
94
|
-
"command": "win",
|
|
95
|
-
"COMMAND": "win",
|
|
96
|
-
"Cmd": "win",
|
|
97
|
-
"cmd": "win",
|
|
98
|
-
"CMD": "win",
|
|
99
|
-
"super": "win",
|
|
100
|
-
"SUPER": "win",
|
|
101
|
-
"meta": "win",
|
|
102
|
-
"META": "win",
|
|
103
|
-
"Super_L": "win",
|
|
104
|
-
"Meta_L": "win",
|
|
105
|
-
"Win": "win",
|
|
106
|
-
"win": "win",
|
|
107
|
-
"WIN": "win",
|
|
108
|
-
"Windows": "win",
|
|
109
|
-
"windows": "win",
|
|
110
|
-
"WINDOWS": "win",
|
|
111
|
-
}
|
|
112
|
-
|
|
113
90
|
|
|
114
91
|
class OpenAIComputerEnvironment(str, Enum):
|
|
115
92
|
MAC = "mac"
|
|
116
|
-
WINDOWS = "windows"
|
|
117
|
-
UBUNTU = "ubuntu"
|
|
118
93
|
|
|
119
94
|
|
|
120
95
|
@dataclass(frozen=True)
|
|
@@ -136,31 +111,6 @@ _DESKTOP_PROFILES = {
|
|
|
136
111
|
os_type="macOS",
|
|
137
112
|
openai_environment=OpenAIComputerEnvironment.MAC,
|
|
138
113
|
),
|
|
139
|
-
Platform.WINDOWS: DesktopProfile(
|
|
140
|
-
platform=Platform.WINDOWS,
|
|
141
|
-
os_type="Windows",
|
|
142
|
-
notes=(
|
|
143
|
-
"Windows notes:\n"
|
|
144
|
-
"- Use Ctrl (not Cmd) for shortcuts: Ctrl+C/V/X copy/paste/cut, "
|
|
145
|
-
"Ctrl+S save, Ctrl+Z undo.\n"
|
|
146
|
-
"- Open apps via Start (Windows key, app name, Enter) or by "
|
|
147
|
-
"double-clicking a file."
|
|
148
|
-
),
|
|
149
|
-
key_aliases=_WINDOWS_KEY_ALIASES,
|
|
150
|
-
openai_environment=OpenAIComputerEnvironment.WINDOWS,
|
|
151
|
-
),
|
|
152
|
-
Platform.UBUNTU: DesktopProfile(
|
|
153
|
-
platform=Platform.UBUNTU,
|
|
154
|
-
os_type="Ubuntu",
|
|
155
|
-
notes=(
|
|
156
|
-
"Ubuntu notes:\n"
|
|
157
|
-
"- Use Ctrl for shortcuts: Ctrl+C/V/X copy/paste/cut, Ctrl+S save, Ctrl+Z undo.\n"
|
|
158
|
-
"- Open apps via Activities, the app launcher, or the file manager.\n"
|
|
159
|
-
"- The home directory is /home/user."
|
|
160
|
-
),
|
|
161
|
-
key_aliases=_WINDOWS_KEY_ALIASES,
|
|
162
|
-
openai_environment=OpenAIComputerEnvironment.UBUNTU,
|
|
163
|
-
),
|
|
164
114
|
}
|
|
165
115
|
|
|
166
116
|
|
|
@@ -172,8 +122,6 @@ def desktop_profile(
|
|
|
172
122
|
openai_environment: OpenAIComputerEnvironment | str | None = None,
|
|
173
123
|
) -> DesktopProfile:
|
|
174
124
|
parsed = Platform.parse(platform)
|
|
175
|
-
if parsed is Platform.IOS:
|
|
176
|
-
raise ValueError("iOS uses IOSAgent, not desktop CUA agents")
|
|
177
125
|
base = _DESKTOP_PROFILES[parsed]
|
|
178
126
|
environment = (
|
|
179
127
|
OpenAIComputerEnvironment(openai_environment)
|
|
@@ -13,34 +13,22 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
from use_computer.core.sandbox import AsyncSandbox
|
|
15
15
|
|
|
16
|
-
_OSWORLD_FFMPEG_RECORDING_ID = "__osworld_ffmpeg__"
|
|
17
|
-
|
|
18
16
|
|
|
19
17
|
class RecordingMixin:
|
|
20
18
|
async def start_recording(self: Any, sandbox: AsyncSandbox) -> None:
|
|
21
|
-
if getattr(getattr(sandbox, "type", None), "value", "") == "ios":
|
|
22
|
-
try:
|
|
23
|
-
await sandbox.screenshot.take_full_screen()
|
|
24
|
-
except Exception as e:
|
|
25
|
-
self.logger.warning(f"iOS recording warm-up screenshot failed: {e}")
|
|
26
19
|
try:
|
|
27
20
|
result = await sandbox.recording.start(name="trial")
|
|
28
21
|
self._recording_id = result.id
|
|
29
22
|
self.logger.info(f"Recording started: {self._recording_id}")
|
|
30
23
|
except Exception as e:
|
|
31
24
|
self.logger.warning(f"Failed to start recording: {e}")
|
|
32
|
-
if await self._start_osworld_ffmpeg_recording(sandbox):
|
|
33
|
-
return
|
|
34
25
|
|
|
35
26
|
async def stop_recording(self: Any, sandbox: AsyncSandbox) -> None:
|
|
36
27
|
if not self._recording_id:
|
|
37
28
|
return
|
|
38
29
|
await asyncio.sleep(3.0)
|
|
39
30
|
rec_path = self.logs_dir / "recording.mp4"
|
|
40
|
-
|
|
41
|
-
await self._stop_osworld_ffmpeg_recording(sandbox, rec_path)
|
|
42
|
-
self._recording_id = None
|
|
43
|
-
return
|
|
31
|
+
|
|
44
32
|
try:
|
|
45
33
|
info = await sandbox.recording.stop(self._recording_id)
|
|
46
34
|
self.logger.info(
|
|
@@ -172,82 +160,3 @@ class RecordingMixin:
|
|
|
172
160
|
except Exception as e:
|
|
173
161
|
self.logger.warning(f"Screenshot recording fallback failed: {type(e).__name__}: {e}")
|
|
174
162
|
return False
|
|
175
|
-
|
|
176
|
-
async def _start_osworld_ffmpeg_recording(self: Any, sandbox: AsyncSandbox) -> bool:
|
|
177
|
-
osworld: Any = getattr(sandbox, "osworld", None)
|
|
178
|
-
if osworld is None:
|
|
179
|
-
return False
|
|
180
|
-
if getattr(getattr(sandbox, "type", None), "value", "") != "ubuntu":
|
|
181
|
-
self.logger.warning("OSWorld ffmpeg recording fallback is only implemented for Ubuntu")
|
|
182
|
-
return False
|
|
183
|
-
|
|
184
|
-
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
185
|
-
script = """
|
|
186
|
-
set -eux
|
|
187
|
-
rm -f /tmp/osworld-recording.mp4 /tmp/osworld-recording.log /tmp/osworld-recording.pid
|
|
188
|
-
export DISPLAY=:0
|
|
189
|
-
nohup ffmpeg -y \
|
|
190
|
-
-video_size 1920x1080 \
|
|
191
|
-
-framerate 2 \
|
|
192
|
-
-f x11grab \
|
|
193
|
-
-i :0 \
|
|
194
|
-
-t 120 \
|
|
195
|
-
-pix_fmt yuv420p \
|
|
196
|
-
/tmp/osworld-recording.mp4 > /tmp/osworld-recording.log 2>&1 &
|
|
197
|
-
echo $! > /tmp/osworld-recording.pid
|
|
198
|
-
cat /tmp/osworld-recording.pid
|
|
199
|
-
"""
|
|
200
|
-
try:
|
|
201
|
-
result = await osworld.execute(["bash", "-lc", script], timeout=10)
|
|
202
|
-
except Exception as e:
|
|
203
|
-
self.logger.warning(f"OSWorld ffmpeg recording start failed: {type(e).__name__}: {e}")
|
|
204
|
-
return False
|
|
205
|
-
|
|
206
|
-
(self.logs_dir / "recording-start.json").write_text(json.dumps(result, indent=2))
|
|
207
|
-
if result.get("returncode") != 0:
|
|
208
|
-
self.logger.warning(f"OSWorld ffmpeg recording start failed: {result}")
|
|
209
|
-
return False
|
|
210
|
-
|
|
211
|
-
self._recording_id = _OSWORLD_FFMPEG_RECORDING_ID
|
|
212
|
-
self.logger.info("OSWorld ffmpeg recording started")
|
|
213
|
-
return True
|
|
214
|
-
|
|
215
|
-
async def _stop_osworld_ffmpeg_recording(self: Any, sandbox: AsyncSandbox, rec_path) -> None:
|
|
216
|
-
osworld: Any = getattr(sandbox, "osworld", None)
|
|
217
|
-
if osworld is None:
|
|
218
|
-
return
|
|
219
|
-
stop_script = """
|
|
220
|
-
set -eux
|
|
221
|
-
if [ -f /tmp/osworld-recording.pid ]; then
|
|
222
|
-
pid="$(cat /tmp/osworld-recording.pid)"
|
|
223
|
-
if kill -0 "$pid" 2>/dev/null; then
|
|
224
|
-
kill -INT "$pid" || true
|
|
225
|
-
fi
|
|
226
|
-
for _ in $(seq 1 40); do
|
|
227
|
-
if kill -0 "$pid" 2>/dev/null; then
|
|
228
|
-
sleep 0.25
|
|
229
|
-
else
|
|
230
|
-
break
|
|
231
|
-
fi
|
|
232
|
-
done
|
|
233
|
-
fi
|
|
234
|
-
ls -lh /tmp/osworld-recording.mp4 /tmp/osworld-recording.log
|
|
235
|
-
tail -80 /tmp/osworld-recording.log
|
|
236
|
-
"""
|
|
237
|
-
t_dl = time.monotonic()
|
|
238
|
-
try:
|
|
239
|
-
meta: dict[str, Any] = {
|
|
240
|
-
"stop": await osworld.execute(["bash", "-lc", stop_script], timeout=30)
|
|
241
|
-
}
|
|
242
|
-
data = await osworld.file("/tmp/osworld-recording.mp4")
|
|
243
|
-
rec_path.write_bytes(data)
|
|
244
|
-
local_size = rec_path.stat().st_size
|
|
245
|
-
meta["downloaded_size"] = local_size
|
|
246
|
-
(self.logs_dir / "recording.json").write_text(json.dumps(meta, indent=2))
|
|
247
|
-
self.logger.info(
|
|
248
|
-
f"OSWorld ffmpeg recording saved: {rec_path} "
|
|
249
|
-
f"local_size={local_size}B ({local_size / (1024 * 1024):.2f} MB) "
|
|
250
|
-
f"download_elapsed={time.monotonic() - t_dl:.1f}s"
|
|
251
|
-
)
|
|
252
|
-
except Exception as e:
|
|
253
|
-
self.logger.warning(f"OSWorld ffmpeg recording save failed: {type(e).__name__}: {e}")
|
|
@@ -15,7 +15,7 @@ def screenshot_cap_for_model(model: str) -> int:
|
|
|
15
15
|
if "claude" in m or "anthropic/" in m:
|
|
16
16
|
return 1568 # Other Anthropic models cap at 1568px
|
|
17
17
|
if "kimi" in m or "fireworks" in m:
|
|
18
|
-
# Kimi: y-coord accuracy degrades at >1024
|
|
18
|
+
# Kimi: y-coord accuracy degrades at >1024
|
|
19
19
|
return 896
|
|
20
20
|
# TODO: probe Qwen 3.6 Plus separately — it inherits Kimi's cap for now
|
|
21
21
|
return 1280 # OpenAI/Gemini fallback
|