droidrun 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- droidrun/__init__.py +1 -10
- droidrun/adb/device.py +101 -71
- droidrun/adb/manager.py +3 -3
- droidrun/agent/codeact/codeact_agent.py +22 -12
- droidrun/agent/context/personas/__init__.py +0 -2
- droidrun/agent/context/personas/default.py +1 -1
- droidrun/agent/droid/droid_agent.py +56 -8
- droidrun/agent/droid/events.py +4 -0
- droidrun/agent/planner/planner_agent.py +32 -12
- droidrun/agent/utils/chat_utils.py +4 -7
- droidrun/agent/utils/llm_picker.py +1 -0
- droidrun/cli/main.py +163 -78
- droidrun/portal.py +139 -0
- droidrun/telemetry/__init__.py +4 -0
- droidrun/telemetry/events.py +27 -0
- droidrun/telemetry/tracker.py +83 -0
- droidrun/tools/adb.py +199 -407
- droidrun/tools/ios.py +10 -5
- droidrun/tools/tools.py +42 -11
- {droidrun-0.3.0.dist-info → droidrun-0.3.2.dist-info}/METADATA +19 -29
- {droidrun-0.3.0.dist-info → droidrun-0.3.2.dist-info}/RECORD +24 -23
- droidrun/agent/context/personas/extractor.py +0 -52
- droidrun/agent/context/todo.txt +0 -4
- droidrun/run.py +0 -105
- {droidrun-0.3.0.dist-info → droidrun-0.3.2.dist-info}/WHEEL +0 -0
- {droidrun-0.3.0.dist-info → droidrun-0.3.2.dist-info}/entry_points.txt +0 -0
- {droidrun-0.3.0.dist-info → droidrun-0.3.2.dist-info}/licenses/LICENSE +0 -0
droidrun/tools/adb.py
CHANGED
@@ -3,27 +3,31 @@ UI Actions - Core UI interaction tools for Android device control.
|
|
3
3
|
"""
|
4
4
|
|
5
5
|
import os
|
6
|
-
import re
|
7
6
|
import json
|
8
7
|
import time
|
9
|
-
import tempfile
|
10
8
|
import asyncio
|
11
|
-
import
|
12
|
-
import
|
13
|
-
from typing import Optional, Dict, Tuple, List, Any
|
9
|
+
import logging
|
10
|
+
from typing import Optional, Dict, Tuple, List, Any, Type, Self
|
14
11
|
from droidrun.adb.device import Device
|
15
12
|
from droidrun.adb.manager import DeviceManager
|
16
13
|
from droidrun.tools.tools import Tools
|
17
14
|
|
15
|
+
logger = logging.getLogger("droidrun-adb-tools")
|
16
|
+
|
18
17
|
|
19
18
|
class AdbTools(Tools):
|
20
19
|
"""Core UI interaction tools for Android device control."""
|
21
20
|
|
22
|
-
def __init__(self, serial: str
|
21
|
+
def __init__(self, serial: str) -> None:
|
22
|
+
"""Initialize the AdbTools instance.
|
23
|
+
|
24
|
+
Args:
|
25
|
+
serial: Device serial number
|
26
|
+
"""
|
27
|
+
self.device_manager = DeviceManager()
|
23
28
|
# Instance‐level cache for clickable elements (index-based tapping)
|
24
29
|
self.clickable_elements_cache: List[Dict[str, Any]] = []
|
25
30
|
self.serial = serial
|
26
|
-
self.device_manager = DeviceManager()
|
27
31
|
self.last_screenshot = None
|
28
32
|
self.reason = None
|
29
33
|
self.success = None
|
@@ -33,19 +37,38 @@ class AdbTools(Tools):
|
|
33
37
|
# Store all screenshots with timestamps
|
34
38
|
self.screenshots: List[Dict[str, Any]] = []
|
35
39
|
|
36
|
-
|
40
|
+
@classmethod
|
41
|
+
async def create(cls: Type[Self], serial: str = None) -> Self:
|
42
|
+
"""Create an AdbTools instance.
|
43
|
+
|
44
|
+
Args:
|
45
|
+
serial: Optional device serial number. If not provided, the first device found will be used.
|
46
|
+
|
47
|
+
Returns:
|
48
|
+
AdbTools instance
|
49
|
+
"""
|
50
|
+
if not serial:
|
51
|
+
dvm = DeviceManager()
|
52
|
+
devices = await dvm.list_devices()
|
53
|
+
if not devices or len(devices) < 1:
|
54
|
+
raise ValueError("No devices found")
|
55
|
+
serial = devices[0].serial
|
56
|
+
|
57
|
+
return AdbTools(serial)
|
58
|
+
|
59
|
+
def _get_device_serial(self) -> str:
|
37
60
|
"""Get the device serial from the instance or environment variable."""
|
38
61
|
# First try using the instance's serial
|
39
62
|
if self.serial:
|
40
63
|
return self.serial
|
41
64
|
|
42
|
-
async def
|
65
|
+
async def _get_device(self) -> Optional[Device]:
|
43
66
|
"""Get the device instance using the instance's serial or from environment variable.
|
44
67
|
|
45
68
|
Returns:
|
46
69
|
Device instance or None if not found
|
47
70
|
"""
|
48
|
-
serial = self.
|
71
|
+
serial = self._get_device_serial()
|
49
72
|
if not serial:
|
50
73
|
raise ValueError("No device serial specified - set device_serial parameter")
|
51
74
|
|
@@ -55,161 +78,52 @@ class AdbTools(Tools):
|
|
55
78
|
|
56
79
|
return device
|
57
80
|
|
58
|
-
def
|
59
|
-
|
81
|
+
def _parse_content_provider_output(
|
82
|
+
self, raw_output: str
|
83
|
+
) -> Optional[Dict[str, Any]]:
|
84
|
+
"""
|
85
|
+
Parse the raw ADB content provider output and extract JSON data.
|
60
86
|
|
61
87
|
Args:
|
62
|
-
|
88
|
+
raw_output (str): Raw output from ADB content query command
|
63
89
|
|
64
90
|
Returns:
|
65
|
-
|
91
|
+
dict: Parsed JSON data or None if parsing failed
|
66
92
|
"""
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
# Format is: "package:/path/to/base.apk=com.package.name"
|
71
|
-
path_and_pkg = line[8:] # Strip "package:"
|
72
|
-
if "=" in path_and_pkg:
|
73
|
-
path, package = path_and_pkg.rsplit("=", 1)
|
74
|
-
apps.append({"package": package.strip(), "path": path.strip()})
|
75
|
-
return apps
|
93
|
+
# The ADB content query output format is: "Row: 0 result={json_data}"
|
94
|
+
# We need to extract the JSON part after "result="
|
95
|
+
lines = raw_output.strip().split("\n")
|
76
96
|
|
77
|
-
|
78
|
-
|
79
|
-
Get all clickable UI elements from the device using the custom TopViewService.
|
97
|
+
for line in lines:
|
98
|
+
line = line.strip()
|
80
99
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
100
|
+
# Look for lines that contain "result=" pattern
|
101
|
+
if "result=" in line:
|
102
|
+
# Extract everything after "result="
|
103
|
+
result_start = line.find("result=") + 7
|
104
|
+
json_str = line[result_start:]
|
85
105
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
try:
|
93
|
-
# Get the device
|
94
|
-
if serial:
|
95
|
-
device_manager = DeviceManager()
|
96
|
-
device = await device_manager.get_device(serial)
|
97
|
-
if not device:
|
98
|
-
raise ValueError(f"Device {serial} not found")
|
99
|
-
else:
|
100
|
-
device = await self.get_device()
|
101
|
-
|
102
|
-
# Create a temporary file for the JSON
|
103
|
-
with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as temp:
|
104
|
-
local_path = temp.name
|
106
|
+
try:
|
107
|
+
# Parse the JSON string
|
108
|
+
json_data = json.loads(json_str)
|
109
|
+
return json_data
|
110
|
+
except json.JSONDecodeError:
|
111
|
+
continue
|
105
112
|
|
106
|
-
try
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
# Check if we've exceeded total time
|
114
|
-
current_time = asyncio.get_event_loop().time()
|
115
|
-
if current_time - start_total_time > max_total_time:
|
116
|
-
raise ValueError(
|
117
|
-
f"Failed to get UI elements after {max_total_time} seconds of retries"
|
118
|
-
)
|
119
|
-
|
120
|
-
# Clear logcat to make it easier to find our output
|
121
|
-
await device._adb.shell(device._serial, "logcat -c")
|
122
|
-
|
123
|
-
# Trigger the custom service via broadcast to get only interactive elements
|
124
|
-
await device._adb.shell(
|
125
|
-
device._serial,
|
126
|
-
"am broadcast -a com.droidrun.portal.GET_ELEMENTS",
|
127
|
-
)
|
128
|
-
|
129
|
-
# Poll for the JSON file path
|
130
|
-
start_time = asyncio.get_event_loop().time()
|
131
|
-
max_wait_time = 10 # Maximum wait time in seconds
|
132
|
-
poll_interval = 0.2 # Check every 200ms
|
133
|
-
|
134
|
-
device_path = None
|
135
|
-
while asyncio.get_event_loop().time() - start_time < max_wait_time:
|
136
|
-
# Check logcat for the file path
|
137
|
-
logcat_output = await device._adb.shell(
|
138
|
-
device._serial,
|
139
|
-
'logcat -d | grep "DROIDRUN_FILE" | grep "JSON data written to" | tail -1',
|
140
|
-
)
|
141
|
-
|
142
|
-
# Parse the file path if present
|
143
|
-
match = re.search(r"JSON data written to: (.*)", logcat_output)
|
144
|
-
if match:
|
145
|
-
device_path = match.group(1).strip()
|
146
|
-
break
|
147
|
-
|
148
|
-
# Wait before polling again
|
149
|
-
await asyncio.sleep(poll_interval)
|
150
|
-
|
151
|
-
# Check if we found the file path
|
152
|
-
if not device_path:
|
153
|
-
await asyncio.sleep(retry_interval)
|
154
|
-
continue
|
155
|
-
|
156
|
-
# Pull the JSON file from the device
|
157
|
-
await device._adb.pull_file(device._serial, device_path, local_path)
|
158
|
-
|
159
|
-
# Read the JSON file
|
160
|
-
async with aiofiles.open(local_path, "r", encoding="utf-8") as f:
|
161
|
-
json_content = await f.read()
|
162
|
-
|
163
|
-
# Try to parse the JSON
|
164
|
-
try:
|
165
|
-
ui_data = json.loads(json_content)
|
166
|
-
|
167
|
-
# Filter out the "type" attribute from all elements
|
168
|
-
filtered_data = []
|
169
|
-
for element in ui_data:
|
170
|
-
# Create a copy of the element without the "type" attribute
|
171
|
-
filtered_element = {
|
172
|
-
k: v for k, v in element.items() if k != "type"
|
173
|
-
}
|
174
|
-
|
175
|
-
# Also filter children if present
|
176
|
-
if "children" in filtered_element:
|
177
|
-
filtered_element["children"] = [
|
178
|
-
{k: v for k, v in child.items() if k != "type"}
|
179
|
-
for child in filtered_element["children"]
|
180
|
-
]
|
181
|
-
|
182
|
-
filtered_data.append(filtered_element)
|
183
|
-
|
184
|
-
# If we got elements, store them and return
|
185
|
-
if filtered_data:
|
186
|
-
# Store the filtered UI data in cache
|
187
|
-
global CLICKABLE_ELEMENTS_CACHE
|
188
|
-
CLICKABLE_ELEMENTS_CACHE = filtered_data
|
189
|
-
|
190
|
-
# Add a small sleep to ensure UI is fully loaded/processed
|
191
|
-
await asyncio.sleep(0.5) # 500ms sleep
|
192
|
-
|
193
|
-
# Convert the dictionary to a JSON string before returning
|
194
|
-
|
195
|
-
return filtered_data
|
196
|
-
|
197
|
-
# If no elements found, wait and retry
|
198
|
-
await asyncio.sleep(retry_interval)
|
199
|
-
|
200
|
-
except json.JSONDecodeError:
|
201
|
-
# If JSON parsing failed, wait and retry
|
202
|
-
await asyncio.sleep(retry_interval)
|
203
|
-
continue
|
204
|
-
|
205
|
-
except Exception as e:
|
206
|
-
# Clean up in case of error
|
207
|
-
with contextlib.suppress(OSError):
|
208
|
-
os.unlink(local_path)
|
209
|
-
raise ValueError(f"Error retrieving clickable elements: {e}")
|
113
|
+
# Fallback: try to parse lines that start with { or [
|
114
|
+
elif line.startswith("{") or line.startswith("["):
|
115
|
+
try:
|
116
|
+
json_data = json.loads(line)
|
117
|
+
return json_data
|
118
|
+
except json.JSONDecodeError:
|
119
|
+
continue
|
210
120
|
|
211
|
-
|
212
|
-
|
121
|
+
# If no valid JSON found in individual lines, try the entire output
|
122
|
+
try:
|
123
|
+
json_data = json.loads(raw_output.strip())
|
124
|
+
return json_data
|
125
|
+
except json.JSONDecodeError:
|
126
|
+
return None
|
213
127
|
|
214
128
|
async def tap_by_index(self, index: int, serial: Optional[str] = None) -> str:
|
215
129
|
"""
|
@@ -250,15 +164,15 @@ class AdbTools(Tools):
|
|
250
164
|
|
251
165
|
try:
|
252
166
|
# Check if we have cached elements
|
253
|
-
if not
|
254
|
-
return "Error: No UI elements cached. Call
|
167
|
+
if not self.clickable_elements_cache:
|
168
|
+
return "Error: No UI elements cached. Call get_state first."
|
255
169
|
|
256
170
|
# Find the element with the given index (including in children)
|
257
|
-
element = find_element_by_index(
|
171
|
+
element = find_element_by_index(self.clickable_elements_cache, index)
|
258
172
|
|
259
173
|
if not element:
|
260
174
|
# List available indices to help the user
|
261
|
-
indices = sorted(collect_all_indices(
|
175
|
+
indices = sorted(collect_all_indices(self.clickable_elements_cache))
|
262
176
|
indices_str = ", ".join(str(idx) for idx in indices[:20])
|
263
177
|
if len(indices) > 20:
|
264
178
|
indices_str += f"... and {len(indices) - 20} more"
|
@@ -285,12 +199,11 @@ class AdbTools(Tools):
|
|
285
199
|
|
286
200
|
# Get the device and tap at the coordinates
|
287
201
|
if serial:
|
288
|
-
|
289
|
-
device = await device_manager.get_device(serial)
|
202
|
+
device = await self.device_manager.get_device(serial)
|
290
203
|
if not device:
|
291
204
|
return f"Error: Device {serial} not found"
|
292
205
|
else:
|
293
|
-
device = await self.
|
206
|
+
device = await self._get_device()
|
294
207
|
|
295
208
|
await device.tap(x, y)
|
296
209
|
|
@@ -333,12 +246,11 @@ class AdbTools(Tools):
|
|
333
246
|
"""
|
334
247
|
try:
|
335
248
|
if self.serial:
|
336
|
-
|
337
|
-
device = await device_manager.get_device(self.serial)
|
249
|
+
device = await self.device_manager.get_device(self.serial)
|
338
250
|
if not device:
|
339
251
|
return f"Error: Device {self.serial} not found"
|
340
252
|
else:
|
341
|
-
device = await self.
|
253
|
+
device = await self._get_device()
|
342
254
|
|
343
255
|
await device.tap(x, y)
|
344
256
|
print(f"Tapped at coordinates ({x}, {y})")
|
@@ -380,16 +292,17 @@ class AdbTools(Tools):
|
|
380
292
|
"""
|
381
293
|
try:
|
382
294
|
if self.serial:
|
383
|
-
|
384
|
-
device = await device_manager.get_device(self.serial)
|
295
|
+
device = await self.device_manager.get_device(self.serial)
|
385
296
|
if not device:
|
386
297
|
return f"Error: Device {self.serial} not found"
|
387
298
|
else:
|
388
|
-
device = await self.
|
299
|
+
device = await self._get_device()
|
389
300
|
|
390
301
|
await device.swipe(start_x, start_y, end_x, end_y, duration_ms)
|
391
302
|
await asyncio.sleep(1)
|
392
|
-
print(
|
303
|
+
print(
|
304
|
+
f"Swiped from ({start_x}, {start_y}) to ({end_x}, {end_y}) in {duration_ms}ms"
|
305
|
+
)
|
393
306
|
return True
|
394
307
|
except ValueError as e:
|
395
308
|
print(f"Error: {str(e)}")
|
@@ -408,12 +321,11 @@ class AdbTools(Tools):
|
|
408
321
|
"""
|
409
322
|
try:
|
410
323
|
if serial:
|
411
|
-
|
412
|
-
device = await device_manager.get_device(serial)
|
324
|
+
device = await self.device_manager.get_device(serial)
|
413
325
|
if not device:
|
414
326
|
return f"Error: Device {serial} not found"
|
415
327
|
else:
|
416
|
-
device = await self.
|
328
|
+
device = await self._get_device()
|
417
329
|
|
418
330
|
# Save the current keyboard
|
419
331
|
original_ime = await device._adb.shell(
|
@@ -432,14 +344,14 @@ class AdbTools(Tools):
|
|
432
344
|
)
|
433
345
|
|
434
346
|
# Wait for keyboard to change
|
435
|
-
await asyncio.sleep(
|
347
|
+
await asyncio.sleep(1)
|
436
348
|
|
437
349
|
# Encode the text to Base64
|
438
350
|
import base64
|
439
351
|
|
440
352
|
encoded_text = base64.b64encode(text.encode()).decode()
|
441
353
|
|
442
|
-
cmd = f'
|
354
|
+
cmd = f'content insert --uri "content://com.droidrun.portal/keyboard/input" --bind base64_text:s:"{encoded_text}"'
|
443
355
|
await device._adb.shell(device._serial, cmd)
|
444
356
|
|
445
357
|
# Wait for text input to complete
|
@@ -462,12 +374,11 @@ class AdbTools(Tools):
|
|
462
374
|
"""
|
463
375
|
try:
|
464
376
|
if self.serial:
|
465
|
-
|
466
|
-
device = await device_manager.get_device(self.serial)
|
377
|
+
device = await self.device_manager.get_device(self.serial)
|
467
378
|
if not device:
|
468
379
|
return f"Error: Device {self.serial} not found"
|
469
380
|
else:
|
470
|
-
device = await self.
|
381
|
+
device = await self._get_device()
|
471
382
|
|
472
383
|
await device.press_key(3)
|
473
384
|
return f"Pressed key BACK"
|
@@ -479,6 +390,7 @@ class AdbTools(Tools):
|
|
479
390
|
Press a key on the Android device.
|
480
391
|
|
481
392
|
Common keycodes:
|
393
|
+
- 3: HOME
|
482
394
|
- 4: BACK
|
483
395
|
- 66: ENTER
|
484
396
|
- 67: DELETE
|
@@ -488,16 +400,16 @@ class AdbTools(Tools):
|
|
488
400
|
"""
|
489
401
|
try:
|
490
402
|
if self.serial:
|
491
|
-
|
492
|
-
device = await device_manager.get_device(self.serial)
|
403
|
+
device = await self.device_manager.get_device(self.serial)
|
493
404
|
if not device:
|
494
405
|
return f"Error: Device {self.serial} not found"
|
495
406
|
else:
|
496
|
-
device = await self.
|
407
|
+
device = await self._get_device()
|
497
408
|
|
498
409
|
key_names = {
|
499
410
|
66: "ENTER",
|
500
411
|
4: "BACK",
|
412
|
+
3: "HOME",
|
501
413
|
67: "DELETE",
|
502
414
|
}
|
503
415
|
key_name = key_names.get(keycode, str(keycode))
|
@@ -517,12 +429,11 @@ class AdbTools(Tools):
|
|
517
429
|
"""
|
518
430
|
try:
|
519
431
|
if self.serial:
|
520
|
-
|
521
|
-
device = await device_manager.get_device(self.serial)
|
432
|
+
device = await self.device_manager.get_device(self.serial)
|
522
433
|
if not device:
|
523
434
|
return f"Error: Device {self.serial} not found"
|
524
435
|
else:
|
525
|
-
device = await self.
|
436
|
+
device = await self._get_device()
|
526
437
|
|
527
438
|
result = await device.start_app(package, activity)
|
528
439
|
return result
|
@@ -542,12 +453,11 @@ class AdbTools(Tools):
|
|
542
453
|
"""
|
543
454
|
try:
|
544
455
|
if self.serial:
|
545
|
-
|
546
|
-
device = await device_manager.get_device(self.serial)
|
456
|
+
device = await self.device_manager.get_device(self.serial)
|
547
457
|
if not device:
|
548
458
|
return f"Error: Device {self.serial} not found"
|
549
459
|
else:
|
550
|
-
device = await self.
|
460
|
+
device = await self._get_device()
|
551
461
|
|
552
462
|
if not os.path.exists(apk_path):
|
553
463
|
return f"Error: APK file not found at {apk_path}"
|
@@ -565,12 +475,11 @@ class AdbTools(Tools):
|
|
565
475
|
"""
|
566
476
|
try:
|
567
477
|
if self.serial:
|
568
|
-
|
569
|
-
device = await device_manager.get_device(self.serial)
|
478
|
+
device = await self.device_manager.get_device(self.serial)
|
570
479
|
if not device:
|
571
480
|
raise ValueError(f"Device {self.serial} not found")
|
572
481
|
else:
|
573
|
-
device = await self.
|
482
|
+
device = await self._get_device()
|
574
483
|
screen_tuple = await device.take_screenshot()
|
575
484
|
self.last_screenshot = screen_tuple[1]
|
576
485
|
|
@@ -598,162 +507,16 @@ class AdbTools(Tools):
|
|
598
507
|
"""
|
599
508
|
try:
|
600
509
|
if self.serial:
|
601
|
-
|
602
|
-
device = await device_manager.get_device(self.serial)
|
510
|
+
device = await self.device_manager.get_device(self.serial)
|
603
511
|
if not device:
|
604
512
|
raise ValueError(f"Device {self.serial} not found")
|
605
513
|
else:
|
606
|
-
device = await self.
|
607
|
-
|
608
|
-
# Use the direct ADB command to get packages with paths
|
609
|
-
cmd = ["pm", "list", "packages", "-f"]
|
610
|
-
if not include_system_apps:
|
611
|
-
cmd.append("-3")
|
514
|
+
device = await self._get_device()
|
612
515
|
|
613
|
-
|
614
|
-
|
615
|
-
# Parse the package list using the function
|
616
|
-
packages = self.parse_package_list(output)
|
617
|
-
# Format package list for better readability
|
618
|
-
package_list = [pack["package"] for pack in packages]
|
619
|
-
print(f"Returning {len(package_list)} packages")
|
620
|
-
return package_list
|
516
|
+
return await device.list_packages(include_system_apps)
|
621
517
|
except ValueError as e:
|
622
518
|
raise ValueError(f"Error listing packages: {str(e)}")
|
623
519
|
|
624
|
-
async def extract(self, filename: Optional[str] = None) -> str:
|
625
|
-
"""Extract and save the current UI state to a JSON file.
|
626
|
-
|
627
|
-
This function captures the current UI state including all UI elements
|
628
|
-
and saves it to a JSON file for later analysis or reference.
|
629
|
-
|
630
|
-
Args:
|
631
|
-
filename: Optional filename to save the UI state (defaults to ui_state_TIMESTAMP.json)
|
632
|
-
|
633
|
-
Returns:
|
634
|
-
Path to the saved JSON file
|
635
|
-
"""
|
636
|
-
try:
|
637
|
-
# Generate default filename if not provided
|
638
|
-
if not filename:
|
639
|
-
timestamp = int(time.time())
|
640
|
-
filename = f"ui_state_{timestamp}.json"
|
641
|
-
|
642
|
-
# Ensure the filename ends with .json
|
643
|
-
if not filename.endswith(".json"):
|
644
|
-
filename += ".json"
|
645
|
-
|
646
|
-
# Get the UI elements
|
647
|
-
ui_elements = await self.get_all_elements(self.serial)
|
648
|
-
|
649
|
-
# Save to file
|
650
|
-
save_path = os.path.abspath(filename)
|
651
|
-
async with aiofiles.open(save_path, "w", encoding="utf-8") as f:
|
652
|
-
await f.write(json.dumps(ui_elements, indent=2))
|
653
|
-
|
654
|
-
return f"UI state extracted and saved to {save_path}"
|
655
|
-
|
656
|
-
except Exception as e:
|
657
|
-
return f"Error extracting UI state: {e}"
|
658
|
-
|
659
|
-
async def get_all_elements(self) -> Dict[str, Any]:
|
660
|
-
"""
|
661
|
-
Get all UI elements from the device, including non-interactive elements.
|
662
|
-
|
663
|
-
This function interacts with the TopViewService app installed on the device
|
664
|
-
to capture all UI elements, even those that are not interactive. This provides
|
665
|
-
a complete view of the UI hierarchy for analysis or debugging purposes.
|
666
|
-
|
667
|
-
Returns:
|
668
|
-
Dictionary containing all UI elements extracted from the device screen
|
669
|
-
"""
|
670
|
-
try:
|
671
|
-
# Get the device
|
672
|
-
device_manager = DeviceManager()
|
673
|
-
device = await device_manager.get_device(self.serial)
|
674
|
-
if not device:
|
675
|
-
raise ValueError(f"Device {self.serial} not found")
|
676
|
-
|
677
|
-
# Create a temporary file for the JSON
|
678
|
-
with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as temp:
|
679
|
-
local_path = temp.name
|
680
|
-
|
681
|
-
try:
|
682
|
-
# Clear logcat to make it easier to find our output
|
683
|
-
await device._adb.shell(device._serial, "logcat -c")
|
684
|
-
|
685
|
-
# Trigger the custom service via broadcast to get ALL elements
|
686
|
-
await device._adb.shell(
|
687
|
-
device._serial,
|
688
|
-
"am broadcast -a com.droidrun.portal.GET_ALL_ELEMENTS",
|
689
|
-
)
|
690
|
-
|
691
|
-
# Poll for the JSON file path
|
692
|
-
start_time = asyncio.get_event_loop().time()
|
693
|
-
max_wait_time = 10 # Maximum wait time in seconds
|
694
|
-
poll_interval = 0.2 # Check every 200ms
|
695
|
-
|
696
|
-
device_path = None
|
697
|
-
while asyncio.get_event_loop().time() - start_time < max_wait_time:
|
698
|
-
# Check logcat for the file path
|
699
|
-
logcat_output = await device._adb.shell(
|
700
|
-
device._serial,
|
701
|
-
'logcat -d | grep "DROIDRUN_FILE" | grep "JSON data written to" | tail -1',
|
702
|
-
)
|
703
|
-
|
704
|
-
# Parse the file path if present
|
705
|
-
match = re.search(r"JSON data written to: (.*)", logcat_output)
|
706
|
-
if match:
|
707
|
-
device_path = match.group(1).strip()
|
708
|
-
break
|
709
|
-
|
710
|
-
# Wait before polling again
|
711
|
-
await asyncio.sleep(poll_interval)
|
712
|
-
|
713
|
-
# Check if we found the file path
|
714
|
-
if not device_path:
|
715
|
-
raise ValueError(
|
716
|
-
f"Failed to find the JSON file path in logcat after {max_wait_time} seconds"
|
717
|
-
)
|
718
|
-
|
719
|
-
# Pull the JSON file from the device
|
720
|
-
await device._adb.pull_file(device._serial, device_path, local_path)
|
721
|
-
|
722
|
-
# Read the JSON file
|
723
|
-
async with aiofiles.open(local_path, "r", encoding="utf-8") as f:
|
724
|
-
json_content = await f.read()
|
725
|
-
|
726
|
-
# Clean up the temporary file
|
727
|
-
with contextlib.suppress(OSError):
|
728
|
-
os.unlink(local_path)
|
729
|
-
|
730
|
-
# Try to parse the JSON
|
731
|
-
import json
|
732
|
-
|
733
|
-
try:
|
734
|
-
ui_data = json.loads(json_content)
|
735
|
-
|
736
|
-
return {
|
737
|
-
"all_elements": ui_data,
|
738
|
-
"count": (
|
739
|
-
len(ui_data)
|
740
|
-
if isinstance(ui_data, list)
|
741
|
-
else sum(1 for _ in ui_data.get("elements", []))
|
742
|
-
),
|
743
|
-
"message": "Retrieved all UI elements from the device screen",
|
744
|
-
}
|
745
|
-
except json.JSONDecodeError:
|
746
|
-
raise ValueError("Failed to parse UI elements JSON data")
|
747
|
-
|
748
|
-
except Exception as e:
|
749
|
-
# Clean up in case of error
|
750
|
-
with contextlib.suppress(OSError):
|
751
|
-
os.unlink(local_path)
|
752
|
-
raise ValueError(f"Error retrieving all UI elements: {e}")
|
753
|
-
|
754
|
-
except Exception as e:
|
755
|
-
raise ValueError(f"Error getting all UI elements: {e}")
|
756
|
-
|
757
520
|
def complete(self, success: bool, reason: str = ""):
|
758
521
|
"""
|
759
522
|
Mark the task as finished.
|
@@ -773,75 +536,6 @@ class AdbTools(Tools):
|
|
773
536
|
self.reason = reason
|
774
537
|
self.finished = True
|
775
538
|
|
776
|
-
async def get_phone_state(self, serial: Optional[str] = None) -> Dict[str, Any]:
|
777
|
-
"""
|
778
|
-
Get the current phone state including current activity and keyboard visibility.
|
779
|
-
|
780
|
-
Args:
|
781
|
-
serial: Optional device serial number
|
782
|
-
|
783
|
-
Returns:
|
784
|
-
Dictionary with current phone state information
|
785
|
-
"""
|
786
|
-
try:
|
787
|
-
# Get the device
|
788
|
-
if serial:
|
789
|
-
device_manager = DeviceManager()
|
790
|
-
device = await device_manager.get_device(serial)
|
791
|
-
if not device:
|
792
|
-
raise ValueError(f"Device {serial} not found")
|
793
|
-
else:
|
794
|
-
device = await self.get_device()
|
795
|
-
|
796
|
-
# Clear logcat to make it easier to find our output
|
797
|
-
await device._adb.shell(device._serial, "logcat -c")
|
798
|
-
|
799
|
-
# Trigger the custom service via broadcast to get phone state
|
800
|
-
await device._adb.shell(
|
801
|
-
device._serial, "am broadcast -a com.droidrun.portal.GET_PHONE_STATE"
|
802
|
-
)
|
803
|
-
|
804
|
-
# Poll for the phone state data in logcat
|
805
|
-
start_time = asyncio.get_event_loop().time()
|
806
|
-
max_wait_time = 10 # Maximum wait time in seconds
|
807
|
-
poll_interval = 0.2 # Check every 200ms
|
808
|
-
|
809
|
-
while asyncio.get_event_loop().time() - start_time < max_wait_time:
|
810
|
-
# Check logcat for the phone state data
|
811
|
-
logcat_output = await device._adb.shell(
|
812
|
-
device._serial,
|
813
|
-
'logcat -d | grep "DROIDRUN_PHONE_STATE_DATA" | tail -1',
|
814
|
-
)
|
815
|
-
|
816
|
-
# Parse the JSON data if present
|
817
|
-
if "CHUNK|" in logcat_output:
|
818
|
-
# Format: DROIDRUN_PHONE_STATE_DATA: CHUNK|0|1|{json_data}
|
819
|
-
# Extract the JSON part after the last |
|
820
|
-
parts = logcat_output.split("|")
|
821
|
-
if len(parts) >= 4:
|
822
|
-
json_data = "|".join(
|
823
|
-
parts[3:]
|
824
|
-
) # In case JSON contains | characters
|
825
|
-
try:
|
826
|
-
phone_state = json.loads(json_data)
|
827
|
-
return phone_state
|
828
|
-
except json.JSONDecodeError:
|
829
|
-
# If JSON parsing failed, wait and retry
|
830
|
-
await asyncio.sleep(poll_interval)
|
831
|
-
continue
|
832
|
-
|
833
|
-
# Wait before polling again
|
834
|
-
await asyncio.sleep(poll_interval)
|
835
|
-
|
836
|
-
# If we couldn't get the phone state, return error
|
837
|
-
return {
|
838
|
-
"error": "Timeout",
|
839
|
-
"message": f"Failed to get phone state data after {max_wait_time} seconds",
|
840
|
-
}
|
841
|
-
|
842
|
-
except Exception as e:
|
843
|
-
return {"error": str(e), "message": f"Error getting phone state: {str(e)}"}
|
844
|
-
|
845
539
|
async def remember(self, information: str) -> str:
|
846
540
|
"""
|
847
541
|
Store important information to remember for future context.
|
@@ -877,3 +571,101 @@ class AdbTools(Tools):
|
|
877
571
|
List of stored memory items
|
878
572
|
"""
|
879
573
|
return self.memory.copy()
|
574
|
+
|
575
|
+
async def get_state(self, serial: Optional[str] = None) -> Dict[str, Any]:
|
576
|
+
"""
|
577
|
+
Get both the a11y tree and phone state in a single call using the combined /state endpoint.
|
578
|
+
|
579
|
+
Args:
|
580
|
+
serial: Optional device serial number
|
581
|
+
|
582
|
+
Returns:
|
583
|
+
Dictionary containing both 'a11y_tree' and 'phone_state' data
|
584
|
+
"""
|
585
|
+
|
586
|
+
try:
|
587
|
+
if serial:
|
588
|
+
device = await self.device_manager.get_device(serial)
|
589
|
+
if not device:
|
590
|
+
raise ValueError(f"Device {serial} not found")
|
591
|
+
else:
|
592
|
+
device = await self._get_device()
|
593
|
+
|
594
|
+
adb_output = await device._adb.shell(
|
595
|
+
device._serial,
|
596
|
+
"content query --uri content://com.droidrun.portal/state",
|
597
|
+
)
|
598
|
+
|
599
|
+
state_data = self._parse_content_provider_output(adb_output)
|
600
|
+
|
601
|
+
if state_data is None:
|
602
|
+
return {
|
603
|
+
"error": "Parse Error",
|
604
|
+
"message": "Failed to parse state data from ContentProvider response",
|
605
|
+
}
|
606
|
+
|
607
|
+
if isinstance(state_data, dict) and "data" in state_data:
|
608
|
+
data_str = state_data["data"]
|
609
|
+
try:
|
610
|
+
combined_data = json.loads(data_str)
|
611
|
+
except json.JSONDecodeError:
|
612
|
+
return {
|
613
|
+
"error": "Parse Error",
|
614
|
+
"message": "Failed to parse JSON data from ContentProvider data field",
|
615
|
+
}
|
616
|
+
else:
|
617
|
+
return {
|
618
|
+
"error": "Format Error",
|
619
|
+
"message": f"Unexpected state data format: {type(state_data)}",
|
620
|
+
}
|
621
|
+
|
622
|
+
# Validate that both a11y_tree and phone_state are present
|
623
|
+
if "a11y_tree" not in combined_data:
|
624
|
+
return {
|
625
|
+
"error": "Missing Data",
|
626
|
+
"message": "a11y_tree not found in combined state data",
|
627
|
+
}
|
628
|
+
|
629
|
+
if "phone_state" not in combined_data:
|
630
|
+
return {
|
631
|
+
"error": "Missing Data",
|
632
|
+
"message": "phone_state not found in combined state data",
|
633
|
+
}
|
634
|
+
|
635
|
+
# Filter out the "type" attribute from all a11y_tree elements
|
636
|
+
elements = combined_data["a11y_tree"]
|
637
|
+
filtered_elements = []
|
638
|
+
for element in elements:
|
639
|
+
# Create a copy of the element without the "type" attribute
|
640
|
+
filtered_element = {k: v for k, v in element.items() if k != "type"}
|
641
|
+
|
642
|
+
# Also filter children if present
|
643
|
+
if "children" in filtered_element:
|
644
|
+
filtered_element["children"] = [
|
645
|
+
{k: v for k, v in child.items() if k != "type"}
|
646
|
+
for child in filtered_element["children"]
|
647
|
+
]
|
648
|
+
|
649
|
+
filtered_elements.append(filtered_element)
|
650
|
+
|
651
|
+
self.clickable_elements_cache = filtered_elements
|
652
|
+
|
653
|
+
return {
|
654
|
+
"a11y_tree": filtered_elements,
|
655
|
+
"phone_state": combined_data["phone_state"],
|
656
|
+
}
|
657
|
+
|
658
|
+
except Exception as e:
|
659
|
+
return {
|
660
|
+
"error": str(e),
|
661
|
+
"message": f"Error getting combined state: {str(e)}",
|
662
|
+
}
|
663
|
+
|
664
|
+
|
665
|
+
if __name__ == "__main__":
|
666
|
+
|
667
|
+
async def main():
|
668
|
+
tools = await AdbTools.create()
|
669
|
+
print(tools.serial)
|
670
|
+
|
671
|
+
asyncio.run(main())
|