tapwright 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tapwright/__init__.py +2 -0
- tapwright/__main__.py +5 -0
- tapwright/airplay.py +103 -0
- tapwright/changes.py +264 -0
- tapwright/cli.py +374 -0
- tapwright/client.py +151 -0
- tapwright/control.py +189 -0
- tapwright/daemon.py +2401 -0
- tapwright/descriptions.py +186 -0
- tapwright/device.py +495 -0
- tapwright/errors.py +13 -0
- tapwright/mcp_server.py +266 -0
- tapwright/mirror.py +91 -0
- tapwright/rtp.py +109 -0
- tapwright/server.py +435 -0
- tapwright/session.py +672 -0
- tapwright/shot.py +42 -0
- tapwright/signing.py +201 -0
- tapwright/skill/SKILL.md +203 -0
- tapwright/skill/agents/openai.yaml +4 -0
- tapwright/skill/references/setup.md +102 -0
- tapwright/skills.py +60 -0
- tapwright/store.py +130 -0
- tapwright/stream.py +405 -0
- tapwright/tapline.py +118 -0
- tapwright/ui.py +1242 -0
- tapwright/video.py +486 -0
- tapwright/viewer.html +2060 -0
- tapwright-1.0.0.dist-info/METADATA +254 -0
- tapwright-1.0.0.dist-info/RECORD +34 -0
- tapwright-1.0.0.dist-info/WHEEL +5 -0
- tapwright-1.0.0.dist-info/entry_points.txt +2 -0
- tapwright-1.0.0.dist-info/licenses/LICENSE +21 -0
- tapwright-1.0.0.dist-info/top_level.txt +1 -0
tapwright/control.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Who acts on the phone, and when: one action at a time, a person before an agent, a pause
|
|
2
|
+
switch, and screen generations that tell an agent its element refs went stale. Also turns a
|
|
3
|
+
person's pointer path from the live view into a tap, long press or drag. Pure logic: no
|
|
4
|
+
phone, no network."""
|
|
5
|
+
import asyncio
|
|
6
|
+
import math
|
|
7
|
+
import time
|
|
8
|
+
from contextlib import asynccontextmanager
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from .stream import Frame
|
|
14
|
+
|
|
15
|
+
PERSON_IDLE_S = 3.0 # a person counts as driving until idle this long
|
|
16
|
+
AGENT_MAX_WAIT_S = 30.0 # an agent action waits at most this long for a person to finish
|
|
17
|
+
LONG_PRESS_MS = 500
|
|
18
|
+
# A tap's finger rests this long: the result showed 75 ms sooner than with 60 ms, and the call took as long
|
|
19
|
+
# (an iPhone 14 Pro Max, iOS 26.5, 2026-11-09). A person's tap is over when it reaches the phone: replayed quick.
|
|
20
|
+
# Through the live view a 20 ms tap showed in 0.27-0.30 s and 60 ms in 0.30-0.38 s; the button's press highlight
|
|
21
|
+
# landed in a frame about as often (6 and 4 of 10), so persons get 20 ms too.
|
|
22
|
+
TAP_MS = 20
|
|
23
|
+
TAP_MOVE_PT = 10.0
|
|
24
|
+
PATH_POINTS = 20 # a drag is replayed through at most this many points
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Refused(Exception):
|
|
28
|
+
"""An action not run, with a machine-readable code (docs/VIEWER-PROTOCOL.md)."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, code: str, message: str) -> None:
|
|
31
|
+
super().__init__(message)
|
|
32
|
+
self.code = code
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class ClientView:
|
|
37
|
+
"""What one agent client knows: the generation of its last look or own action, and its elements."""
|
|
38
|
+
seen: int = -1
|
|
39
|
+
items: list = field(default_factory=list)
|
|
40
|
+
read_at: float = float("-inf") # the clock when `items` was read
|
|
41
|
+
moved: bool = False # an action that returned no list ran since: the screen has probably changed
|
|
42
|
+
unlooked: int = 0 # how many answers since the last list or change block carried none (`look: false`)
|
|
43
|
+
shown_off: list = field(default_factory=list) # what lay off screen at the agent's last answer (for `~` lines)
|
|
44
|
+
dropped: bool = False # the connection dropped since `items` was read
|
|
45
|
+
shown: list = field(default_factory=list) # the list as of the agent's last answer: what changes are measured against
|
|
46
|
+
top_ref: int = 0 # the highest ref number this agent was given: numbers are never reused until a full list
|
|
47
|
+
app: tuple[str, str] | None = None # (name, bundle id) of the foreground app at the last read
|
|
48
|
+
alert: tuple[str, tuple[str, ...]] | None = None # (title, buttons) of the OS alert at the last answer
|
|
49
|
+
keyboard: bool = False # whether the keyboard was shown at the last answer
|
|
50
|
+
read_keyboard: bool = False # whether the keyboard showed on the newest read of any kind (answers and silent reads)
|
|
51
|
+
alert_now: tuple[str, tuple[str, ...]] | None = None # (title, buttons) of the OS alert on the newest read of any kind
|
|
52
|
+
read_frame: "Frame | None" = None # the stream frame at that moment: the picture that matches the list
|
|
53
|
+
read_mark: tuple[int, "Frame | None"] = (0, None) # (seq, the newest stream frame) when `items` was read
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class Controller:
|
|
57
|
+
def __init__(self, clock=time.monotonic, sleep=asyncio.sleep) -> None:
|
|
58
|
+
self.clock, self.sleep = clock, sleep
|
|
59
|
+
self.paused = False
|
|
60
|
+
self.generation = 0 # +1 after every action that may have changed the screen
|
|
61
|
+
self.person_generation = -1 # the generation a person's last action produced
|
|
62
|
+
self.clients: dict[str, ClientView] = {}
|
|
63
|
+
self._lock = asyncio.Lock()
|
|
64
|
+
self._person_at: float | None = None
|
|
65
|
+
self._people_waiting = 0 # people queued for the lock: agents step aside while any wait
|
|
66
|
+
self._agent_at: float | None = None
|
|
67
|
+
self._agent_running = False
|
|
68
|
+
|
|
69
|
+
def client(self, client_id: str) -> ClientView:
|
|
70
|
+
return self.clients.setdefault(client_id, ClientView())
|
|
71
|
+
|
|
72
|
+
def _recent(self, at: float | None) -> bool:
|
|
73
|
+
return at is not None and self.clock() - at < PERSON_IDLE_S
|
|
74
|
+
|
|
75
|
+
def person_active(self) -> bool:
|
|
76
|
+
return self._people_waiting > 0 or self._recent(self._person_at)
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def driver(self) -> str | None:
|
|
80
|
+
if self.person_active():
|
|
81
|
+
return "person"
|
|
82
|
+
if self._agent_running or self._recent(self._agent_at):
|
|
83
|
+
return "agent"
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
def set_paused(self, on: bool) -> None:
|
|
87
|
+
self.paused = bool(on)
|
|
88
|
+
|
|
89
|
+
async def person(self, fn):
|
|
90
|
+
"""A person's action: runs as soon as the current action ends, ahead of waiting agents."""
|
|
91
|
+
self._person_at = self.clock()
|
|
92
|
+
self._people_waiting += 1
|
|
93
|
+
try:
|
|
94
|
+
await self._lock.acquire()
|
|
95
|
+
finally:
|
|
96
|
+
self._people_waiting -= 1
|
|
97
|
+
try:
|
|
98
|
+
return await fn()
|
|
99
|
+
finally:
|
|
100
|
+
self.generation += 1
|
|
101
|
+
self.person_generation = self.generation
|
|
102
|
+
self._person_at = self.clock()
|
|
103
|
+
self._lock.release()
|
|
104
|
+
|
|
105
|
+
async def agent(self, fn):
|
|
106
|
+
"""An agent's action: refused while paused, waits while a person drives."""
|
|
107
|
+
start = self.clock()
|
|
108
|
+
while True:
|
|
109
|
+
if self.paused:
|
|
110
|
+
raise Refused("paused", "paused by the user in the live view; ask them to resume")
|
|
111
|
+
if self.person_active():
|
|
112
|
+
if self.clock() - start >= AGENT_MAX_WAIT_S:
|
|
113
|
+
raise Refused("busy", "a person is using the phone; try again in a moment")
|
|
114
|
+
await self.sleep(0.25)
|
|
115
|
+
continue
|
|
116
|
+
async with self._lock:
|
|
117
|
+
if self.paused or self.person_active():
|
|
118
|
+
continue # a person got in first: let go of the queue and wait again
|
|
119
|
+
self._agent_running = True
|
|
120
|
+
refused = False
|
|
121
|
+
try:
|
|
122
|
+
return await fn()
|
|
123
|
+
except Refused:
|
|
124
|
+
refused = True
|
|
125
|
+
raise
|
|
126
|
+
finally:
|
|
127
|
+
self._agent_running = False
|
|
128
|
+
if not refused:
|
|
129
|
+
self.generation += 1
|
|
130
|
+
self._agent_at = self.clock()
|
|
131
|
+
|
|
132
|
+
@asynccontextmanager
|
|
133
|
+
async def exclusive(self):
|
|
134
|
+
"""Work that must not overlap any action or read: reconnecting."""
|
|
135
|
+
async with self._lock:
|
|
136
|
+
yield
|
|
137
|
+
|
|
138
|
+
async def read(self, fn):
|
|
139
|
+
"""A read (a look, the app list, settings): queued with actions, never refused or delayed."""
|
|
140
|
+
async with self._lock:
|
|
141
|
+
return await fn()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(frozen=True)
|
|
145
|
+
class Gesture:
|
|
146
|
+
kind: str # "tap", "longPress" or "drag"
|
|
147
|
+
points: tuple # ((t_ms from 0, x, y), ...) in points
|
|
148
|
+
hold_ms: int = 0
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def to_points(path, screen: tuple[float, float]) -> list[tuple[int, float, float]]:
|
|
152
|
+
"""[[t_ms, x, y], ...] with x, y in 0..1 of the frame -> (t_ms from 0, x, y) in points."""
|
|
153
|
+
if not path:
|
|
154
|
+
raise ValueError("a gesture needs at least one point")
|
|
155
|
+
w, h = screen
|
|
156
|
+
out: list[tuple[float, float, float]] = []
|
|
157
|
+
for point in path:
|
|
158
|
+
try:
|
|
159
|
+
t, x, y = (float(v) for v in point)
|
|
160
|
+
except (TypeError, ValueError) as e:
|
|
161
|
+
raise ValueError("each gesture point must be [t_ms, x, y] numbers") from e
|
|
162
|
+
if not all(math.isfinite(v) for v in (t, x, y)):
|
|
163
|
+
raise ValueError("gesture numbers must be finite")
|
|
164
|
+
if out:
|
|
165
|
+
t = max(t, out[-1][0]) # times never go backwards
|
|
166
|
+
out.append((t, min(max(x, 0.0), 1.0) * w, min(max(y, 0.0), 1.0) * h))
|
|
167
|
+
t0 = out[0][0]
|
|
168
|
+
return [(int(round(t - t0)), x, y) for t, x, y in out]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def thin(points: list, limit: int) -> list:
|
|
172
|
+
"""At most `limit` points, the first and last kept, evenly spread between."""
|
|
173
|
+
if len(points) <= limit:
|
|
174
|
+
return list(points)
|
|
175
|
+
step = (len(points) - 1) / (limit - 1)
|
|
176
|
+
return [points[round(i * step)] for i in range(limit)]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def classify(path, screen: tuple[float, float]) -> Gesture:
|
|
180
|
+
"""A pointer path from the live view as a tap, a long press, or a drag along the path."""
|
|
181
|
+
pts = to_points(path, screen)
|
|
182
|
+
_, x0, y0 = pts[0]
|
|
183
|
+
moved = max(math.hypot(x - x0, y - y0) for _, x, y in pts)
|
|
184
|
+
duration = pts[-1][0]
|
|
185
|
+
if moved < TAP_MOVE_PT:
|
|
186
|
+
if duration >= LONG_PRESS_MS:
|
|
187
|
+
return Gesture("longPress", (pts[0],), hold_ms=duration)
|
|
188
|
+
return Gesture("tap", (pts[0],), hold_ms=TAP_MS)
|
|
189
|
+
return Gesture("drag", tuple(thin(pts, PATH_POINTS)))
|