mobster-cli 0.4.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1009) hide show
  1. mobile_agent/__init__.py +4 -0
  2. mobile_agent/__main__.py +598 -0
  3. mobile_agent/accounts/__init__.py +1 -0
  4. mobile_agent/accounts/context.py +80 -0
  5. mobile_agent/accounts/mcp.py +55 -0
  6. mobile_agent/accounts/pacing.py +318 -0
  7. mobile_agent/accounts/register.py +78 -0
  8. mobile_agent/accounts/routes.py +212 -0
  9. mobile_agent/accounts/routing.py +69 -0
  10. mobile_agent/accounts/screen.py +273 -0
  11. mobile_agent/accounts/stats.py +54 -0
  12. mobile_agent/accounts/store.py +394 -0
  13. mobile_agent/agent_hooks.py +123 -0
  14. mobile_agent/ai_plans/__init__.py +7 -0
  15. mobile_agent/ai_plans/agents.py +96 -0
  16. mobile_agent/ai_plans/claude_code.py +208 -0
  17. mobile_agent/ai_plans/cli.py +400 -0
  18. mobile_agent/ai_plans/codex.py +258 -0
  19. mobile_agent/ai_plans/compat_chat.py +307 -0
  20. mobile_agent/ai_plans/device_login.py +164 -0
  21. mobile_agent/ai_plans/draft_client.py +68 -0
  22. mobile_agent/ai_plans/gemini.py +137 -0
  23. mobile_agent/ai_plans/instructions.py +65 -0
  24. mobile_agent/ai_plans/mcp_http.py +160 -0
  25. mobile_agent/ai_plans/phone_tools.py +892 -0
  26. mobile_agent/ai_plans/providers.py +235 -0
  27. mobile_agent/ai_plans/routes.py +160 -0
  28. mobile_agent/ai_plans/runner.py +400 -0
  29. mobile_agent/ai_plans/vault.py +114 -0
  30. mobile_agent/alerts.py +337 -0
  31. mobile_agent/android/__init__.py +18 -0
  32. mobile_agent/android/adb.py +268 -0
  33. mobile_agent/android/conditions.py +128 -0
  34. mobile_agent/android/doctor.py +54 -0
  35. mobile_agent/android/driver.py +137 -0
  36. mobile_agent/android/emulator.py +216 -0
  37. mobile_agent/android/logs.py +85 -0
  38. mobile_agent/android/manager.py +225 -0
  39. mobile_agent/android/records.py +31 -0
  40. mobile_agent/android/tree.py +96 -0
  41. mobile_agent/answer_shapes.py +230 -0
  42. mobile_agent/api_devices.py +333 -0
  43. mobile_agent/api_errors.py +17 -0
  44. mobile_agent/api_get.py +249 -0
  45. mobile_agent/api_http.py +467 -0
  46. mobile_agent/api_post.py +453 -0
  47. mobile_agent/api_routes.py +173 -0
  48. mobile_agent/app_api/__init__.py +4 -0
  49. mobile_agent/app_api/cli.py +151 -0
  50. mobile_agent/app_api/endpoints.py +193 -0
  51. mobile_agent/app_api/mcp.py +54 -0
  52. mobile_agent/app_api/openapi.py +146 -0
  53. mobile_agent/app_api/reading.py +172 -0
  54. mobile_agent/app_api/register.py +10 -0
  55. mobile_agent/app_api/screen.py +216 -0
  56. mobile_agent/app_api/service.py +382 -0
  57. mobile_agent/app_api/sessions.py +404 -0
  58. mobile_agent/app_api/tokens.py +167 -0
  59. mobile_agent/app_choice.py +131 -0
  60. mobile_agent/app_icons.py +183 -0
  61. mobile_agent/app_killer.py +69 -0
  62. mobile_agent/artifacts.py +606 -0
  63. mobile_agent/attach/__init__.py +13 -0
  64. mobile_agent/attach/cli.py +46 -0
  65. mobile_agent/attach/flutter.py +28 -0
  66. mobile_agent/attach/score.py +119 -0
  67. mobile_agent/attach/web.py +13 -0
  68. mobile_agent/attachments/__init__.py +1 -0
  69. mobile_agent/attachments/agent.py +418 -0
  70. mobile_agent/attachments/extract.py +217 -0
  71. mobile_agent/attachments/mcp.py +104 -0
  72. mobile_agent/attachments/register.py +73 -0
  73. mobile_agent/attachments/routes.py +326 -0
  74. mobile_agent/attachments/service.py +170 -0
  75. mobile_agent/attachments/sniff.py +152 -0
  76. mobile_agent/attachments/store.py +441 -0
  77. mobile_agent/audit_verify.py +436 -0
  78. mobile_agent/bench/README.md +410 -0
  79. mobile_agent/bench/__init__.py +0 -0
  80. mobile_agent/bench/__main__.py +394 -0
  81. mobile_agent/bench/ab.py +168 -0
  82. mobile_agent/bench/actuation.py +368 -0
  83. mobile_agent/bench/agents/__init__.py +9 -0
  84. mobile_agent/bench/agents/base.py +105 -0
  85. mobile_agent/bench/agents/plan_clients.py +245 -0
  86. mobile_agent/bench/agents/smart.py +533 -0
  87. mobile_agent/bench/checks.py +358 -0
  88. mobile_agent/bench/decide_gate.py +214 -0
  89. mobile_agent/bench/diagnose.py +181 -0
  90. mobile_agent/bench/evals/__init__.py +14 -0
  91. mobile_agent/bench/evals/build.py +565 -0
  92. mobile_agent/bench/evals/checkpoint.py +214 -0
  93. mobile_agent/bench/evals/irt.py +219 -0
  94. mobile_agent/bench/evals/l0.py +647 -0
  95. mobile_agent/bench/evals/l1.py +866 -0
  96. mobile_agent/bench/evals/models.py +124 -0
  97. mobile_agent/bench/evals/pairwise.py +189 -0
  98. mobile_agent/bench/evals/proxy.py +90 -0
  99. mobile_agent/bench/evals/sets.py +149 -0
  100. mobile_agent/bench/evals/traces.py +446 -0
  101. mobile_agent/bench/fixtures.py +176 -0
  102. mobile_agent/bench/guard_replay.py +129 -0
  103. mobile_agent/bench/h2-manifest.json +20 -0
  104. mobile_agent/bench/h3-manifest.json +20 -0
  105. mobile_agent/bench/heldout-manifest.json +32 -0
  106. mobile_agent/bench/heldout.py +375 -0
  107. mobile_agent/bench/iosworld.py +901 -0
  108. mobile_agent/bench/iosworld_stats.py +510 -0
  109. mobile_agent/bench/llm_units.py +116 -0
  110. mobile_agent/bench/manifest.json +93 -0
  111. mobile_agent/bench/metrics.py +66 -0
  112. mobile_agent/bench/phone.py +44 -0
  113. mobile_agent/bench/replay.py +172 -0
  114. mobile_agent/bench/report.py +207 -0
  115. mobile_agent/bench/runner.py +456 -0
  116. mobile_agent/bench/safety.py +310 -0
  117. mobile_agent/bench/stream_timing.py +213 -0
  118. mobile_agent/bench/suite.py +444 -0
  119. mobile_agent/bench/sweep.py +176 -0
  120. mobile_agent/bench/tape.py +798 -0
  121. mobile_agent/bench/tape_replay.py +88 -0
  122. mobile_agent/bench/trace_evals.py +128 -0
  123. mobile_agent/bench/trace_view.py +81 -0
  124. mobile_agent/bench/truth/iphone15pro.json +138 -0
  125. mobile_agent/bench/truth.py +340 -0
  126. mobile_agent/bench/tuning.py +543 -0
  127. mobile_agent/bench/tuning_analysis.py +656 -0
  128. mobile_agent/bench/vague-heldout-manifest.json +12 -0
  129. mobile_agent/bench/vague_rubric.py +135 -0
  130. mobile_agent/bench/vague_run.py +243 -0
  131. mobile_agent/brand.py +454 -0
  132. mobile_agent/brand_fonts.py +756 -0
  133. mobile_agent/build_identity.py +21 -0
  134. mobile_agent/cable_bt.py +623 -0
  135. mobile_agent/cable_bt_runtime.py +128 -0
  136. mobile_agent/catalog.py +69 -0
  137. mobile_agent/chart_svg.py +415 -0
  138. mobile_agent/chat_here.py +189 -0
  139. mobile_agent/cli_constants.py +5 -0
  140. mobile_agent/cli_parser.py +611 -0
  141. mobile_agent/cli_run.py +115 -0
  142. mobile_agent/compiled_read/__init__.py +21 -0
  143. mobile_agent/compiled_read/delta.py +89 -0
  144. mobile_agent/compiled_read/extractors.py +202 -0
  145. mobile_agent/compiled_read/routes.py +56 -0
  146. mobile_agent/completion.py +267 -0
  147. mobile_agent/compose.py +48 -0
  148. mobile_agent/config.py +114 -0
  149. mobile_agent/connectors/__init__.py +1 -0
  150. mobile_agent/connectors/api_session.py +76 -0
  151. mobile_agent/connectors/callback_page.py +146 -0
  152. mobile_agent/connectors/client.py +582 -0
  153. mobile_agent/connectors/github_app.py +55 -0
  154. mobile_agent/connectors/graph_api.py +396 -0
  155. mobile_agent/connectors/imports.py +170 -0
  156. mobile_agent/connectors/issue.py +66 -0
  157. mobile_agent/connectors/keystore.py +85 -0
  158. mobile_agent/connectors/marks.py +222 -0
  159. mobile_agent/connectors/oauth.py +476 -0
  160. mobile_agent/connectors/picker.py +228 -0
  161. mobile_agent/connectors/presets.py +591 -0
  162. mobile_agent/connectors/register.py +48 -0
  163. mobile_agent/connectors/routes.py +153 -0
  164. mobile_agent/connectors/service.py +966 -0
  165. mobile_agent/connectors/shared.py +136 -0
  166. mobile_agent/connectors/skill.py +446 -0
  167. mobile_agent/connectors/slack_api.py +269 -0
  168. mobile_agent/connectors/steps.py +92 -0
  169. mobile_agent/connectors/store.py +144 -0
  170. mobile_agent/connectors/vault.py +72 -0
  171. mobile_agent/console.py +178 -0
  172. mobile_agent/content_queue/__init__.py +1 -0
  173. mobile_agent/content_queue/client.py +263 -0
  174. mobile_agent/content_queue/importing.py +153 -0
  175. mobile_agent/content_queue/media_check.py +224 -0
  176. mobile_agent/content_queue/prompt.py +64 -0
  177. mobile_agent/content_queue/register.py +91 -0
  178. mobile_agent/content_queue/routes.py +370 -0
  179. mobile_agent/content_queue/runner.py +475 -0
  180. mobile_agent/content_queue/store.py +521 -0
  181. mobile_agent/contract.py +996 -0
  182. mobile_agent/contract_asks.py +248 -0
  183. mobile_agent/contract_prompt.py +45 -0
  184. mobile_agent/contract_screen.py +981 -0
  185. mobile_agent/contract_view.py +84 -0
  186. mobile_agent/corpus/__init__.py +26 -0
  187. mobile_agent/corpus/cli.py +63 -0
  188. mobile_agent/corpus/consent.py +55 -0
  189. mobile_agent/corpus/listener.py +76 -0
  190. mobile_agent/corpus/register.py +9 -0
  191. mobile_agent/corpus/skeleton.py +72 -0
  192. mobile_agent/corpus/store.py +143 -0
  193. mobile_agent/costs.py +161 -0
  194. mobile_agent/decider_scope.py +40 -0
  195. mobile_agent/decision_provider.py +189 -0
  196. mobile_agent/deeplinks.py +352 -0
  197. mobile_agent/device_info.py +116 -0
  198. mobile_agent/device_manager.py +978 -0
  199. mobile_agent/device_status.py +195 -0
  200. mobile_agent/device_targets.py +217 -0
  201. mobile_agent/devices.py +676 -0
  202. mobile_agent/devkit/__init__.py +64 -0
  203. mobile_agent/devkit/account_cli.py +90 -0
  204. mobile_agent/devkit/asc.py +562 -0
  205. mobile_agent/devkit/asc_cli.py +214 -0
  206. mobile_agent/devkit/ci.py +737 -0
  207. mobile_agent/devkit/ci_cli.py +196 -0
  208. mobile_agent/devkit/ci_hosts.py +194 -0
  209. mobile_agent/devkit/es256.py +197 -0
  210. mobile_agent/devkit/expo.py +440 -0
  211. mobile_agent/devkit/expo_cli.py +405 -0
  212. mobile_agent/devkit/expo_dev.py +203 -0
  213. mobile_agent/devkit/expo_routes.py +90 -0
  214. mobile_agent/devkit/fixtures_cli.py +133 -0
  215. mobile_agent/devkit/fixtures_routes.py +82 -0
  216. mobile_agent/devkit/links.py +375 -0
  217. mobile_agent/devkit/links_cli.py +123 -0
  218. mobile_agent/devkit/links_routes.py +83 -0
  219. mobile_agent/devkit/nightly.py +152 -0
  220. mobile_agent/devkit/shots.py +203 -0
  221. mobile_agent/devkit/testflight.py +155 -0
  222. mobile_agent/devkit/testflight_cli.py +127 -0
  223. mobile_agent/devkit/testflight_routes.py +111 -0
  224. mobile_agent/devkit/xcode.py +236 -0
  225. mobile_agent/devkit/xcode_cli.py +101 -0
  226. mobile_agent/devtools.py +419 -0
  227. mobile_agent/doctor.py +509 -0
  228. mobile_agent/doctor_fix.py +467 -0
  229. mobile_agent/downloads.py +55 -0
  230. mobile_agent/drivers.py +793 -0
  231. mobile_agent/dry_run.py +102 -0
  232. mobile_agent/effect_ledger.py +83 -0
  233. mobile_agent/engine_health.py +378 -0
  234. mobile_agent/engine_hooks.py +96 -0
  235. mobile_agent/engines.py +408 -0
  236. mobile_agent/env_file.py +63 -0
  237. mobile_agent/env_secrets.py +149 -0
  238. mobile_agent/errors.py +15 -0
  239. mobile_agent/evals/__init__.py +3 -0
  240. mobile_agent/evals/app_survey.py +149 -0
  241. mobile_agent/evals/development/__init__.py +1 -0
  242. mobile_agent/evals/development/score.py +276 -0
  243. mobile_agent/evals/latency_report.py +312 -0
  244. mobile_agent/evals/offline_report.py +374 -0
  245. mobile_agent/evals/oracles.py +298 -0
  246. mobile_agent/evals/redteam/__init__.py +15 -0
  247. mobile_agent/evals/redteam/__main__.py +72 -0
  248. mobile_agent/evals/redteam/cases.py +730 -0
  249. mobile_agent/evals/redteam/harness.py +604 -0
  250. mobile_agent/evals/redteam/live.py +139 -0
  251. mobile_agent/evals/redteam/plain.py +85 -0
  252. mobile_agent/evals/vision/__init__.py +0 -0
  253. mobile_agent/evals/vision/images/dog_001.jpg +0 -0
  254. mobile_agent/evals/vision/images/dog_002.jpg +0 -0
  255. mobile_agent/evals/vision/images/dog_003.jpg +0 -0
  256. mobile_agent/evals/vision/images/dog_004.jpg +0 -0
  257. mobile_agent/evals/vision/images/dog_005.jpg +0 -0
  258. mobile_agent/evals/vision/images/dog_006.jpg +0 -0
  259. mobile_agent/evals/vision/images/dog_007.jpg +0 -0
  260. mobile_agent/evals/vision/images/dog_010.jpg +0 -0
  261. mobile_agent/evals/vision/images/dog_011.jpg +0 -0
  262. mobile_agent/evals/vision/images/dog_013.jpg +0 -0
  263. mobile_agent/evals/vision/images/dog_014.jpg +0 -0
  264. mobile_agent/evals/vision/images/dog_015.jpg +0 -0
  265. mobile_agent/evals/vision/images/dog_016.jpg +0 -0
  266. mobile_agent/evals/vision/images/dog_017.jpg +0 -0
  267. mobile_agent/evals/vision/images/dog_019.jpg +0 -0
  268. mobile_agent/evals/vision/images/dog_020.jpg +0 -0
  269. mobile_agent/evals/vision/images/dog_021.jpg +0 -0
  270. mobile_agent/evals/vision/images/dog_022.jpg +0 -0
  271. mobile_agent/evals/vision/images/dog_023.jpg +0 -0
  272. mobile_agent/evals/vision/images/dog_024.jpg +0 -0
  273. mobile_agent/evals/vision/images/dog_026.jpg +0 -0
  274. mobile_agent/evals/vision/images/dog_027.jpg +0 -0
  275. mobile_agent/evals/vision/images/dog_034.jpg +0 -0
  276. mobile_agent/evals/vision/images/dog_035.jpg +0 -0
  277. mobile_agent/evals/vision/images/dog_038.jpg +0 -0
  278. mobile_agent/evals/vision/images/dog_042.jpg +0 -0
  279. mobile_agent/evals/vision/images/dog_047.jpg +0 -0
  280. mobile_agent/evals/vision/images/dog_052.jpg +0 -0
  281. mobile_agent/evals/vision/images/dog_058.jpg +0 -0
  282. mobile_agent/evals/vision/images/dog_062.jpg +0 -0
  283. mobile_agent/evals/vision/images/eyes_000.jpg +0 -0
  284. mobile_agent/evals/vision/images/eyes_010.jpg +0 -0
  285. mobile_agent/evals/vision/images/eyes_011.jpg +0 -0
  286. mobile_agent/evals/vision/images/eyes_015.jpg +0 -0
  287. mobile_agent/evals/vision/images/eyes_016.jpg +0 -0
  288. mobile_agent/evals/vision/images/eyes_017.jpg +0 -0
  289. mobile_agent/evals/vision/images/eyes_019.jpg +0 -0
  290. mobile_agent/evals/vision/images/eyes_021.jpg +0 -0
  291. mobile_agent/evals/vision/images/eyes_025.jpg +0 -0
  292. mobile_agent/evals/vision/images/eyes_026.jpg +0 -0
  293. mobile_agent/evals/vision/images/eyes_027.jpg +0 -0
  294. mobile_agent/evals/vision/images/eyes_030.jpg +0 -0
  295. mobile_agent/evals/vision/images/eyes_032.jpg +0 -0
  296. mobile_agent/evals/vision/images/eyes_033.jpg +0 -0
  297. mobile_agent/evals/vision/images/eyes_034.jpg +0 -0
  298. mobile_agent/evals/vision/images/eyes_035.jpg +0 -0
  299. mobile_agent/evals/vision/images/eyes_036.jpg +0 -0
  300. mobile_agent/evals/vision/images/eyes_037.jpg +0 -0
  301. mobile_agent/evals/vision/images/eyes_039.jpg +0 -0
  302. mobile_agent/evals/vision/images/eyes_040.jpg +0 -0
  303. mobile_agent/evals/vision/images/eyes_041.jpg +0 -0
  304. mobile_agent/evals/vision/images/eyes_042.jpg +0 -0
  305. mobile_agent/evals/vision/images/eyes_043.jpg +0 -0
  306. mobile_agent/evals/vision/images/eyes_044.jpg +0 -0
  307. mobile_agent/evals/vision/images/eyes_046.jpg +0 -0
  308. mobile_agent/evals/vision/images/eyes_047.jpg +0 -0
  309. mobile_agent/evals/vision/images/eyes_048.jpg +0 -0
  310. mobile_agent/evals/vision/images/eyes_049.jpg +0 -0
  311. mobile_agent/evals/vision/images/eyes_051.jpg +0 -0
  312. mobile_agent/evals/vision/images/eyes_053.jpg +0 -0
  313. mobile_agent/evals/vision/images/eyes_054.jpg +0 -0
  314. mobile_agent/evals/vision/images/eyes_055.jpg +0 -0
  315. mobile_agent/evals/vision/images/eyes_058.jpg +0 -0
  316. mobile_agent/evals/vision/images/eyes_059.jpg +0 -0
  317. mobile_agent/evals/vision/images/receipt_002.jpg +0 -0
  318. mobile_agent/evals/vision/images/receipt_003.jpg +0 -0
  319. mobile_agent/evals/vision/images/receipt_004.jpg +0 -0
  320. mobile_agent/evals/vision/images/receipt_007.jpg +0 -0
  321. mobile_agent/evals/vision/images/receipt_009.jpg +0 -0
  322. mobile_agent/evals/vision/images/receipt_010.jpg +0 -0
  323. mobile_agent/evals/vision/images/receipt_011.jpg +0 -0
  324. mobile_agent/evals/vision/images/receipt_012.jpg +0 -0
  325. mobile_agent/evals/vision/images/receipt_013.jpg +0 -0
  326. mobile_agent/evals/vision/images/receipt_014.jpg +0 -0
  327. mobile_agent/evals/vision/images/receipt_016.jpg +0 -0
  328. mobile_agent/evals/vision/images/receipt_017.jpg +0 -0
  329. mobile_agent/evals/vision/images/receipt_023.jpg +0 -0
  330. mobile_agent/evals/vision/images/receipt_027.jpg +0 -0
  331. mobile_agent/evals/vision/images/receipt_029.jpg +0 -0
  332. mobile_agent/evals/vision/images/receipt_032.jpg +0 -0
  333. mobile_agent/evals/vision/images/receipt_033.jpg +0 -0
  334. mobile_agent/evals/vision/images/receipt_036.jpg +0 -0
  335. mobile_agent/evals/vision/images/receipt_041.jpg +0 -0
  336. mobile_agent/evals/vision/images/receipt_043.jpg +0 -0
  337. mobile_agent/evals/vision/images/receipt_047.jpg +0 -0
  338. mobile_agent/evals/vision/images/receipt_049.jpg +0 -0
  339. mobile_agent/evals/vision/images/receipt_051.jpg +0 -0
  340. mobile_agent/evals/vision/images/receipt_053.jpg +0 -0
  341. mobile_agent/evals/vision/images/receipt_056.jpg +0 -0
  342. mobile_agent/evals/vision/images/receipt_057.jpg +0 -0
  343. mobile_agent/evals/vision/images/receipt_058.jpg +0 -0
  344. mobile_agent/evals/vision/images/receipt_059.jpg +0 -0
  345. mobile_agent/evals/vision/images/receipt_060.jpg +0 -0
  346. mobile_agent/evals/vision/images/receipt_061.jpg +0 -0
  347. mobile_agent/evals/vision/images/receipt_062.jpg +0 -0
  348. mobile_agent/evals/vision/images/receipt_063.jpg +0 -0
  349. mobile_agent/evals/vision/images/receipt_064.jpg +0 -0
  350. mobile_agent/evals/vision/images/receipt_065.jpg +0 -0
  351. mobile_agent/evals/vision/images/receipt_066.jpg +0 -0
  352. mobile_agent/evals/vision/images/receipt_067.jpg +0 -0
  353. mobile_agent/evals/vision/images/receipt_068.jpg +0 -0
  354. mobile_agent/evals/vision/images/receipt_069.jpg +0 -0
  355. mobile_agent/evals/vision/images/receipt_070.jpg +0 -0
  356. mobile_agent/evals/vision/images/receipt_071.jpg +0 -0
  357. mobile_agent/evals/vision/images/sunglasses_000.jpg +0 -0
  358. mobile_agent/evals/vision/images/sunglasses_001.jpg +0 -0
  359. mobile_agent/evals/vision/images/sunglasses_002.jpg +0 -0
  360. mobile_agent/evals/vision/images/sunglasses_004.jpg +0 -0
  361. mobile_agent/evals/vision/images/sunglasses_005.jpg +0 -0
  362. mobile_agent/evals/vision/images/sunglasses_008.jpg +0 -0
  363. mobile_agent/evals/vision/images/sunglasses_009.jpg +0 -0
  364. mobile_agent/evals/vision/images/sunglasses_010.jpg +0 -0
  365. mobile_agent/evals/vision/images/sunglasses_012.jpg +0 -0
  366. mobile_agent/evals/vision/images/sunglasses_013.jpg +0 -0
  367. mobile_agent/evals/vision/images/sunglasses_015.jpg +0 -0
  368. mobile_agent/evals/vision/images/sunglasses_016.jpg +0 -0
  369. mobile_agent/evals/vision/images/sunglasses_017.jpg +0 -0
  370. mobile_agent/evals/vision/images/sunglasses_018.jpg +0 -0
  371. mobile_agent/evals/vision/images/sunglasses_021.jpg +0 -0
  372. mobile_agent/evals/vision/images/sunglasses_022.jpg +0 -0
  373. mobile_agent/evals/vision/images/sunglasses_023.jpg +0 -0
  374. mobile_agent/evals/vision/images/sunglasses_024.jpg +0 -0
  375. mobile_agent/evals/vision/images/sunglasses_030.jpg +0 -0
  376. mobile_agent/evals/vision/images/sunglasses_031.jpg +0 -0
  377. mobile_agent/evals/vision/images/sunglasses_032.jpg +0 -0
  378. mobile_agent/evals/vision/images/sunglasses_033.jpg +0 -0
  379. mobile_agent/evals/vision/images/sunglasses_034.jpg +0 -0
  380. mobile_agent/evals/vision/images/sunglasses_036.jpg +0 -0
  381. mobile_agent/evals/vision/images/sunglasses_038.jpg +0 -0
  382. mobile_agent/evals/vision/images/sunglasses_039.jpg +0 -0
  383. mobile_agent/evals/vision/images/sunglasses_040.jpg +0 -0
  384. mobile_agent/evals/vision/images/sunglasses_041.jpg +0 -0
  385. mobile_agent/evals/vision/manifest.json +2355 -0
  386. mobile_agent/eventbus.py +110 -0
  387. mobile_agent/explain.py +34 -0
  388. mobile_agent/failure_line.py +197 -0
  389. mobile_agent/fanout.py +663 -0
  390. mobile_agent/feedback/__init__.py +2 -0
  391. mobile_agent/feedback/bundle.py +204 -0
  392. mobile_agent/feedback/outbox.py +196 -0
  393. mobile_agent/feedback/register.py +9 -0
  394. mobile_agent/feedback/routes.py +177 -0
  395. mobile_agent/fleet.py +841 -0
  396. mobile_agent/fleet_health.py +729 -0
  397. mobile_agent/fleet_phone.py +186 -0
  398. mobile_agent/frontier/__init__.py +79 -0
  399. mobile_agent/frontier/act.py +380 -0
  400. mobile_agent/frontier/actions.py +70 -0
  401. mobile_agent/frontier/agent.py +256 -0
  402. mobile_agent/frontier/approvals.py +366 -0
  403. mobile_agent/frontier/hooks.py +199 -0
  404. mobile_agent/frontier/interrupts.py +125 -0
  405. mobile_agent/frontier/observe.py +331 -0
  406. mobile_agent/frontier/phases.py +381 -0
  407. mobile_agent/frontier/pipeline.py +61 -0
  408. mobile_agent/frontier/prompt.py +384 -0
  409. mobile_agent/frontier/protocol.py +91 -0
  410. mobile_agent/frontier/safety.py +467 -0
  411. mobile_agent/frontier/saved.py +375 -0
  412. mobile_agent/frontier/screen.py +419 -0
  413. mobile_agent/frontier/skills.py +147 -0
  414. mobile_agent/frontier/stages/__init__.py +1 -0
  415. mobile_agent/frontier/stages/after.py +192 -0
  416. mobile_agent/frontier/stages/approve.py +138 -0
  417. mobile_agent/frontier/stages/controls.py +115 -0
  418. mobile_agent/frontier/stages/ends.py +102 -0
  419. mobile_agent/frontier/stages/gate.py +92 -0
  420. mobile_agent/frontier/stages/guards.py +132 -0
  421. mobile_agent/frontier/stages/points.py +155 -0
  422. mobile_agent/frontier/stages/send.py +177 -0
  423. mobile_agent/frontier/turn.py +151 -0
  424. mobile_agent/full_disk_access.py +83 -0
  425. mobile_agent/gemini.py +69 -0
  426. mobile_agent/genuine.py +120 -0
  427. mobile_agent/graph/__init__.py +4 -0
  428. mobile_agent/graph/catalog.py +789 -0
  429. mobile_agent/graph/chain.py +306 -0
  430. mobile_agent/graph/dispatch.py +98 -0
  431. mobile_agent/graph/draft.py +373 -0
  432. mobile_agent/graph/edit.py +76 -0
  433. mobile_agent/graph/estimate.py +189 -0
  434. mobile_agent/graph/expr.py +562 -0
  435. mobile_agent/graph/functions.py +271 -0
  436. mobile_agent/graph/infer.py +144 -0
  437. mobile_agent/graph/nodes.py +936 -0
  438. mobile_agent/graph/phone_steps.py +178 -0
  439. mobile_agent/graph/rerun.py +327 -0
  440. mobile_agent/graph/run.py +899 -0
  441. mobile_agent/graph/schema.py +842 -0
  442. mobile_agent/graph/state.py +161 -0
  443. mobile_agent/graph/values.py +198 -0
  444. mobile_agent/harness/__init__.py +2 -0
  445. mobile_agent/harness/checkpoints.py +468 -0
  446. mobile_agent/harness/control.py +101 -0
  447. mobile_agent/harness/prewarm.py +213 -0
  448. mobile_agent/harness/register.py +43 -0
  449. mobile_agent/harness/routes.py +114 -0
  450. mobile_agent/harness/settings.py +66 -0
  451. mobile_agent/harness/tools/__init__.py +67 -0
  452. mobile_agent/harness/tools/ask_user.py +102 -0
  453. mobile_agent/harness/tools/watch.py +120 -0
  454. mobile_agent/harness_api.py +815 -0
  455. mobile_agent/hedge.py +203 -0
  456. mobile_agent/help_topics.py +305 -0
  457. mobile_agent/hints.py +248 -0
  458. mobile_agent/history.py +69 -0
  459. mobile_agent/home.py +214 -0
  460. mobile_agent/http_pool.py +175 -0
  461. mobile_agent/image_policy.py +139 -0
  462. mobile_agent/inbox.py +148 -0
  463. mobile_agent/installed_apps.py +344 -0
  464. mobile_agent/integrations/__init__.py +1 -0
  465. mobile_agent/integrations/cli.py +472 -0
  466. mobile_agent/integrations/client_files.py +65 -0
  467. mobile_agent/integrations/clients.py +998 -0
  468. mobile_agent/integrations/doctor.py +185 -0
  469. mobile_agent/integrations/jsonc.py +383 -0
  470. mobile_agent/integrations/routes.py +135 -0
  471. mobile_agent/integrations/rules.py +112 -0
  472. mobile_agent/integrations/server_options.py +46 -0
  473. mobile_agent/integrations/skill.py +152 -0
  474. mobile_agent/integrations/skill_files.py +10 -0
  475. mobile_agent/integrations/textedit.py +309 -0
  476. mobile_agent/intents/__init__.py +17 -0
  477. mobile_agent/intents/atlas.py +216 -0
  478. mobile_agent/intents/callback.py +44 -0
  479. mobile_agent/intents/cli.py +55 -0
  480. mobile_agent/intents/harvest.py +91 -0
  481. mobile_agent/intents/plan.py +93 -0
  482. mobile_agent/ios/__init__.py +1 -0
  483. mobile_agent/ios/guide.py +16 -0
  484. mobile_agent/ios/kb.py +212 -0
  485. mobile_agent/ios/kb_data.py +1588 -0
  486. mobile_agent/ios/mcp.py +44 -0
  487. mobile_agent/ios/register.py +8 -0
  488. mobile_agent/ios/system.py +369 -0
  489. mobile_agent/journal.py +531 -0
  490. mobile_agent/keychain_acl.py +374 -0
  491. mobile_agent/keys.py +395 -0
  492. mobile_agent/latency_overlap.py +36 -0
  493. mobile_agent/latency_trace.py +170 -0
  494. mobile_agent/learned/__init__.py +15 -0
  495. mobile_agent/learned/app_path.py +160 -0
  496. mobile_agent/learned/code_read.py +38 -0
  497. mobile_agent/learned/parts.py +546 -0
  498. mobile_agent/learned/programs.py +691 -0
  499. mobile_agent/learned/register.py +171 -0
  500. mobile_agent/learned/routes.py +61 -0
  501. mobile_agent/learned/taught.py +329 -0
  502. mobile_agent/learned/teach.py +538 -0
  503. mobile_agent/learned/workflows.py +647 -0
  504. mobile_agent/learned/zero.py +41 -0
  505. mobile_agent/links.py +37 -0
  506. mobile_agent/listen.py +607 -0
  507. mobile_agent/lock_spike.py +278 -0
  508. mobile_agent/lockscreen.py +526 -0
  509. mobile_agent/login.py +354 -0
  510. mobile_agent/longform/__init__.py +15 -0
  511. mobile_agent/longform/ads.py +191 -0
  512. mobile_agent/longform/blocks.py +117 -0
  513. mobile_agent/longform/budget.py +265 -0
  514. mobile_agent/longform/context.py +285 -0
  515. mobile_agent/longform/costs.py +93 -0
  516. mobile_agent/longform/digest.py +145 -0
  517. mobile_agent/longform/extract.py +508 -0
  518. mobile_agent/longform/feed.py +505 -0
  519. mobile_agent/longform/followup.py +168 -0
  520. mobile_agent/longform/frames.py +82 -0
  521. mobile_agent/longform/identity.py +87 -0
  522. mobile_agent/longform/inbox.py +221 -0
  523. mobile_agent/longform/items.py +425 -0
  524. mobile_agent/longform/models.py +115 -0
  525. mobile_agent/longform/notebook.py +329 -0
  526. mobile_agent/longform/plan.py +614 -0
  527. mobile_agent/longform/recall.py +195 -0
  528. mobile_agent/longform/receipts.py +81 -0
  529. mobile_agent/longform/register.py +124 -0
  530. mobile_agent/longform/report.py +343 -0
  531. mobile_agent/longform/routes.py +102 -0
  532. mobile_agent/longform/session.py +644 -0
  533. mobile_agent/longform/store.py +146 -0
  534. mobile_agent/longform/synthesis.py +347 -0
  535. mobile_agent/mac_tools.py +223 -0
  536. mobile_agent/manual_control.py +143 -0
  537. mobile_agent/maps.py +149 -0
  538. mobile_agent/marks.py +92 -0
  539. mobile_agent/mcp_server/__init__.py +3 -0
  540. mobile_agent/mcp_server/__main__.py +29 -0
  541. mobile_agent/mcp_server/action_journal.py +24 -0
  542. mobile_agent/mcp_server/action_tools.py +638 -0
  543. mobile_agent/mcp_server/approvals.py +219 -0
  544. mobile_agent/mcp_server/catalog.py +92 -0
  545. mobile_agent/mcp_server/cli.py +194 -0
  546. mobile_agent/mcp_server/codes.py +147 -0
  547. mobile_agent/mcp_server/device_tools.py +306 -0
  548. mobile_agent/mcp_server/direct.py +272 -0
  549. mobile_agent/mcp_server/expo_tools.py +105 -0
  550. mobile_agent/mcp_server/phone_tools.py +510 -0
  551. mobile_agent/mcp_server/protocol.py +364 -0
  552. mobile_agent/mcp_server/registry.py +55 -0
  553. mobile_agent/mcp_server/run_tools.py +531 -0
  554. mobile_agent/mcp_server/schemas.py +425 -0
  555. mobile_agent/mcp_server/screen_tools.py +168 -0
  556. mobile_agent/mcp_server/session.py +469 -0
  557. mobile_agent/mcp_server/support.py +145 -0
  558. mobile_agent/mcp_server/tools.py +493 -0
  559. mobile_agent/memory/__init__.py +9 -0
  560. mobile_agent/memory/cli.py +207 -0
  561. mobile_agent/memory/extract.py +191 -0
  562. mobile_agent/memory/listener.py +100 -0
  563. mobile_agent/memory/model.py +144 -0
  564. mobile_agent/memory/project.py +96 -0
  565. mobile_agent/memory/provider.py +127 -0
  566. mobile_agent/memory/register.py +21 -0
  567. mobile_agent/memory/retrieval.py +172 -0
  568. mobile_agent/memory/routes.py +216 -0
  569. mobile_agent/memory/runs.py +124 -0
  570. mobile_agent/memory/store.py +806 -0
  571. mobile_agent/memory/writer.py +278 -0
  572. mobile_agent/mirroring.py +585 -0
  573. mobile_agent/mirroring_runtime.py +221 -0
  574. mobile_agent/model_clients.py +604 -0
  575. mobile_agent/narrate.py +813 -0
  576. mobile_agent/native_helpers.py +84 -0
  577. mobile_agent/netxray/__init__.py +1 -0
  578. mobile_agent/netxray/analyze.py +322 -0
  579. mobile_agent/netxray/attribution.py +62 -0
  580. mobile_agent/netxray/capture.py +88 -0
  581. mobile_agent/netxray/cli.py +142 -0
  582. mobile_agent/netxray/job.py +219 -0
  583. mobile_agent/netxray/mobster-pcap.c +560 -0
  584. mobile_agent/netxray/packets.py +410 -0
  585. mobile_agent/netxray/pcapng.py +223 -0
  586. mobile_agent/netxray/register.py +93 -0
  587. mobile_agent/netxray/report.py +214 -0
  588. mobile_agent/netxray/store.py +48 -0
  589. mobile_agent/netxray/summary.py +157 -0
  590. mobile_agent/netxray/target.py +67 -0
  591. mobile_agent/netxray/tool.py +66 -0
  592. mobile_agent/netxray/trackers.py +194 -0
  593. mobile_agent/notify_triggers/__init__.py +1 -0
  594. mobile_agent/notify_triggers/rules.py +39 -0
  595. mobile_agent/notify_triggers/sources.py +315 -0
  596. mobile_agent/notify_triggers/watch.py +135 -0
  597. mobile_agent/offers.py +86 -0
  598. mobile_agent/orchestrate.py +215 -0
  599. mobile_agent/outlets/__init__.py +1 -0
  600. mobile_agent/outlets/connect.py +249 -0
  601. mobile_agent/outlets/folder.py +180 -0
  602. mobile_agent/outlets/hooks.py +453 -0
  603. mobile_agent/outlets/inbound.py +472 -0
  604. mobile_agent/outlets/inbound_reply.py +57 -0
  605. mobile_agent/outlets/inbound_routes.py +81 -0
  606. mobile_agent/outlets/kinds.py +308 -0
  607. mobile_agent/outlets/mail.py +110 -0
  608. mobile_agent/outlets/net.py +86 -0
  609. mobile_agent/outlets/notion.py +244 -0
  610. mobile_agent/outlets/register.py +56 -0
  611. mobile_agent/outlets/routes.py +135 -0
  612. mobile_agent/outlets/service.py +580 -0
  613. mobile_agent/outlets/sheets.py +169 -0
  614. mobile_agent/outlets/slack.py +177 -0
  615. mobile_agent/outlets/store.py +194 -0
  616. mobile_agent/outlets/vault.py +80 -0
  617. mobile_agent/outlets/webhook.py +316 -0
  618. mobile_agent/output_contract.py +141 -0
  619. mobile_agent/owner_presence.py +211 -0
  620. mobile_agent/ownership.py +219 -0
  621. mobile_agent/packs/__init__.py +1 -0
  622. mobile_agent/packs/calendar.py +243 -0
  623. mobile_agent/packs/context.py +68 -0
  624. mobile_agent/packs/core.py +757 -0
  625. mobile_agent/packs/front_door.py +316 -0
  626. mobile_agent/packs/heal.py +214 -0
  627. mobile_agent/packs/health.py +122 -0
  628. mobile_agent/packs/insights.py +76 -0
  629. mobile_agent/packs/instagram.py +640 -0
  630. mobile_agent/packs/linkedin.py +106 -0
  631. mobile_agent/packs/maps.py +196 -0
  632. mobile_agent/packs/mcp.py +149 -0
  633. mobile_agent/packs/messages.py +270 -0
  634. mobile_agent/packs/notes.py +226 -0
  635. mobile_agent/packs/photos.py +166 -0
  636. mobile_agent/packs/pins.py +47 -0
  637. mobile_agent/packs/reddit.py +110 -0
  638. mobile_agent/packs/register.py +276 -0
  639. mobile_agent/packs/rows.py +108 -0
  640. mobile_agent/packs/safari.py +143 -0
  641. mobile_agent/packs/settings.py +253 -0
  642. mobile_agent/packs/social.py +300 -0
  643. mobile_agent/packs/threads.py +97 -0
  644. mobile_agent/packs/tiktok.py +455 -0
  645. mobile_agent/packs/tools.py +290 -0
  646. mobile_agent/packs/x.py +127 -0
  647. mobile_agent/packs/youtube.py +199 -0
  648. mobile_agent/parity.py +270 -0
  649. mobile_agent/paths.py +99 -0
  650. mobile_agent/perception.py +470 -0
  651. mobile_agent/permissions/__init__.py +32 -0
  652. mobile_agent/permissions/consequence.py +535 -0
  653. mobile_agent/permissions/exceptions.py +99 -0
  654. mobile_agent/permissions/gate.py +316 -0
  655. mobile_agent/permissions/outside.py +316 -0
  656. mobile_agent/permissions/policy.py +288 -0
  657. mobile_agent/permissions/project_policy.py +411 -0
  658. mobile_agent/permissions/register.py +148 -0
  659. mobile_agent/permissions/rules.py +291 -0
  660. mobile_agent/phone_io/__init__.py +146 -0
  661. mobile_agent/phone_io/cli.py +427 -0
  662. mobile_agent/phone_io/clipboard.py +78 -0
  663. mobile_agent/phone_io/controls.py +78 -0
  664. mobile_agent/phone_io/files.py +289 -0
  665. mobile_agent/phone_io/handoff.py +123 -0
  666. mobile_agent/phone_io/icloud.py +118 -0
  667. mobile_agent/phone_io/install.py +188 -0
  668. mobile_agent/phone_io/network_capture.py +188 -0
  669. mobile_agent/phone_io/network_cli.py +138 -0
  670. mobile_agent/phone_io/network_profiles.py +138 -0
  671. mobile_agent/phone_io/photos.py +512 -0
  672. mobile_agent/phone_io/recording.py +115 -0
  673. mobile_agent/phone_io/secrets.py +146 -0
  674. mobile_agent/phone_io/transfer.py +895 -0
  675. mobile_agent/phone_io/tunnel.py +129 -0
  676. mobile_agent/phone_ready.py +91 -0
  677. mobile_agent/pinning.py +158 -0
  678. mobile_agent/plan_gate.py +807 -0
  679. mobile_agent/plugins/__init__.py +3 -0
  680. mobile_agent/plugins/install.py +163 -0
  681. mobile_agent/plugins/manifest.py +242 -0
  682. mobile_agent/plugins/register.py +15 -0
  683. mobile_agent/plugins/routes.py +82 -0
  684. mobile_agent/plugins/service.py +322 -0
  685. mobile_agent/plugins/skills.py +76 -0
  686. mobile_agent/plugins/store.py +79 -0
  687. mobile_agent/pool.py +285 -0
  688. mobile_agent/power_watch.py +275 -0
  689. mobile_agent/premium_model.py +22 -0
  690. mobile_agent/project_env.py +291 -0
  691. mobile_agent/project_env_cli.py +74 -0
  692. mobile_agent/project_env_routes.py +55 -0
  693. mobile_agent/projects/__init__.py +27 -0
  694. mobile_agent/projects/ci.py +332 -0
  695. mobile_agent/projects/clone.py +339 -0
  696. mobile_agent/projects/drafts.py +158 -0
  697. mobile_agent/projects/folder.py +436 -0
  698. mobile_agent/projects/github.py +131 -0
  699. mobile_agent/projects/register.py +31 -0
  700. mobile_agent/projects/revisions.py +253 -0
  701. mobile_agent/projects/routes.py +234 -0
  702. mobile_agent/projects/runs.py +297 -0
  703. mobile_agent/projects/service.py +695 -0
  704. mobile_agent/projects/setup.py +84 -0
  705. mobile_agent/projects/store.py +379 -0
  706. mobile_agent/projects/sync.py +380 -0
  707. mobile_agent/projects/workflow_files.py +270 -0
  708. mobile_agent/replay_core/__init__.py +26 -0
  709. mobile_agent/replay_core/checks.py +594 -0
  710. mobile_agent/replay_core/engine.py +596 -0
  711. mobile_agent/replay_core/heal.py +364 -0
  712. mobile_agent/replay_core/path.py +240 -0
  713. mobile_agent/replay_core/recorder.py +206 -0
  714. mobile_agent/replay_core/store.py +171 -0
  715. mobile_agent/result_blocks.py +941 -0
  716. mobile_agent/run.py +642 -0
  717. mobile_agent/run_delivery.py +123 -0
  718. mobile_agent/run_executor.py +200 -0
  719. mobile_agent/run_export.py +284 -0
  720. mobile_agent/run_gif.py +891 -0
  721. mobile_agent/runner_hints.py +86 -0
  722. mobile_agent/runner_renewal.py +205 -0
  723. mobile_agent/runtime_create.py +549 -0
  724. mobile_agent/runtime_driver.py +458 -0
  725. mobile_agent/runtime_limits.py +319 -0
  726. mobile_agent/runtime_queue.py +394 -0
  727. mobile_agent/runtime_runs.py +383 -0
  728. mobile_agent/runtime_status.py +181 -0
  729. mobile_agent/runtime_teach.py +295 -0
  730. mobile_agent/runtime_work.py +599 -0
  731. mobile_agent/screen_text.py +101 -0
  732. mobile_agent/secret_filter.py +113 -0
  733. mobile_agent/serve.py +160 -0
  734. mobile_agent/server.py +363 -0
  735. mobile_agent/service_support.py +42 -0
  736. mobile_agent/settle/__init__.py +1 -0
  737. mobile_agent/settle/predict.py +102 -0
  738. mobile_agent/settle/replay.py +92 -0
  739. mobile_agent/settle/store.py +93 -0
  740. mobile_agent/settle/stream.py +43 -0
  741. mobile_agent/setup_cli.py +225 -0
  742. mobile_agent/setup_errors.py +87 -0
  743. mobile_agent/setup_service.py +527 -0
  744. mobile_agent/share.py +104 -0
  745. mobile_agent/signing.py +184 -0
  746. mobile_agent/sim/__init__.py +6 -0
  747. mobile_agent/sim/__main__.py +26 -0
  748. mobile_agent/sim/api.py +952 -0
  749. mobile_agent/sim/apps.py +172 -0
  750. mobile_agent/sim/biometrics.py +84 -0
  751. mobile_agent/sim/cli.py +271 -0
  752. mobile_agent/sim/doctor.py +293 -0
  753. mobile_agent/sim/governor.py +90 -0
  754. mobile_agent/sim/holders.py +88 -0
  755. mobile_agent/sim/models.py +62 -0
  756. mobile_agent/sim/registry.py +161 -0
  757. mobile_agent/sim/simctl.py +269 -0
  758. mobile_agent/sim/wda.py +618 -0
  759. mobile_agent/sim/xcode.py +74 -0
  760. mobile_agent/skills/__init__.py +12 -0
  761. mobile_agent/skills/codes.py +597 -0
  762. mobile_agent/skills/notifications.py +281 -0
  763. mobile_agent/skills/open_url.py +127 -0
  764. mobile_agent/smart_events.py +588 -0
  765. mobile_agent/smart_result.py +490 -0
  766. mobile_agent/smart_run.py +413 -0
  767. mobile_agent/state.py +662 -0
  768. mobile_agent/states.py +217 -0
  769. mobile_agent/steering.py +70 -0
  770. mobile_agent/style.py +178 -0
  771. mobile_agent/supervised.py +234 -0
  772. mobile_agent/swift/helper_bt_hid.swift +534 -0
  773. mobile_agent/swift/helper_doc_text.swift +184 -0
  774. mobile_agent/swift/helper_draft_reply.swift +170 -0
  775. mobile_agent/swift/helper_map.swift +110 -0
  776. mobile_agent/swift/helper_mirroring.swift +1099 -0
  777. mobile_agent/swift/helper_mjpeg_mp4.swift +103 -0
  778. mobile_agent/swift/helper_pdf.swift +100 -0
  779. mobile_agent/swift/helper_phone_audio.swift +993 -0
  780. mobile_agent/swift/helper_phone_screen.swift +416 -0
  781. mobile_agent/swift/helper_screen_text.swift +24 -0
  782. mobile_agent/swift/helper_transfer.swift +246 -0
  783. mobile_agent/swift/ocr.swift +22 -0
  784. mobile_agent/task_policy.py +337 -0
  785. mobile_agent/team_signing/__init__.py +34 -0
  786. mobile_agent/team_signing/apple_ca.py +37 -0
  787. mobile_agent/team_signing/bundle.py +136 -0
  788. mobile_agent/team_signing/cli.py +208 -0
  789. mobile_agent/team_signing/identity.py +248 -0
  790. mobile_agent/team_signing/ledger.py +105 -0
  791. mobile_agent/team_signing/prebuilt.py +61 -0
  792. mobile_agent/team_signing/resign.py +256 -0
  793. mobile_agent/team_signing/service.py +382 -0
  794. mobile_agent/team_signing/tooling.py +243 -0
  795. mobile_agent/team_signing/writes.py +268 -0
  796. mobile_agent/terminal_image.py +81 -0
  797. mobile_agent/testkit/__init__.py +1 -0
  798. mobile_agent/testkit/a11y_baseline.py +132 -0
  799. mobile_agent/testkit/api.py +403 -0
  800. mobile_agent/testkit/capsule.py +400 -0
  801. mobile_agent/testkit/capsule_cli.py +57 -0
  802. mobile_agent/testkit/cli.py +948 -0
  803. mobile_agent/testkit/cli_arguments.py +193 -0
  804. mobile_agent/testkit/comparison.py +288 -0
  805. mobile_agent/testkit/comparison_cli.py +41 -0
  806. mobile_agent/testkit/discover.py +257 -0
  807. mobile_agent/testkit/editor.py +243 -0
  808. mobile_agent/testkit/execute.py +515 -0
  809. mobile_agent/testkit/github_checks.py +248 -0
  810. mobile_agent/testkit/gitref.py +66 -0
  811. mobile_agent/testkit/heal_review.py +176 -0
  812. mobile_agent/testkit/health.py +480 -0
  813. mobile_agent/testkit/history.py +255 -0
  814. mobile_agent/testkit/hosted_results.py +321 -0
  815. mobile_agent/testkit/input_files.py +173 -0
  816. mobile_agent/testkit/jobs.py +285 -0
  817. mobile_agent/testkit/junit_verdict.py +33 -0
  818. mobile_agent/testkit/maestro.py +802 -0
  819. mobile_agent/testkit/mcp.py +562 -0
  820. mobile_agent/testkit/mcp_comparison.py +88 -0
  821. mobile_agent/testkit/minimize.py +471 -0
  822. mobile_agent/testkit/minimizer_cli.py +48 -0
  823. mobile_agent/testkit/notify.py +212 -0
  824. mobile_agent/testkit/phone_video.py +191 -0
  825. mobile_agent/testkit/pr.py +491 -0
  826. mobile_agent/testkit/pr_report.py +562 -0
  827. mobile_agent/testkit/prbuild.py +148 -0
  828. mobile_agent/testkit/proof_report.py +55 -0
  829. mobile_agent/testkit/provenance.py +119 -0
  830. mobile_agent/testkit/prrun.py +244 -0
  831. mobile_agent/testkit/recent.py +166 -0
  832. mobile_agent/testkit/recipes.py +146 -0
  833. mobile_agent/testkit/record.py +204 -0
  834. mobile_agent/testkit/register.py +288 -0
  835. mobile_agent/testkit/reports.py +998 -0
  836. mobile_agent/testkit/repro.py +432 -0
  837. mobile_agent/testkit/scenario.py +440 -0
  838. mobile_agent/testkit/scenario_cli.py +37 -0
  839. mobile_agent/testkit/snapshots_cli.py +60 -0
  840. mobile_agent/testkit/suite.py +670 -0
  841. mobile_agent/testkit/targets.py +497 -0
  842. mobile_agent/testkit/xctest.py +378 -0
  843. mobile_agent/threads/__init__.py +1 -0
  844. mobile_agent/threads/chat_once.py +264 -0
  845. mobile_agent/threads/cli.py +740 -0
  846. mobile_agent/threads/client.py +326 -0
  847. mobile_agent/threads/context.py +246 -0
  848. mobile_agent/threads/mcp.py +739 -0
  849. mobile_agent/threads/recent.py +34 -0
  850. mobile_agent/threads/register.py +54 -0
  851. mobile_agent/threads/routes.py +188 -0
  852. mobile_agent/threads/service.py +975 -0
  853. mobile_agent/threads/store.py +361 -0
  854. mobile_agent/threads/summary.py +117 -0
  855. mobile_agent/threads/term.py +101 -0
  856. mobile_agent/threads/workflow_cli.py +104 -0
  857. mobile_agent/tls.py +36 -0
  858. mobile_agent/traces.py +271 -0
  859. mobile_agent/tracks.py +235 -0
  860. mobile_agent/transport.py +271 -0
  861. mobile_agent/triggers/__init__.py +1 -0
  862. mobile_agent/triggers/actions.py +204 -0
  863. mobile_agent/triggers/model.py +146 -0
  864. mobile_agent/triggers/register.py +29 -0
  865. mobile_agent/triggers/routes.py +101 -0
  866. mobile_agent/triggers/service.py +387 -0
  867. mobile_agent/triggers/store.py +142 -0
  868. mobile_agent/tui/__init__.py +89 -0
  869. mobile_agent/tui/app.py +582 -0
  870. mobile_agent/tui/chrome.py +321 -0
  871. mobile_agent/tui/commands.py +375 -0
  872. mobile_agent/tui/demo_phone.py +369 -0
  873. mobile_agent/tui/images.py +29 -0
  874. mobile_agent/tui/keys.py +64 -0
  875. mobile_agent/tui/prompt.py +195 -0
  876. mobile_agent/tui/render.py +395 -0
  877. mobile_agent/tui/screens.py +180 -0
  878. mobile_agent/tui/session.py +308 -0
  879. mobile_agent/tui/tasks.py +287 -0
  880. mobile_agent/tui/theme.py +118 -0
  881. mobile_agent/tui/widgets.py +272 -0
  882. mobile_agent/update_check.py +213 -0
  883. mobile_agent/usage.py +184 -0
  884. mobile_agent/verify/__init__.py +0 -0
  885. mobile_agent/verify/__main__.py +41 -0
  886. mobile_agent/verify/a11y.py +189 -0
  887. mobile_agent/verify/actions.py +290 -0
  888. mobile_agent/verify/assertions.py +570 -0
  889. mobile_agent/verify/audit.py +312 -0
  890. mobile_agent/verify/business.py +221 -0
  891. mobile_agent/verify/business_run.py +131 -0
  892. mobile_agent/verify/business_state.py +168 -0
  893. mobile_agent/verify/check_schema.py +398 -0
  894. mobile_agent/verify/checks.py +614 -0
  895. mobile_agent/verify/cli.py +423 -0
  896. mobile_agent/verify/conditions.py +483 -0
  897. mobile_agent/verify/crashes.py +321 -0
  898. mobile_agent/verify/device_audit.py +246 -0
  899. mobile_agent/verify/dialogs.py +287 -0
  900. mobile_agent/verify/evidence.py +705 -0
  901. mobile_agent/verify/fixtures.py +305 -0
  902. mobile_agent/verify/flow.py +213 -0
  903. mobile_agent/verify/l10n.py +141 -0
  904. mobile_agent/verify/netcerts.py +67 -0
  905. mobile_agent/verify/netproxy.py +356 -0
  906. mobile_agent/verify/netrun.py +81 -0
  907. mobile_agent/verify/netshim.m +28 -0
  908. mobile_agent/verify/netshim.py +48 -0
  909. mobile_agent/verify/netstubs.py +132 -0
  910. mobile_agent/verify/overlay.py +174 -0
  911. mobile_agent/verify/perf.py +239 -0
  912. mobile_agent/verify/proof.py +192 -0
  913. mobile_agent/verify/replay_admission.py +48 -0
  914. mobile_agent/verify/report.py +952 -0
  915. mobile_agent/verify/run_cleanup.py +67 -0
  916. mobile_agent/verify/run_links.py +48 -0
  917. mobile_agent/verify/run_perf.py +44 -0
  918. mobile_agent/verify/run_secrets.py +39 -0
  919. mobile_agent/verify/runner.py +976 -0
  920. mobile_agent/verify/runs.py +57 -0
  921. mobile_agent/verify/screen_coverage.py +181 -0
  922. mobile_agent/verify/smart.py +106 -0
  923. mobile_agent/verify/snapshot_run.py +114 -0
  924. mobile_agent/verify/snapshots.py +306 -0
  925. mobile_agent/verify/sound.py +335 -0
  926. mobile_agent/verify/source_observation.py +130 -0
  927. mobile_agent/verify/system_surfaces/__init__.py +7 -0
  928. mobile_agent/verify/system_surfaces/play.py +106 -0
  929. mobile_agent/verify/system_surfaces/spec.py +139 -0
  930. mobile_agent/verify/system_surfaces/stage.py +138 -0
  931. mobile_agent/verify/system_surfaces/surfaces.py +121 -0
  932. mobile_agent/verify/tree.py +223 -0
  933. mobile_agent/verify/verdict.py +377 -0
  934. mobile_agent/verify/wire.py +392 -0
  935. mobile_agent/verify/wirestore.py +176 -0
  936. mobile_agent/waiting/__init__.py +1 -0
  937. mobile_agent/waiting/contacts.py +73 -0
  938. mobile_agent/waiting/drafts.py +136 -0
  939. mobile_agent/waiting/job.py +145 -0
  940. mobile_agent/waiting/messages.py +174 -0
  941. mobile_agent/waiting/rank.py +115 -0
  942. mobile_agent/waiting/register.py +26 -0
  943. mobile_agent/watch.py +502 -0
  944. mobile_agent/wda/__init__.py +2 -0
  945. mobile_agent/wda/frame_clock.py +646 -0
  946. mobile_agent/wda/frame_source.py +325 -0
  947. mobile_agent/wda/frame_store.py +115 -0
  948. mobile_agent/wda/gate.py +163 -0
  949. mobile_agent/wda/gestures.py +291 -0
  950. mobile_agent/wda/guards.py +94 -0
  951. mobile_agent/wda/identity.py +55 -0
  952. mobile_agent/wda/observe.py +884 -0
  953. mobile_agent/wda/patches.py +856 -0
  954. mobile_agent/wda/point_gestures.py +150 -0
  955. mobile_agent/wda/source.py +32 -0
  956. mobile_agent/wda/system_dialogs.py +571 -0
  957. mobile_agent/wda/text.py +694 -0
  958. mobile_agent/wda/timeouts.py +77 -0
  959. mobile_agent/wda/video.py +505 -0
  960. mobile_agent/wda/watchdog.py +243 -0
  961. mobile_agent/wireless/__init__.py +15 -0
  962. mobile_agent/wireless/api.py +354 -0
  963. mobile_agent/wireless/cli.py +287 -0
  964. mobile_agent/wireless/devicectl.py +229 -0
  965. mobile_agent/wireless/exposure.py +33 -0
  966. mobile_agent/wireless/mobster-wifi-pair.c +122 -0
  967. mobile_agent/wireless/monitor.py +332 -0
  968. mobile_agent/wireless/network.py +334 -0
  969. mobile_agent/wireless/onboarding.py +293 -0
  970. mobile_agent/wireless/pair.py +124 -0
  971. mobile_agent/wireless/probe.py +322 -0
  972. mobile_agent/wireless/register.py +8 -0
  973. mobile_agent/wireless/relay.py +179 -0
  974. mobile_agent/wireless/result.py +87 -0
  975. mobile_agent/wireless/store.py +85 -0
  976. mobile_agent/wireless/switch.py +77 -0
  977. mobile_agent/wireless/transport.py +118 -0
  978. mobile_agent/wireless/words.py +81 -0
  979. mobile_agent/wireless/xctestrun.py +83 -0
  980. mobile_agent/workflow_tools.py +59 -0
  981. mobile_agent/workflows/__init__.py +2 -0
  982. mobile_agent/workflows/brief.py +201 -0
  983. mobile_agent/workflows/cron.py +110 -0
  984. mobile_agent/workflows/health.py +257 -0
  985. mobile_agent/workflows/history.py +126 -0
  986. mobile_agent/workflows/records.py +442 -0
  987. mobile_agent/workflows/runs.py +1000 -0
  988. mobile_agent/workflows/scheduler.py +134 -0
  989. mobile_agent/workflows/service.py +973 -0
  990. mobile_agent/workflows/starters.py +285 -0
  991. mobile_agent/workflows/triggers.py +248 -0
  992. mobile_agent/xray/__init__.py +1 -0
  993. mobile_agent/xray/app_cli.py +116 -0
  994. mobile_agent/xray/cli.py +21 -0
  995. mobile_agent/xray/crawl.py +754 -0
  996. mobile_agent/xray/diff.py +171 -0
  997. mobile_agent/xray/diff_cli.py +43 -0
  998. mobile_agent/xray/job.py +184 -0
  999. mobile_agent/xray/paywall.py +188 -0
  1000. mobile_agent/xray/policy.py +305 -0
  1001. mobile_agent/xray/register.py +43 -0
  1002. mobile_agent/xray/report.py +283 -0
  1003. mobile_agent/xray/screens.py +117 -0
  1004. mobster_cli-0.4.3.dist-info/METADATA +93 -0
  1005. mobster_cli-0.4.3.dist-info/RECORD +1009 -0
  1006. mobster_cli-0.4.3.dist-info/WHEEL +5 -0
  1007. mobster_cli-0.4.3.dist-info/entry_points.txt +3 -0
  1008. mobster_cli-0.4.3.dist-info/licenses/LICENSE +21 -0
  1009. mobster_cli-0.4.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,993 @@
1
+ // phone-audio: hears what a USB iPhone plays, on this Mac, for Mobster's WATCH and MCP's listen (mobile_agent/listen.py).
2
+ //
3
+ // phone-audio devices
4
+ // The iPhones this Mac can hear now: {"ok":true,"devices":[{"id","name","model"}]}.
5
+ // phone-audio listen --seconds N [--udid UDID [--name NAME] [--phones K]] [--locale en-US] [--work DIR]
6
+ // [--keep PATH] [--live [--frames]]
7
+ // Records N seconds (1 to 120) of the iPhone's audio, then turns its speech into text on this Mac:
8
+ // {"ok":true,"text","segments","engine","locale","seconds","recordedSeconds","findMs","startMs","recordMs",
9
+ // "transcribeMs","peakDb","match","soundWindows","classifyMs"}. The iPhone is the capture device whose uniqueID
10
+ // is UDID, else the one named NAME (the phone's name as the device layer reads it), else the only iOS capture
11
+ // device when K, the USB iPhones the device layer sees, is 1; with no UDID, the only one. match says which: udid,
12
+ // name or only. segments are [{"start","end","text"}], in seconds from the recording's start; soundWindows are
13
+ // the sounds told apart in it (see below).
14
+ // The recording is a private file in DIR (else the temporary folder), deleted before this exits, unless --keep
15
+ // names a file to keep it in ("kept").
16
+ // With --live (WATCH, mobile_agent/watch.py): {"event":"recording"} once the recording has started, then
17
+ // {"event":"level","t","db"} every half second (the audio's average power), and the recording ends early when
18
+ // stdin reads a line "stop" or closes. With --frames too, the phone's picture from the same capture:
19
+ // {"event":"frame","seq","path","t"} about 4 times a second, a 590-pixel-wide JPEG in the private folder (only
20
+ // the newest 8 kept), so WATCH needn't read the screen through WebDriverAgent while the capture runs (opening
21
+ // it resets the phone's USB network link, which ends WebDriverAgent's runner). The answer is still the last
22
+ // line. Before the first capture macOS asks whether Mobster may use the camera and the microphone (see
23
+ // authorize); while it asks, {"event":"asking","service"} comes first.
24
+ // phone-audio record-video --seconds N --udid UDID [--name NAME] [--phones K] --work DIR
25
+ // Records 1 to 300 seconds of the iPhone's screen and the audio it offers into DIR/video.mov, picking the
26
+ // capture device as listen does: {"ok":true,"seconds","durationSeconds","audio","bytes","match"}. DIR must be a
27
+ // private folder (mode 0700, yours); the caller deletes it, even if this process is killed. A capture device
28
+ // another app is using is never taken over: capture_busy.
29
+ // phone-audio match [--udid UDID] [--name NAME] [--phones K] < [{"id","name","model"}]
30
+ // Which device listen would pick from that list: {"ok":true,"index","match"}, or listen's failure. For tests and
31
+ // diagnosis; nothing is captured.
32
+ // phone-audio transcribe FILE [--locale en-US]
33
+ // The speech in an audio file, the same way: {"ok":true,"text","segments","engine","locale","transcribeMs"}.
34
+ // phone-audio classify FILE
35
+ // The sounds in an audio file, the way listen classifies them: {"ok":true,"soundWindows","classifyMs"}. For tests
36
+ // and diagnosis; nothing is captured.
37
+ // phone-audio status [--locale en-US]
38
+ // What this Mac can understand, without asking for anything: {"ok":true,"speechAnalyzer","locale","sfSpeech":
39
+ // {"authorization","onDevice"},"capture":{"camera","microphone"}}. capture is granted, denied, restricted or
40
+ // not_determined for each. speechAnalyzer is installed, supported (its model isn't downloaded),
41
+ // downloading, unsupported (not this language) or unavailable (before macOS 26).
42
+ // phone-audio prepare [--locale en-US]
43
+ // Installs macOS's own on-device speech model for the language (SpeechAnalyzer's, the one Voice uses), once:
44
+ // {"ok":true,"speechAnalyzer":"installed","locale"}. Nothing else is downloaded, ever.
45
+ //
46
+ // Failures are {"ok":false,"error","message"}: no_device (no USB iPhone to hear), ambiguous_device (several iPhones,
47
+ // and none is surely the one asked for: it never guesses), permission (macOS doesn't let Mobster capture the iPhone:
48
+ // denied, refused without asking, or its prompt unanswered for ASK_SECONDS), capture_failed, no_engine (no on-device speech recognition for that language), speech_denied,
49
+ // transcribe_failed. Exit 0 either way; 2 for a usage error.
50
+ //
51
+ // How it hears: CoreMediaIO's kCMIOHardwarePropertyAllowScreenCaptureDevices, turned on in this process only, makes a
52
+ // USB iPhone appear as a capture device: the one QuickTime Player records from. AVCaptureAudioFileOutput records its
53
+ // audio track as 16-bit mono WAV. Nothing is tapped on the phone, and nothing is captured outside the N seconds.
54
+ //
55
+ // How it understands: SpeechAnalyzer with SpeechTranscriber on macOS 26 and later, else SFSpeechRecognizer with
56
+ // requiresOnDeviceRecognition. Both run on this Mac. SpeechAnalyzer downloads Apple's speech model once when it is
57
+ // missing; if installation fails, on-device SFSpeechRecognizer is tried. There is no server recognition fallback:
58
+ // without an available on-device model, it answers no_engine. What it hears is data: it goes to stdout as text.
59
+ //
60
+ // How it tells sounds apart: SoundAnalysis's built-in classifier (SNClassifySoundRequest, 303 labels: music and its
61
+ // instruments, laughter, applause, a dog, a siren), on this Mac, alongside the transcription and on the same
62
+ // recording. soundWindows are [{"t","d","labels":{label: confidence}}]: a 3-second window every 1.5 s (the whole
63
+ // file when it is shorter), each with its labels at least SOUND_FLOOR sure, at most SOUND_LABELS of them.
64
+ // mobile_agent/listen.py collapses them into timed sounds. A classifier failure leaves soundWindows out and says why
65
+ // in soundsError; the words still stand.
66
+
67
+ import AVFoundation
68
+ import CoreImage
69
+ import CoreMediaIO
70
+ import Foundation
71
+ import SoundAnalysis
72
+ import Speech
73
+
74
+ // MARK: - Output
75
+
76
+ func emit(_ value: [String: Any]) {
77
+ if let data = try? JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]),
78
+ let line = String(data: data, encoding: .utf8) {
79
+ print(line)
80
+ fflush(stdout)
81
+ }
82
+ }
83
+
84
+ /// Folders to delete before exiting, whichever way it exits: the recording never outlives this process unasked.
85
+ var cleanup: [URL] = []
86
+
87
+ func removeTemporary() {
88
+ for url in cleanup { try? FileManager.default.removeItem(at: url) }
89
+ cleanup = []
90
+ }
91
+
92
+ func fail(_ error: String, _ message: String) -> Never {
93
+ removeTemporary()
94
+ emit(["ok": false, "error": error, "message": message])
95
+ exit(0)
96
+ }
97
+
98
+ /// Turns the main run loop until `done()` or `timeout` seconds: AVFoundation and Speech deliver here.
99
+ @discardableResult
100
+ func pump(timeout: TimeInterval, until done: () -> Bool) -> Bool {
101
+ let deadline = Date().addingTimeInterval(timeout)
102
+ while !done() {
103
+ if Date() >= deadline { return false }
104
+ RunLoop.current.run(until: Date().addingTimeInterval(0.02))
105
+ }
106
+ return true
107
+ }
108
+
109
+ func usage() -> Never {
110
+ FileHandle.standardError.write(Data(("usage: phone-audio devices | phone-audio listen --seconds N [--udid UDID] [--name NAME] "
111
+ + "[--phones K] [--locale L] [--work DIR] [--keep PATH] [--live [--frames]] | phone-audio match [--udid UDID] "
112
+ + "[--name NAME] [--phones K] < devices.json | phone-audio transcribe FILE [--locale L] "
113
+ + "| phone-audio classify FILE | phone-audio record-video --seconds N --udid UDID [--name NAME] "
114
+ + "[--phones K] --work DIR\n").utf8))
115
+ exit(2)
116
+ }
117
+
118
+ func milliseconds(since start: Date) -> Int { Int((Date().timeIntervalSince(start) * 1000).rounded()) }
119
+
120
+ // MARK: - Arguments
121
+
122
+ var arguments = Array(CommandLine.arguments.dropFirst())
123
+ guard let command = arguments.first else { usage() }
124
+ arguments.removeFirst()
125
+
126
+ func option(_ name: String) -> String? {
127
+ guard let index = arguments.firstIndex(of: name), index + 1 < arguments.count else { return nil }
128
+ return arguments[index + 1]
129
+ }
130
+
131
+ // MARK: - The iPhone as a capture device
132
+
133
+ /// Lets this process see screen-capture devices (USB iPhones). Only this process: QuickTime's own setting is untouched.
134
+ func allowScreenCaptureDevices() {
135
+ var address = CMIOObjectPropertyAddress(
136
+ mSelector: CMIOObjectPropertySelector(kCMIOHardwarePropertyAllowScreenCaptureDevices),
137
+ mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
138
+ mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
139
+ var allow: UInt32 = 1
140
+ CMIOObjectSetPropertyData(CMIOObjectID(kCMIOObjectSystemObject), &address, 0, nil,
141
+ UInt32(MemoryLayout<UInt32>.size), &allow)
142
+ }
143
+
144
+ /// The iPhones this Mac can capture: external muxed devices made by Apple.
145
+ func iphones() -> [AVCaptureDevice] {
146
+ let types: [AVCaptureDevice.DeviceType]
147
+ if #available(macOS 14.0, *) { types = [.external] } else { types = [.externalUnknown] }
148
+ return AVCaptureDevice.DiscoverySession(deviceTypes: types, mediaType: .muxed, position: .unspecified).devices
149
+ }
150
+
151
+ /// A UDID as the capture device's uniqueID may spell it: no dashes, upper case.
152
+ func plain(_ id: String) -> String { id.replacingOccurrences(of: "-", with: "").uppercased() }
153
+
154
+ /// A name compared the way people read it: its words, any case.
155
+ func folded(_ name: String) -> String {
156
+ name.split(whereSeparator: \.isWhitespace).joined(separator: " ").folding(options: [.caseInsensitive,
157
+ .diacriticInsensitive],
158
+ locale: nil)
159
+ }
160
+
161
+ /// What the matching reads of a capture device: AVCaptureDevice's uniqueID, localizedName and modelID.
162
+ struct Candidate { let id: String; let name: String; let model: String }
163
+
164
+ /// Which iPhone to hear, and how it was told: "udid", "name" or "only" (the only iOS capture device, on the only
165
+ /// USB iPhone). ambiguous(n): n iOS capture devices and none is the one asked for. none: no iOS capture device fits.
166
+ enum Pick: Equatable { case found(Int, String), ambiguous(Int), none }
167
+
168
+ /// An iPhone or iPad as a capture device: AVFoundation's modelID for one is "iOS Device".
169
+ func looksLikeIOS(_ device: Candidate) -> Bool {
170
+ let model = device.model.lowercased()
171
+ return model.contains("ios") || model.hasPrefix("iphone") || model.hasPrefix("ipad")
172
+ }
173
+
174
+ /// The capture device for the iPhone asked for. On some Macs an iPhone's capture device has a uniqueID of its own,
175
+ /// not its UDID (measured 13 Oct 2026), so after the UDID it tries the phone's name as the device layer reads it
176
+ /// (`name`, passed only when no other USB iPhone shares it), then the only iOS capture device, when the device layer
177
+ /// sees one USB iPhone (`phones`) or no UDID was asked for. Never a guess between two phones.
178
+ func pick(_ found: [Candidate], udid: String?, name: String?, phones: Int?) -> Pick {
179
+ if let udid = udid, let index = found.firstIndex(where: { plain($0.id) == plain(udid) }) {
180
+ return .found(index, "udid")
181
+ }
182
+ let ios = found.indices.filter { looksLikeIOS(found[$0]) }
183
+ if let name = name, !folded(name).isEmpty {
184
+ let named = ios.filter { folded(found[$0].name) == folded(name) }
185
+ if named.count == 1 { return .found(named[0], "name") }
186
+ if named.count > 1 { return .ambiguous(named.count) }
187
+ }
188
+ if ios.count == 1 && (udid == nil || phones == 1) { return .found(ios[0], "only") }
189
+ if ios.count > 1 || (ios.count == 1 && (phones ?? 0) > 1) { return .ambiguous(max(ios.count, phones ?? 0)) }
190
+ return .none
191
+ }
192
+
193
+ func candidates(_ devices: [AVCaptureDevice]) -> [Candidate] {
194
+ devices.map { Candidate(id: $0.uniqueID, name: $0.localizedName, model: $0.modelID) }
195
+ }
196
+
197
+ /// The iPhone to hear (see pick). They appear a moment after the property is set, so look for up to `timeout`
198
+ /// seconds; the answer then is the last look's.
199
+ func findPhone(udid: String?, name: String?, phones: Int?, timeout: TimeInterval) -> (AVCaptureDevice?, Pick) {
200
+ let deadline = Date().addingTimeInterval(timeout)
201
+ var last = Pick.none
202
+ repeat {
203
+ let found = iphones()
204
+ last = pick(candidates(found), udid: udid, name: name, phones: phones)
205
+ if case .found(let index, _) = last { return (found[index], last) }
206
+ RunLoop.current.run(until: Date().addingTimeInterval(0.1))
207
+ } while Date() < deadline
208
+ return (nil, last)
209
+ }
210
+
211
+ /// Why no iPhone was picked, as listen's failure: what happened, then what to do. It names the phone asked for.
212
+ func refuse(_ result: Pick, name: String?) -> Never {
213
+ let phone = name.map { "\u{201C}\($0)\u{201D}" }
214
+ if case .ambiguous(let count) = result {
215
+ fail("ambiguous_device", phone.map {
216
+ "\(count) iPhones are plugged in, and Mobster can't tell which one is \($0). Unplug the others, then try again."
217
+ } ?? "\(count) iPhones are plugged in, and Mobster can't tell which one to hear. Unplug the others, then try again.")
218
+ }
219
+ fail("no_device", phone.map {
220
+ "No iPhone to hear: macOS shows no capture device for \($0). Plug it in with a cable, unlock it, then try again."
221
+ } ?? "No iPhone to hear: plug one in with a cable and unlock it.")
222
+ }
223
+
224
+ // MARK: - Permission
225
+
226
+ /// What macOS says when Mobster may not capture the iPhone: listen's failure "permission".
227
+ let PERMISSION_MESSAGE = "Mobster needs permission to hear your iPhone. Allow it in System Settings › Privacy & Security › Microphone/Camera."
228
+ /// How long the person has to answer macOS's prompt before listen gives up (the prompt stays up; the next try hears).
229
+ let ASK_SECONDS = 60.0
230
+
231
+ func authorizationName(_ status: AVAuthorizationStatus) -> String {
232
+ switch status {
233
+ case .authorized: return "granted"
234
+ case .denied: return "denied"
235
+ case .restricted: return "restricted"
236
+ case .notDetermined: return "not_determined"
237
+ @unknown default: return "unknown"
238
+ }
239
+ }
240
+
241
+ /// Makes sure macOS lets this process capture the iPhone, before anything is opened. macOS counts the iPhone's
242
+ /// capture device as a camera with a microphone and asks for both, attributed to the app that started this helper
243
+ /// (Mobster.app, which must hold the camera and audio-input entitlements: tccd checks the responsible app, not this
244
+ /// helper). Granted: returns. Denied, or refused without a prompt (tccd's policy: the status then stays
245
+ /// not_determined): fails "permission". Not asked yet: asks, and with `live` says {"event":"asking","service"} first,
246
+ /// so WATCH can wait for the answer; `stop` (WATCH gave up waiting) ends the wait as an unanswered prompt.
247
+ func authorize(live: Bool, stop: StopSignal? = nil) {
248
+ for (media, service) in [(AVMediaType.video, "camera"), (AVMediaType.audio, "microphone")] {
249
+ switch AVCaptureDevice.authorizationStatus(for: media) {
250
+ case .authorized:
251
+ continue
252
+ case .notDetermined:
253
+ if live { emit(["event": "asking", "service": service]) }
254
+ let answer = SendableBox<Bool>()
255
+ AVCaptureDevice.requestAccess(for: media) { answer.value = $0 }
256
+ pump(timeout: ASK_SECONDS) { answer.value != nil || stop?.isSet == true }
257
+ guard answer.value != nil else {
258
+ fail("permission", "macOS is asking whether Mobster may hear your iPhone. Answer its prompt, then try again.")
259
+ }
260
+ if answer.value != true { fail("permission", PERMISSION_MESSAGE) }
261
+ default:
262
+ fail("permission", PERMISSION_MESSAGE)
263
+ }
264
+ }
265
+ }
266
+
267
+ // MARK: - Recording
268
+
269
+ final class Recorder: NSObject, AVCaptureFileOutputRecordingDelegate {
270
+ private let lock = NSLock()
271
+ private var finished = false
272
+ private var started = false
273
+ private(set) var error: Error?
274
+ var done: Bool { lock.lock(); defer { lock.unlock() }; return finished }
275
+ var recording: Bool { lock.lock(); defer { lock.unlock() }; return started }
276
+
277
+ func fileOutput(_ output: AVCaptureFileOutput, didStartRecordingTo fileURL: URL, from connections: [AVCaptureConnection]) {
278
+ lock.lock()
279
+ started = true
280
+ lock.unlock()
281
+ }
282
+
283
+ func fileOutput(_ output: AVCaptureFileOutput, didFinishRecordingTo outputFileURL: URL,
284
+ from connections: [AVCaptureConnection], error: Error?) {
285
+ // Reaching the maximum duration is how a recording ends here, and AVFoundation reports it as an error.
286
+ lock.lock()
287
+ if let error = error as NSError?, !(error.userInfo[AVErrorRecordingSuccessfullyFinishedKey] as? Bool ?? false) {
288
+ self.error = error
289
+ }
290
+ finished = true
291
+ lock.unlock()
292
+ }
293
+ }
294
+
295
+ /// Records the selected USB phone's screen, with its audio when it offers one, into `url`. A capture device that
296
+ /// another app is using is never taken over. Returns whether the movie has an audio track.
297
+ func recordVideo(_ device: AVCaptureDevice, seconds: Double, to url: URL) -> Bool {
298
+ guard !device.isInUseByAnotherApplication, !device.isSuspended else {
299
+ fail("capture_busy", "The iPhone's capture device is busy or suspended. Close its other capture first.")
300
+ }
301
+ let session = AVCaptureSession()
302
+ let input: AVCaptureDeviceInput
303
+ do { input = try AVCaptureDeviceInput(device: device) } catch {
304
+ if (error as NSError).code == AVError.applicationIsNotAuthorizedToUseDevice.rawValue {
305
+ fail("permission", PERMISSION_MESSAGE)
306
+ }
307
+ fail("capture_failed", "Mobster couldn't open the iPhone's video (\(error.localizedDescription)).")
308
+ }
309
+ guard session.canAddInput(input) else { fail("capture_failed", "The iPhone offered no capture input.") }
310
+ session.addInput(input)
311
+ let output = AVCaptureMovieFileOutput()
312
+ output.maxRecordedDuration = CMTime(seconds: seconds, preferredTimescale: 600)
313
+ guard session.canAddOutput(output) else { fail("capture_failed", "The iPhone offered no movie output.") }
314
+ session.addOutput(output)
315
+ guard output.connection(with: .video) != nil else {
316
+ fail("capture_failed", "The iPhone offered no video to record. Unlock it and check the USB cable.")
317
+ }
318
+ let audio = output.connection(with: .audio) != nil
319
+ let cancelled = SendableBox<Bool>()
320
+ // The signal sources run on the main queue, outside the signal handler. The movie is always finished first.
321
+ let signals = [SIGINT, SIGTERM].map { number -> DispatchSourceSignal in
322
+ signal(number, SIG_IGN)
323
+ let source = DispatchSource.makeSignalSource(signal: number, queue: .main)
324
+ source.setEventHandler { cancelled.value = true }
325
+ source.resume()
326
+ return source
327
+ }
328
+ defer { for source in signals { source.cancel() } }
329
+ session.startRunning()
330
+ let recorder = Recorder()
331
+ output.startRecording(to: url, recordingDelegate: recorder)
332
+ let completed = pump(timeout: seconds + 10) { recorder.done || cancelled.value == true }
333
+ if !recorder.done {
334
+ output.stopRecording()
335
+ pump(timeout: 5) { recorder.done }
336
+ }
337
+ session.stopRunning()
338
+ if cancelled.value == true { fail("cancelled", "The recording was cancelled.") }
339
+ guard completed, recorder.done else { fail("capture_failed", "The iPhone's video capture didn't finish in time.") }
340
+ if let error = recorder.error {
341
+ fail("capture_failed", "The video recording stopped early (\(error.localizedDescription)).")
342
+ }
343
+ return audio
344
+ }
345
+
346
+ /// Set once stdin reads "stop" or closes (--live): the recording then ends early, and what it has is kept.
347
+ final class StopSignal: @unchecked Sendable {
348
+ private let lock = NSLock()
349
+ private var stopped = false
350
+ var isSet: Bool { lock.lock(); defer { lock.unlock() }; return stopped }
351
+ func set() { lock.lock(); stopped = true; lock.unlock() }
352
+ }
353
+
354
+ func watchStdin(_ signal: StopSignal) {
355
+ Thread.detachNewThread {
356
+ while let line = readLine() {
357
+ if line.trimmingCharacters(in: .whitespaces) == "stop" { break }
358
+ }
359
+ signal.set()
360
+ }
361
+ }
362
+
363
+ /// The phone's picture from the same capture (--frames DIR, WATCH): while it runs, WebDriverAgent can't be relied on
364
+ /// (opening the capture resets the phone's USB network link, which ends its runner), so WATCH reads the screen here.
365
+ /// FRAME_HZ JPEGs a second, FRAME_WIDTH pixels wide, as DIR/frame-<seq>.jpg; only the newest FRAMES_KEPT stay.
366
+ let FRAME_HZ = 4.0
367
+ let FRAME_WIDTH = 590.0
368
+ let FRAMES_KEPT = 8
369
+
370
+ final class FrameWriter: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, @unchecked Sendable {
371
+ private let lock = NSLock()
372
+ private let folder: URL
373
+ private let context = CIContext()
374
+ private var last = 0.0
375
+ private var written: [URL] = []
376
+ private(set) var seq = 0
377
+ private var newest: (Int, String)?
378
+ init(folder: URL) { self.folder = folder }
379
+
380
+ /// The newest frame written since the last call, as (seq, path), else nil.
381
+ func take() -> (Int, String)? { lock.lock(); defer { lock.unlock() }; let value = newest; newest = nil; return value }
382
+
383
+ func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
384
+ let now = Date().timeIntervalSinceReferenceDate
385
+ guard now - last >= 1 / FRAME_HZ, let pixels = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
386
+ last = now
387
+ var image = CIImage(cvPixelBuffer: pixels)
388
+ let width = image.extent.width
389
+ if width > FRAME_WIDTH { image = image.transformed(by: CGAffineTransform(scaleX: FRAME_WIDTH / width, y: FRAME_WIDTH / width)) }
390
+ guard let space = CGColorSpace(name: CGColorSpace.sRGB),
391
+ let data = context.jpegRepresentation(of: image, colorSpace: space,
392
+ options: [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 0.7])
393
+ else { return }
394
+ lock.lock()
395
+ seq += 1
396
+ let url = folder.appendingPathComponent("frame-\(seq).jpg")
397
+ lock.unlock()
398
+ guard (try? data.write(to: url, options: .atomic)) != nil else { return }
399
+ lock.lock()
400
+ written.append(url)
401
+ let old = written.count > FRAMES_KEPT ? written.removeFirst() : nil
402
+ newest = (seq, url.path)
403
+ lock.unlock()
404
+ if let old = old { try? FileManager.default.removeItem(at: old) }
405
+ }
406
+ }
407
+
408
+ /// The sound through a data output, for --frames: AVCaptureAudioFileOutput refuses to record ("Cannot Record") once a
409
+ /// video output shares the session (measured on the test device, 14 Oct 2026), so the samples are written to the same WAV
410
+ /// (16-bit mono at 16 kHz) here. It stops taking samples at `limit` seconds, and keeps the newest level in dB.
411
+ final class AudioWriter: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate, @unchecked Sendable {
412
+ private let lock = NSLock()
413
+ private var file: AVAudioFile?
414
+ private let limit: Double
415
+ private var frames: AVAudioFramePosition = 0
416
+ private var started = false
417
+ private var power = -160.0
418
+ private(set) var failure: String?
419
+ let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16_000, channels: 1, interleaved: true)!
420
+
421
+ init(url: URL, limit: Double) {
422
+ self.limit = limit
423
+ var made: AVAudioFile?
424
+ do {
425
+ made = try AVAudioFile(forWriting: url, settings: [AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: 16_000,
426
+ AVNumberOfChannelsKey: 1, AVLinearPCMBitDepthKey: 16,
427
+ AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false],
428
+ commonFormat: .pcmFormatInt16, interleaved: true)
429
+ } catch { failure = error.localizedDescription }
430
+ file = made
431
+ }
432
+
433
+ var recording: Bool { lock.lock(); defer { lock.unlock() }; return started }
434
+ var seconds: Double { lock.lock(); defer { lock.unlock() }; return Double(frames) / 16_000 }
435
+ var full: Bool { seconds >= limit }
436
+ var level: Double { lock.lock(); defer { lock.unlock() }; return power }
437
+
438
+ /// Closes the WAV (its header is written when the file goes), so it can be read.
439
+ func finish() { lock.lock(); file = nil; lock.unlock() }
440
+
441
+ func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
442
+ lock.lock(); let file = self.file; lock.unlock()
443
+ guard let file = file, !full else { return }
444
+ let count = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
445
+ guard count > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: count) else { return }
446
+ buffer.frameLength = count
447
+ let status = CMSampleBufferCopyPCMDataIntoAudioBufferList(sampleBuffer, at: 0, frameCount: Int32(count),
448
+ into: buffer.mutableAudioBufferList)
449
+ guard status == noErr, let samples = buffer.int16ChannelData?[0] else { return }
450
+ var sum = 0.0
451
+ for index in 0..<Int(count) { let value = Double(samples[index]) / 32768; sum += value * value }
452
+ let rms = (sum / Double(count)).squareRoot()
453
+ do { try file.write(from: buffer) } catch {
454
+ lock.lock(); failure = error.localizedDescription; lock.unlock(); return
455
+ }
456
+ lock.lock()
457
+ frames += AVAudioFramePosition(count)
458
+ started = true
459
+ power = rms > 0 ? max(-160, 20 * log10(rms)) : -160
460
+ lock.unlock()
461
+ }
462
+ }
463
+
464
+ /// Records `seconds` of the phone's audio into `url` (WAV, 16-bit mono at 16 kHz). Returns how long the capture took
465
+ /// to start, in milliseconds, and how many seconds were recorded. With `live`, it reports the start and the level as
466
+ /// it goes, and `stop` ends it early. With `frames`, it also writes the phone's picture there and says
467
+ /// {"event":"frame","seq","path","t"} for each (FrameWriter).
468
+ func record(_ device: AVCaptureDevice, seconds: Double, to url: URL, live: Bool = false,
469
+ stop: StopSignal? = nil, frames: URL? = nil) -> (Int, Double) {
470
+ let session = AVCaptureSession()
471
+ let input: AVCaptureDeviceInput
472
+ do {
473
+ input = try AVCaptureDeviceInput(device: device)
474
+ } catch {
475
+ let code = (error as NSError).code
476
+ if code == AVError.applicationIsNotAuthorizedToUseDevice.rawValue { fail("permission", PERMISSION_MESSAGE) }
477
+ fail("capture_failed", "Mobster couldn't open the iPhone's audio (\(error.localizedDescription)).")
478
+ }
479
+ guard session.canAddInput(input) else { fail("capture_failed", "Mobster couldn't open the iPhone's audio.") }
480
+ session.addInput(input)
481
+ if let frames = frames {
482
+ return recordWithFrames(session, seconds: seconds, to: url, live: live, stop: stop, frames: frames)
483
+ }
484
+ let output = AVCaptureAudioFileOutput()
485
+ output.audioSettings = [AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: 16_000, AVNumberOfChannelsKey: 1,
486
+ AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
487
+ AVLinearPCMIsNonInterleaved: false]
488
+ output.maxRecordedDuration = CMTime(seconds: seconds, preferredTimescale: 600)
489
+ guard session.canAddOutput(output) else { fail("capture_failed", "Mobster couldn't record the iPhone's audio.") }
490
+ session.addOutput(output)
491
+ guard output.connection(with: .audio) != nil else {
492
+ fail("capture_failed", "The iPhone offered no audio to record.")
493
+ }
494
+ let started = Date()
495
+ session.startRunning()
496
+ let recorder = Recorder()
497
+ output.startRecording(to: url, outputFileType: .wav, recordingDelegate: recorder)
498
+ let waited = milliseconds(since: started)
499
+ var announced = false
500
+ var stopping = false
501
+ var nextLevel = Date()
502
+ let finished = pump(timeout: seconds + 10) {
503
+ if recorder.done { return true }
504
+ if live && !announced && recorder.recording {
505
+ announced = true
506
+ emit(["event": "recording", "startMs": milliseconds(since: started)])
507
+ }
508
+ if live && announced && Date() >= nextLevel {
509
+ nextLevel = Date().addingTimeInterval(0.5)
510
+ let channels = output.connection(with: .audio)?.audioChannels ?? []
511
+ let level = channels.map { Double($0.averagePowerLevel) }.max() ?? -160
512
+ emit(["event": "level", "t": (output.recordedDuration.seconds * 10).rounded() / 10,
513
+ "db": (max(-160, level) * 10).rounded() / 10])
514
+ }
515
+ if let stop = stop, stop.isSet, !stopping {
516
+ stopping = true
517
+ output.stopRecording()
518
+ }
519
+ return false
520
+ }
521
+ if !finished {
522
+ output.stopRecording()
523
+ pump(timeout: 3) { recorder.done }
524
+ }
525
+ let recorded = output.recordedDuration.seconds
526
+ session.stopRunning()
527
+ if let error = recorder.error {
528
+ fail("capture_failed", "The recording stopped early (\(error.localizedDescription)).")
529
+ }
530
+ return (waited, recorded.isFinite ? (recorded * 10).rounded() / 10 : seconds)
531
+ }
532
+
533
+ /// record() for --frames: the sound through AudioWriter and the picture through FrameWriter, on one session. Says
534
+ /// {"event":"recording"} at the first sound, {"event":"level"} every half second and {"event":"frame"} for each picture.
535
+ func recordWithFrames(_ session: AVCaptureSession, seconds: Double, to url: URL, live: Bool, stop: StopSignal?,
536
+ frames: URL) -> (Int, Double) {
537
+ let audio = AVCaptureAudioDataOutput()
538
+ audio.audioSettings = [AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: 16_000, AVNumberOfChannelsKey: 1,
539
+ AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
540
+ AVLinearPCMIsNonInterleaved: false]
541
+ let sound = AudioWriter(url: url, limit: seconds)
542
+ if let failure = sound.failure { fail("capture_failed", "Mobster couldn't record the iPhone's audio (\(failure)).") }
543
+ guard session.canAddOutput(audio) else { fail("capture_failed", "Mobster couldn't record the iPhone's audio.") }
544
+ session.addOutput(audio)
545
+ audio.setSampleBufferDelegate(sound, queue: DispatchQueue(label: "phone-audio.sound"))
546
+ // The picture is a bonus: without it WATCH falls back to WebDriverAgent's frames, and the sound still records.
547
+ let video = AVCaptureVideoDataOutput()
548
+ video.alwaysDiscardsLateVideoFrames = true
549
+ video.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
550
+ let writer = FrameWriter(folder: frames)
551
+ if session.canAddOutput(video) {
552
+ session.addOutput(video)
553
+ video.setSampleBufferDelegate(writer, queue: DispatchQueue(label: "phone-audio.frames"))
554
+ }
555
+ let started = Date()
556
+ session.startRunning()
557
+ var waited: Int?
558
+ var nextLevel = Date()
559
+ pump(timeout: seconds + 10) {
560
+ if waited == nil && sound.recording {
561
+ waited = milliseconds(since: started)
562
+ if live { emit(["event": "recording", "startMs": waited!]) }
563
+ }
564
+ if live, let (seq, path) = writer.take() {
565
+ emit(["event": "frame", "seq": seq, "path": path, "t": (Date().timeIntervalSince(started) * 100).rounded() / 100])
566
+ }
567
+ if live && waited != nil && Date() >= nextLevel {
568
+ nextLevel = Date().addingTimeInterval(0.5)
569
+ emit(["event": "level", "t": (sound.seconds * 10).rounded() / 10, "db": (sound.level * 10).rounded() / 10])
570
+ }
571
+ return sound.full || stop?.isSet == true || sound.failure != nil
572
+ }
573
+ session.stopRunning()
574
+ sound.finish()
575
+ if let failure = sound.failure { fail("capture_failed", "The recording stopped early (\(failure)).") }
576
+ return (waited ?? milliseconds(since: started), (sound.seconds * 10).rounded() / 10)
577
+ }
578
+
579
+ /// The loudest sample in decibels below full scale, so a caller can tell silence (nothing played) from speech it
580
+ /// couldn't understand. nil when the file can't be read.
581
+ func peakDecibels(_ url: URL) -> Double? {
582
+ guard let file = try? AVAudioFile(forReading: url),
583
+ let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: AVAudioFrameCount(file.length)),
584
+ (try? file.read(into: buffer)) != nil, let channel = buffer.floatChannelData?[0] else { return nil }
585
+ var peak: Float = 0
586
+ for index in 0..<Int(buffer.frameLength) { peak = max(peak, abs(channel[index])) }
587
+ return peak > 0 ? (Double(20 * log10(peak)) * 10).rounded() / 10 : -160
588
+ }
589
+
590
+ // MARK: - Speech
591
+
592
+ /// What was said: the text, and the same text in timed pieces ("start" and "end" in seconds from the audio's start).
593
+ struct Heard { let text: String; let segments: [[String: Any]]; let engine: String; let locale: String }
594
+
595
+ func segment(_ start: Double, _ end: Double, _ text: String) -> [String: Any] {
596
+ ["start": (start * 10).rounded() / 10, "end": (end * 10).rounded() / 10,
597
+ "text": text.split(whereSeparator: \.isWhitespace).joined(separator: " ")]
598
+ }
599
+
600
+ enum SpeechFailure: Error { case noEngine(String), denied, failed(String) }
601
+
602
+ func wanted(_ locale: String?) -> Locale { locale.map { Locale(identifier: $0) } ?? Locale.current }
603
+
604
+ @available(macOS 26.0, *)
605
+ func analyze(_ url: URL, locale: Locale) async throws -> Heard? {
606
+ guard SpeechTranscriber.isAvailable,
607
+ let resolved = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { return nil }
608
+ let transcriber = SpeechTranscriber(locale: resolved, preset: .transcription)
609
+ switch await AssetInventory.status(forModules: [transcriber]) {
610
+ case .installed:
611
+ break
612
+ case .supported, .downloading:
613
+ // macOS's own model for the language, as Voice uses: usually already on this Mac, so this only reserves it
614
+ // (measured 0.3 s on 11 Oct 2026). A missing model is downloaded from Apple; if that fails, the on-device
615
+ // SFSpeechRecognizer below is tried.
616
+ guard let request = try? await AssetInventory.assetInstallationRequest(supporting: [transcriber]),
617
+ (try? await request.downloadAndInstall()) != nil else {
618
+ if await AssetInventory.status(forModules: [transcriber]) != .installed { return nil }
619
+ break
620
+ }
621
+ default:
622
+ return nil
623
+ }
624
+ let analyzer = SpeechAnalyzer(modules: [transcriber])
625
+ let collect = Task { () -> [(String, CMTimeRange)] in
626
+ var pieces: [(String, CMTimeRange)] = []
627
+ for try await result in transcriber.results where result.isFinal {
628
+ pieces.append((String(result.text.characters), result.range))
629
+ }
630
+ return pieces
631
+ }
632
+ let file = try AVAudioFile(forReading: url)
633
+ if let last = try await analyzer.analyzeSequence(from: file) {
634
+ try await analyzer.finalizeAndFinish(through: last)
635
+ } else {
636
+ await analyzer.cancelAndFinishNow()
637
+ }
638
+ let pieces = try await collect.value
639
+ let text = pieces.map { $0.0 }.joined(separator: " ")
640
+ let segments = pieces.filter { !$0.0.trimmingCharacters(in: .whitespaces).isEmpty }.map { piece in
641
+ segment(piece.1.start.seconds.isFinite ? piece.1.start.seconds : 0,
642
+ piece.1.end.seconds.isFinite ? piece.1.end.seconds : 0, piece.0)
643
+ }
644
+ return Heard(text: text.split(whereSeparator: \.isWhitespace).joined(separator: " "), segments: segments,
645
+ engine: "speechanalyzer", locale: resolved.identifier(.bcp47))
646
+ }
647
+
648
+ func recognize(_ url: URL, locale: Locale) async throws -> Heard {
649
+ guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.supportsOnDeviceRecognition else {
650
+ throw SpeechFailure.noEngine("This Mac has no on-device speech recognition for \(locale.identifier).")
651
+ }
652
+ var status = SFSpeechRecognizer.authorizationStatus()
653
+ if status == .notDetermined {
654
+ status = await withCheckedContinuation { (done: CheckedContinuation<SFSpeechRecognizerAuthorizationStatus, Never>) in
655
+ SFSpeechRecognizer.requestAuthorization { done.resume(returning: $0) }
656
+ }
657
+ }
658
+ guard status == .authorized else { throw SpeechFailure.denied }
659
+ let request = SFSpeechURLRecognitionRequest(url: url)
660
+ request.requiresOnDeviceRecognition = true
661
+ request.shouldReportPartialResults = false
662
+ request.addsPunctuation = true
663
+ let heard: (String, [[String: Any]]) = try await withCheckedThrowingContinuation { done in
664
+ var finished = false
665
+ recognizer.recognitionTask(with: request) { result, error in
666
+ guard !finished else { return }
667
+ if let result = result, result.isFinal {
668
+ finished = true
669
+ done.resume(returning: (result.bestTranscription.formattedString,
670
+ phrases(result.bestTranscription.segments)))
671
+ } else if let error = error as NSError? {
672
+ finished = true
673
+ // "No speech detected" is an answer: nothing was said.
674
+ if error.domain == "kAFAssistantErrorDomain" && error.code == 1110 {
675
+ done.resume(returning: ("", []))
676
+ } else {
677
+ done.resume(throwing: SpeechFailure.failed(error.localizedDescription))
678
+ }
679
+ }
680
+ }
681
+ }
682
+ return Heard(text: heard.0, segments: heard.1, engine: "sfspeech", locale: locale.identifier)
683
+ }
684
+
685
+ /// SFSpeech's words, timed, as phrases: a new one after a pause of 0.7 s or more, or after 14 words.
686
+ func phrases(_ words: [SFTranscriptionSegment]) -> [[String: Any]] {
687
+ var out: [[String: Any]] = []
688
+ var current: [SFTranscriptionSegment] = []
689
+ func close() {
690
+ guard let first = current.first, let last = current.last else { return }
691
+ out.append(segment(first.timestamp, last.timestamp + last.duration,
692
+ current.map { $0.substring }.joined(separator: " ")))
693
+ current = []
694
+ }
695
+ for word in words {
696
+ if let last = current.last, word.timestamp - (last.timestamp + last.duration) >= 0.7 || current.count >= 14 {
697
+ close()
698
+ }
699
+ current.append(word)
700
+ }
701
+ close()
702
+ return out
703
+ }
704
+
705
+ func transcribe(_ url: URL, locale: String?) async throws -> Heard {
706
+ if #available(macOS 26.0, *), let heard = try await analyze(url, locale: wanted(locale)) {
707
+ return heard
708
+ }
709
+ return try await recognize(url, locale: wanted(locale))
710
+ }
711
+
712
+ func transcribed(_ url: URL, locale: String?) -> (Heard, Int) {
713
+ let started = Date()
714
+ let box = SendableBox<Result<Heard, Error>>()
715
+ Task.detached {
716
+ do { box.value = .success(try await transcribe(url, locale: locale)) } catch { box.value = .failure(error) }
717
+ }
718
+ pump(timeout: 300) { box.value != nil }
719
+ switch box.value {
720
+ case .success(let heard)?: return (heard, milliseconds(since: started))
721
+ case .failure(SpeechFailure.noEngine(let message))?: fail("no_engine", message)
722
+ case .failure(SpeechFailure.denied)?:
723
+ fail("speech_denied", "macOS didn't let Mobster use speech recognition. Allow it in System Settings › Privacy & Security › Speech Recognition.")
724
+ case .failure(let error)?: fail("transcribe_failed", "Mobster couldn't turn the audio into text (\(error.localizedDescription)).")
725
+ case nil: fail("transcribe_failed", "Mobster couldn't turn the audio into text.")
726
+ }
727
+ }
728
+
729
+ final class SendableBox<Value>: @unchecked Sendable {
730
+ private let lock = NSLock()
731
+ private var stored: Value?
732
+ var value: Value? {
733
+ get { lock.lock(); defer { lock.unlock() }; return stored }
734
+ set { lock.lock(); stored = newValue; lock.unlock() }
735
+ }
736
+ }
737
+
738
+ // MARK: - Sounds
739
+
740
+ /// A label below this confidence isn't reported; at most this many per window, the surest first.
741
+ let SOUND_FLOOR = 0.3
742
+ let SOUND_LABELS = 6
743
+ /// The classifier's window and how far it moves (half a window: SoundAnalysis's default overlap).
744
+ let SOUND_WINDOW = 3.0
745
+
746
+ /// The classifier's windows as they come: start, length and the labels at least SOUND_FLOOR sure.
747
+ final class SoundWindows: NSObject, SNResultsObserving, @unchecked Sendable {
748
+ private let lock = NSLock()
749
+ private var rows: [[String: Any]] = []
750
+ private var failed: String?
751
+ var windows: [[String: Any]] { lock.lock(); defer { lock.unlock() }; return rows }
752
+ var failure: String? { lock.lock(); defer { lock.unlock() }; return failed }
753
+
754
+ func request(_ request: SNRequest, didProduce result: SNResult) {
755
+ guard let result = result as? SNClassificationResult else { return }
756
+ var labels: [String: Double] = [:]
757
+ for item in result.classifications.prefix(SOUND_LABELS) where item.confidence >= SOUND_FLOOR {
758
+ labels[item.identifier] = (item.confidence * 100).rounded() / 100
759
+ }
760
+ let start = result.timeRange.start.seconds, length = result.timeRange.duration.seconds
761
+ guard start.isFinite, length.isFinite else { return }
762
+ lock.lock()
763
+ rows.append(["t": (start * 10).rounded() / 10, "d": (length * 10).rounded() / 10, "labels": labels])
764
+ lock.unlock()
765
+ }
766
+
767
+ func request(_ request: SNRequest, didFailWithError error: Error) {
768
+ lock.lock()
769
+ failed = error.localizedDescription
770
+ lock.unlock()
771
+ }
772
+ }
773
+
774
+ /// The sounds in `url`, classified on this Mac: {"soundWindows","classifyMs"}, or {"soundsError","classifyMs"}.
775
+ /// Synchronous: listen runs it on its own thread while the speech is transcribed.
776
+ func classifySounds(_ url: URL) -> [String: Any] {
777
+ let started = Date()
778
+ guard #available(macOS 12.0, *) else {
779
+ return ["soundsError": "Telling sounds apart needs macOS 12 or later.", "classifyMs": 0]
780
+ }
781
+ do {
782
+ let file = try AVAudioFile(forReading: url)
783
+ let rate = file.processingFormat.sampleRate
784
+ let seconds = rate > 0 ? Double(file.length) / rate : 0
785
+ guard seconds >= 0.5 else { return ["soundWindows": [], "classifyMs": milliseconds(since: started)] }
786
+ let request = try SNClassifySoundRequest(classifierIdentifier: .version1)
787
+ request.windowDuration = CMTime(seconds: min(SOUND_WINDOW, seconds), preferredTimescale: 48_000)
788
+ request.overlapFactor = 0.5
789
+ let analyzer = try SNAudioFileAnalyzer(url: url)
790
+ let windows = SoundWindows()
791
+ try analyzer.add(request, withObserver: windows)
792
+ analyzer.analyze()
793
+ if let failure = windows.failure {
794
+ return ["soundsError": "The sound classifier failed (\(failure)).",
795
+ "classifyMs": milliseconds(since: started)]
796
+ }
797
+ return ["soundWindows": windows.windows, "classifyMs": milliseconds(since: started)]
798
+ } catch {
799
+ return ["soundsError": "The sound classifier couldn't read the audio (\(error.localizedDescription)).",
800
+ "classifyMs": milliseconds(since: started)]
801
+ }
802
+ }
803
+
804
+ /// classifySounds on a thread of its own; the box holds its answer when it is done.
805
+ func classifyingSounds(_ url: URL) -> SendableBox<[String: Any]> {
806
+ let box = SendableBox<[String: Any]>()
807
+ Thread.detachNewThread { box.value = classifySounds(url) }
808
+ return box
809
+ }
810
+
811
+ // MARK: - Commands
812
+
813
+ switch command {
814
+ case "record-video":
815
+ guard let text = option("--seconds"), let seconds = Double(text), seconds.isFinite,
816
+ seconds >= 1, seconds <= 300, let udid = option("--udid"), !udid.isEmpty,
817
+ let work = option("--work") else { usage() }
818
+ let folder = URL(fileURLWithPath: work)
819
+ let attributes = try? FileManager.default.attributesOfItem(atPath: folder.path)
820
+ guard attributes?[.type] as? FileAttributeType == .typeDirectory,
821
+ attributes?[.posixPermissions] as? Int == 0o700,
822
+ attributes?[.ownerAccountID] as? UInt32 == getuid() else {
823
+ fail("capture_failed", "The video working folder must be owned by you with permissions 0700.")
824
+ }
825
+ let video = folder.appendingPathComponent("video.mov")
826
+ guard !FileManager.default.fileExists(atPath: video.path) else {
827
+ fail("capture_failed", "The video working file already exists. Choose an empty private folder.")
828
+ }
829
+ cleanup.append(video)
830
+ allowScreenCaptureDevices()
831
+ let name = option("--name")
832
+ let (found, picked) = findPhone(udid: udid, name: name, phones: option("--phones").flatMap { Int($0) }, timeout: 5)
833
+ guard let phone = found, case .found(_, let match) = picked else { refuse(picked, name: name) }
834
+ authorize(live: false)
835
+ let audio = recordVideo(phone, seconds: seconds, to: video)
836
+ guard let size = (try? FileManager.default.attributesOfItem(atPath: video.path))?[.size] as? UInt64,
837
+ size > 0 else { fail("capture_failed", "The iPhone produced no video recording.") }
838
+ let durationBox = SendableBox<Double>()
839
+ Task.detached {
840
+ do { durationBox.value = try await AVURLAsset(url: video).load(.duration).seconds } catch { durationBox.value = -1 }
841
+ }
842
+ guard pump(timeout: 5, until: { durationBox.value != nil }), let duration = durationBox.value,
843
+ duration.isFinite, duration > 0 else { fail("capture_failed", "The recorded movie has no readable duration.") }
844
+ do { try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: video.path) } catch {
845
+ fail("capture_failed", "Mobster couldn't protect the recorded video file.")
846
+ }
847
+ cleanup = []
848
+ emit(["ok": true, "seconds": seconds, "durationSeconds": duration, "audio": audio, "bytes": size, "match": match])
849
+
850
+ case "devices":
851
+ allowScreenCaptureDevices()
852
+ pump(timeout: 2) { false } // give them a moment to appear
853
+ emit(["ok": true, "devices": iphones().map { ["id": $0.uniqueID, "name": $0.localizedName, "model": $0.modelID] }])
854
+
855
+ case "match":
856
+ // Which device listen would pick from the list on stdin ([{"id","name","model"}]), by the same rules: for tests
857
+ // and diagnosis. Nothing is captured.
858
+ let input = FileHandle.standardInput.readDataToEndOfFile()
859
+ guard let rows = (try? JSONSerialization.jsonObject(with: input)) as? [[String: Any]] else { usage() }
860
+ let listed = rows.map { Candidate(id: $0["id"] as? String ?? "", name: $0["name"] as? String ?? "",
861
+ model: $0["model"] as? String ?? "") }
862
+ let name = option("--name")
863
+ switch pick(listed, udid: option("--udid"), name: name, phones: option("--phones").flatMap { Int($0) }) {
864
+ case .found(let index, let how): emit(["ok": true, "index": index, "match": how])
865
+ case let other: refuse(other, name: name)
866
+ }
867
+
868
+ case "listen":
869
+ guard let text = option("--seconds"), let seconds = Double(text), seconds >= 1, seconds <= 120 else { usage() }
870
+ let keep = option("--keep")
871
+ let live = arguments.contains("--live")
872
+ let stop = StopSignal()
873
+ if live { watchStdin(stop) }
874
+ let looking = Date()
875
+ allowScreenCaptureDevices()
876
+ let name = option("--name")
877
+ let (found, picked) = findPhone(udid: option("--udid"), name: name, phones: option("--phones").flatMap { Int($0) },
878
+ timeout: 5)
879
+ guard let phone = found, case .found(_, let match) = picked else { refuse(picked, name: name) }
880
+ let findMs = milliseconds(since: looking)
881
+ authorize(live: live, stop: live ? stop : nil)
882
+ // The caller's private folder (--work), which it deletes itself, even if this process is killed; else our own.
883
+ let folder: URL
884
+ if let work = option("--work") {
885
+ folder = URL(fileURLWithPath: work).appendingPathComponent("capture-\(UUID().uuidString)")
886
+ } else {
887
+ folder = FileManager.default.temporaryDirectory.appendingPathComponent("mobster-listen-\(UUID().uuidString)")
888
+ }
889
+ try? FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true,
890
+ attributes: [.posixPermissions: 0o700])
891
+ cleanup.append(folder)
892
+ let audio = folder.appendingPathComponent("audio.wav")
893
+ let started = Date()
894
+ var frames: URL?
895
+ if live, arguments.contains("--frames") {
896
+ frames = folder.appendingPathComponent("frames")
897
+ try? FileManager.default.createDirectory(at: frames!, withIntermediateDirectories: true,
898
+ attributes: [.posixPermissions: 0o700])
899
+ }
900
+ let (waited, recorded) = record(phone, seconds: seconds, to: audio, live: live, stop: stop, frames: frames)
901
+ let recordMs = milliseconds(since: started)
902
+ let peak = peakDecibels(audio)
903
+ let sounds = classifyingSounds(audio)
904
+ let (heard, transcribeMs) = transcribed(audio, locale: option("--locale"))
905
+ pump(timeout: 60) { sounds.value != nil }
906
+ var answer: [String: Any] = ["ok": true, "text": heard.text, "segments": heard.segments, "engine": heard.engine,
907
+ "locale": heard.locale, "seconds": seconds, "recordedSeconds": recorded,
908
+ "recordMs": recordMs, "startMs": waited, "findMs": findMs,
909
+ "transcribeMs": transcribeMs, "match": match]
910
+ if let peak = peak { answer["peakDb"] = peak }
911
+ answer.merge(sounds.value ?? ["soundsError": "The sound classifier took too long."]) { mine, _ in mine }
912
+ if let keep = keep {
913
+ let target = URL(fileURLWithPath: keep)
914
+ try? FileManager.default.removeItem(at: target)
915
+ if (try? FileManager.default.moveItem(at: audio, to: target)) != nil { answer["kept"] = target.path }
916
+ }
917
+ removeTemporary()
918
+ emit(answer)
919
+
920
+ case "transcribe":
921
+ guard let path = arguments.first, !path.hasPrefix("--") else { usage() }
922
+ let url = URL(fileURLWithPath: path)
923
+ guard FileManager.default.isReadableFile(atPath: url.path) else { fail("transcribe_failed", "No audio file at \(path).") }
924
+ let (heard, transcribeMs) = transcribed(url, locale: option("--locale"))
925
+ emit(["ok": true, "text": heard.text, "segments": heard.segments, "engine": heard.engine, "locale": heard.locale,
926
+ "transcribeMs": transcribeMs])
927
+
928
+ case "classify":
929
+ guard let path = arguments.first, !path.hasPrefix("--") else { usage() }
930
+ guard FileManager.default.isReadableFile(atPath: path) else { fail("classify_failed", "No audio file at \(path).") }
931
+ var answer: [String: Any] = ["ok": true]
932
+ answer.merge(classifySounds(URL(fileURLWithPath: path))) { mine, _ in mine }
933
+ emit(answer)
934
+
935
+ case "prepare":
936
+ let locale = wanted(option("--locale"))
937
+ let box = SendableBox<[String: Any]>()
938
+ Task.detached {
939
+ guard #available(macOS 26.0, *), SpeechTranscriber.isAvailable else {
940
+ box.value = ["ok": false, "error": "no_engine", "message": "On-device speech models need macOS 26 or later."]
941
+ return
942
+ }
943
+ guard let resolved = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else {
944
+ box.value = ["ok": false, "error": "no_engine", "message": "This language isn't available on this Mac."]
945
+ return
946
+ }
947
+ let transcriber = SpeechTranscriber(locale: resolved, preset: .transcription)
948
+ do {
949
+ if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
950
+ try await request.downloadAndInstall()
951
+ }
952
+ box.value = ["ok": true, "speechAnalyzer": "installed", "locale": resolved.identifier(.bcp47)]
953
+ } catch {
954
+ box.value = ["ok": false, "error": "prepare_failed",
955
+ "message": "macOS couldn't install its speech model (\(error.localizedDescription))."]
956
+ }
957
+ }
958
+ pump(timeout: 900) { box.value != nil }
959
+ emit(box.value ?? ["ok": false, "error": "prepare_failed", "message": "macOS took too long to install its speech model."])
960
+
961
+ case "status":
962
+ let locale = wanted(option("--locale"))
963
+ let box = SendableBox<[String: Any]>()
964
+ Task.detached {
965
+ var answer: [String: Any] = ["ok": true, "speechAnalyzer": "unavailable"]
966
+ if #available(macOS 26.0, *), SpeechTranscriber.isAvailable {
967
+ if let resolved = await SpeechTranscriber.supportedLocale(equivalentTo: locale) {
968
+ let transcriber = SpeechTranscriber(locale: resolved, preset: .transcription)
969
+ switch await AssetInventory.status(forModules: [transcriber]) {
970
+ case .installed: answer["speechAnalyzer"] = "installed"
971
+ case .downloading: answer["speechAnalyzer"] = "downloading"
972
+ case .supported: answer["speechAnalyzer"] = "supported"
973
+ default: answer["speechAnalyzer"] = "unsupported"
974
+ }
975
+ answer["locale"] = resolved.identifier(.bcp47)
976
+ } else {
977
+ answer["speechAnalyzer"] = "unsupported"
978
+ }
979
+ }
980
+ let names: [SFSpeechRecognizerAuthorizationStatus: String] = [.authorized: "granted", .denied: "denied",
981
+ .restricted: "restricted", .notDetermined: "not_determined"]
982
+ answer["sfSpeech"] = ["authorization": names[SFSpeechRecognizer.authorizationStatus()] ?? "unknown",
983
+ "onDevice": SFSpeechRecognizer(locale: locale)?.supportsOnDeviceRecognition ?? false]
984
+ answer["capture"] = ["camera": authorizationName(AVCaptureDevice.authorizationStatus(for: .video)),
985
+ "microphone": authorizationName(AVCaptureDevice.authorizationStatus(for: .audio))]
986
+ box.value = answer
987
+ }
988
+ pump(timeout: 30) { box.value != nil }
989
+ emit(box.value ?? ["ok": false, "error": "status_failed", "message": "Mobster couldn't read this Mac's speech recognition."])
990
+
991
+ default:
992
+ usage()
993
+ }