mobster-cli 0.4.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobile_agent/__init__.py +4 -0
- mobile_agent/__main__.py +598 -0
- mobile_agent/accounts/__init__.py +1 -0
- mobile_agent/accounts/context.py +80 -0
- mobile_agent/accounts/mcp.py +55 -0
- mobile_agent/accounts/pacing.py +318 -0
- mobile_agent/accounts/register.py +78 -0
- mobile_agent/accounts/routes.py +212 -0
- mobile_agent/accounts/routing.py +69 -0
- mobile_agent/accounts/screen.py +273 -0
- mobile_agent/accounts/stats.py +54 -0
- mobile_agent/accounts/store.py +394 -0
- mobile_agent/agent_hooks.py +123 -0
- mobile_agent/ai_plans/__init__.py +7 -0
- mobile_agent/ai_plans/agents.py +96 -0
- mobile_agent/ai_plans/claude_code.py +208 -0
- mobile_agent/ai_plans/cli.py +400 -0
- mobile_agent/ai_plans/codex.py +258 -0
- mobile_agent/ai_plans/compat_chat.py +307 -0
- mobile_agent/ai_plans/device_login.py +164 -0
- mobile_agent/ai_plans/draft_client.py +68 -0
- mobile_agent/ai_plans/gemini.py +137 -0
- mobile_agent/ai_plans/instructions.py +65 -0
- mobile_agent/ai_plans/mcp_http.py +160 -0
- mobile_agent/ai_plans/phone_tools.py +892 -0
- mobile_agent/ai_plans/providers.py +235 -0
- mobile_agent/ai_plans/routes.py +160 -0
- mobile_agent/ai_plans/runner.py +400 -0
- mobile_agent/ai_plans/vault.py +114 -0
- mobile_agent/alerts.py +337 -0
- mobile_agent/android/__init__.py +18 -0
- mobile_agent/android/adb.py +268 -0
- mobile_agent/android/conditions.py +128 -0
- mobile_agent/android/doctor.py +54 -0
- mobile_agent/android/driver.py +137 -0
- mobile_agent/android/emulator.py +216 -0
- mobile_agent/android/logs.py +85 -0
- mobile_agent/android/manager.py +225 -0
- mobile_agent/android/records.py +31 -0
- mobile_agent/android/tree.py +96 -0
- mobile_agent/answer_shapes.py +230 -0
- mobile_agent/api_devices.py +333 -0
- mobile_agent/api_errors.py +17 -0
- mobile_agent/api_get.py +249 -0
- mobile_agent/api_http.py +467 -0
- mobile_agent/api_post.py +453 -0
- mobile_agent/api_routes.py +173 -0
- mobile_agent/app_api/__init__.py +4 -0
- mobile_agent/app_api/cli.py +151 -0
- mobile_agent/app_api/endpoints.py +193 -0
- mobile_agent/app_api/mcp.py +54 -0
- mobile_agent/app_api/openapi.py +146 -0
- mobile_agent/app_api/reading.py +172 -0
- mobile_agent/app_api/register.py +10 -0
- mobile_agent/app_api/screen.py +216 -0
- mobile_agent/app_api/service.py +382 -0
- mobile_agent/app_api/sessions.py +404 -0
- mobile_agent/app_api/tokens.py +167 -0
- mobile_agent/app_choice.py +131 -0
- mobile_agent/app_icons.py +183 -0
- mobile_agent/app_killer.py +69 -0
- mobile_agent/artifacts.py +606 -0
- mobile_agent/attach/__init__.py +13 -0
- mobile_agent/attach/cli.py +46 -0
- mobile_agent/attach/flutter.py +28 -0
- mobile_agent/attach/score.py +119 -0
- mobile_agent/attach/web.py +13 -0
- mobile_agent/attachments/__init__.py +1 -0
- mobile_agent/attachments/agent.py +418 -0
- mobile_agent/attachments/extract.py +217 -0
- mobile_agent/attachments/mcp.py +104 -0
- mobile_agent/attachments/register.py +73 -0
- mobile_agent/attachments/routes.py +326 -0
- mobile_agent/attachments/service.py +170 -0
- mobile_agent/attachments/sniff.py +152 -0
- mobile_agent/attachments/store.py +441 -0
- mobile_agent/audit_verify.py +436 -0
- mobile_agent/bench/README.md +410 -0
- mobile_agent/bench/__init__.py +0 -0
- mobile_agent/bench/__main__.py +394 -0
- mobile_agent/bench/ab.py +168 -0
- mobile_agent/bench/actuation.py +368 -0
- mobile_agent/bench/agents/__init__.py +9 -0
- mobile_agent/bench/agents/base.py +105 -0
- mobile_agent/bench/agents/plan_clients.py +245 -0
- mobile_agent/bench/agents/smart.py +533 -0
- mobile_agent/bench/checks.py +358 -0
- mobile_agent/bench/decide_gate.py +214 -0
- mobile_agent/bench/diagnose.py +181 -0
- mobile_agent/bench/evals/__init__.py +14 -0
- mobile_agent/bench/evals/build.py +565 -0
- mobile_agent/bench/evals/checkpoint.py +214 -0
- mobile_agent/bench/evals/irt.py +219 -0
- mobile_agent/bench/evals/l0.py +647 -0
- mobile_agent/bench/evals/l1.py +866 -0
- mobile_agent/bench/evals/models.py +124 -0
- mobile_agent/bench/evals/pairwise.py +189 -0
- mobile_agent/bench/evals/proxy.py +90 -0
- mobile_agent/bench/evals/sets.py +149 -0
- mobile_agent/bench/evals/traces.py +446 -0
- mobile_agent/bench/fixtures.py +176 -0
- mobile_agent/bench/guard_replay.py +129 -0
- mobile_agent/bench/h2-manifest.json +20 -0
- mobile_agent/bench/h3-manifest.json +20 -0
- mobile_agent/bench/heldout-manifest.json +32 -0
- mobile_agent/bench/heldout.py +375 -0
- mobile_agent/bench/iosworld.py +901 -0
- mobile_agent/bench/iosworld_stats.py +510 -0
- mobile_agent/bench/llm_units.py +116 -0
- mobile_agent/bench/manifest.json +93 -0
- mobile_agent/bench/metrics.py +66 -0
- mobile_agent/bench/phone.py +44 -0
- mobile_agent/bench/replay.py +172 -0
- mobile_agent/bench/report.py +207 -0
- mobile_agent/bench/runner.py +456 -0
- mobile_agent/bench/safety.py +310 -0
- mobile_agent/bench/stream_timing.py +213 -0
- mobile_agent/bench/suite.py +444 -0
- mobile_agent/bench/sweep.py +176 -0
- mobile_agent/bench/tape.py +798 -0
- mobile_agent/bench/tape_replay.py +88 -0
- mobile_agent/bench/trace_evals.py +128 -0
- mobile_agent/bench/trace_view.py +81 -0
- mobile_agent/bench/truth/iphone15pro.json +138 -0
- mobile_agent/bench/truth.py +340 -0
- mobile_agent/bench/tuning.py +543 -0
- mobile_agent/bench/tuning_analysis.py +656 -0
- mobile_agent/bench/vague-heldout-manifest.json +12 -0
- mobile_agent/bench/vague_rubric.py +135 -0
- mobile_agent/bench/vague_run.py +243 -0
- mobile_agent/brand.py +454 -0
- mobile_agent/brand_fonts.py +756 -0
- mobile_agent/build_identity.py +21 -0
- mobile_agent/cable_bt.py +623 -0
- mobile_agent/cable_bt_runtime.py +128 -0
- mobile_agent/catalog.py +69 -0
- mobile_agent/chart_svg.py +415 -0
- mobile_agent/chat_here.py +189 -0
- mobile_agent/cli_constants.py +5 -0
- mobile_agent/cli_parser.py +611 -0
- mobile_agent/cli_run.py +115 -0
- mobile_agent/compiled_read/__init__.py +21 -0
- mobile_agent/compiled_read/delta.py +89 -0
- mobile_agent/compiled_read/extractors.py +202 -0
- mobile_agent/compiled_read/routes.py +56 -0
- mobile_agent/completion.py +267 -0
- mobile_agent/compose.py +48 -0
- mobile_agent/config.py +114 -0
- mobile_agent/connectors/__init__.py +1 -0
- mobile_agent/connectors/api_session.py +76 -0
- mobile_agent/connectors/callback_page.py +146 -0
- mobile_agent/connectors/client.py +582 -0
- mobile_agent/connectors/github_app.py +55 -0
- mobile_agent/connectors/graph_api.py +396 -0
- mobile_agent/connectors/imports.py +170 -0
- mobile_agent/connectors/issue.py +66 -0
- mobile_agent/connectors/keystore.py +85 -0
- mobile_agent/connectors/marks.py +222 -0
- mobile_agent/connectors/oauth.py +476 -0
- mobile_agent/connectors/picker.py +228 -0
- mobile_agent/connectors/presets.py +591 -0
- mobile_agent/connectors/register.py +48 -0
- mobile_agent/connectors/routes.py +153 -0
- mobile_agent/connectors/service.py +966 -0
- mobile_agent/connectors/shared.py +136 -0
- mobile_agent/connectors/skill.py +446 -0
- mobile_agent/connectors/slack_api.py +269 -0
- mobile_agent/connectors/steps.py +92 -0
- mobile_agent/connectors/store.py +144 -0
- mobile_agent/connectors/vault.py +72 -0
- mobile_agent/console.py +178 -0
- mobile_agent/content_queue/__init__.py +1 -0
- mobile_agent/content_queue/client.py +263 -0
- mobile_agent/content_queue/importing.py +153 -0
- mobile_agent/content_queue/media_check.py +224 -0
- mobile_agent/content_queue/prompt.py +64 -0
- mobile_agent/content_queue/register.py +91 -0
- mobile_agent/content_queue/routes.py +370 -0
- mobile_agent/content_queue/runner.py +475 -0
- mobile_agent/content_queue/store.py +521 -0
- mobile_agent/contract.py +996 -0
- mobile_agent/contract_asks.py +248 -0
- mobile_agent/contract_prompt.py +45 -0
- mobile_agent/contract_screen.py +981 -0
- mobile_agent/contract_view.py +84 -0
- mobile_agent/corpus/__init__.py +26 -0
- mobile_agent/corpus/cli.py +63 -0
- mobile_agent/corpus/consent.py +55 -0
- mobile_agent/corpus/listener.py +76 -0
- mobile_agent/corpus/register.py +9 -0
- mobile_agent/corpus/skeleton.py +72 -0
- mobile_agent/corpus/store.py +143 -0
- mobile_agent/costs.py +161 -0
- mobile_agent/decider_scope.py +40 -0
- mobile_agent/decision_provider.py +189 -0
- mobile_agent/deeplinks.py +352 -0
- mobile_agent/device_info.py +116 -0
- mobile_agent/device_manager.py +978 -0
- mobile_agent/device_status.py +195 -0
- mobile_agent/device_targets.py +217 -0
- mobile_agent/devices.py +676 -0
- mobile_agent/devkit/__init__.py +64 -0
- mobile_agent/devkit/account_cli.py +90 -0
- mobile_agent/devkit/asc.py +562 -0
- mobile_agent/devkit/asc_cli.py +214 -0
- mobile_agent/devkit/ci.py +737 -0
- mobile_agent/devkit/ci_cli.py +196 -0
- mobile_agent/devkit/ci_hosts.py +194 -0
- mobile_agent/devkit/es256.py +197 -0
- mobile_agent/devkit/expo.py +440 -0
- mobile_agent/devkit/expo_cli.py +405 -0
- mobile_agent/devkit/expo_dev.py +203 -0
- mobile_agent/devkit/expo_routes.py +90 -0
- mobile_agent/devkit/fixtures_cli.py +133 -0
- mobile_agent/devkit/fixtures_routes.py +82 -0
- mobile_agent/devkit/links.py +375 -0
- mobile_agent/devkit/links_cli.py +123 -0
- mobile_agent/devkit/links_routes.py +83 -0
- mobile_agent/devkit/nightly.py +152 -0
- mobile_agent/devkit/shots.py +203 -0
- mobile_agent/devkit/testflight.py +155 -0
- mobile_agent/devkit/testflight_cli.py +127 -0
- mobile_agent/devkit/testflight_routes.py +111 -0
- mobile_agent/devkit/xcode.py +236 -0
- mobile_agent/devkit/xcode_cli.py +101 -0
- mobile_agent/devtools.py +419 -0
- mobile_agent/doctor.py +509 -0
- mobile_agent/doctor_fix.py +467 -0
- mobile_agent/downloads.py +55 -0
- mobile_agent/drivers.py +793 -0
- mobile_agent/dry_run.py +102 -0
- mobile_agent/effect_ledger.py +83 -0
- mobile_agent/engine_health.py +378 -0
- mobile_agent/engine_hooks.py +96 -0
- mobile_agent/engines.py +408 -0
- mobile_agent/env_file.py +63 -0
- mobile_agent/env_secrets.py +149 -0
- mobile_agent/errors.py +15 -0
- mobile_agent/evals/__init__.py +3 -0
- mobile_agent/evals/app_survey.py +149 -0
- mobile_agent/evals/development/__init__.py +1 -0
- mobile_agent/evals/development/score.py +276 -0
- mobile_agent/evals/latency_report.py +312 -0
- mobile_agent/evals/offline_report.py +374 -0
- mobile_agent/evals/oracles.py +298 -0
- mobile_agent/evals/redteam/__init__.py +15 -0
- mobile_agent/evals/redteam/__main__.py +72 -0
- mobile_agent/evals/redteam/cases.py +730 -0
- mobile_agent/evals/redteam/harness.py +604 -0
- mobile_agent/evals/redteam/live.py +139 -0
- mobile_agent/evals/redteam/plain.py +85 -0
- mobile_agent/evals/vision/__init__.py +0 -0
- mobile_agent/evals/vision/images/dog_001.jpg +0 -0
- mobile_agent/evals/vision/images/dog_002.jpg +0 -0
- mobile_agent/evals/vision/images/dog_003.jpg +0 -0
- mobile_agent/evals/vision/images/dog_004.jpg +0 -0
- mobile_agent/evals/vision/images/dog_005.jpg +0 -0
- mobile_agent/evals/vision/images/dog_006.jpg +0 -0
- mobile_agent/evals/vision/images/dog_007.jpg +0 -0
- mobile_agent/evals/vision/images/dog_010.jpg +0 -0
- mobile_agent/evals/vision/images/dog_011.jpg +0 -0
- mobile_agent/evals/vision/images/dog_013.jpg +0 -0
- mobile_agent/evals/vision/images/dog_014.jpg +0 -0
- mobile_agent/evals/vision/images/dog_015.jpg +0 -0
- mobile_agent/evals/vision/images/dog_016.jpg +0 -0
- mobile_agent/evals/vision/images/dog_017.jpg +0 -0
- mobile_agent/evals/vision/images/dog_019.jpg +0 -0
- mobile_agent/evals/vision/images/dog_020.jpg +0 -0
- mobile_agent/evals/vision/images/dog_021.jpg +0 -0
- mobile_agent/evals/vision/images/dog_022.jpg +0 -0
- mobile_agent/evals/vision/images/dog_023.jpg +0 -0
- mobile_agent/evals/vision/images/dog_024.jpg +0 -0
- mobile_agent/evals/vision/images/dog_026.jpg +0 -0
- mobile_agent/evals/vision/images/dog_027.jpg +0 -0
- mobile_agent/evals/vision/images/dog_034.jpg +0 -0
- mobile_agent/evals/vision/images/dog_035.jpg +0 -0
- mobile_agent/evals/vision/images/dog_038.jpg +0 -0
- mobile_agent/evals/vision/images/dog_042.jpg +0 -0
- mobile_agent/evals/vision/images/dog_047.jpg +0 -0
- mobile_agent/evals/vision/images/dog_052.jpg +0 -0
- mobile_agent/evals/vision/images/dog_058.jpg +0 -0
- mobile_agent/evals/vision/images/dog_062.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_000.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_010.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_011.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_015.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_016.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_017.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_019.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_021.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_025.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_026.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_027.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_030.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_032.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_033.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_034.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_035.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_036.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_037.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_039.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_040.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_041.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_042.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_043.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_044.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_046.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_047.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_048.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_049.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_051.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_053.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_054.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_055.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_058.jpg +0 -0
- mobile_agent/evals/vision/images/eyes_059.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_002.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_003.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_004.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_007.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_009.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_010.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_011.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_012.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_013.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_014.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_016.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_017.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_023.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_027.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_029.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_032.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_033.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_036.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_041.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_043.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_047.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_049.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_051.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_053.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_056.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_057.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_058.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_059.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_060.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_061.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_062.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_063.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_064.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_065.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_066.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_067.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_068.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_069.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_070.jpg +0 -0
- mobile_agent/evals/vision/images/receipt_071.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_000.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_001.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_002.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_004.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_005.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_008.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_009.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_010.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_012.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_013.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_015.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_016.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_017.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_018.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_021.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_022.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_023.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_024.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_030.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_031.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_032.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_033.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_034.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_036.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_038.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_039.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_040.jpg +0 -0
- mobile_agent/evals/vision/images/sunglasses_041.jpg +0 -0
- mobile_agent/evals/vision/manifest.json +2355 -0
- mobile_agent/eventbus.py +110 -0
- mobile_agent/explain.py +34 -0
- mobile_agent/failure_line.py +197 -0
- mobile_agent/fanout.py +663 -0
- mobile_agent/feedback/__init__.py +2 -0
- mobile_agent/feedback/bundle.py +204 -0
- mobile_agent/feedback/outbox.py +196 -0
- mobile_agent/feedback/register.py +9 -0
- mobile_agent/feedback/routes.py +177 -0
- mobile_agent/fleet.py +841 -0
- mobile_agent/fleet_health.py +729 -0
- mobile_agent/fleet_phone.py +186 -0
- mobile_agent/frontier/__init__.py +79 -0
- mobile_agent/frontier/act.py +380 -0
- mobile_agent/frontier/actions.py +70 -0
- mobile_agent/frontier/agent.py +256 -0
- mobile_agent/frontier/approvals.py +366 -0
- mobile_agent/frontier/hooks.py +199 -0
- mobile_agent/frontier/interrupts.py +125 -0
- mobile_agent/frontier/observe.py +331 -0
- mobile_agent/frontier/phases.py +381 -0
- mobile_agent/frontier/pipeline.py +61 -0
- mobile_agent/frontier/prompt.py +384 -0
- mobile_agent/frontier/protocol.py +91 -0
- mobile_agent/frontier/safety.py +467 -0
- mobile_agent/frontier/saved.py +375 -0
- mobile_agent/frontier/screen.py +419 -0
- mobile_agent/frontier/skills.py +147 -0
- mobile_agent/frontier/stages/__init__.py +1 -0
- mobile_agent/frontier/stages/after.py +192 -0
- mobile_agent/frontier/stages/approve.py +138 -0
- mobile_agent/frontier/stages/controls.py +115 -0
- mobile_agent/frontier/stages/ends.py +102 -0
- mobile_agent/frontier/stages/gate.py +92 -0
- mobile_agent/frontier/stages/guards.py +132 -0
- mobile_agent/frontier/stages/points.py +155 -0
- mobile_agent/frontier/stages/send.py +177 -0
- mobile_agent/frontier/turn.py +151 -0
- mobile_agent/full_disk_access.py +83 -0
- mobile_agent/gemini.py +69 -0
- mobile_agent/genuine.py +120 -0
- mobile_agent/graph/__init__.py +4 -0
- mobile_agent/graph/catalog.py +789 -0
- mobile_agent/graph/chain.py +306 -0
- mobile_agent/graph/dispatch.py +98 -0
- mobile_agent/graph/draft.py +373 -0
- mobile_agent/graph/edit.py +76 -0
- mobile_agent/graph/estimate.py +189 -0
- mobile_agent/graph/expr.py +562 -0
- mobile_agent/graph/functions.py +271 -0
- mobile_agent/graph/infer.py +144 -0
- mobile_agent/graph/nodes.py +936 -0
- mobile_agent/graph/phone_steps.py +178 -0
- mobile_agent/graph/rerun.py +327 -0
- mobile_agent/graph/run.py +899 -0
- mobile_agent/graph/schema.py +842 -0
- mobile_agent/graph/state.py +161 -0
- mobile_agent/graph/values.py +198 -0
- mobile_agent/harness/__init__.py +2 -0
- mobile_agent/harness/checkpoints.py +468 -0
- mobile_agent/harness/control.py +101 -0
- mobile_agent/harness/prewarm.py +213 -0
- mobile_agent/harness/register.py +43 -0
- mobile_agent/harness/routes.py +114 -0
- mobile_agent/harness/settings.py +66 -0
- mobile_agent/harness/tools/__init__.py +67 -0
- mobile_agent/harness/tools/ask_user.py +102 -0
- mobile_agent/harness/tools/watch.py +120 -0
- mobile_agent/harness_api.py +815 -0
- mobile_agent/hedge.py +203 -0
- mobile_agent/help_topics.py +305 -0
- mobile_agent/hints.py +248 -0
- mobile_agent/history.py +69 -0
- mobile_agent/home.py +214 -0
- mobile_agent/http_pool.py +175 -0
- mobile_agent/image_policy.py +139 -0
- mobile_agent/inbox.py +148 -0
- mobile_agent/installed_apps.py +344 -0
- mobile_agent/integrations/__init__.py +1 -0
- mobile_agent/integrations/cli.py +472 -0
- mobile_agent/integrations/client_files.py +65 -0
- mobile_agent/integrations/clients.py +998 -0
- mobile_agent/integrations/doctor.py +185 -0
- mobile_agent/integrations/jsonc.py +383 -0
- mobile_agent/integrations/routes.py +135 -0
- mobile_agent/integrations/rules.py +112 -0
- mobile_agent/integrations/server_options.py +46 -0
- mobile_agent/integrations/skill.py +152 -0
- mobile_agent/integrations/skill_files.py +10 -0
- mobile_agent/integrations/textedit.py +309 -0
- mobile_agent/intents/__init__.py +17 -0
- mobile_agent/intents/atlas.py +216 -0
- mobile_agent/intents/callback.py +44 -0
- mobile_agent/intents/cli.py +55 -0
- mobile_agent/intents/harvest.py +91 -0
- mobile_agent/intents/plan.py +93 -0
- mobile_agent/ios/__init__.py +1 -0
- mobile_agent/ios/guide.py +16 -0
- mobile_agent/ios/kb.py +212 -0
- mobile_agent/ios/kb_data.py +1588 -0
- mobile_agent/ios/mcp.py +44 -0
- mobile_agent/ios/register.py +8 -0
- mobile_agent/ios/system.py +369 -0
- mobile_agent/journal.py +531 -0
- mobile_agent/keychain_acl.py +374 -0
- mobile_agent/keys.py +395 -0
- mobile_agent/latency_overlap.py +36 -0
- mobile_agent/latency_trace.py +170 -0
- mobile_agent/learned/__init__.py +15 -0
- mobile_agent/learned/app_path.py +160 -0
- mobile_agent/learned/code_read.py +38 -0
- mobile_agent/learned/parts.py +546 -0
- mobile_agent/learned/programs.py +691 -0
- mobile_agent/learned/register.py +171 -0
- mobile_agent/learned/routes.py +61 -0
- mobile_agent/learned/taught.py +329 -0
- mobile_agent/learned/teach.py +538 -0
- mobile_agent/learned/workflows.py +647 -0
- mobile_agent/learned/zero.py +41 -0
- mobile_agent/links.py +37 -0
- mobile_agent/listen.py +607 -0
- mobile_agent/lock_spike.py +278 -0
- mobile_agent/lockscreen.py +526 -0
- mobile_agent/login.py +354 -0
- mobile_agent/longform/__init__.py +15 -0
- mobile_agent/longform/ads.py +191 -0
- mobile_agent/longform/blocks.py +117 -0
- mobile_agent/longform/budget.py +265 -0
- mobile_agent/longform/context.py +285 -0
- mobile_agent/longform/costs.py +93 -0
- mobile_agent/longform/digest.py +145 -0
- mobile_agent/longform/extract.py +508 -0
- mobile_agent/longform/feed.py +505 -0
- mobile_agent/longform/followup.py +168 -0
- mobile_agent/longform/frames.py +82 -0
- mobile_agent/longform/identity.py +87 -0
- mobile_agent/longform/inbox.py +221 -0
- mobile_agent/longform/items.py +425 -0
- mobile_agent/longform/models.py +115 -0
- mobile_agent/longform/notebook.py +329 -0
- mobile_agent/longform/plan.py +614 -0
- mobile_agent/longform/recall.py +195 -0
- mobile_agent/longform/receipts.py +81 -0
- mobile_agent/longform/register.py +124 -0
- mobile_agent/longform/report.py +343 -0
- mobile_agent/longform/routes.py +102 -0
- mobile_agent/longform/session.py +644 -0
- mobile_agent/longform/store.py +146 -0
- mobile_agent/longform/synthesis.py +347 -0
- mobile_agent/mac_tools.py +223 -0
- mobile_agent/manual_control.py +143 -0
- mobile_agent/maps.py +149 -0
- mobile_agent/marks.py +92 -0
- mobile_agent/mcp_server/__init__.py +3 -0
- mobile_agent/mcp_server/__main__.py +29 -0
- mobile_agent/mcp_server/action_journal.py +24 -0
- mobile_agent/mcp_server/action_tools.py +638 -0
- mobile_agent/mcp_server/approvals.py +219 -0
- mobile_agent/mcp_server/catalog.py +92 -0
- mobile_agent/mcp_server/cli.py +194 -0
- mobile_agent/mcp_server/codes.py +147 -0
- mobile_agent/mcp_server/device_tools.py +306 -0
- mobile_agent/mcp_server/direct.py +272 -0
- mobile_agent/mcp_server/expo_tools.py +105 -0
- mobile_agent/mcp_server/phone_tools.py +510 -0
- mobile_agent/mcp_server/protocol.py +364 -0
- mobile_agent/mcp_server/registry.py +55 -0
- mobile_agent/mcp_server/run_tools.py +531 -0
- mobile_agent/mcp_server/schemas.py +425 -0
- mobile_agent/mcp_server/screen_tools.py +168 -0
- mobile_agent/mcp_server/session.py +469 -0
- mobile_agent/mcp_server/support.py +145 -0
- mobile_agent/mcp_server/tools.py +493 -0
- mobile_agent/memory/__init__.py +9 -0
- mobile_agent/memory/cli.py +207 -0
- mobile_agent/memory/extract.py +191 -0
- mobile_agent/memory/listener.py +100 -0
- mobile_agent/memory/model.py +144 -0
- mobile_agent/memory/project.py +96 -0
- mobile_agent/memory/provider.py +127 -0
- mobile_agent/memory/register.py +21 -0
- mobile_agent/memory/retrieval.py +172 -0
- mobile_agent/memory/routes.py +216 -0
- mobile_agent/memory/runs.py +124 -0
- mobile_agent/memory/store.py +806 -0
- mobile_agent/memory/writer.py +278 -0
- mobile_agent/mirroring.py +585 -0
- mobile_agent/mirroring_runtime.py +221 -0
- mobile_agent/model_clients.py +604 -0
- mobile_agent/narrate.py +813 -0
- mobile_agent/native_helpers.py +84 -0
- mobile_agent/netxray/__init__.py +1 -0
- mobile_agent/netxray/analyze.py +322 -0
- mobile_agent/netxray/attribution.py +62 -0
- mobile_agent/netxray/capture.py +88 -0
- mobile_agent/netxray/cli.py +142 -0
- mobile_agent/netxray/job.py +219 -0
- mobile_agent/netxray/mobster-pcap.c +560 -0
- mobile_agent/netxray/packets.py +410 -0
- mobile_agent/netxray/pcapng.py +223 -0
- mobile_agent/netxray/register.py +93 -0
- mobile_agent/netxray/report.py +214 -0
- mobile_agent/netxray/store.py +48 -0
- mobile_agent/netxray/summary.py +157 -0
- mobile_agent/netxray/target.py +67 -0
- mobile_agent/netxray/tool.py +66 -0
- mobile_agent/netxray/trackers.py +194 -0
- mobile_agent/notify_triggers/__init__.py +1 -0
- mobile_agent/notify_triggers/rules.py +39 -0
- mobile_agent/notify_triggers/sources.py +315 -0
- mobile_agent/notify_triggers/watch.py +135 -0
- mobile_agent/offers.py +86 -0
- mobile_agent/orchestrate.py +215 -0
- mobile_agent/outlets/__init__.py +1 -0
- mobile_agent/outlets/connect.py +249 -0
- mobile_agent/outlets/folder.py +180 -0
- mobile_agent/outlets/hooks.py +453 -0
- mobile_agent/outlets/inbound.py +472 -0
- mobile_agent/outlets/inbound_reply.py +57 -0
- mobile_agent/outlets/inbound_routes.py +81 -0
- mobile_agent/outlets/kinds.py +308 -0
- mobile_agent/outlets/mail.py +110 -0
- mobile_agent/outlets/net.py +86 -0
- mobile_agent/outlets/notion.py +244 -0
- mobile_agent/outlets/register.py +56 -0
- mobile_agent/outlets/routes.py +135 -0
- mobile_agent/outlets/service.py +580 -0
- mobile_agent/outlets/sheets.py +169 -0
- mobile_agent/outlets/slack.py +177 -0
- mobile_agent/outlets/store.py +194 -0
- mobile_agent/outlets/vault.py +80 -0
- mobile_agent/outlets/webhook.py +316 -0
- mobile_agent/output_contract.py +141 -0
- mobile_agent/owner_presence.py +211 -0
- mobile_agent/ownership.py +219 -0
- mobile_agent/packs/__init__.py +1 -0
- mobile_agent/packs/calendar.py +243 -0
- mobile_agent/packs/context.py +68 -0
- mobile_agent/packs/core.py +757 -0
- mobile_agent/packs/front_door.py +316 -0
- mobile_agent/packs/heal.py +214 -0
- mobile_agent/packs/health.py +122 -0
- mobile_agent/packs/insights.py +76 -0
- mobile_agent/packs/instagram.py +640 -0
- mobile_agent/packs/linkedin.py +106 -0
- mobile_agent/packs/maps.py +196 -0
- mobile_agent/packs/mcp.py +149 -0
- mobile_agent/packs/messages.py +270 -0
- mobile_agent/packs/notes.py +226 -0
- mobile_agent/packs/photos.py +166 -0
- mobile_agent/packs/pins.py +47 -0
- mobile_agent/packs/reddit.py +110 -0
- mobile_agent/packs/register.py +276 -0
- mobile_agent/packs/rows.py +108 -0
- mobile_agent/packs/safari.py +143 -0
- mobile_agent/packs/settings.py +253 -0
- mobile_agent/packs/social.py +300 -0
- mobile_agent/packs/threads.py +97 -0
- mobile_agent/packs/tiktok.py +455 -0
- mobile_agent/packs/tools.py +290 -0
- mobile_agent/packs/x.py +127 -0
- mobile_agent/packs/youtube.py +199 -0
- mobile_agent/parity.py +270 -0
- mobile_agent/paths.py +99 -0
- mobile_agent/perception.py +470 -0
- mobile_agent/permissions/__init__.py +32 -0
- mobile_agent/permissions/consequence.py +535 -0
- mobile_agent/permissions/exceptions.py +99 -0
- mobile_agent/permissions/gate.py +316 -0
- mobile_agent/permissions/outside.py +316 -0
- mobile_agent/permissions/policy.py +288 -0
- mobile_agent/permissions/project_policy.py +411 -0
- mobile_agent/permissions/register.py +148 -0
- mobile_agent/permissions/rules.py +291 -0
- mobile_agent/phone_io/__init__.py +146 -0
- mobile_agent/phone_io/cli.py +427 -0
- mobile_agent/phone_io/clipboard.py +78 -0
- mobile_agent/phone_io/controls.py +78 -0
- mobile_agent/phone_io/files.py +289 -0
- mobile_agent/phone_io/handoff.py +123 -0
- mobile_agent/phone_io/icloud.py +118 -0
- mobile_agent/phone_io/install.py +188 -0
- mobile_agent/phone_io/network_capture.py +188 -0
- mobile_agent/phone_io/network_cli.py +138 -0
- mobile_agent/phone_io/network_profiles.py +138 -0
- mobile_agent/phone_io/photos.py +512 -0
- mobile_agent/phone_io/recording.py +115 -0
- mobile_agent/phone_io/secrets.py +146 -0
- mobile_agent/phone_io/transfer.py +895 -0
- mobile_agent/phone_io/tunnel.py +129 -0
- mobile_agent/phone_ready.py +91 -0
- mobile_agent/pinning.py +158 -0
- mobile_agent/plan_gate.py +807 -0
- mobile_agent/plugins/__init__.py +3 -0
- mobile_agent/plugins/install.py +163 -0
- mobile_agent/plugins/manifest.py +242 -0
- mobile_agent/plugins/register.py +15 -0
- mobile_agent/plugins/routes.py +82 -0
- mobile_agent/plugins/service.py +322 -0
- mobile_agent/plugins/skills.py +76 -0
- mobile_agent/plugins/store.py +79 -0
- mobile_agent/pool.py +285 -0
- mobile_agent/power_watch.py +275 -0
- mobile_agent/premium_model.py +22 -0
- mobile_agent/project_env.py +291 -0
- mobile_agent/project_env_cli.py +74 -0
- mobile_agent/project_env_routes.py +55 -0
- mobile_agent/projects/__init__.py +27 -0
- mobile_agent/projects/ci.py +332 -0
- mobile_agent/projects/clone.py +339 -0
- mobile_agent/projects/drafts.py +158 -0
- mobile_agent/projects/folder.py +436 -0
- mobile_agent/projects/github.py +131 -0
- mobile_agent/projects/register.py +31 -0
- mobile_agent/projects/revisions.py +253 -0
- mobile_agent/projects/routes.py +234 -0
- mobile_agent/projects/runs.py +297 -0
- mobile_agent/projects/service.py +695 -0
- mobile_agent/projects/setup.py +84 -0
- mobile_agent/projects/store.py +379 -0
- mobile_agent/projects/sync.py +380 -0
- mobile_agent/projects/workflow_files.py +270 -0
- mobile_agent/replay_core/__init__.py +26 -0
- mobile_agent/replay_core/checks.py +594 -0
- mobile_agent/replay_core/engine.py +596 -0
- mobile_agent/replay_core/heal.py +364 -0
- mobile_agent/replay_core/path.py +240 -0
- mobile_agent/replay_core/recorder.py +206 -0
- mobile_agent/replay_core/store.py +171 -0
- mobile_agent/result_blocks.py +941 -0
- mobile_agent/run.py +642 -0
- mobile_agent/run_delivery.py +123 -0
- mobile_agent/run_executor.py +200 -0
- mobile_agent/run_export.py +284 -0
- mobile_agent/run_gif.py +891 -0
- mobile_agent/runner_hints.py +86 -0
- mobile_agent/runner_renewal.py +205 -0
- mobile_agent/runtime_create.py +549 -0
- mobile_agent/runtime_driver.py +458 -0
- mobile_agent/runtime_limits.py +319 -0
- mobile_agent/runtime_queue.py +394 -0
- mobile_agent/runtime_runs.py +383 -0
- mobile_agent/runtime_status.py +181 -0
- mobile_agent/runtime_teach.py +295 -0
- mobile_agent/runtime_work.py +599 -0
- mobile_agent/screen_text.py +101 -0
- mobile_agent/secret_filter.py +113 -0
- mobile_agent/serve.py +160 -0
- mobile_agent/server.py +363 -0
- mobile_agent/service_support.py +42 -0
- mobile_agent/settle/__init__.py +1 -0
- mobile_agent/settle/predict.py +102 -0
- mobile_agent/settle/replay.py +92 -0
- mobile_agent/settle/store.py +93 -0
- mobile_agent/settle/stream.py +43 -0
- mobile_agent/setup_cli.py +225 -0
- mobile_agent/setup_errors.py +87 -0
- mobile_agent/setup_service.py +527 -0
- mobile_agent/share.py +104 -0
- mobile_agent/signing.py +184 -0
- mobile_agent/sim/__init__.py +6 -0
- mobile_agent/sim/__main__.py +26 -0
- mobile_agent/sim/api.py +952 -0
- mobile_agent/sim/apps.py +172 -0
- mobile_agent/sim/biometrics.py +84 -0
- mobile_agent/sim/cli.py +271 -0
- mobile_agent/sim/doctor.py +293 -0
- mobile_agent/sim/governor.py +90 -0
- mobile_agent/sim/holders.py +88 -0
- mobile_agent/sim/models.py +62 -0
- mobile_agent/sim/registry.py +161 -0
- mobile_agent/sim/simctl.py +269 -0
- mobile_agent/sim/wda.py +618 -0
- mobile_agent/sim/xcode.py +74 -0
- mobile_agent/skills/__init__.py +12 -0
- mobile_agent/skills/codes.py +597 -0
- mobile_agent/skills/notifications.py +281 -0
- mobile_agent/skills/open_url.py +127 -0
- mobile_agent/smart_events.py +588 -0
- mobile_agent/smart_result.py +490 -0
- mobile_agent/smart_run.py +413 -0
- mobile_agent/state.py +662 -0
- mobile_agent/states.py +217 -0
- mobile_agent/steering.py +70 -0
- mobile_agent/style.py +178 -0
- mobile_agent/supervised.py +234 -0
- mobile_agent/swift/helper_bt_hid.swift +534 -0
- mobile_agent/swift/helper_doc_text.swift +184 -0
- mobile_agent/swift/helper_draft_reply.swift +170 -0
- mobile_agent/swift/helper_map.swift +110 -0
- mobile_agent/swift/helper_mirroring.swift +1099 -0
- mobile_agent/swift/helper_mjpeg_mp4.swift +103 -0
- mobile_agent/swift/helper_pdf.swift +100 -0
- mobile_agent/swift/helper_phone_audio.swift +993 -0
- mobile_agent/swift/helper_phone_screen.swift +416 -0
- mobile_agent/swift/helper_screen_text.swift +24 -0
- mobile_agent/swift/helper_transfer.swift +246 -0
- mobile_agent/swift/ocr.swift +22 -0
- mobile_agent/task_policy.py +337 -0
- mobile_agent/team_signing/__init__.py +34 -0
- mobile_agent/team_signing/apple_ca.py +37 -0
- mobile_agent/team_signing/bundle.py +136 -0
- mobile_agent/team_signing/cli.py +208 -0
- mobile_agent/team_signing/identity.py +248 -0
- mobile_agent/team_signing/ledger.py +105 -0
- mobile_agent/team_signing/prebuilt.py +61 -0
- mobile_agent/team_signing/resign.py +256 -0
- mobile_agent/team_signing/service.py +382 -0
- mobile_agent/team_signing/tooling.py +243 -0
- mobile_agent/team_signing/writes.py +268 -0
- mobile_agent/terminal_image.py +81 -0
- mobile_agent/testkit/__init__.py +1 -0
- mobile_agent/testkit/a11y_baseline.py +132 -0
- mobile_agent/testkit/api.py +403 -0
- mobile_agent/testkit/capsule.py +400 -0
- mobile_agent/testkit/capsule_cli.py +57 -0
- mobile_agent/testkit/cli.py +948 -0
- mobile_agent/testkit/cli_arguments.py +193 -0
- mobile_agent/testkit/comparison.py +288 -0
- mobile_agent/testkit/comparison_cli.py +41 -0
- mobile_agent/testkit/discover.py +257 -0
- mobile_agent/testkit/editor.py +243 -0
- mobile_agent/testkit/execute.py +515 -0
- mobile_agent/testkit/github_checks.py +248 -0
- mobile_agent/testkit/gitref.py +66 -0
- mobile_agent/testkit/heal_review.py +176 -0
- mobile_agent/testkit/health.py +480 -0
- mobile_agent/testkit/history.py +255 -0
- mobile_agent/testkit/hosted_results.py +321 -0
- mobile_agent/testkit/input_files.py +173 -0
- mobile_agent/testkit/jobs.py +285 -0
- mobile_agent/testkit/junit_verdict.py +33 -0
- mobile_agent/testkit/maestro.py +802 -0
- mobile_agent/testkit/mcp.py +562 -0
- mobile_agent/testkit/mcp_comparison.py +88 -0
- mobile_agent/testkit/minimize.py +471 -0
- mobile_agent/testkit/minimizer_cli.py +48 -0
- mobile_agent/testkit/notify.py +212 -0
- mobile_agent/testkit/phone_video.py +191 -0
- mobile_agent/testkit/pr.py +491 -0
- mobile_agent/testkit/pr_report.py +562 -0
- mobile_agent/testkit/prbuild.py +148 -0
- mobile_agent/testkit/proof_report.py +55 -0
- mobile_agent/testkit/provenance.py +119 -0
- mobile_agent/testkit/prrun.py +244 -0
- mobile_agent/testkit/recent.py +166 -0
- mobile_agent/testkit/recipes.py +146 -0
- mobile_agent/testkit/record.py +204 -0
- mobile_agent/testkit/register.py +288 -0
- mobile_agent/testkit/reports.py +998 -0
- mobile_agent/testkit/repro.py +432 -0
- mobile_agent/testkit/scenario.py +440 -0
- mobile_agent/testkit/scenario_cli.py +37 -0
- mobile_agent/testkit/snapshots_cli.py +60 -0
- mobile_agent/testkit/suite.py +670 -0
- mobile_agent/testkit/targets.py +497 -0
- mobile_agent/testkit/xctest.py +378 -0
- mobile_agent/threads/__init__.py +1 -0
- mobile_agent/threads/chat_once.py +264 -0
- mobile_agent/threads/cli.py +740 -0
- mobile_agent/threads/client.py +326 -0
- mobile_agent/threads/context.py +246 -0
- mobile_agent/threads/mcp.py +739 -0
- mobile_agent/threads/recent.py +34 -0
- mobile_agent/threads/register.py +54 -0
- mobile_agent/threads/routes.py +188 -0
- mobile_agent/threads/service.py +975 -0
- mobile_agent/threads/store.py +361 -0
- mobile_agent/threads/summary.py +117 -0
- mobile_agent/threads/term.py +101 -0
- mobile_agent/threads/workflow_cli.py +104 -0
- mobile_agent/tls.py +36 -0
- mobile_agent/traces.py +271 -0
- mobile_agent/tracks.py +235 -0
- mobile_agent/transport.py +271 -0
- mobile_agent/triggers/__init__.py +1 -0
- mobile_agent/triggers/actions.py +204 -0
- mobile_agent/triggers/model.py +146 -0
- mobile_agent/triggers/register.py +29 -0
- mobile_agent/triggers/routes.py +101 -0
- mobile_agent/triggers/service.py +387 -0
- mobile_agent/triggers/store.py +142 -0
- mobile_agent/tui/__init__.py +89 -0
- mobile_agent/tui/app.py +582 -0
- mobile_agent/tui/chrome.py +321 -0
- mobile_agent/tui/commands.py +375 -0
- mobile_agent/tui/demo_phone.py +369 -0
- mobile_agent/tui/images.py +29 -0
- mobile_agent/tui/keys.py +64 -0
- mobile_agent/tui/prompt.py +195 -0
- mobile_agent/tui/render.py +395 -0
- mobile_agent/tui/screens.py +180 -0
- mobile_agent/tui/session.py +308 -0
- mobile_agent/tui/tasks.py +287 -0
- mobile_agent/tui/theme.py +118 -0
- mobile_agent/tui/widgets.py +272 -0
- mobile_agent/update_check.py +213 -0
- mobile_agent/usage.py +184 -0
- mobile_agent/verify/__init__.py +0 -0
- mobile_agent/verify/__main__.py +41 -0
- mobile_agent/verify/a11y.py +189 -0
- mobile_agent/verify/actions.py +290 -0
- mobile_agent/verify/assertions.py +570 -0
- mobile_agent/verify/audit.py +312 -0
- mobile_agent/verify/business.py +221 -0
- mobile_agent/verify/business_run.py +131 -0
- mobile_agent/verify/business_state.py +168 -0
- mobile_agent/verify/check_schema.py +398 -0
- mobile_agent/verify/checks.py +614 -0
- mobile_agent/verify/cli.py +423 -0
- mobile_agent/verify/conditions.py +483 -0
- mobile_agent/verify/crashes.py +321 -0
- mobile_agent/verify/device_audit.py +246 -0
- mobile_agent/verify/dialogs.py +287 -0
- mobile_agent/verify/evidence.py +705 -0
- mobile_agent/verify/fixtures.py +305 -0
- mobile_agent/verify/flow.py +213 -0
- mobile_agent/verify/l10n.py +141 -0
- mobile_agent/verify/netcerts.py +67 -0
- mobile_agent/verify/netproxy.py +356 -0
- mobile_agent/verify/netrun.py +81 -0
- mobile_agent/verify/netshim.m +28 -0
- mobile_agent/verify/netshim.py +48 -0
- mobile_agent/verify/netstubs.py +132 -0
- mobile_agent/verify/overlay.py +174 -0
- mobile_agent/verify/perf.py +239 -0
- mobile_agent/verify/proof.py +192 -0
- mobile_agent/verify/replay_admission.py +48 -0
- mobile_agent/verify/report.py +952 -0
- mobile_agent/verify/run_cleanup.py +67 -0
- mobile_agent/verify/run_links.py +48 -0
- mobile_agent/verify/run_perf.py +44 -0
- mobile_agent/verify/run_secrets.py +39 -0
- mobile_agent/verify/runner.py +976 -0
- mobile_agent/verify/runs.py +57 -0
- mobile_agent/verify/screen_coverage.py +181 -0
- mobile_agent/verify/smart.py +106 -0
- mobile_agent/verify/snapshot_run.py +114 -0
- mobile_agent/verify/snapshots.py +306 -0
- mobile_agent/verify/sound.py +335 -0
- mobile_agent/verify/source_observation.py +130 -0
- mobile_agent/verify/system_surfaces/__init__.py +7 -0
- mobile_agent/verify/system_surfaces/play.py +106 -0
- mobile_agent/verify/system_surfaces/spec.py +139 -0
- mobile_agent/verify/system_surfaces/stage.py +138 -0
- mobile_agent/verify/system_surfaces/surfaces.py +121 -0
- mobile_agent/verify/tree.py +223 -0
- mobile_agent/verify/verdict.py +377 -0
- mobile_agent/verify/wire.py +392 -0
- mobile_agent/verify/wirestore.py +176 -0
- mobile_agent/waiting/__init__.py +1 -0
- mobile_agent/waiting/contacts.py +73 -0
- mobile_agent/waiting/drafts.py +136 -0
- mobile_agent/waiting/job.py +145 -0
- mobile_agent/waiting/messages.py +174 -0
- mobile_agent/waiting/rank.py +115 -0
- mobile_agent/waiting/register.py +26 -0
- mobile_agent/watch.py +502 -0
- mobile_agent/wda/__init__.py +2 -0
- mobile_agent/wda/frame_clock.py +646 -0
- mobile_agent/wda/frame_source.py +325 -0
- mobile_agent/wda/frame_store.py +115 -0
- mobile_agent/wda/gate.py +163 -0
- mobile_agent/wda/gestures.py +291 -0
- mobile_agent/wda/guards.py +94 -0
- mobile_agent/wda/identity.py +55 -0
- mobile_agent/wda/observe.py +884 -0
- mobile_agent/wda/patches.py +856 -0
- mobile_agent/wda/point_gestures.py +150 -0
- mobile_agent/wda/source.py +32 -0
- mobile_agent/wda/system_dialogs.py +571 -0
- mobile_agent/wda/text.py +694 -0
- mobile_agent/wda/timeouts.py +77 -0
- mobile_agent/wda/video.py +505 -0
- mobile_agent/wda/watchdog.py +243 -0
- mobile_agent/wireless/__init__.py +15 -0
- mobile_agent/wireless/api.py +354 -0
- mobile_agent/wireless/cli.py +287 -0
- mobile_agent/wireless/devicectl.py +229 -0
- mobile_agent/wireless/exposure.py +33 -0
- mobile_agent/wireless/mobster-wifi-pair.c +122 -0
- mobile_agent/wireless/monitor.py +332 -0
- mobile_agent/wireless/network.py +334 -0
- mobile_agent/wireless/onboarding.py +293 -0
- mobile_agent/wireless/pair.py +124 -0
- mobile_agent/wireless/probe.py +322 -0
- mobile_agent/wireless/register.py +8 -0
- mobile_agent/wireless/relay.py +179 -0
- mobile_agent/wireless/result.py +87 -0
- mobile_agent/wireless/store.py +85 -0
- mobile_agent/wireless/switch.py +77 -0
- mobile_agent/wireless/transport.py +118 -0
- mobile_agent/wireless/words.py +81 -0
- mobile_agent/wireless/xctestrun.py +83 -0
- mobile_agent/workflow_tools.py +59 -0
- mobile_agent/workflows/__init__.py +2 -0
- mobile_agent/workflows/brief.py +201 -0
- mobile_agent/workflows/cron.py +110 -0
- mobile_agent/workflows/health.py +257 -0
- mobile_agent/workflows/history.py +126 -0
- mobile_agent/workflows/records.py +442 -0
- mobile_agent/workflows/runs.py +1000 -0
- mobile_agent/workflows/scheduler.py +134 -0
- mobile_agent/workflows/service.py +973 -0
- mobile_agent/workflows/starters.py +285 -0
- mobile_agent/workflows/triggers.py +248 -0
- mobile_agent/xray/__init__.py +1 -0
- mobile_agent/xray/app_cli.py +116 -0
- mobile_agent/xray/cli.py +21 -0
- mobile_agent/xray/crawl.py +754 -0
- mobile_agent/xray/diff.py +171 -0
- mobile_agent/xray/diff_cli.py +43 -0
- mobile_agent/xray/job.py +184 -0
- mobile_agent/xray/paywall.py +188 -0
- mobile_agent/xray/policy.py +305 -0
- mobile_agent/xray/register.py +43 -0
- mobile_agent/xray/report.py +283 -0
- mobile_agent/xray/screens.py +117 -0
- mobster_cli-0.4.3.dist-info/METADATA +93 -0
- mobster_cli-0.4.3.dist-info/RECORD +1009 -0
- mobster_cli-0.4.3.dist-info/WHEEL +5 -0
- mobster_cli-0.4.3.dist-info/entry_points.txt +3 -0
- mobster_cli-0.4.3.dist-info/licenses/LICENSE +21 -0
- mobster_cli-0.4.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,993 @@
|
|
|
1
|
+
// phone-audio: hears what a USB iPhone plays, on this Mac, for Mobster's WATCH and MCP's listen (mobile_agent/listen.py).
|
|
2
|
+
//
|
|
3
|
+
// phone-audio devices
|
|
4
|
+
// The iPhones this Mac can hear now: {"ok":true,"devices":[{"id","name","model"}]}.
|
|
5
|
+
// phone-audio listen --seconds N [--udid UDID [--name NAME] [--phones K]] [--locale en-US] [--work DIR]
|
|
6
|
+
// [--keep PATH] [--live [--frames]]
|
|
7
|
+
// Records N seconds (1 to 120) of the iPhone's audio, then turns its speech into text on this Mac:
|
|
8
|
+
// {"ok":true,"text","segments","engine","locale","seconds","recordedSeconds","findMs","startMs","recordMs",
|
|
9
|
+
// "transcribeMs","peakDb","match","soundWindows","classifyMs"}. The iPhone is the capture device whose uniqueID
|
|
10
|
+
// is UDID, else the one named NAME (the phone's name as the device layer reads it), else the only iOS capture
|
|
11
|
+
// device when K, the USB iPhones the device layer sees, is 1; with no UDID, the only one. match says which: udid,
|
|
12
|
+
// name or only. segments are [{"start","end","text"}], in seconds from the recording's start; soundWindows are
|
|
13
|
+
// the sounds told apart in it (see below).
|
|
14
|
+
// The recording is a private file in DIR (else the temporary folder), deleted before this exits, unless --keep
|
|
15
|
+
// names a file to keep it in ("kept").
|
|
16
|
+
// With --live (WATCH, mobile_agent/watch.py): {"event":"recording"} once the recording has started, then
|
|
17
|
+
// {"event":"level","t","db"} every half second (the audio's average power), and the recording ends early when
|
|
18
|
+
// stdin reads a line "stop" or closes. With --frames too, the phone's picture from the same capture:
|
|
19
|
+
// {"event":"frame","seq","path","t"} about 4 times a second, a 590-pixel-wide JPEG in the private folder (only
|
|
20
|
+
// the newest 8 kept), so WATCH needn't read the screen through WebDriverAgent while the capture runs (opening
|
|
21
|
+
// it resets the phone's USB network link, which ends WebDriverAgent's runner). The answer is still the last
|
|
22
|
+
// line. Before the first capture macOS asks whether Mobster may use the camera and the microphone (see
|
|
23
|
+
// authorize); while it asks, {"event":"asking","service"} comes first.
|
|
24
|
+
// phone-audio record-video --seconds N --udid UDID [--name NAME] [--phones K] --work DIR
|
|
25
|
+
// Records 1 to 300 seconds of the iPhone's screen and the audio it offers into DIR/video.mov, picking the
|
|
26
|
+
// capture device as listen does: {"ok":true,"seconds","durationSeconds","audio","bytes","match"}. DIR must be a
|
|
27
|
+
// private folder (mode 0700, yours); the caller deletes it, even if this process is killed. A capture device
|
|
28
|
+
// another app is using is never taken over: capture_busy.
|
|
29
|
+
// phone-audio match [--udid UDID] [--name NAME] [--phones K] < [{"id","name","model"}]
|
|
30
|
+
// Which device listen would pick from that list: {"ok":true,"index","match"}, or listen's failure. For tests and
|
|
31
|
+
// diagnosis; nothing is captured.
|
|
32
|
+
// phone-audio transcribe FILE [--locale en-US]
|
|
33
|
+
// The speech in an audio file, the same way: {"ok":true,"text","segments","engine","locale","transcribeMs"}.
|
|
34
|
+
// phone-audio classify FILE
|
|
35
|
+
// The sounds in an audio file, the way listen classifies them: {"ok":true,"soundWindows","classifyMs"}. For tests
|
|
36
|
+
// and diagnosis; nothing is captured.
|
|
37
|
+
// phone-audio status [--locale en-US]
|
|
38
|
+
// What this Mac can understand, without asking for anything: {"ok":true,"speechAnalyzer","locale","sfSpeech":
|
|
39
|
+
// {"authorization","onDevice"},"capture":{"camera","microphone"}}. capture is granted, denied, restricted or
|
|
40
|
+
// not_determined for each. speechAnalyzer is installed, supported (its model isn't downloaded),
|
|
41
|
+
// downloading, unsupported (not this language) or unavailable (before macOS 26).
|
|
42
|
+
// phone-audio prepare [--locale en-US]
|
|
43
|
+
// Installs macOS's own on-device speech model for the language (SpeechAnalyzer's, the one Voice uses), once:
|
|
44
|
+
// {"ok":true,"speechAnalyzer":"installed","locale"}. Nothing else is downloaded, ever.
|
|
45
|
+
//
|
|
46
|
+
// Failures are {"ok":false,"error","message"}: no_device (no USB iPhone to hear), ambiguous_device (several iPhones,
|
|
47
|
+
// and none is surely the one asked for: it never guesses), permission (macOS doesn't let Mobster capture the iPhone:
|
|
48
|
+
// denied, refused without asking, or its prompt unanswered for ASK_SECONDS), capture_failed, no_engine (no on-device speech recognition for that language), speech_denied,
|
|
49
|
+
// transcribe_failed. Exit 0 either way; 2 for a usage error.
|
|
50
|
+
//
|
|
51
|
+
// How it hears: CoreMediaIO's kCMIOHardwarePropertyAllowScreenCaptureDevices, turned on in this process only, makes a
|
|
52
|
+
// USB iPhone appear as a capture device: the one QuickTime Player records from. AVCaptureAudioFileOutput records its
|
|
53
|
+
// audio track as 16-bit mono WAV. Nothing is tapped on the phone, and nothing is captured outside the N seconds.
|
|
54
|
+
//
|
|
55
|
+
// How it understands: SpeechAnalyzer with SpeechTranscriber on macOS 26 and later, else SFSpeechRecognizer with
|
|
56
|
+
// requiresOnDeviceRecognition. Both run on this Mac. SpeechAnalyzer downloads Apple's speech model once when it is
|
|
57
|
+
// missing; if installation fails, on-device SFSpeechRecognizer is tried. There is no server recognition fallback:
|
|
58
|
+
// without an available on-device model, it answers no_engine. What it hears is data: it goes to stdout as text.
|
|
59
|
+
//
|
|
60
|
+
// How it tells sounds apart: SoundAnalysis's built-in classifier (SNClassifySoundRequest, 303 labels: music and its
|
|
61
|
+
// instruments, laughter, applause, a dog, a siren), on this Mac, alongside the transcription and on the same
|
|
62
|
+
// recording. soundWindows are [{"t","d","labels":{label: confidence}}]: a 3-second window every 1.5 s (the whole
|
|
63
|
+
// file when it is shorter), each with its labels at least SOUND_FLOOR sure, at most SOUND_LABELS of them.
|
|
64
|
+
// mobile_agent/listen.py collapses them into timed sounds. A classifier failure leaves soundWindows out and says why
|
|
65
|
+
// in soundsError; the words still stand.
|
|
66
|
+
|
|
67
|
+
import AVFoundation
|
|
68
|
+
import CoreImage
|
|
69
|
+
import CoreMediaIO
|
|
70
|
+
import Foundation
|
|
71
|
+
import SoundAnalysis
|
|
72
|
+
import Speech
|
|
73
|
+
|
|
74
|
+
// MARK: - Output
|
|
75
|
+
|
|
76
|
+
func emit(_ value: [String: Any]) {
|
|
77
|
+
if let data = try? JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]),
|
|
78
|
+
let line = String(data: data, encoding: .utf8) {
|
|
79
|
+
print(line)
|
|
80
|
+
fflush(stdout)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// Folders to delete before exiting, whichever way it exits: the recording never outlives this process unasked.
|
|
85
|
+
var cleanup: [URL] = []
|
|
86
|
+
|
|
87
|
+
func removeTemporary() {
|
|
88
|
+
for url in cleanup { try? FileManager.default.removeItem(at: url) }
|
|
89
|
+
cleanup = []
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
func fail(_ error: String, _ message: String) -> Never {
|
|
93
|
+
removeTemporary()
|
|
94
|
+
emit(["ok": false, "error": error, "message": message])
|
|
95
|
+
exit(0)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/// Turns the main run loop until `done()` or `timeout` seconds: AVFoundation and Speech deliver here.
|
|
99
|
+
@discardableResult
|
|
100
|
+
func pump(timeout: TimeInterval, until done: () -> Bool) -> Bool {
|
|
101
|
+
let deadline = Date().addingTimeInterval(timeout)
|
|
102
|
+
while !done() {
|
|
103
|
+
if Date() >= deadline { return false }
|
|
104
|
+
RunLoop.current.run(until: Date().addingTimeInterval(0.02))
|
|
105
|
+
}
|
|
106
|
+
return true
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
func usage() -> Never {
|
|
110
|
+
FileHandle.standardError.write(Data(("usage: phone-audio devices | phone-audio listen --seconds N [--udid UDID] [--name NAME] "
|
|
111
|
+
+ "[--phones K] [--locale L] [--work DIR] [--keep PATH] [--live [--frames]] | phone-audio match [--udid UDID] "
|
|
112
|
+
+ "[--name NAME] [--phones K] < devices.json | phone-audio transcribe FILE [--locale L] "
|
|
113
|
+
+ "| phone-audio classify FILE | phone-audio record-video --seconds N --udid UDID [--name NAME] "
|
|
114
|
+
+ "[--phones K] --work DIR\n").utf8))
|
|
115
|
+
exit(2)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
func milliseconds(since start: Date) -> Int { Int((Date().timeIntervalSince(start) * 1000).rounded()) }
|
|
119
|
+
|
|
120
|
+
// MARK: - Arguments
|
|
121
|
+
|
|
122
|
+
var arguments = Array(CommandLine.arguments.dropFirst())
|
|
123
|
+
guard let command = arguments.first else { usage() }
|
|
124
|
+
arguments.removeFirst()
|
|
125
|
+
|
|
126
|
+
func option(_ name: String) -> String? {
|
|
127
|
+
guard let index = arguments.firstIndex(of: name), index + 1 < arguments.count else { return nil }
|
|
128
|
+
return arguments[index + 1]
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// MARK: - The iPhone as a capture device
|
|
132
|
+
|
|
133
|
+
/// Lets this process see screen-capture devices (USB iPhones). Only this process: QuickTime's own setting is untouched.
|
|
134
|
+
func allowScreenCaptureDevices() {
|
|
135
|
+
var address = CMIOObjectPropertyAddress(
|
|
136
|
+
mSelector: CMIOObjectPropertySelector(kCMIOHardwarePropertyAllowScreenCaptureDevices),
|
|
137
|
+
mScope: CMIOObjectPropertyScope(kCMIOObjectPropertyScopeGlobal),
|
|
138
|
+
mElement: CMIOObjectPropertyElement(kCMIOObjectPropertyElementMain))
|
|
139
|
+
var allow: UInt32 = 1
|
|
140
|
+
CMIOObjectSetPropertyData(CMIOObjectID(kCMIOObjectSystemObject), &address, 0, nil,
|
|
141
|
+
UInt32(MemoryLayout<UInt32>.size), &allow)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/// The iPhones this Mac can capture: external muxed devices made by Apple.
|
|
145
|
+
func iphones() -> [AVCaptureDevice] {
|
|
146
|
+
let types: [AVCaptureDevice.DeviceType]
|
|
147
|
+
if #available(macOS 14.0, *) { types = [.external] } else { types = [.externalUnknown] }
|
|
148
|
+
return AVCaptureDevice.DiscoverySession(deviceTypes: types, mediaType: .muxed, position: .unspecified).devices
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/// A UDID as the capture device's uniqueID may spell it: no dashes, upper case.
|
|
152
|
+
func plain(_ id: String) -> String { id.replacingOccurrences(of: "-", with: "").uppercased() }
|
|
153
|
+
|
|
154
|
+
/// A name compared the way people read it: its words, any case.
|
|
155
|
+
func folded(_ name: String) -> String {
|
|
156
|
+
name.split(whereSeparator: \.isWhitespace).joined(separator: " ").folding(options: [.caseInsensitive,
|
|
157
|
+
.diacriticInsensitive],
|
|
158
|
+
locale: nil)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/// What the matching reads of a capture device: AVCaptureDevice's uniqueID, localizedName and modelID.
|
|
162
|
+
struct Candidate { let id: String; let name: String; let model: String }
|
|
163
|
+
|
|
164
|
+
/// Which iPhone to hear, and how it was told: "udid", "name" or "only" (the only iOS capture device, on the only
|
|
165
|
+
/// USB iPhone). ambiguous(n): n iOS capture devices and none is the one asked for. none: no iOS capture device fits.
|
|
166
|
+
enum Pick: Equatable { case found(Int, String), ambiguous(Int), none }
|
|
167
|
+
|
|
168
|
+
/// An iPhone or iPad as a capture device: AVFoundation's modelID for one is "iOS Device".
|
|
169
|
+
func looksLikeIOS(_ device: Candidate) -> Bool {
|
|
170
|
+
let model = device.model.lowercased()
|
|
171
|
+
return model.contains("ios") || model.hasPrefix("iphone") || model.hasPrefix("ipad")
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/// The capture device for the iPhone asked for. On some Macs an iPhone's capture device has a uniqueID of its own,
|
|
175
|
+
/// not its UDID (measured 13 Oct 2026), so after the UDID it tries the phone's name as the device layer reads it
|
|
176
|
+
/// (`name`, passed only when no other USB iPhone shares it), then the only iOS capture device, when the device layer
|
|
177
|
+
/// sees one USB iPhone (`phones`) or no UDID was asked for. Never a guess between two phones.
|
|
178
|
+
func pick(_ found: [Candidate], udid: String?, name: String?, phones: Int?) -> Pick {
|
|
179
|
+
if let udid = udid, let index = found.firstIndex(where: { plain($0.id) == plain(udid) }) {
|
|
180
|
+
return .found(index, "udid")
|
|
181
|
+
}
|
|
182
|
+
let ios = found.indices.filter { looksLikeIOS(found[$0]) }
|
|
183
|
+
if let name = name, !folded(name).isEmpty {
|
|
184
|
+
let named = ios.filter { folded(found[$0].name) == folded(name) }
|
|
185
|
+
if named.count == 1 { return .found(named[0], "name") }
|
|
186
|
+
if named.count > 1 { return .ambiguous(named.count) }
|
|
187
|
+
}
|
|
188
|
+
if ios.count == 1 && (udid == nil || phones == 1) { return .found(ios[0], "only") }
|
|
189
|
+
if ios.count > 1 || (ios.count == 1 && (phones ?? 0) > 1) { return .ambiguous(max(ios.count, phones ?? 0)) }
|
|
190
|
+
return .none
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
func candidates(_ devices: [AVCaptureDevice]) -> [Candidate] {
|
|
194
|
+
devices.map { Candidate(id: $0.uniqueID, name: $0.localizedName, model: $0.modelID) }
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/// The iPhone to hear (see pick). They appear a moment after the property is set, so look for up to `timeout`
|
|
198
|
+
/// seconds; the answer then is the last look's.
|
|
199
|
+
func findPhone(udid: String?, name: String?, phones: Int?, timeout: TimeInterval) -> (AVCaptureDevice?, Pick) {
|
|
200
|
+
let deadline = Date().addingTimeInterval(timeout)
|
|
201
|
+
var last = Pick.none
|
|
202
|
+
repeat {
|
|
203
|
+
let found = iphones()
|
|
204
|
+
last = pick(candidates(found), udid: udid, name: name, phones: phones)
|
|
205
|
+
if case .found(let index, _) = last { return (found[index], last) }
|
|
206
|
+
RunLoop.current.run(until: Date().addingTimeInterval(0.1))
|
|
207
|
+
} while Date() < deadline
|
|
208
|
+
return (nil, last)
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/// Why no iPhone was picked, as listen's failure: what happened, then what to do. It names the phone asked for.
|
|
212
|
+
func refuse(_ result: Pick, name: String?) -> Never {
|
|
213
|
+
let phone = name.map { "\u{201C}\($0)\u{201D}" }
|
|
214
|
+
if case .ambiguous(let count) = result {
|
|
215
|
+
fail("ambiguous_device", phone.map {
|
|
216
|
+
"\(count) iPhones are plugged in, and Mobster can't tell which one is \($0). Unplug the others, then try again."
|
|
217
|
+
} ?? "\(count) iPhones are plugged in, and Mobster can't tell which one to hear. Unplug the others, then try again.")
|
|
218
|
+
}
|
|
219
|
+
fail("no_device", phone.map {
|
|
220
|
+
"No iPhone to hear: macOS shows no capture device for \($0). Plug it in with a cable, unlock it, then try again."
|
|
221
|
+
} ?? "No iPhone to hear: plug one in with a cable and unlock it.")
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// MARK: - Permission
|
|
225
|
+
|
|
226
|
+
/// What macOS says when Mobster may not capture the iPhone: listen's failure "permission".
|
|
227
|
+
let PERMISSION_MESSAGE = "Mobster needs permission to hear your iPhone. Allow it in System Settings › Privacy & Security › Microphone/Camera."
|
|
228
|
+
/// How long the person has to answer macOS's prompt before listen gives up (the prompt stays up; the next try hears).
|
|
229
|
+
let ASK_SECONDS = 60.0
|
|
230
|
+
|
|
231
|
+
func authorizationName(_ status: AVAuthorizationStatus) -> String {
|
|
232
|
+
switch status {
|
|
233
|
+
case .authorized: return "granted"
|
|
234
|
+
case .denied: return "denied"
|
|
235
|
+
case .restricted: return "restricted"
|
|
236
|
+
case .notDetermined: return "not_determined"
|
|
237
|
+
@unknown default: return "unknown"
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/// Makes sure macOS lets this process capture the iPhone, before anything is opened. macOS counts the iPhone's
|
|
242
|
+
/// capture device as a camera with a microphone and asks for both, attributed to the app that started this helper
|
|
243
|
+
/// (Mobster.app, which must hold the camera and audio-input entitlements: tccd checks the responsible app, not this
|
|
244
|
+
/// helper). Granted: returns. Denied, or refused without a prompt (tccd's policy: the status then stays
|
|
245
|
+
/// not_determined): fails "permission". Not asked yet: asks, and with `live` says {"event":"asking","service"} first,
|
|
246
|
+
/// so WATCH can wait for the answer; `stop` (WATCH gave up waiting) ends the wait as an unanswered prompt.
|
|
247
|
+
func authorize(live: Bool, stop: StopSignal? = nil) {
|
|
248
|
+
for (media, service) in [(AVMediaType.video, "camera"), (AVMediaType.audio, "microphone")] {
|
|
249
|
+
switch AVCaptureDevice.authorizationStatus(for: media) {
|
|
250
|
+
case .authorized:
|
|
251
|
+
continue
|
|
252
|
+
case .notDetermined:
|
|
253
|
+
if live { emit(["event": "asking", "service": service]) }
|
|
254
|
+
let answer = SendableBox<Bool>()
|
|
255
|
+
AVCaptureDevice.requestAccess(for: media) { answer.value = $0 }
|
|
256
|
+
pump(timeout: ASK_SECONDS) { answer.value != nil || stop?.isSet == true }
|
|
257
|
+
guard answer.value != nil else {
|
|
258
|
+
fail("permission", "macOS is asking whether Mobster may hear your iPhone. Answer its prompt, then try again.")
|
|
259
|
+
}
|
|
260
|
+
if answer.value != true { fail("permission", PERMISSION_MESSAGE) }
|
|
261
|
+
default:
|
|
262
|
+
fail("permission", PERMISSION_MESSAGE)
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// MARK: - Recording
|
|
268
|
+
|
|
269
|
+
final class Recorder: NSObject, AVCaptureFileOutputRecordingDelegate {
|
|
270
|
+
private let lock = NSLock()
|
|
271
|
+
private var finished = false
|
|
272
|
+
private var started = false
|
|
273
|
+
private(set) var error: Error?
|
|
274
|
+
var done: Bool { lock.lock(); defer { lock.unlock() }; return finished }
|
|
275
|
+
var recording: Bool { lock.lock(); defer { lock.unlock() }; return started }
|
|
276
|
+
|
|
277
|
+
func fileOutput(_ output: AVCaptureFileOutput, didStartRecordingTo fileURL: URL, from connections: [AVCaptureConnection]) {
|
|
278
|
+
lock.lock()
|
|
279
|
+
started = true
|
|
280
|
+
lock.unlock()
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
func fileOutput(_ output: AVCaptureFileOutput, didFinishRecordingTo outputFileURL: URL,
|
|
284
|
+
from connections: [AVCaptureConnection], error: Error?) {
|
|
285
|
+
// Reaching the maximum duration is how a recording ends here, and AVFoundation reports it as an error.
|
|
286
|
+
lock.lock()
|
|
287
|
+
if let error = error as NSError?, !(error.userInfo[AVErrorRecordingSuccessfullyFinishedKey] as? Bool ?? false) {
|
|
288
|
+
self.error = error
|
|
289
|
+
}
|
|
290
|
+
finished = true
|
|
291
|
+
lock.unlock()
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/// Records the selected USB phone's screen, with its audio when it offers one, into `url`. A capture device that
|
|
296
|
+
/// another app is using is never taken over. Returns whether the movie has an audio track.
|
|
297
|
+
func recordVideo(_ device: AVCaptureDevice, seconds: Double, to url: URL) -> Bool {
|
|
298
|
+
guard !device.isInUseByAnotherApplication, !device.isSuspended else {
|
|
299
|
+
fail("capture_busy", "The iPhone's capture device is busy or suspended. Close its other capture first.")
|
|
300
|
+
}
|
|
301
|
+
let session = AVCaptureSession()
|
|
302
|
+
let input: AVCaptureDeviceInput
|
|
303
|
+
do { input = try AVCaptureDeviceInput(device: device) } catch {
|
|
304
|
+
if (error as NSError).code == AVError.applicationIsNotAuthorizedToUseDevice.rawValue {
|
|
305
|
+
fail("permission", PERMISSION_MESSAGE)
|
|
306
|
+
}
|
|
307
|
+
fail("capture_failed", "Mobster couldn't open the iPhone's video (\(error.localizedDescription)).")
|
|
308
|
+
}
|
|
309
|
+
guard session.canAddInput(input) else { fail("capture_failed", "The iPhone offered no capture input.") }
|
|
310
|
+
session.addInput(input)
|
|
311
|
+
let output = AVCaptureMovieFileOutput()
|
|
312
|
+
output.maxRecordedDuration = CMTime(seconds: seconds, preferredTimescale: 600)
|
|
313
|
+
guard session.canAddOutput(output) else { fail("capture_failed", "The iPhone offered no movie output.") }
|
|
314
|
+
session.addOutput(output)
|
|
315
|
+
guard output.connection(with: .video) != nil else {
|
|
316
|
+
fail("capture_failed", "The iPhone offered no video to record. Unlock it and check the USB cable.")
|
|
317
|
+
}
|
|
318
|
+
let audio = output.connection(with: .audio) != nil
|
|
319
|
+
let cancelled = SendableBox<Bool>()
|
|
320
|
+
// The signal sources run on the main queue, outside the signal handler. The movie is always finished first.
|
|
321
|
+
let signals = [SIGINT, SIGTERM].map { number -> DispatchSourceSignal in
|
|
322
|
+
signal(number, SIG_IGN)
|
|
323
|
+
let source = DispatchSource.makeSignalSource(signal: number, queue: .main)
|
|
324
|
+
source.setEventHandler { cancelled.value = true }
|
|
325
|
+
source.resume()
|
|
326
|
+
return source
|
|
327
|
+
}
|
|
328
|
+
defer { for source in signals { source.cancel() } }
|
|
329
|
+
session.startRunning()
|
|
330
|
+
let recorder = Recorder()
|
|
331
|
+
output.startRecording(to: url, recordingDelegate: recorder)
|
|
332
|
+
let completed = pump(timeout: seconds + 10) { recorder.done || cancelled.value == true }
|
|
333
|
+
if !recorder.done {
|
|
334
|
+
output.stopRecording()
|
|
335
|
+
pump(timeout: 5) { recorder.done }
|
|
336
|
+
}
|
|
337
|
+
session.stopRunning()
|
|
338
|
+
if cancelled.value == true { fail("cancelled", "The recording was cancelled.") }
|
|
339
|
+
guard completed, recorder.done else { fail("capture_failed", "The iPhone's video capture didn't finish in time.") }
|
|
340
|
+
if let error = recorder.error {
|
|
341
|
+
fail("capture_failed", "The video recording stopped early (\(error.localizedDescription)).")
|
|
342
|
+
}
|
|
343
|
+
return audio
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/// Set once stdin reads "stop" or closes (--live): the recording then ends early, and what it has is kept.
|
|
347
|
+
final class StopSignal: @unchecked Sendable {
|
|
348
|
+
private let lock = NSLock()
|
|
349
|
+
private var stopped = false
|
|
350
|
+
var isSet: Bool { lock.lock(); defer { lock.unlock() }; return stopped }
|
|
351
|
+
func set() { lock.lock(); stopped = true; lock.unlock() }
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
func watchStdin(_ signal: StopSignal) {
|
|
355
|
+
Thread.detachNewThread {
|
|
356
|
+
while let line = readLine() {
|
|
357
|
+
if line.trimmingCharacters(in: .whitespaces) == "stop" { break }
|
|
358
|
+
}
|
|
359
|
+
signal.set()
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/// The phone's picture from the same capture (--frames DIR, WATCH): while it runs, WebDriverAgent can't be relied on
|
|
364
|
+
/// (opening the capture resets the phone's USB network link, which ends its runner), so WATCH reads the screen here.
|
|
365
|
+
/// FRAME_HZ JPEGs a second, FRAME_WIDTH pixels wide, as DIR/frame-<seq>.jpg; only the newest FRAMES_KEPT stay.
|
|
366
|
+
let FRAME_HZ = 4.0
|
|
367
|
+
let FRAME_WIDTH = 590.0
|
|
368
|
+
let FRAMES_KEPT = 8
|
|
369
|
+
|
|
370
|
+
final class FrameWriter: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, @unchecked Sendable {
|
|
371
|
+
private let lock = NSLock()
|
|
372
|
+
private let folder: URL
|
|
373
|
+
private let context = CIContext()
|
|
374
|
+
private var last = 0.0
|
|
375
|
+
private var written: [URL] = []
|
|
376
|
+
private(set) var seq = 0
|
|
377
|
+
private var newest: (Int, String)?
|
|
378
|
+
init(folder: URL) { self.folder = folder }
|
|
379
|
+
|
|
380
|
+
/// The newest frame written since the last call, as (seq, path), else nil.
|
|
381
|
+
func take() -> (Int, String)? { lock.lock(); defer { lock.unlock() }; let value = newest; newest = nil; return value }
|
|
382
|
+
|
|
383
|
+
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
|
|
384
|
+
let now = Date().timeIntervalSinceReferenceDate
|
|
385
|
+
guard now - last >= 1 / FRAME_HZ, let pixels = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
|
386
|
+
last = now
|
|
387
|
+
var image = CIImage(cvPixelBuffer: pixels)
|
|
388
|
+
let width = image.extent.width
|
|
389
|
+
if width > FRAME_WIDTH { image = image.transformed(by: CGAffineTransform(scaleX: FRAME_WIDTH / width, y: FRAME_WIDTH / width)) }
|
|
390
|
+
guard let space = CGColorSpace(name: CGColorSpace.sRGB),
|
|
391
|
+
let data = context.jpegRepresentation(of: image, colorSpace: space,
|
|
392
|
+
options: [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 0.7])
|
|
393
|
+
else { return }
|
|
394
|
+
lock.lock()
|
|
395
|
+
seq += 1
|
|
396
|
+
let url = folder.appendingPathComponent("frame-\(seq).jpg")
|
|
397
|
+
lock.unlock()
|
|
398
|
+
guard (try? data.write(to: url, options: .atomic)) != nil else { return }
|
|
399
|
+
lock.lock()
|
|
400
|
+
written.append(url)
|
|
401
|
+
let old = written.count > FRAMES_KEPT ? written.removeFirst() : nil
|
|
402
|
+
newest = (seq, url.path)
|
|
403
|
+
lock.unlock()
|
|
404
|
+
if let old = old { try? FileManager.default.removeItem(at: old) }
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/// The sound through a data output, for --frames: AVCaptureAudioFileOutput refuses to record ("Cannot Record") once a
|
|
409
|
+
/// video output shares the session (measured on the test device, 14 Oct 2026), so the samples are written to the same WAV
|
|
410
|
+
/// (16-bit mono at 16 kHz) here. It stops taking samples at `limit` seconds, and keeps the newest level in dB.
|
|
411
|
+
final class AudioWriter: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate, @unchecked Sendable {
|
|
412
|
+
private let lock = NSLock()
|
|
413
|
+
private var file: AVAudioFile?
|
|
414
|
+
private let limit: Double
|
|
415
|
+
private var frames: AVAudioFramePosition = 0
|
|
416
|
+
private var started = false
|
|
417
|
+
private var power = -160.0
|
|
418
|
+
private(set) var failure: String?
|
|
419
|
+
let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16_000, channels: 1, interleaved: true)!
|
|
420
|
+
|
|
421
|
+
init(url: URL, limit: Double) {
|
|
422
|
+
self.limit = limit
|
|
423
|
+
var made: AVAudioFile?
|
|
424
|
+
do {
|
|
425
|
+
made = try AVAudioFile(forWriting: url, settings: [AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: 16_000,
|
|
426
|
+
AVNumberOfChannelsKey: 1, AVLinearPCMBitDepthKey: 16,
|
|
427
|
+
AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false],
|
|
428
|
+
commonFormat: .pcmFormatInt16, interleaved: true)
|
|
429
|
+
} catch { failure = error.localizedDescription }
|
|
430
|
+
file = made
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
var recording: Bool { lock.lock(); defer { lock.unlock() }; return started }
|
|
434
|
+
var seconds: Double { lock.lock(); defer { lock.unlock() }; return Double(frames) / 16_000 }
|
|
435
|
+
var full: Bool { seconds >= limit }
|
|
436
|
+
var level: Double { lock.lock(); defer { lock.unlock() }; return power }
|
|
437
|
+
|
|
438
|
+
/// Closes the WAV (its header is written when the file goes), so it can be read.
|
|
439
|
+
func finish() { lock.lock(); file = nil; lock.unlock() }
|
|
440
|
+
|
|
441
|
+
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
|
|
442
|
+
lock.lock(); let file = self.file; lock.unlock()
|
|
443
|
+
guard let file = file, !full else { return }
|
|
444
|
+
let count = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
|
|
445
|
+
guard count > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: count) else { return }
|
|
446
|
+
buffer.frameLength = count
|
|
447
|
+
let status = CMSampleBufferCopyPCMDataIntoAudioBufferList(sampleBuffer, at: 0, frameCount: Int32(count),
|
|
448
|
+
into: buffer.mutableAudioBufferList)
|
|
449
|
+
guard status == noErr, let samples = buffer.int16ChannelData?[0] else { return }
|
|
450
|
+
var sum = 0.0
|
|
451
|
+
for index in 0..<Int(count) { let value = Double(samples[index]) / 32768; sum += value * value }
|
|
452
|
+
let rms = (sum / Double(count)).squareRoot()
|
|
453
|
+
do { try file.write(from: buffer) } catch {
|
|
454
|
+
lock.lock(); failure = error.localizedDescription; lock.unlock(); return
|
|
455
|
+
}
|
|
456
|
+
lock.lock()
|
|
457
|
+
frames += AVAudioFramePosition(count)
|
|
458
|
+
started = true
|
|
459
|
+
power = rms > 0 ? max(-160, 20 * log10(rms)) : -160
|
|
460
|
+
lock.unlock()
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/// Records `seconds` of the phone's audio into `url` (WAV, 16-bit mono at 16 kHz). Returns how long the capture took
|
|
465
|
+
/// to start, in milliseconds, and how many seconds were recorded. With `live`, it reports the start and the level as
|
|
466
|
+
/// it goes, and `stop` ends it early. With `frames`, it also writes the phone's picture there and says
|
|
467
|
+
/// {"event":"frame","seq","path","t"} for each (FrameWriter).
|
|
468
|
+
func record(_ device: AVCaptureDevice, seconds: Double, to url: URL, live: Bool = false,
|
|
469
|
+
stop: StopSignal? = nil, frames: URL? = nil) -> (Int, Double) {
|
|
470
|
+
let session = AVCaptureSession()
|
|
471
|
+
let input: AVCaptureDeviceInput
|
|
472
|
+
do {
|
|
473
|
+
input = try AVCaptureDeviceInput(device: device)
|
|
474
|
+
} catch {
|
|
475
|
+
let code = (error as NSError).code
|
|
476
|
+
if code == AVError.applicationIsNotAuthorizedToUseDevice.rawValue { fail("permission", PERMISSION_MESSAGE) }
|
|
477
|
+
fail("capture_failed", "Mobster couldn't open the iPhone's audio (\(error.localizedDescription)).")
|
|
478
|
+
}
|
|
479
|
+
guard session.canAddInput(input) else { fail("capture_failed", "Mobster couldn't open the iPhone's audio.") }
|
|
480
|
+
session.addInput(input)
|
|
481
|
+
if let frames = frames {
|
|
482
|
+
return recordWithFrames(session, seconds: seconds, to: url, live: live, stop: stop, frames: frames)
|
|
483
|
+
}
|
|
484
|
+
let output = AVCaptureAudioFileOutput()
|
|
485
|
+
output.audioSettings = [AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: 16_000, AVNumberOfChannelsKey: 1,
|
|
486
|
+
AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
|
|
487
|
+
AVLinearPCMIsNonInterleaved: false]
|
|
488
|
+
output.maxRecordedDuration = CMTime(seconds: seconds, preferredTimescale: 600)
|
|
489
|
+
guard session.canAddOutput(output) else { fail("capture_failed", "Mobster couldn't record the iPhone's audio.") }
|
|
490
|
+
session.addOutput(output)
|
|
491
|
+
guard output.connection(with: .audio) != nil else {
|
|
492
|
+
fail("capture_failed", "The iPhone offered no audio to record.")
|
|
493
|
+
}
|
|
494
|
+
let started = Date()
|
|
495
|
+
session.startRunning()
|
|
496
|
+
let recorder = Recorder()
|
|
497
|
+
output.startRecording(to: url, outputFileType: .wav, recordingDelegate: recorder)
|
|
498
|
+
let waited = milliseconds(since: started)
|
|
499
|
+
var announced = false
|
|
500
|
+
var stopping = false
|
|
501
|
+
var nextLevel = Date()
|
|
502
|
+
let finished = pump(timeout: seconds + 10) {
|
|
503
|
+
if recorder.done { return true }
|
|
504
|
+
if live && !announced && recorder.recording {
|
|
505
|
+
announced = true
|
|
506
|
+
emit(["event": "recording", "startMs": milliseconds(since: started)])
|
|
507
|
+
}
|
|
508
|
+
if live && announced && Date() >= nextLevel {
|
|
509
|
+
nextLevel = Date().addingTimeInterval(0.5)
|
|
510
|
+
let channels = output.connection(with: .audio)?.audioChannels ?? []
|
|
511
|
+
let level = channels.map { Double($0.averagePowerLevel) }.max() ?? -160
|
|
512
|
+
emit(["event": "level", "t": (output.recordedDuration.seconds * 10).rounded() / 10,
|
|
513
|
+
"db": (max(-160, level) * 10).rounded() / 10])
|
|
514
|
+
}
|
|
515
|
+
if let stop = stop, stop.isSet, !stopping {
|
|
516
|
+
stopping = true
|
|
517
|
+
output.stopRecording()
|
|
518
|
+
}
|
|
519
|
+
return false
|
|
520
|
+
}
|
|
521
|
+
if !finished {
|
|
522
|
+
output.stopRecording()
|
|
523
|
+
pump(timeout: 3) { recorder.done }
|
|
524
|
+
}
|
|
525
|
+
let recorded = output.recordedDuration.seconds
|
|
526
|
+
session.stopRunning()
|
|
527
|
+
if let error = recorder.error {
|
|
528
|
+
fail("capture_failed", "The recording stopped early (\(error.localizedDescription)).")
|
|
529
|
+
}
|
|
530
|
+
return (waited, recorded.isFinite ? (recorded * 10).rounded() / 10 : seconds)
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/// record() for --frames: the sound through AudioWriter and the picture through FrameWriter, on one session. Says
|
|
534
|
+
/// {"event":"recording"} at the first sound, {"event":"level"} every half second and {"event":"frame"} for each picture.
|
|
535
|
+
func recordWithFrames(_ session: AVCaptureSession, seconds: Double, to url: URL, live: Bool, stop: StopSignal?,
|
|
536
|
+
frames: URL) -> (Int, Double) {
|
|
537
|
+
let audio = AVCaptureAudioDataOutput()
|
|
538
|
+
audio.audioSettings = [AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: 16_000, AVNumberOfChannelsKey: 1,
|
|
539
|
+
AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
|
|
540
|
+
AVLinearPCMIsNonInterleaved: false]
|
|
541
|
+
let sound = AudioWriter(url: url, limit: seconds)
|
|
542
|
+
if let failure = sound.failure { fail("capture_failed", "Mobster couldn't record the iPhone's audio (\(failure)).") }
|
|
543
|
+
guard session.canAddOutput(audio) else { fail("capture_failed", "Mobster couldn't record the iPhone's audio.") }
|
|
544
|
+
session.addOutput(audio)
|
|
545
|
+
audio.setSampleBufferDelegate(sound, queue: DispatchQueue(label: "phone-audio.sound"))
|
|
546
|
+
// The picture is a bonus: without it WATCH falls back to WebDriverAgent's frames, and the sound still records.
|
|
547
|
+
let video = AVCaptureVideoDataOutput()
|
|
548
|
+
video.alwaysDiscardsLateVideoFrames = true
|
|
549
|
+
video.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
|
|
550
|
+
let writer = FrameWriter(folder: frames)
|
|
551
|
+
if session.canAddOutput(video) {
|
|
552
|
+
session.addOutput(video)
|
|
553
|
+
video.setSampleBufferDelegate(writer, queue: DispatchQueue(label: "phone-audio.frames"))
|
|
554
|
+
}
|
|
555
|
+
let started = Date()
|
|
556
|
+
session.startRunning()
|
|
557
|
+
var waited: Int?
|
|
558
|
+
var nextLevel = Date()
|
|
559
|
+
pump(timeout: seconds + 10) {
|
|
560
|
+
if waited == nil && sound.recording {
|
|
561
|
+
waited = milliseconds(since: started)
|
|
562
|
+
if live { emit(["event": "recording", "startMs": waited!]) }
|
|
563
|
+
}
|
|
564
|
+
if live, let (seq, path) = writer.take() {
|
|
565
|
+
emit(["event": "frame", "seq": seq, "path": path, "t": (Date().timeIntervalSince(started) * 100).rounded() / 100])
|
|
566
|
+
}
|
|
567
|
+
if live && waited != nil && Date() >= nextLevel {
|
|
568
|
+
nextLevel = Date().addingTimeInterval(0.5)
|
|
569
|
+
emit(["event": "level", "t": (sound.seconds * 10).rounded() / 10, "db": (sound.level * 10).rounded() / 10])
|
|
570
|
+
}
|
|
571
|
+
return sound.full || stop?.isSet == true || sound.failure != nil
|
|
572
|
+
}
|
|
573
|
+
session.stopRunning()
|
|
574
|
+
sound.finish()
|
|
575
|
+
if let failure = sound.failure { fail("capture_failed", "The recording stopped early (\(failure)).") }
|
|
576
|
+
return (waited ?? milliseconds(since: started), (sound.seconds * 10).rounded() / 10)
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/// The loudest sample in decibels below full scale, so a caller can tell silence (nothing played) from speech it
|
|
580
|
+
/// couldn't understand. nil when the file can't be read.
|
|
581
|
+
func peakDecibels(_ url: URL) -> Double? {
|
|
582
|
+
guard let file = try? AVAudioFile(forReading: url),
|
|
583
|
+
let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: AVAudioFrameCount(file.length)),
|
|
584
|
+
(try? file.read(into: buffer)) != nil, let channel = buffer.floatChannelData?[0] else { return nil }
|
|
585
|
+
var peak: Float = 0
|
|
586
|
+
for index in 0..<Int(buffer.frameLength) { peak = max(peak, abs(channel[index])) }
|
|
587
|
+
return peak > 0 ? (Double(20 * log10(peak)) * 10).rounded() / 10 : -160
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// MARK: - Speech
|
|
591
|
+
|
|
592
|
+
/// What was said: the text, and the same text in timed pieces ("start" and "end" in seconds from the audio's start).
|
|
593
|
+
struct Heard { let text: String; let segments: [[String: Any]]; let engine: String; let locale: String }
|
|
594
|
+
|
|
595
|
+
func segment(_ start: Double, _ end: Double, _ text: String) -> [String: Any] {
|
|
596
|
+
["start": (start * 10).rounded() / 10, "end": (end * 10).rounded() / 10,
|
|
597
|
+
"text": text.split(whereSeparator: \.isWhitespace).joined(separator: " ")]
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
enum SpeechFailure: Error { case noEngine(String), denied, failed(String) }
|
|
601
|
+
|
|
602
|
+
func wanted(_ locale: String?) -> Locale { locale.map { Locale(identifier: $0) } ?? Locale.current }
|
|
603
|
+
|
|
604
|
+
@available(macOS 26.0, *)
|
|
605
|
+
func analyze(_ url: URL, locale: Locale) async throws -> Heard? {
|
|
606
|
+
guard SpeechTranscriber.isAvailable,
|
|
607
|
+
let resolved = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { return nil }
|
|
608
|
+
let transcriber = SpeechTranscriber(locale: resolved, preset: .transcription)
|
|
609
|
+
switch await AssetInventory.status(forModules: [transcriber]) {
|
|
610
|
+
case .installed:
|
|
611
|
+
break
|
|
612
|
+
case .supported, .downloading:
|
|
613
|
+
// macOS's own model for the language, as Voice uses: usually already on this Mac, so this only reserves it
|
|
614
|
+
// (measured 0.3 s on 11 Oct 2026). A missing model is downloaded from Apple; if that fails, the on-device
|
|
615
|
+
// SFSpeechRecognizer below is tried.
|
|
616
|
+
guard let request = try? await AssetInventory.assetInstallationRequest(supporting: [transcriber]),
|
|
617
|
+
(try? await request.downloadAndInstall()) != nil else {
|
|
618
|
+
if await AssetInventory.status(forModules: [transcriber]) != .installed { return nil }
|
|
619
|
+
break
|
|
620
|
+
}
|
|
621
|
+
default:
|
|
622
|
+
return nil
|
|
623
|
+
}
|
|
624
|
+
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
|
625
|
+
let collect = Task { () -> [(String, CMTimeRange)] in
|
|
626
|
+
var pieces: [(String, CMTimeRange)] = []
|
|
627
|
+
for try await result in transcriber.results where result.isFinal {
|
|
628
|
+
pieces.append((String(result.text.characters), result.range))
|
|
629
|
+
}
|
|
630
|
+
return pieces
|
|
631
|
+
}
|
|
632
|
+
let file = try AVAudioFile(forReading: url)
|
|
633
|
+
if let last = try await analyzer.analyzeSequence(from: file) {
|
|
634
|
+
try await analyzer.finalizeAndFinish(through: last)
|
|
635
|
+
} else {
|
|
636
|
+
await analyzer.cancelAndFinishNow()
|
|
637
|
+
}
|
|
638
|
+
let pieces = try await collect.value
|
|
639
|
+
let text = pieces.map { $0.0 }.joined(separator: " ")
|
|
640
|
+
let segments = pieces.filter { !$0.0.trimmingCharacters(in: .whitespaces).isEmpty }.map { piece in
|
|
641
|
+
segment(piece.1.start.seconds.isFinite ? piece.1.start.seconds : 0,
|
|
642
|
+
piece.1.end.seconds.isFinite ? piece.1.end.seconds : 0, piece.0)
|
|
643
|
+
}
|
|
644
|
+
return Heard(text: text.split(whereSeparator: \.isWhitespace).joined(separator: " "), segments: segments,
|
|
645
|
+
engine: "speechanalyzer", locale: resolved.identifier(.bcp47))
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
func recognize(_ url: URL, locale: Locale) async throws -> Heard {
|
|
649
|
+
guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.supportsOnDeviceRecognition else {
|
|
650
|
+
throw SpeechFailure.noEngine("This Mac has no on-device speech recognition for \(locale.identifier).")
|
|
651
|
+
}
|
|
652
|
+
var status = SFSpeechRecognizer.authorizationStatus()
|
|
653
|
+
if status == .notDetermined {
|
|
654
|
+
status = await withCheckedContinuation { (done: CheckedContinuation<SFSpeechRecognizerAuthorizationStatus, Never>) in
|
|
655
|
+
SFSpeechRecognizer.requestAuthorization { done.resume(returning: $0) }
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
guard status == .authorized else { throw SpeechFailure.denied }
|
|
659
|
+
let request = SFSpeechURLRecognitionRequest(url: url)
|
|
660
|
+
request.requiresOnDeviceRecognition = true
|
|
661
|
+
request.shouldReportPartialResults = false
|
|
662
|
+
request.addsPunctuation = true
|
|
663
|
+
let heard: (String, [[String: Any]]) = try await withCheckedThrowingContinuation { done in
|
|
664
|
+
var finished = false
|
|
665
|
+
recognizer.recognitionTask(with: request) { result, error in
|
|
666
|
+
guard !finished else { return }
|
|
667
|
+
if let result = result, result.isFinal {
|
|
668
|
+
finished = true
|
|
669
|
+
done.resume(returning: (result.bestTranscription.formattedString,
|
|
670
|
+
phrases(result.bestTranscription.segments)))
|
|
671
|
+
} else if let error = error as NSError? {
|
|
672
|
+
finished = true
|
|
673
|
+
// "No speech detected" is an answer: nothing was said.
|
|
674
|
+
if error.domain == "kAFAssistantErrorDomain" && error.code == 1110 {
|
|
675
|
+
done.resume(returning: ("", []))
|
|
676
|
+
} else {
|
|
677
|
+
done.resume(throwing: SpeechFailure.failed(error.localizedDescription))
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
return Heard(text: heard.0, segments: heard.1, engine: "sfspeech", locale: locale.identifier)
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/// SFSpeech's words, timed, as phrases: a new one after a pause of 0.7 s or more, or after 14 words.
|
|
686
|
+
func phrases(_ words: [SFTranscriptionSegment]) -> [[String: Any]] {
|
|
687
|
+
var out: [[String: Any]] = []
|
|
688
|
+
var current: [SFTranscriptionSegment] = []
|
|
689
|
+
func close() {
|
|
690
|
+
guard let first = current.first, let last = current.last else { return }
|
|
691
|
+
out.append(segment(first.timestamp, last.timestamp + last.duration,
|
|
692
|
+
current.map { $0.substring }.joined(separator: " ")))
|
|
693
|
+
current = []
|
|
694
|
+
}
|
|
695
|
+
for word in words {
|
|
696
|
+
if let last = current.last, word.timestamp - (last.timestamp + last.duration) >= 0.7 || current.count >= 14 {
|
|
697
|
+
close()
|
|
698
|
+
}
|
|
699
|
+
current.append(word)
|
|
700
|
+
}
|
|
701
|
+
close()
|
|
702
|
+
return out
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
func transcribe(_ url: URL, locale: String?) async throws -> Heard {
|
|
706
|
+
if #available(macOS 26.0, *), let heard = try await analyze(url, locale: wanted(locale)) {
|
|
707
|
+
return heard
|
|
708
|
+
}
|
|
709
|
+
return try await recognize(url, locale: wanted(locale))
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
func transcribed(_ url: URL, locale: String?) -> (Heard, Int) {
|
|
713
|
+
let started = Date()
|
|
714
|
+
let box = SendableBox<Result<Heard, Error>>()
|
|
715
|
+
Task.detached {
|
|
716
|
+
do { box.value = .success(try await transcribe(url, locale: locale)) } catch { box.value = .failure(error) }
|
|
717
|
+
}
|
|
718
|
+
pump(timeout: 300) { box.value != nil }
|
|
719
|
+
switch box.value {
|
|
720
|
+
case .success(let heard)?: return (heard, milliseconds(since: started))
|
|
721
|
+
case .failure(SpeechFailure.noEngine(let message))?: fail("no_engine", message)
|
|
722
|
+
case .failure(SpeechFailure.denied)?:
|
|
723
|
+
fail("speech_denied", "macOS didn't let Mobster use speech recognition. Allow it in System Settings › Privacy & Security › Speech Recognition.")
|
|
724
|
+
case .failure(let error)?: fail("transcribe_failed", "Mobster couldn't turn the audio into text (\(error.localizedDescription)).")
|
|
725
|
+
case nil: fail("transcribe_failed", "Mobster couldn't turn the audio into text.")
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
final class SendableBox<Value>: @unchecked Sendable {
|
|
730
|
+
private let lock = NSLock()
|
|
731
|
+
private var stored: Value?
|
|
732
|
+
var value: Value? {
|
|
733
|
+
get { lock.lock(); defer { lock.unlock() }; return stored }
|
|
734
|
+
set { lock.lock(); stored = newValue; lock.unlock() }
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// MARK: - Sounds
|
|
739
|
+
|
|
740
|
+
/// A label below this confidence isn't reported; at most this many per window, the surest first.
|
|
741
|
+
let SOUND_FLOOR = 0.3
|
|
742
|
+
let SOUND_LABELS = 6
|
|
743
|
+
/// The classifier's window and how far it moves (half a window: SoundAnalysis's default overlap).
|
|
744
|
+
let SOUND_WINDOW = 3.0
|
|
745
|
+
|
|
746
|
+
/// The classifier's windows as they come: start, length and the labels at least SOUND_FLOOR sure.
|
|
747
|
+
final class SoundWindows: NSObject, SNResultsObserving, @unchecked Sendable {
|
|
748
|
+
private let lock = NSLock()
|
|
749
|
+
private var rows: [[String: Any]] = []
|
|
750
|
+
private var failed: String?
|
|
751
|
+
var windows: [[String: Any]] { lock.lock(); defer { lock.unlock() }; return rows }
|
|
752
|
+
var failure: String? { lock.lock(); defer { lock.unlock() }; return failed }
|
|
753
|
+
|
|
754
|
+
func request(_ request: SNRequest, didProduce result: SNResult) {
|
|
755
|
+
guard let result = result as? SNClassificationResult else { return }
|
|
756
|
+
var labels: [String: Double] = [:]
|
|
757
|
+
for item in result.classifications.prefix(SOUND_LABELS) where item.confidence >= SOUND_FLOOR {
|
|
758
|
+
labels[item.identifier] = (item.confidence * 100).rounded() / 100
|
|
759
|
+
}
|
|
760
|
+
let start = result.timeRange.start.seconds, length = result.timeRange.duration.seconds
|
|
761
|
+
guard start.isFinite, length.isFinite else { return }
|
|
762
|
+
lock.lock()
|
|
763
|
+
rows.append(["t": (start * 10).rounded() / 10, "d": (length * 10).rounded() / 10, "labels": labels])
|
|
764
|
+
lock.unlock()
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
func request(_ request: SNRequest, didFailWithError error: Error) {
|
|
768
|
+
lock.lock()
|
|
769
|
+
failed = error.localizedDescription
|
|
770
|
+
lock.unlock()
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
/// The sounds in `url`, classified on this Mac: {"soundWindows","classifyMs"}, or {"soundsError","classifyMs"}.
|
|
775
|
+
/// Synchronous: listen runs it on its own thread while the speech is transcribed.
|
|
776
|
+
func classifySounds(_ url: URL) -> [String: Any] {
|
|
777
|
+
let started = Date()
|
|
778
|
+
guard #available(macOS 12.0, *) else {
|
|
779
|
+
return ["soundsError": "Telling sounds apart needs macOS 12 or later.", "classifyMs": 0]
|
|
780
|
+
}
|
|
781
|
+
do {
|
|
782
|
+
let file = try AVAudioFile(forReading: url)
|
|
783
|
+
let rate = file.processingFormat.sampleRate
|
|
784
|
+
let seconds = rate > 0 ? Double(file.length) / rate : 0
|
|
785
|
+
guard seconds >= 0.5 else { return ["soundWindows": [], "classifyMs": milliseconds(since: started)] }
|
|
786
|
+
let request = try SNClassifySoundRequest(classifierIdentifier: .version1)
|
|
787
|
+
request.windowDuration = CMTime(seconds: min(SOUND_WINDOW, seconds), preferredTimescale: 48_000)
|
|
788
|
+
request.overlapFactor = 0.5
|
|
789
|
+
let analyzer = try SNAudioFileAnalyzer(url: url)
|
|
790
|
+
let windows = SoundWindows()
|
|
791
|
+
try analyzer.add(request, withObserver: windows)
|
|
792
|
+
analyzer.analyze()
|
|
793
|
+
if let failure = windows.failure {
|
|
794
|
+
return ["soundsError": "The sound classifier failed (\(failure)).",
|
|
795
|
+
"classifyMs": milliseconds(since: started)]
|
|
796
|
+
}
|
|
797
|
+
return ["soundWindows": windows.windows, "classifyMs": milliseconds(since: started)]
|
|
798
|
+
} catch {
|
|
799
|
+
return ["soundsError": "The sound classifier couldn't read the audio (\(error.localizedDescription)).",
|
|
800
|
+
"classifyMs": milliseconds(since: started)]
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
/// classifySounds on a thread of its own; the box holds its answer when it is done.
|
|
805
|
+
func classifyingSounds(_ url: URL) -> SendableBox<[String: Any]> {
|
|
806
|
+
let box = SendableBox<[String: Any]>()
|
|
807
|
+
Thread.detachNewThread { box.value = classifySounds(url) }
|
|
808
|
+
return box
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// MARK: - Commands
|
|
812
|
+
|
|
813
|
+
switch command {
|
|
814
|
+
case "record-video":
|
|
815
|
+
guard let text = option("--seconds"), let seconds = Double(text), seconds.isFinite,
|
|
816
|
+
seconds >= 1, seconds <= 300, let udid = option("--udid"), !udid.isEmpty,
|
|
817
|
+
let work = option("--work") else { usage() }
|
|
818
|
+
let folder = URL(fileURLWithPath: work)
|
|
819
|
+
let attributes = try? FileManager.default.attributesOfItem(atPath: folder.path)
|
|
820
|
+
guard attributes?[.type] as? FileAttributeType == .typeDirectory,
|
|
821
|
+
attributes?[.posixPermissions] as? Int == 0o700,
|
|
822
|
+
attributes?[.ownerAccountID] as? UInt32 == getuid() else {
|
|
823
|
+
fail("capture_failed", "The video working folder must be owned by you with permissions 0700.")
|
|
824
|
+
}
|
|
825
|
+
let video = folder.appendingPathComponent("video.mov")
|
|
826
|
+
guard !FileManager.default.fileExists(atPath: video.path) else {
|
|
827
|
+
fail("capture_failed", "The video working file already exists. Choose an empty private folder.")
|
|
828
|
+
}
|
|
829
|
+
cleanup.append(video)
|
|
830
|
+
allowScreenCaptureDevices()
|
|
831
|
+
let name = option("--name")
|
|
832
|
+
let (found, picked) = findPhone(udid: udid, name: name, phones: option("--phones").flatMap { Int($0) }, timeout: 5)
|
|
833
|
+
guard let phone = found, case .found(_, let match) = picked else { refuse(picked, name: name) }
|
|
834
|
+
authorize(live: false)
|
|
835
|
+
let audio = recordVideo(phone, seconds: seconds, to: video)
|
|
836
|
+
guard let size = (try? FileManager.default.attributesOfItem(atPath: video.path))?[.size] as? UInt64,
|
|
837
|
+
size > 0 else { fail("capture_failed", "The iPhone produced no video recording.") }
|
|
838
|
+
let durationBox = SendableBox<Double>()
|
|
839
|
+
Task.detached {
|
|
840
|
+
do { durationBox.value = try await AVURLAsset(url: video).load(.duration).seconds } catch { durationBox.value = -1 }
|
|
841
|
+
}
|
|
842
|
+
guard pump(timeout: 5, until: { durationBox.value != nil }), let duration = durationBox.value,
|
|
843
|
+
duration.isFinite, duration > 0 else { fail("capture_failed", "The recorded movie has no readable duration.") }
|
|
844
|
+
do { try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: video.path) } catch {
|
|
845
|
+
fail("capture_failed", "Mobster couldn't protect the recorded video file.")
|
|
846
|
+
}
|
|
847
|
+
cleanup = []
|
|
848
|
+
emit(["ok": true, "seconds": seconds, "durationSeconds": duration, "audio": audio, "bytes": size, "match": match])
|
|
849
|
+
|
|
850
|
+
case "devices":
|
|
851
|
+
allowScreenCaptureDevices()
|
|
852
|
+
pump(timeout: 2) { false } // give them a moment to appear
|
|
853
|
+
emit(["ok": true, "devices": iphones().map { ["id": $0.uniqueID, "name": $0.localizedName, "model": $0.modelID] }])
|
|
854
|
+
|
|
855
|
+
case "match":
|
|
856
|
+
// Which device listen would pick from the list on stdin ([{"id","name","model"}]), by the same rules: for tests
|
|
857
|
+
// and diagnosis. Nothing is captured.
|
|
858
|
+
let input = FileHandle.standardInput.readDataToEndOfFile()
|
|
859
|
+
guard let rows = (try? JSONSerialization.jsonObject(with: input)) as? [[String: Any]] else { usage() }
|
|
860
|
+
let listed = rows.map { Candidate(id: $0["id"] as? String ?? "", name: $0["name"] as? String ?? "",
|
|
861
|
+
model: $0["model"] as? String ?? "") }
|
|
862
|
+
let name = option("--name")
|
|
863
|
+
switch pick(listed, udid: option("--udid"), name: name, phones: option("--phones").flatMap { Int($0) }) {
|
|
864
|
+
case .found(let index, let how): emit(["ok": true, "index": index, "match": how])
|
|
865
|
+
case let other: refuse(other, name: name)
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
case "listen":
|
|
869
|
+
guard let text = option("--seconds"), let seconds = Double(text), seconds >= 1, seconds <= 120 else { usage() }
|
|
870
|
+
let keep = option("--keep")
|
|
871
|
+
let live = arguments.contains("--live")
|
|
872
|
+
let stop = StopSignal()
|
|
873
|
+
if live { watchStdin(stop) }
|
|
874
|
+
let looking = Date()
|
|
875
|
+
allowScreenCaptureDevices()
|
|
876
|
+
let name = option("--name")
|
|
877
|
+
let (found, picked) = findPhone(udid: option("--udid"), name: name, phones: option("--phones").flatMap { Int($0) },
|
|
878
|
+
timeout: 5)
|
|
879
|
+
guard let phone = found, case .found(_, let match) = picked else { refuse(picked, name: name) }
|
|
880
|
+
let findMs = milliseconds(since: looking)
|
|
881
|
+
authorize(live: live, stop: live ? stop : nil)
|
|
882
|
+
// The caller's private folder (--work), which it deletes itself, even if this process is killed; else our own.
|
|
883
|
+
let folder: URL
|
|
884
|
+
if let work = option("--work") {
|
|
885
|
+
folder = URL(fileURLWithPath: work).appendingPathComponent("capture-\(UUID().uuidString)")
|
|
886
|
+
} else {
|
|
887
|
+
folder = FileManager.default.temporaryDirectory.appendingPathComponent("mobster-listen-\(UUID().uuidString)")
|
|
888
|
+
}
|
|
889
|
+
try? FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true,
|
|
890
|
+
attributes: [.posixPermissions: 0o700])
|
|
891
|
+
cleanup.append(folder)
|
|
892
|
+
let audio = folder.appendingPathComponent("audio.wav")
|
|
893
|
+
let started = Date()
|
|
894
|
+
var frames: URL?
|
|
895
|
+
if live, arguments.contains("--frames") {
|
|
896
|
+
frames = folder.appendingPathComponent("frames")
|
|
897
|
+
try? FileManager.default.createDirectory(at: frames!, withIntermediateDirectories: true,
|
|
898
|
+
attributes: [.posixPermissions: 0o700])
|
|
899
|
+
}
|
|
900
|
+
let (waited, recorded) = record(phone, seconds: seconds, to: audio, live: live, stop: stop, frames: frames)
|
|
901
|
+
let recordMs = milliseconds(since: started)
|
|
902
|
+
let peak = peakDecibels(audio)
|
|
903
|
+
let sounds = classifyingSounds(audio)
|
|
904
|
+
let (heard, transcribeMs) = transcribed(audio, locale: option("--locale"))
|
|
905
|
+
pump(timeout: 60) { sounds.value != nil }
|
|
906
|
+
var answer: [String: Any] = ["ok": true, "text": heard.text, "segments": heard.segments, "engine": heard.engine,
|
|
907
|
+
"locale": heard.locale, "seconds": seconds, "recordedSeconds": recorded,
|
|
908
|
+
"recordMs": recordMs, "startMs": waited, "findMs": findMs,
|
|
909
|
+
"transcribeMs": transcribeMs, "match": match]
|
|
910
|
+
if let peak = peak { answer["peakDb"] = peak }
|
|
911
|
+
answer.merge(sounds.value ?? ["soundsError": "The sound classifier took too long."]) { mine, _ in mine }
|
|
912
|
+
if let keep = keep {
|
|
913
|
+
let target = URL(fileURLWithPath: keep)
|
|
914
|
+
try? FileManager.default.removeItem(at: target)
|
|
915
|
+
if (try? FileManager.default.moveItem(at: audio, to: target)) != nil { answer["kept"] = target.path }
|
|
916
|
+
}
|
|
917
|
+
removeTemporary()
|
|
918
|
+
emit(answer)
|
|
919
|
+
|
|
920
|
+
case "transcribe":
|
|
921
|
+
guard let path = arguments.first, !path.hasPrefix("--") else { usage() }
|
|
922
|
+
let url = URL(fileURLWithPath: path)
|
|
923
|
+
guard FileManager.default.isReadableFile(atPath: url.path) else { fail("transcribe_failed", "No audio file at \(path).") }
|
|
924
|
+
let (heard, transcribeMs) = transcribed(url, locale: option("--locale"))
|
|
925
|
+
emit(["ok": true, "text": heard.text, "segments": heard.segments, "engine": heard.engine, "locale": heard.locale,
|
|
926
|
+
"transcribeMs": transcribeMs])
|
|
927
|
+
|
|
928
|
+
case "classify":
|
|
929
|
+
guard let path = arguments.first, !path.hasPrefix("--") else { usage() }
|
|
930
|
+
guard FileManager.default.isReadableFile(atPath: path) else { fail("classify_failed", "No audio file at \(path).") }
|
|
931
|
+
var answer: [String: Any] = ["ok": true]
|
|
932
|
+
answer.merge(classifySounds(URL(fileURLWithPath: path))) { mine, _ in mine }
|
|
933
|
+
emit(answer)
|
|
934
|
+
|
|
935
|
+
case "prepare":
|
|
936
|
+
let locale = wanted(option("--locale"))
|
|
937
|
+
let box = SendableBox<[String: Any]>()
|
|
938
|
+
Task.detached {
|
|
939
|
+
guard #available(macOS 26.0, *), SpeechTranscriber.isAvailable else {
|
|
940
|
+
box.value = ["ok": false, "error": "no_engine", "message": "On-device speech models need macOS 26 or later."]
|
|
941
|
+
return
|
|
942
|
+
}
|
|
943
|
+
guard let resolved = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else {
|
|
944
|
+
box.value = ["ok": false, "error": "no_engine", "message": "This language isn't available on this Mac."]
|
|
945
|
+
return
|
|
946
|
+
}
|
|
947
|
+
let transcriber = SpeechTranscriber(locale: resolved, preset: .transcription)
|
|
948
|
+
do {
|
|
949
|
+
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
|
950
|
+
try await request.downloadAndInstall()
|
|
951
|
+
}
|
|
952
|
+
box.value = ["ok": true, "speechAnalyzer": "installed", "locale": resolved.identifier(.bcp47)]
|
|
953
|
+
} catch {
|
|
954
|
+
box.value = ["ok": false, "error": "prepare_failed",
|
|
955
|
+
"message": "macOS couldn't install its speech model (\(error.localizedDescription))."]
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
pump(timeout: 900) { box.value != nil }
|
|
959
|
+
emit(box.value ?? ["ok": false, "error": "prepare_failed", "message": "macOS took too long to install its speech model."])
|
|
960
|
+
|
|
961
|
+
case "status":
|
|
962
|
+
let locale = wanted(option("--locale"))
|
|
963
|
+
let box = SendableBox<[String: Any]>()
|
|
964
|
+
Task.detached {
|
|
965
|
+
var answer: [String: Any] = ["ok": true, "speechAnalyzer": "unavailable"]
|
|
966
|
+
if #available(macOS 26.0, *), SpeechTranscriber.isAvailable {
|
|
967
|
+
if let resolved = await SpeechTranscriber.supportedLocale(equivalentTo: locale) {
|
|
968
|
+
let transcriber = SpeechTranscriber(locale: resolved, preset: .transcription)
|
|
969
|
+
switch await AssetInventory.status(forModules: [transcriber]) {
|
|
970
|
+
case .installed: answer["speechAnalyzer"] = "installed"
|
|
971
|
+
case .downloading: answer["speechAnalyzer"] = "downloading"
|
|
972
|
+
case .supported: answer["speechAnalyzer"] = "supported"
|
|
973
|
+
default: answer["speechAnalyzer"] = "unsupported"
|
|
974
|
+
}
|
|
975
|
+
answer["locale"] = resolved.identifier(.bcp47)
|
|
976
|
+
} else {
|
|
977
|
+
answer["speechAnalyzer"] = "unsupported"
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
let names: [SFSpeechRecognizerAuthorizationStatus: String] = [.authorized: "granted", .denied: "denied",
|
|
981
|
+
.restricted: "restricted", .notDetermined: "not_determined"]
|
|
982
|
+
answer["sfSpeech"] = ["authorization": names[SFSpeechRecognizer.authorizationStatus()] ?? "unknown",
|
|
983
|
+
"onDevice": SFSpeechRecognizer(locale: locale)?.supportsOnDeviceRecognition ?? false]
|
|
984
|
+
answer["capture"] = ["camera": authorizationName(AVCaptureDevice.authorizationStatus(for: .video)),
|
|
985
|
+
"microphone": authorizationName(AVCaptureDevice.authorizationStatus(for: .audio))]
|
|
986
|
+
box.value = answer
|
|
987
|
+
}
|
|
988
|
+
pump(timeout: 30) { box.value != nil }
|
|
989
|
+
emit(box.value ?? ["ok": false, "error": "status_failed", "message": "Mobster couldn't read this Mac's speech recognition."])
|
|
990
|
+
|
|
991
|
+
default:
|
|
992
|
+
usage()
|
|
993
|
+
}
|