qa-agent-kit 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qa_agent_kit-0.6.0.dist-info/METADATA +157 -0
- qa_agent_kit-0.6.0.dist-info/RECORD +32 -0
- qa_agent_kit-0.6.0.dist-info/WHEEL +4 -0
- qa_agent_kit-0.6.0.dist-info/entry_points.txt +6 -0
- qa_agent_kit-0.6.0.dist-info/licenses/LICENSE +21 -0
- qakit/AGENTS.md +48 -0
- qakit/__init__.py +21 -0
- qakit/_cli.py +46 -0
- qakit/docs/README.md +14 -0
- qakit/docs/agents.md +40 -0
- qakit/docs/case-studies.md +146 -0
- qakit/docs/device-qa-protocol.md +426 -0
- qakit/docs/getting-started.md +100 -0
- qakit/docs/ko/device-qa-protocol.md +433 -0
- qakit/docs/manifest.md +96 -0
- qakit/docs/mutation-testing.md +69 -0
- qakit/docs/prose-qa.md +49 -0
- qakit/docs/pypi.md +40 -0
- qakit/docs/report.md +64 -0
- qakit/skills/README.md +51 -0
- qakit/skills/device-qa/SKILL.md +87 -0
- qakit/skills/prose-qa/SKILL.md +49 -0
- qakit/skills/qa-rigor/SKILL.md +105 -0
- qakit/tools/device_lock.py +151 -0
- qakit/tools/device_qa.py +838 -0
- qakit/tools/install_skills.py +120 -0
- qakit/tools/mutate.py +684 -0
- qakit/tools/prose_check.py +291 -0
- qakit/tools/recording_frames.swift +78 -0
- qakit/tools/release_checks.py +287 -0
- qakit/tools/screenshot_contrast.swift +42 -0
- qakit/tools/shots_gif.py +81 -0
qakit/tools/device_qa.py
ADDED
|
@@ -0,0 +1,838 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Real-device QA runner: readiness check, Developer Mode, test runs under device-wide variants, screenshots, crashes, hangs,
|
|
3
|
+
upgrades, the Release archive, trait changes, and one lock per phone for many sessions.
|
|
4
|
+
|
|
5
|
+
The test iPhone is USB-attached to the Mac that runs this script. Protocol and what to run when: docs/device-qa-protocol.md.
|
|
6
|
+
Per-app manifest (docs/manifest.md): `--manifest path/to/device_qa.json`, or `--app <slug>` which resolves
|
|
7
|
+
`<apps-root>/<slug>/release/testing/device_qa.json` (apps-root: --apps-root, $QAKIT_APPS_ROOT, else <git top>/apps).
|
|
8
|
+
|
|
9
|
+
python3 tools/device_qa.py doctor --app myapp
|
|
10
|
+
python3 tools/device_qa.py devmode reveal # Settings > Privacy & Security > Developer Mode appears
|
|
11
|
+
python3 tools/device_qa.py test --app myapp --suite ui --variant dark --variant large-text
|
|
12
|
+
python3 tools/device_qa.py test --app myapp --suite smoke --suite ui --configuration Release
|
|
13
|
+
python3 tools/device_qa.py shots <bundle.xcresult> --out <dir>
|
|
14
|
+
python3 tools/device_qa.py crashes --app myapp --since-minutes 60 --out <dir>
|
|
15
|
+
python3 tools/device_qa.py perf --app myapp --seconds 40 # Time Profiler attached: potential hangs, hang risks
|
|
16
|
+
python3 tools/device_qa.py upgrade --app myapp --from myapp/ios/v1.0 # previous release's data survives an in-place upgrade
|
|
17
|
+
python3 tools/device_qa.py release --app myapp --archive <xcarchive> # the Release archive, cold-launched with no debugger
|
|
18
|
+
python3 tools/device_qa.py traits --app myapp --screen map # text size and dark mode changed in front and in background
|
|
19
|
+
python3 tools/device_qa.py lock status # who holds the test phone, who waits
|
|
20
|
+
python3 tools/device_qa.py lock run --note "<who>" -- xcodebuild ... # raw phone work, in turn with the other sessions
|
|
21
|
+
|
|
22
|
+
One phone, many sessions: every command that drives the phone waits its turn (device_lock.py). QAKIT_DEVICE_NOTE names the
|
|
23
|
+
holder for the others; --no-wait fails fast instead. Needs Xcode 27 or later (`devicectl device settings appearance`).
|
|
24
|
+
Artifacts go to --out, else $QAKIT_ARTIFACTS/<app>/<stamp>, else /tmp/qakit/<app>/<stamp>.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import datetime as dt
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
import re
|
|
34
|
+
import subprocess
|
|
35
|
+
import sys
|
|
36
|
+
import tempfile # mutation-ok: stdlib import
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
from typing import Any
|
|
39
|
+
|
|
40
|
+
import hashlib
|
|
41
|
+
|
|
42
|
+
from device_lock import DeviceBusy, DeviceLock # same folder: on sys.path as a script and through the tests
|
|
43
|
+
|
|
44
|
+
XCODE = os.environ.get("DEVELOPER_DIR", "/Applications/Xcode.app/Contents/Developer") # xcode-select may point at CommandLineTools
|
|
45
|
+
ARTIFACTS = Path(os.environ.get("QAKIT_ARTIFACTS", "/tmp/qakit"))
|
|
46
|
+
VARIANT_KEYS = {"appearance", "text_size", "location", "voiceover", "env"}
|
|
47
|
+
APPEARANCES = {"light", "dark"}
|
|
48
|
+
TEXT_SIZES = {"extra-small", "small", "medium", "large", "extra-large", "extra-extra-large", "extra-extra-extra-large",
|
|
49
|
+
"accessibility-medium", "accessibility-large", "accessibility-extra-large", "accessibility-extra-extra-large",
|
|
50
|
+
"accessibility-extra-extra-extra-large"}
|
|
51
|
+
DEFAULTS = {"appearance": "light", "text_size": "large"} # what a variant run restores, the iOS defaults
|
|
52
|
+
# What a user changes in Settings or Control Center while an app is open or waiting: smallest text, the largest accessibility size,
|
|
53
|
+
# a large standard size, the default, then dark and back. `traits` plays it with each screen in front, then again in the background.
|
|
54
|
+
TRAIT_STEPS = [("text_size", "extra-small"), ("text_size", "accessibility-extra-extra-extra-large"), ("text_size", "extra-extra-extra-large"),
|
|
55
|
+
("text_size", "large"), ("appearance", "dark"), ("appearance", "light")]
|
|
56
|
+
SUMMARY = re.compile(r"Executed (\d+) tests?, with (?:(\d+) tests? skipped and )?(\d+) failures?")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def tool_env(extra: dict[str, str] | None = None) -> dict[str, str]:
|
|
60
|
+
"""The environment the tools run in: DEVELOPER_DIR set, and that Xcode's usr/bin first on PATH, so the xcrun that xcodebuild
|
|
61
|
+
itself spawns finds devicectl (its device diagnostics step printed "unable to find utility devicectl" on a Mac whose
|
|
62
|
+
xcode-select points at CommandLineTools, Daynest 2026-10-29)."""
|
|
63
|
+
env = os.environ.copy()
|
|
64
|
+
env.setdefault("DEVELOPER_DIR", XCODE)
|
|
65
|
+
usr_bin = str(Path(env["DEVELOPER_DIR"]) / "usr/bin")
|
|
66
|
+
if usr_bin not in env.get("PATH", "").split(":"):
|
|
67
|
+
env["PATH"] = usr_bin + ":" + env.get("PATH", "")
|
|
68
|
+
env.update(extra or {})
|
|
69
|
+
return env
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def run(cmd: list[str], *, env: dict[str, str] | None = None, timeout: float | None = 120, log: Path | None = None) -> subprocess.CompletedProcess:
|
|
73
|
+
"""Run a tool and capture (or stream to `log`) its output. A tool that does not answer within `timeout` comes back as a
|
|
74
|
+
failed CompletedProcess (rc 124) with a one-line reason, never as a traceback: `xcrun devicectl list devices` hung for
|
|
75
|
+
the full two minutes on a Mac whose CoreSimulator was older than Xcode (2026-10-29)."""
|
|
76
|
+
try:
|
|
77
|
+
if log:
|
|
78
|
+
with log.open("w", encoding="utf-8") as handle:
|
|
79
|
+
return subprocess.run(cmd, env=env or tool_env(), stdout=handle, stderr=subprocess.STDOUT, timeout=timeout, check=False)
|
|
80
|
+
return subprocess.run(cmd, env=env or tool_env(), capture_output=True, text=True, timeout=timeout, check=False)
|
|
81
|
+
except subprocess.TimeoutExpired:
|
|
82
|
+
reason = f"{cmd[0]} did not answer in {timeout:.0f} s" + (" (xcrun devicectl hangs when CoreSimulator is older than Xcode: update Xcode or reboot)"
|
|
83
|
+
if "devicectl" in cmd else "")
|
|
84
|
+
if log:
|
|
85
|
+
log.write_text(reason + "\n", encoding="utf-8")
|
|
86
|
+
return subprocess.CompletedProcess(cmd, 124, "", reason)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def devicectl_json(args: list[str], timeout: float = 120) -> dict[str, Any]:
|
|
90
|
+
"""`devicectl ... --json-output` as a dict; {} when it failed, {"error": reason} when it hung (the caller shows the reason)."""
|
|
91
|
+
with tempfile.NamedTemporaryFile(suffix=".json") as out:
|
|
92
|
+
done = run(["xcrun", "devicectl", *args, "--json-output", out.name], timeout=timeout)
|
|
93
|
+
if done.returncode == 124:
|
|
94
|
+
return {"error": done.stderr}
|
|
95
|
+
try:
|
|
96
|
+
return json.loads(Path(out.name).read_text(encoding="utf-8") or "{}").get("result", {})
|
|
97
|
+
except json.JSONDecodeError:
|
|
98
|
+
return {}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# ---- pure helpers (agent/tests/test_device_qa.py) ----
|
|
102
|
+
|
|
103
|
+
def pick_device(devices: list[dict[str, Any]], udid: str | None = None) -> dict[str, str] | None:
|
|
104
|
+
"""The attached test iPhone from `devicectl list devices` JSON: physical, paired, wired first; `udid` pins one."""
|
|
105
|
+
found = []
|
|
106
|
+
for device in devices:
|
|
107
|
+
hardware, connection = device.get("hardwareProperties", {}), device.get("connectionProperties", {})
|
|
108
|
+
if hardware.get("reality") == "simulated" or hardware.get("platform", "iOS") != "iOS":
|
|
109
|
+
continue # `reality` is missing until a tunnel opens, so only an explicit "simulated" excludes
|
|
110
|
+
if connection.get("pairingState") != "paired" or not hardware.get("udid"):
|
|
111
|
+
continue
|
|
112
|
+
if udid and hardware["udid"] != udid:
|
|
113
|
+
continue
|
|
114
|
+
found.append({"udid": hardware["udid"], "name": device.get("deviceProperties", {}).get("name", ""),
|
|
115
|
+
"transport": connection.get("transportType", "")})
|
|
116
|
+
found.sort(key=lambda d: d["transport"] != "wired")
|
|
117
|
+
return found[0] if found else None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def validate_manifest(data: dict[str, Any]) -> list[str]:
|
|
121
|
+
errors = [f"{key} must be a non-empty string" for key in ("schema_version", "app", "scheme", "process_name")
|
|
122
|
+
if not isinstance(data.get(key), str) or not data[key].strip()]
|
|
123
|
+
if not any(isinstance(data.get(key), str) and data[key].strip() for key in ("project", "workspace")):
|
|
124
|
+
errors.append("project (or workspace) must be a non-empty string")
|
|
125
|
+
suites = data.get("suites")
|
|
126
|
+
if not isinstance(suites, dict) or not suites or not all(isinstance(v, list) and v and all(isinstance(t, str) for t in v) for v in suites.values()):
|
|
127
|
+
errors.append("suites must map a name to a non-empty list of -only-testing identifiers")
|
|
128
|
+
for name, variant in (data.get("variants") or {}).items():
|
|
129
|
+
unknown = set(variant) - VARIANT_KEYS
|
|
130
|
+
if unknown:
|
|
131
|
+
errors.append(f"variant {name}: unknown keys {sorted(unknown)}")
|
|
132
|
+
if "appearance" in variant and variant["appearance"] not in APPEARANCES:
|
|
133
|
+
errors.append(f"variant {name}: appearance must be one of {sorted(APPEARANCES)}")
|
|
134
|
+
if "text_size" in variant and variant["text_size"] not in TEXT_SIZES:
|
|
135
|
+
errors.append(f"variant {name}: unknown text_size {variant['text_size']}")
|
|
136
|
+
location = variant.get("location")
|
|
137
|
+
if location is not None and not (isinstance(location, list) and len(location) == 2 and all(isinstance(v, (int, float)) for v in location)):
|
|
138
|
+
errors.append(f"variant {name}: location must be [latitude, longitude]")
|
|
139
|
+
if not isinstance(variant.get("env", {}), dict):
|
|
140
|
+
errors.append(f"variant {name}: env must be an object")
|
|
141
|
+
screens = (data.get("traits") or {}).get("screens", {})
|
|
142
|
+
if not isinstance(screens, dict) or not all(isinstance(v, list) and all(isinstance(a, str) for a in v) for v in screens.values()):
|
|
143
|
+
errors.append("traits.screens must map a screen name to its launch arguments")
|
|
144
|
+
return errors
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def variant_commands(udid: str, variant: dict[str, Any]) -> list[list[str]]:
|
|
148
|
+
"""devicectl calls that put the whole phone into the variant (every app, system alerts included)."""
|
|
149
|
+
base = ["xcrun", "devicectl", "device"]
|
|
150
|
+
cmds = []
|
|
151
|
+
if "appearance" in variant or "text_size" in variant:
|
|
152
|
+
cmd = [*base, "settings", "appearance", "--device", udid]
|
|
153
|
+
if "appearance" in variant:
|
|
154
|
+
cmd += ["--mode", variant["appearance"]]
|
|
155
|
+
if "text_size" in variant:
|
|
156
|
+
# Accessibility sizes only apply with Larger Accessibility Sizes on (devicectl help, 2026-10-28).
|
|
157
|
+
larger = "on" if variant["text_size"].startswith("accessibility-") else "off"
|
|
158
|
+
cmd += ["--text-size", variant["text_size"], "--larger-accessibility-sizes", larger]
|
|
159
|
+
cmds.append(cmd)
|
|
160
|
+
if variant.get("voiceover"):
|
|
161
|
+
cmds.append([*base, "settings", "voiceover", "--device", udid, "--enable"])
|
|
162
|
+
if "location" in variant:
|
|
163
|
+
lat, lng = variant["location"]
|
|
164
|
+
cmds.append([*base, "simulate", "location", "coordinate", "--device", udid, "--latitude", str(lat), "--longitude", str(lng)])
|
|
165
|
+
return cmds
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def restore_commands(udid: str, variant: dict[str, Any]) -> list[list[str]]:
|
|
169
|
+
"""Undo `variant_commands`, so the next run (or a human) finds the phone in iOS defaults."""
|
|
170
|
+
base = ["xcrun", "devicectl", "device"]
|
|
171
|
+
cmds = []
|
|
172
|
+
if "appearance" in variant or "text_size" in variant:
|
|
173
|
+
cmds.append([*base, "settings", "appearance", "--device", udid, "--mode", DEFAULTS["appearance"], "--text-size", DEFAULTS["text_size"],
|
|
174
|
+
"--larger-accessibility-sizes", "off"])
|
|
175
|
+
if variant.get("voiceover"):
|
|
176
|
+
cmds.append([*base, "settings", "voiceover", "--device", udid, "--disable"])
|
|
177
|
+
if "location" in variant:
|
|
178
|
+
cmds.append([*base, "simulate", "location", "clear", "--device", udid])
|
|
179
|
+
return cmds
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def trait_commands(udid: str) -> list[list[str]]:
|
|
183
|
+
"""One devicectl call per TRAIT_STEPS change (the largest sizes need Larger Accessibility Sizes on, as in variant_commands)."""
|
|
184
|
+
base = ["xcrun", "devicectl", "device", "settings", "appearance", "--device", udid]
|
|
185
|
+
return [[*base, "--text-size", value, "--larger-accessibility-sizes", "on" if value.startswith("accessibility-") else "off"]
|
|
186
|
+
if kind == "text_size" else [*base, "--mode", value] for kind, value in TRAIT_STEPS]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def runner_env(name: str, variant: dict[str, Any]) -> dict[str, str]:
|
|
190
|
+
"""xcodebuild forwards TEST_RUNNER_<KEY> to the UI-test runner as <KEY>; QAKit reads QA_VARIANT, QA_LANG, ..."""
|
|
191
|
+
env = {"TEST_RUNNER_QA_VARIANT": name}
|
|
192
|
+
env.update({f"TEST_RUNNER_{key}": str(value) for key, value in (variant.get("env") or {}).items()})
|
|
193
|
+
return env
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def summarize_log(text: str) -> dict[str, Any]:
|
|
197
|
+
"""The last (outermost) XCTest summary line, and the overall verdict."""
|
|
198
|
+
hits = SUMMARY.findall(text)
|
|
199
|
+
executed, skipped, failures = (int(hits[-1][0]), int(hits[-1][1] or 0), int(hits[-1][2])) if hits else (0, 0, 0)
|
|
200
|
+
failed_cases = sorted(set(re.findall(r"Test Case '-\[(\S+ \S+)\]' failed", text)))
|
|
201
|
+
succeeded = "** TEST SUCCEEDED **" in text and not failures
|
|
202
|
+
return {"executed": executed, "skipped": skipped, "failures": failures, "failed_cases": failed_cases, "succeeded": succeeded}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def build_index(manifest: list[dict[str, Any]], title: str) -> str:
|
|
206
|
+
"""Markdown contact sheet from `xcresulttool export attachments` manifest.json: one section per test, files in order."""
|
|
207
|
+
lines = [f"# {title}", ""]
|
|
208
|
+
for test in manifest:
|
|
209
|
+
lines.append(f"## {test.get('testIdentifier', '?')}")
|
|
210
|
+
for item in test.get("attachments", []):
|
|
211
|
+
name = item.get("exportedFileName", "")
|
|
212
|
+
# Xcode 27.2's manifest suffixes the readable name with `_<n>_<UUID>.<ext>` (Daynest round 2); the attachment's own name is the step
|
|
213
|
+
shown = re.sub(r"_\d+_[0-9A-Fa-f-]{36}(\.[A-Za-z0-9]+)?$", "", item.get("suggestedHumanReadableName", name))
|
|
214
|
+
lines.append(f"- {shown}: `{name}`" + (" (failure)" if item.get("isAssociatedWithFailure") else ""))
|
|
215
|
+
lines.append("")
|
|
216
|
+
return "\n".join(lines)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def crash_candidates(files: list[dict[str, Any]], process: str | list[str], since: dt.datetime) -> list[str]:
|
|
220
|
+
"""Crash/jetsam reports written since `since` that may concern the app or its extensions (`process` may list several:
|
|
221
|
+
the app and e.g. its widget extension); jetsam files are checked for the names after copying."""
|
|
222
|
+
names = [process] if isinstance(process, str) else list(process)
|
|
223
|
+
picked = []
|
|
224
|
+
for item in files:
|
|
225
|
+
path = item.get("relativePath", "")
|
|
226
|
+
stamp = item.get("metadata", {}).get("lastModDate", "")
|
|
227
|
+
try:
|
|
228
|
+
when = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
|
229
|
+
except ValueError:
|
|
230
|
+
continue
|
|
231
|
+
if when >= since and path.endswith(".ips") and (any(path.startswith(n) for n in names) or path.startswith("JetsamEvent")):
|
|
232
|
+
picked.append(path)
|
|
233
|
+
return picked
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
# ---- commands ----
|
|
237
|
+
|
|
238
|
+
def git_top(start: Path) -> Path:
|
|
239
|
+
result = subprocess.run(["git", "-C", str(start), "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=False)
|
|
240
|
+
return Path(result.stdout.strip()) if result.returncode == 0 else start
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def manifest_path_for(args: argparse.Namespace) -> Path:
|
|
244
|
+
if getattr(args, "manifest", None):
|
|
245
|
+
return Path(args.manifest).resolve()
|
|
246
|
+
if not getattr(args, "app", None):
|
|
247
|
+
raise SystemExit("give --manifest <device_qa.json> or --app <slug>")
|
|
248
|
+
root = Path(getattr(args, "apps_root", None) or os.environ.get("QAKIT_APPS_ROOT") or git_top(Path.cwd()) / "apps")
|
|
249
|
+
return (root / args.app / "release/testing/device_qa.json").resolve()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def load_manifest(path: Path) -> dict[str, Any]:
|
|
253
|
+
if not path.is_file():
|
|
254
|
+
raise SystemExit(f"no manifest at {path} (docs/manifest.md)")
|
|
255
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
256
|
+
errors = validate_manifest(data)
|
|
257
|
+
if errors:
|
|
258
|
+
raise SystemExit(f"{path}: " + "; ".join(errors))
|
|
259
|
+
return data
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def load_app(args: argparse.Namespace) -> tuple[Path, dict[str, Any]]:
|
|
263
|
+
path = manifest_path_for(args)
|
|
264
|
+
return path, load_manifest(path)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def store_config(manifest_path: Path, data: dict[str, Any]) -> dict[str, Any]:
|
|
268
|
+
"""The app's store config JSON (`appstore_config`), {} when the manifest has none."""
|
|
269
|
+
if not data.get("appstore_config"):
|
|
270
|
+
return {}
|
|
271
|
+
return json.loads((manifest_path.parent / data["appstore_config"]).resolve().read_text(encoding="utf-8"))
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def bundle_id(manifest_path: Path, data: dict[str, Any]) -> str:
|
|
275
|
+
"""Manifest `bundle_id`, else the store config's; the installed app's identity for launch, install and uninstall."""
|
|
276
|
+
value = data.get("bundle_id") or store_config(manifest_path, data).get("bundle_id")
|
|
277
|
+
if not value:
|
|
278
|
+
raise SystemExit("the manifest needs bundle_id (or an appstore_config with one)")
|
|
279
|
+
return value
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def team_id(manifest_path: Path, data: dict[str, Any]) -> str:
|
|
283
|
+
value = data.get("team_id") or store_config(manifest_path, data).get("team_id")
|
|
284
|
+
if not value:
|
|
285
|
+
raise SystemExit("the manifest needs team_id (or an appstore_config with one)")
|
|
286
|
+
return value
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def project_args(manifest_path: Path, data: dict[str, Any]) -> list[str]:
|
|
290
|
+
if data.get("workspace"):
|
|
291
|
+
return ["-workspace", str((manifest_path.parent / data["workspace"]).resolve())]
|
|
292
|
+
return ["-project", str((manifest_path.parent / data["project"]).resolve())]
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def app_root(manifest_path: Path, data: dict[str, Any]) -> Path:
|
|
296
|
+
"""The app's folder: what the project or workspace lives in (build output and the source fingerprint are relative to it)."""
|
|
297
|
+
return (manifest_path.parent / (data.get("workspace") or data["project"])).resolve().parent
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def derived_data(manifest_path: Path, data: dict[str, Any], suffix: str = "") -> Path:
|
|
301
|
+
base = (manifest_path.parent / data["derived_data"]).resolve() if data.get("derived_data") else app_root(manifest_path, data) / "build/DerivedData-device"
|
|
302
|
+
return Path(str(base) + suffix)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def signing_args(manifest_path: Path, data: dict[str, Any]) -> list[str]:
|
|
306
|
+
"""Automatic development signing. With an App Store Connect API key (manifest `signing` block, or the `api` block of the
|
|
307
|
+
`appstore_config` JSON) xcodebuild registers the phone and mints profiles with no Xcode login; without one it relies on
|
|
308
|
+
the Mac's Xcode account. The key path is absolute, or relative to the manifest's git top level, else under $QAKIT_SECRETS_ROOT."""
|
|
309
|
+
args = ["-allowProvisioningUpdates", "-allowProvisioningDeviceRegistration"]
|
|
310
|
+
signing = data.get("signing")
|
|
311
|
+
if not signing and data.get("appstore_config"):
|
|
312
|
+
api = store_config(manifest_path, data).get("api", {})
|
|
313
|
+
signing = {"key_path": api.get("private_key_path"), "key_id": api.get("key_id"), "issuer_id": api.get("issuer_id")}
|
|
314
|
+
if not signing or not signing.get("key_path"):
|
|
315
|
+
return args
|
|
316
|
+
key = Path(signing["key_path"]).expanduser()
|
|
317
|
+
if not key.is_absolute():
|
|
318
|
+
top = git_top(manifest_path.parent)
|
|
319
|
+
key = top / key
|
|
320
|
+
if not key.is_file() and os.environ.get("QAKIT_SECRETS_ROOT"):
|
|
321
|
+
key = Path(os.environ["QAKIT_SECRETS_ROOT"]) / signing["key_path"]
|
|
322
|
+
return [*args, "-authenticationKeyPath", str(key), "-authenticationKeyID", signing["key_id"], "-authenticationKeyIssuerID", signing["issuer_id"]]
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
LAST_DEVICE_ERROR = "" # why the last `find_device` found nothing when devicectl itself failed (a hang), for doctor's message
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def manual_signing_problems(manifest_path: Path, data: dict[str, Any]) -> list[str]:
|
|
329
|
+
"""A project that pins App Store provisioning profiles (manual signing) cannot be signed for a test phone with the
|
|
330
|
+
development certificate the ASC key mints: Daynest's first device run died on "profile doesn't include signing
|
|
331
|
+
certificate Apple Development" (2026-10-29). The fix is build-setting overrides in the manifest's `xcodebuild_args`."""
|
|
332
|
+
if any(str(a).startswith("CODE_SIGN_STYLE=") for a in data.get("xcodebuild_args", [])):
|
|
333
|
+
return []
|
|
334
|
+
project = (manifest_path.parent / (data.get("workspace") or data["project"])).resolve()
|
|
335
|
+
pbxproj = project / "project.pbxproj" if project.suffix == ".xcodeproj" else None
|
|
336
|
+
if not pbxproj or not pbxproj.is_file():
|
|
337
|
+
return []
|
|
338
|
+
text = pbxproj.read_text(encoding="utf-8", errors="replace")
|
|
339
|
+
if "CODE_SIGN_STYLE = Manual" in text and "PROVISIONING_PROFILE_SPECIFIER = \"" in text:
|
|
340
|
+
return ['the project pins provisioning profiles (CODE_SIGN_STYLE = Manual): add to the manifest '
|
|
341
|
+
'"xcodebuild_args": ["CODE_SIGN_STYLE=Automatic", "PROVISIONING_PROFILE_SPECIFIER=", '
|
|
342
|
+
'"CODE_SIGN_IDENTITY=Apple Development", "DEVELOPMENT_TEAM=<team id>"] so device runs sign with a development profile']
|
|
343
|
+
return []
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def find_device(udid: str | None) -> dict[str, str] | None:
|
|
347
|
+
global LAST_DEVICE_ERROR
|
|
348
|
+
listing = devicectl_json(["list", "devices"])
|
|
349
|
+
LAST_DEVICE_ERROR = str(listing.get("error") or "")
|
|
350
|
+
return pick_device(listing.get("devices", []), udid)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def cmd_doctor(args: argparse.Namespace) -> int:
|
|
354
|
+
problems: list[str] = []
|
|
355
|
+
device = find_device(args.udid)
|
|
356
|
+
if not device:
|
|
357
|
+
print(LAST_DEVICE_ERROR or "device: none paired. Plug the iPhone into this Mac, unlock it, tap Trust.")
|
|
358
|
+
if args.app or args.manifest:
|
|
359
|
+
manifest_path = manifest_path_for(args)
|
|
360
|
+
load_manifest(manifest_path) # a manifest mistake shows even without a phone
|
|
361
|
+
print(f"manifest: {manifest_path} loads")
|
|
362
|
+
return 1
|
|
363
|
+
details = devicectl_json(["device", "info", "details", "--device", device["udid"]]).get("deviceProperties", {})
|
|
364
|
+
lock = run(["xcrun", "devicectl", "device", "info", "lockState", "--device", device["udid"]]).stdout
|
|
365
|
+
print(f"device: {device['name']} {device['udid']} {device['transport']} iOS {details.get('osVersionNumber')}")
|
|
366
|
+
if details.get("developerModeStatus") != "enabled":
|
|
367
|
+
problems.append("Developer Mode is off: `device_qa.py devmode reveal`, then Settings > Privacy & Security > Developer Mode")
|
|
368
|
+
if "passcodeRequired: true" in lock:
|
|
369
|
+
problems.append("passcode is set: UI automation will stop for it; turn the passcode off on a test phone")
|
|
370
|
+
if not details.get("ddiServicesAvailable"):
|
|
371
|
+
problems.append("developer disk image not mounted yet (it mounts once Developer Mode is on and the phone is unlocked)")
|
|
372
|
+
identities = run(["security", "find-identity", "-v", "-p", "codesigning"]).stdout
|
|
373
|
+
if "Apple Development" not in identities:
|
|
374
|
+
problems.append("no Apple Development identity in the login keychain")
|
|
375
|
+
keychain = run(["security", "show-keychain-info", str(Path.home() / "Library/Keychains/login.keychain-db")])
|
|
376
|
+
if "User interaction is not allowed" in keychain.stdout + keychain.stderr:
|
|
377
|
+
# codesign answers errSecInternalComponent from an ssh or cron session whose login keychain is locked (Daynest, 2026-10-29)
|
|
378
|
+
problems.append("the login keychain is locked for this session (codesign will fail with errSecInternalComponent): run from the "
|
|
379
|
+
"user's GUI session, e.g. a `launchctl bootstrap gui/$UID` LaunchAgent, or unlock the keychain")
|
|
380
|
+
if args.app or args.manifest:
|
|
381
|
+
manifest_path, data = load_app(args)
|
|
382
|
+
problems += manual_signing_problems(manifest_path, data)
|
|
383
|
+
signing = signing_args(manifest_path, data)
|
|
384
|
+
if "-authenticationKeyPath" in signing:
|
|
385
|
+
key = Path(signing[signing.index("-authenticationKeyPath") + 1])
|
|
386
|
+
if not key.is_file():
|
|
387
|
+
problems.append(f"App Store Connect key missing: {key}")
|
|
388
|
+
else:
|
|
389
|
+
print("signing: no App Store Connect key in the manifest; xcodebuild will use the Mac's Xcode account")
|
|
390
|
+
for line in problems:
|
|
391
|
+
print("problem:", line)
|
|
392
|
+
print("ready" if not problems else f"{len(problems)} problem(s)")
|
|
393
|
+
return 0 if not problems else 1
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def cmd_devmode(args: argparse.Namespace) -> int:
|
|
397
|
+
device = find_device(args.udid)
|
|
398
|
+
if not device:
|
|
399
|
+
print("no paired device")
|
|
400
|
+
return 1
|
|
401
|
+
result = run(["idevicedevmodectl", "-u", device["udid"], args.action], timeout=300) # brew install libimobiledevice
|
|
402
|
+
print((result.stdout + result.stderr).strip())
|
|
403
|
+
return result.returncode
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def cmd_test(args: argparse.Namespace) -> int:
|
|
407
|
+
manifest_path, data = load_app(args)
|
|
408
|
+
device = find_device(args.udid)
|
|
409
|
+
if not device:
|
|
410
|
+
print("no paired device; run doctor")
|
|
411
|
+
return 1
|
|
412
|
+
variants = data.get("variants", {})
|
|
413
|
+
names = args.variant or ["default"]
|
|
414
|
+
stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
415
|
+
out = Path(args.out or ARTIFACTS / data["app"] / stamp)
|
|
416
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
417
|
+
started = dt.datetime.now(dt.timezone.utc)
|
|
418
|
+
build_dirs = [derived_data(manifest_path, data), derived_data(manifest_path, data, "-release")]
|
|
419
|
+
tree_before = tree_fingerprint(fingerprint_root(manifest_path, data), build_dirs)
|
|
420
|
+
report: list[dict[str, Any]] = []
|
|
421
|
+
for name in names:
|
|
422
|
+
variant = variants.get(name, {}) if name != "default" else variants.get("default", {})
|
|
423
|
+
if name != "default" and name not in variants:
|
|
424
|
+
raise SystemExit(f"unknown variant {name}; known: {sorted(variants)}")
|
|
425
|
+
for cmd in variant_commands(device["udid"], variant):
|
|
426
|
+
run(cmd)
|
|
427
|
+
try:
|
|
428
|
+
for suite in (["only"] if args.only else args.suite or ["ui"]):
|
|
429
|
+
if suite != "only" and suite not in data["suites"]:
|
|
430
|
+
raise SystemExit(f"unknown suite {suite}; known: {sorted(data['suites'])}")
|
|
431
|
+
tests = args.only if suite == "only" else data["suites"][suite] # --only: single tests while iterating on one defect
|
|
432
|
+
only = [f"-only-testing:{t}" for t in tests]
|
|
433
|
+
bundle = out / f"{name}-{suite}.xcresult"
|
|
434
|
+
log = out / f"{name}-{suite}.log"
|
|
435
|
+
release = args.configuration == "Release" # exit criterion 2: the optimized build users get, not only Debug
|
|
436
|
+
cmd = ["xcodebuild", *project_args(manifest_path, data), "-scheme", data["scheme"], "-destination", f"id={device['udid']}",
|
|
437
|
+
"-configuration", args.configuration,
|
|
438
|
+
"-derivedDataPath", str(derived_data(manifest_path, data, "-release" if release else "")),
|
|
439
|
+
"-resultBundlePath", str(bundle), *signing_args(manifest_path, data), *data.get("xcodebuild_args", []), *only, "test",
|
|
440
|
+
*(["ENABLE_TESTABILITY=YES"] if release else [])] # the scheme builds the @testable unit target too
|
|
441
|
+
run(cmd, env=tool_env(runner_env(name, variant)), timeout=args.timeout, log=log)
|
|
442
|
+
summary = summarize_log(log.read_text(encoding="utf-8", errors="replace"))
|
|
443
|
+
shots = export_shots(bundle, out / f"{name}-{suite}-shots", f"{data['app']} {name} {suite}") if bundle.exists() else 0
|
|
444
|
+
report.append({"variant": name, "suite": suite, **summary, "screenshots": shots, "log": str(log), "xcresult": str(bundle)})
|
|
445
|
+
print(f"{name}/{suite}: executed {summary['executed']} skipped {summary['skipped']} failures {summary['failures']} shots {shots}"
|
|
446
|
+
+ ("" if summary["succeeded"] else " FAILED " + ", ".join(summary["failed_cases"][:5])))
|
|
447
|
+
finally:
|
|
448
|
+
for cmd in restore_commands(device["udid"], variant):
|
|
449
|
+
run(cmd)
|
|
450
|
+
crashes = pull_crashes(device["udid"], [data["process_name"], *data.get("extension_processes", [])], started, out / "crashes")
|
|
451
|
+
tainted = tree_changed(fingerprint_root(manifest_path, data), tree_before, "test run", build_dirs)
|
|
452
|
+
verdict = all(r["succeeded"] for r in report) and not crashes and not tainted
|
|
453
|
+
(out / "summary.json").write_text(json.dumps({"schema": "qakit.device-qa.summary.v1", "app": data["app"], "device": device,
|
|
454
|
+
"started": started.isoformat(), "runs": report, "crashes": crashes,
|
|
455
|
+
"sources_changed_during_run": tainted, "succeeded": verdict},
|
|
456
|
+
ensure_ascii=False, indent=2), encoding="utf-8")
|
|
457
|
+
print(f"crashes: {len(crashes)} artifacts: {out}")
|
|
458
|
+
return 0 if verdict else 1
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def fingerprint_root(manifest_path: Path, data: dict[str, Any]) -> Path:
|
|
462
|
+
"""Where a source edit counts as a changed tree: manifest `fingerprint_root` (an app whose core lives beside the project
|
|
463
|
+
folder, Elsewhen 2026-10-28), else the project folder."""
|
|
464
|
+
return (manifest_path.parent / data["fingerprint_root"]).resolve() if data.get("fingerprint_root") else app_root(manifest_path, data)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def tree_fingerprint(app_dir: Path, exclude: list[Path] = ()) -> str:
|
|
468
|
+
"""What the app's sources look like right now: tracked changes against HEAD plus untracked files. A phone round whose
|
|
469
|
+
fingerprint differs at the end compiled a mix of before and after (an edit landed mid-build and the round died with a
|
|
470
|
+
compile error twice on 2026-10-28), so its results describe no single tree."""
|
|
471
|
+
root = git_top(app_dir)
|
|
472
|
+
rel = str(app_dir.resolve().relative_to(root)) if app_dir.resolve().is_relative_to(root) else str(app_dir)
|
|
473
|
+
# Markdown never compiles: a QA ledger written during a round (docs/qa/QA_*.md) does not make it a mixed round (2026-10-28).
|
|
474
|
+
spec = ["--", rel, f":(glob,exclude){rel}/**/*.md"]
|
|
475
|
+
for folder in exclude: # the run's own build output (DerivedData next to the project, when it is not gitignored) is not a source edit
|
|
476
|
+
folder = folder.resolve()
|
|
477
|
+
if folder.is_relative_to(root):
|
|
478
|
+
spec.append(f":(exclude){folder.relative_to(root)}")
|
|
479
|
+
status = subprocess.run(["git", "-C", str(root), "status", "--porcelain", *spec], capture_output=True, text=True).stdout
|
|
480
|
+
diff = subprocess.run(["git", "-C", str(root), "diff", "HEAD", *spec], capture_output=True, text=True).stdout
|
|
481
|
+
return hashlib.sha1((status + diff).encode("utf-8", "replace")).hexdigest()
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def tree_changed(app_dir: Path, before: str, what: str, exclude: list[Path] = ()) -> bool:
|
|
485
|
+
"""True (and a loud warning) when the sources changed while `what` was running."""
|
|
486
|
+
if tree_fingerprint(app_dir, exclude) == before:
|
|
487
|
+
return False
|
|
488
|
+
print(f"WARNING: sources under {app_dir.name} changed while the {what} was running; its results mix two trees. Re-run.")
|
|
489
|
+
return True
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def export_shots(bundle: Path, out: Path, title: str) -> int:
|
|
493
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
494
|
+
run(["xcrun", "xcresulttool", "export", "attachments", "--path", str(bundle), "--output-path", str(out)], timeout=600)
|
|
495
|
+
manifest_file = out / "manifest.json"
|
|
496
|
+
manifest = json.loads(manifest_file.read_text(encoding="utf-8")) if manifest_file.is_file() else []
|
|
497
|
+
(out / "index.md").write_text(build_index(manifest, title), encoding="utf-8")
|
|
498
|
+
return sum(len(t.get("attachments", [])) for t in manifest)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def pull_crashes(udid: str, process: str | list[str], since: dt.datetime, out: Path) -> list[str]:
|
|
502
|
+
names = [process] if isinstance(process, str) else list(process)
|
|
503
|
+
files = devicectl_json(["device", "info", "files", "--device", udid, "--domain-type", "systemCrashLogs"]).get("files", [])
|
|
504
|
+
kept = []
|
|
505
|
+
for relative in crash_candidates(files, process, since):
|
|
506
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
507
|
+
target = out / Path(relative).name
|
|
508
|
+
run(["xcrun", "devicectl", "device", "copy", "from", "--device", udid, "--domain-type", "systemCrashLogs",
|
|
509
|
+
"--source", relative, "--destination", str(target)])
|
|
510
|
+
text = target.read_text(encoding="utf-8", errors="replace") if target.is_file() else ""
|
|
511
|
+
if any(relative.startswith(n) or f'"{n}"' in text for n in names): # a jetsam report counts only when it names one
|
|
512
|
+
kept.append(str(target))
|
|
513
|
+
elif target.is_file():
|
|
514
|
+
target.unlink()
|
|
515
|
+
return kept
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def cmd_shots(args: argparse.Namespace) -> int:
|
|
519
|
+
count = export_shots(Path(args.xcresult), Path(args.out), Path(args.xcresult).stem)
|
|
520
|
+
print(f"{count} attachments -> {args.out}/index.md")
|
|
521
|
+
return 0
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def trace_rows(xml_text: str) -> list[dict[str, str]]:
|
|
525
|
+
"""Rows of one exported Instruments table (`xctrace export --xpath ...table[@schema=X]`), refs resolved, as {column: fmt}."""
|
|
526
|
+
import xml.etree.ElementTree as ET
|
|
527
|
+
root = ET.fromstring(xml_text) if xml_text.strip() else None
|
|
528
|
+
if root is None:
|
|
529
|
+
return []
|
|
530
|
+
columns = [c.findtext("mnemonic") or "" for c in root.iter("col")]
|
|
531
|
+
seen: dict[str, str] = {}
|
|
532
|
+
for el in root.iter():
|
|
533
|
+
if el.get("id"):
|
|
534
|
+
seen[el.get("id")] = el.get("fmt") or (el.text or "")
|
|
535
|
+
rows = []
|
|
536
|
+
for row in root.iter("row"):
|
|
537
|
+
values = [c.get("fmt") or seen.get(c.get("ref") or "", "") or (c.text or "") for c in row]
|
|
538
|
+
rows.append(dict(zip(columns, values)))
|
|
539
|
+
return rows
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def cmd_perf(args: argparse.Namespace) -> int:
|
|
543
|
+
"""PERF-05: launch the installed app, attach Time Profiler for --seconds, report Instruments' potential hangs and hang risks.
|
|
544
|
+
`xctrace record --launch` waits for a 'boot' that never comes on an iOS newer than Xcode's SDK (27.2 vs 27.1), so it attaches."""
|
|
545
|
+
manifest_path, data = load_app(args)
|
|
546
|
+
device = find_device(args.udid)
|
|
547
|
+
if not device:
|
|
548
|
+
print("no paired device")
|
|
549
|
+
return 1
|
|
550
|
+
bundle = bundle_id(manifest_path, data)
|
|
551
|
+
out = Path(args.out or ARTIFACTS / data["app"] / ("perf-" + dt.datetime.now().strftime("%Y%m%d-%H%M%S")))
|
|
552
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
553
|
+
trace = out / "launch.trace"
|
|
554
|
+
variant = data.get("variants", {}).get(args.variant, {}) if args.variant else {}
|
|
555
|
+
if args.variant and not variant:
|
|
556
|
+
raise SystemExit(f"unknown variant {args.variant}")
|
|
557
|
+
for cmd in variant_commands(device["udid"], variant): # e.g. VoiceOver on: an assistive technology walks the whole tree
|
|
558
|
+
run(cmd)
|
|
559
|
+
try:
|
|
560
|
+
launch_args = ["--", *args.arg] if args.arg else []
|
|
561
|
+
run(["xcrun", "devicectl", "device", "process", "launch", "--device", device["udid"], "--terminate-existing", bundle, *launch_args])
|
|
562
|
+
run(["xcrun", "xctrace", "record", "--template", "Time Profiler", "--device", device["udid"], "--time-limit", f"{args.seconds}s",
|
|
563
|
+
"--output", str(trace), "--attach", data["process_name"]], timeout=args.seconds + 180)
|
|
564
|
+
finally:
|
|
565
|
+
for cmd in restore_commands(device["udid"], variant):
|
|
566
|
+
run(cmd)
|
|
567
|
+
report = {}
|
|
568
|
+
for schema in ("potential-hangs", "hang-risks"):
|
|
569
|
+
xml_text = run(["xcrun", "xctrace", "export", "--input", str(trace), "--xpath", f'/trace-toc/run/data/table[@schema="{schema}"]'],
|
|
570
|
+
timeout=600).stdout
|
|
571
|
+
report[schema] = trace_rows(xml_text)
|
|
572
|
+
print(f"{schema}: {len(report[schema])}" + "".join(f"\n {r.get('duration', '')} {r.get('hang-type', '')} {r.get('thread', '')[:60]}" for r in report[schema][:10]))
|
|
573
|
+
(out / "perf.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
574
|
+
print(f"trace: {trace}")
|
|
575
|
+
return 1 if report["potential-hangs"] else 0
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def cmd_upgrade(args: argparse.Namespace) -> int:
|
|
579
|
+
"""BUILD-04: the previous release's data survives an in-place upgrade. A temporary worktree at --from builds the old app, the
|
|
580
|
+
manifest's seed test (copied in, black-box) leaves real-user data on the phone, this build installs over it and the check test
|
|
581
|
+
runs with QA_UPGRADE=1. The app is uninstalled first so the seed starts from nothing; the worktree is removed afterwards."""
|
|
582
|
+
manifest_path, data = load_app(args)
|
|
583
|
+
cfg = data["upgrade"]
|
|
584
|
+
device = find_device(args.udid)
|
|
585
|
+
if not device:
|
|
586
|
+
print("no paired device")
|
|
587
|
+
return 1
|
|
588
|
+
app_dir = app_root(manifest_path, data)
|
|
589
|
+
root = git_top(app_dir)
|
|
590
|
+
project_rel = (manifest_path.parent / (data.get("workspace") or data["project"])).resolve().relative_to(app_dir)
|
|
591
|
+
bundle = bundle_id(manifest_path, data)
|
|
592
|
+
out = Path(args.out or ARTIFACTS / data["app"] / ("upgrade-" + dt.datetime.now().strftime("%Y%m%d-%H%M%S")))
|
|
593
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
594
|
+
old_repo = out / "old-repo"
|
|
595
|
+
added = run(["git", "-C", str(root), "worktree", "add", "--detach", str(old_repo), args.from_ref], timeout=600)
|
|
596
|
+
if added.returncode:
|
|
597
|
+
print(added.stderr.strip())
|
|
598
|
+
return 1
|
|
599
|
+
try:
|
|
600
|
+
old_app = old_repo / app_dir.relative_to(root)
|
|
601
|
+
seed_src = (manifest_path.parent / cfg["seed_file"]).resolve()
|
|
602
|
+
(old_app / cfg["uitests_dir"] / seed_src.name).write_text(seed_src.read_text(encoding="utf-8"), encoding="utf-8")
|
|
603
|
+
for rel in cfg.get("copy", []): # gitignored files the old build needs (keys)
|
|
604
|
+
if (app_dir / rel).is_file():
|
|
605
|
+
(old_app / rel).write_bytes((app_dir / rel).read_bytes())
|
|
606
|
+
if cfg.get("generator"):
|
|
607
|
+
run([sys.executable, str(old_app / cfg["generator"])], timeout=300)
|
|
608
|
+
run(["xcrun", "devicectl", "device", "uninstall", "app", "--device", device["udid"], bundle])
|
|
609
|
+
steps = [("old-seed", old_app / project_rel, out / "dd-old", cfg["seed"], {}),
|
|
610
|
+
("new-check", app_dir / project_rel, derived_data(manifest_path, data), cfg["check"], {"TEST_RUNNER_QA_UPGRADE": "1"})]
|
|
611
|
+
ok = True
|
|
612
|
+
for name, project, derived, test, env in steps:
|
|
613
|
+
bundle_path, log = out / f"{name}.xcresult", out / f"{name}.log"
|
|
614
|
+
flag = "-workspace" if project.suffix == ".xcworkspace" else "-project"
|
|
615
|
+
run(["xcodebuild", flag, str(project), "-scheme", data["scheme"], "-destination", f"id={device['udid']}",
|
|
616
|
+
"-derivedDataPath", str(derived), "-resultBundlePath", str(bundle_path), *signing_args(manifest_path, data),
|
|
617
|
+
*data.get("xcodebuild_args", []), f"-only-testing:{test}", "test"], env=tool_env(env), timeout=args.timeout, log=log)
|
|
618
|
+
summary = summarize_log(log.read_text(encoding="utf-8", errors="replace"))
|
|
619
|
+
shots = export_shots(bundle_path, out / f"{name}-shots", f"{data['app']} upgrade {name}") if bundle_path.exists() else 0
|
|
620
|
+
print(f"{name} ({test}): executed {summary['executed']} skipped {summary['skipped']} failures {summary['failures']} shots {shots}")
|
|
621
|
+
ok = ok and summary["succeeded"] and summary["executed"] > summary["skipped"] # a skipped check is not a pass
|
|
622
|
+
if not ok:
|
|
623
|
+
break
|
|
624
|
+
finally:
|
|
625
|
+
run(["git", "-C", str(root), "worktree", "remove", "--force", str(old_repo)], timeout=600)
|
|
626
|
+
print(("upgrade kept the data" if ok else "UPGRADE FAILED") + f" artifacts: {out}")
|
|
627
|
+
return 0 if ok else 1
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def export_options(team_id: str, cloudkit_env: str) -> dict[str, Any]:
|
|
631
|
+
"""ExportOptions for installing the exact Release archive on the test phone: development signing ('debugging', the old
|
|
632
|
+
'development'), the CloudKit environment chosen explicitly so a production round trip can be tested before TestFlight."""
|
|
633
|
+
return {"method": "debugging", "signingStyle": "automatic", "teamID": team_id, "iCloudContainerEnvironment": cloudkit_env,
|
|
634
|
+
"compileBitcode": False, "stripSwiftSymbols": True}
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def app_pid(process_list: str, process: str) -> str | None:
|
|
638
|
+
"""`devicectl device info processes` text: the pid of `<process>.app/<process>`, None when it is not running."""
|
|
639
|
+
return next((line.split()[0] for line in process_list.splitlines() if line.rstrip().endswith(f"/{process}.app/{process}")), None)
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def app_running(process_list: str, process: str) -> bool:
|
|
643
|
+
return app_pid(process_list, process) is not None
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def cmd_release(args: argparse.Namespace) -> int:
|
|
647
|
+
"""BUILD-01: the Release archive itself, exported for the test phone, cold-launched --launches times with no debugger attached.
|
|
648
|
+
Each launch: alive after --wait seconds, a screenshot; then crash/jetsam reports since the first launch."""
|
|
649
|
+
import plistlib
|
|
650
|
+
import time
|
|
651
|
+
manifest_path, data = load_app(args)
|
|
652
|
+
device = find_device(args.udid)
|
|
653
|
+
if not device:
|
|
654
|
+
print("no paired device")
|
|
655
|
+
return 1
|
|
656
|
+
bundle = bundle_id(manifest_path, data)
|
|
657
|
+
out = Path(args.out or ARTIFACTS / data["app"] / ("release-" + dt.datetime.now().strftime("%Y%m%d-%H%M%S")))
|
|
658
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
659
|
+
options = out / "ExportOptions-debugging.plist"
|
|
660
|
+
options.write_bytes(plistlib.dumps(export_options(team_id(manifest_path, data), args.cloudkit)))
|
|
661
|
+
exported = run(["xcodebuild", "-exportArchive", "-archivePath", args.archive, "-exportPath", str(out / "export"),
|
|
662
|
+
"-exportOptionsPlist", str(options), *signing_args(manifest_path, data)], timeout=1800, log=out / "export.log")
|
|
663
|
+
ipa = next((out / "export").glob("*.ipa"), None)
|
|
664
|
+
if exported.returncode or not ipa:
|
|
665
|
+
print(f"export failed: {out / 'export.log'}")
|
|
666
|
+
return 1
|
|
667
|
+
installed = run(["xcrun", "devicectl", "device", "install", "app", "--device", device["udid"], str(ipa)], timeout=600)
|
|
668
|
+
if installed.returncode:
|
|
669
|
+
print((installed.stdout + installed.stderr).strip()[-400:])
|
|
670
|
+
return 1
|
|
671
|
+
started = dt.datetime.now(dt.timezone.utc)
|
|
672
|
+
alive = []
|
|
673
|
+
for i in range(1, args.launches + 1):
|
|
674
|
+
run(["xcrun", "devicectl", "device", "process", "launch", "--device", device["udid"], "--terminate-existing", bundle])
|
|
675
|
+
time.sleep(args.wait)
|
|
676
|
+
processes = run(["xcrun", "devicectl", "device", "info", "processes", "--device", device["udid"]]).stdout
|
|
677
|
+
alive.append(app_running(processes, data["process_name"]))
|
|
678
|
+
run(["xcrun", "devicectl", "device", "capture", "screenshot", "--device", device["udid"], "--destination", str(out / f"launch-{i}.png")])
|
|
679
|
+
print(f"launch {i}: {'running' if alive[-1] else 'NOT RUNNING'} screenshot {out / f'launch-{i}.png'}")
|
|
680
|
+
time.sleep(5) # ReportCrash writes a few seconds after the process dies
|
|
681
|
+
crashes = pull_crashes(device["udid"], [data["process_name"], *data.get("extension_processes", [])], started, out / "crashes")
|
|
682
|
+
print(f"{sum(alive)}/{len(alive)} launches alive, crashes {len(crashes)} ipa {ipa.name} artifacts {out}")
|
|
683
|
+
return 0 if all(alive) and not crashes else 1
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def cmd_traits(args: argparse.Namespace) -> int:
|
|
687
|
+
"""INT-06: for each manifest screen, TRAIT_STEPS with the screen in front, then again with Settings in front. iOS asks a waiting
|
|
688
|
+
app to redraw its snapshot after each change and kills it when that takes over 10 s (0x8BADF00D scene-update): Pawday's map
|
|
689
|
+
looped there on the largest text size (2026-10-28). A screen passes when the process that took the changes is still the one
|
|
690
|
+
running on return; the run passes when no crash report was written either. The phone goes back to light and large text."""
|
|
691
|
+
import time
|
|
692
|
+
manifest_path, data = load_app(args)
|
|
693
|
+
device = find_device(args.udid)
|
|
694
|
+
if not device:
|
|
695
|
+
print("no paired device")
|
|
696
|
+
return 1
|
|
697
|
+
udid, process = device["udid"], data["process_name"]
|
|
698
|
+
bundle = bundle_id(manifest_path, data)
|
|
699
|
+
screens = (data.get("traits") or {}).get("screens") or {"launch": []}
|
|
700
|
+
screens = {k: v for k, v in screens.items() if not args.screen or k in args.screen}
|
|
701
|
+
out = Path(args.out or ARTIFACTS / data["app"] / ("traits-" + dt.datetime.now().strftime("%Y%m%d-%H%M%S")))
|
|
702
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
703
|
+
pid = lambda: app_pid(run(["xcrun", "devicectl", "device", "info", "processes", "--device", udid]).stdout, process)
|
|
704
|
+
started = dt.datetime.now(dt.timezone.utc)
|
|
705
|
+
failed = []
|
|
706
|
+
for name, launch_args in screens.items():
|
|
707
|
+
run(["xcrun", "devicectl", "device", "process", "launch", "--device", udid, "--terminate-existing", bundle, *(["--", *launch_args] if launch_args else [])])
|
|
708
|
+
time.sleep(args.wait)
|
|
709
|
+
before = pid()
|
|
710
|
+
for phase in ("front", "background"):
|
|
711
|
+
if phase == "background":
|
|
712
|
+
run(["xcrun", "devicectl", "device", "process", "launch", "--device", udid, "com.apple.Preferences"])
|
|
713
|
+
time.sleep(4)
|
|
714
|
+
for cmd in trait_commands(udid):
|
|
715
|
+
run(cmd)
|
|
716
|
+
time.sleep(args.step)
|
|
717
|
+
run(["xcrun", "devicectl", "device", "process", "launch", "--device", udid, bundle]) # back in front, no relaunch
|
|
718
|
+
time.sleep(4)
|
|
719
|
+
after = pid()
|
|
720
|
+
run(["xcrun", "devicectl", "device", "capture", "screenshot", "--device", udid, "--destination", str(out / f"{name}.png")])
|
|
721
|
+
survived = before is not None and before == after
|
|
722
|
+
print(f"{name}: {'survived' if survived else 'KILLED (relaunched or gone)'} pid {before} -> {after}")
|
|
723
|
+
if not survived:
|
|
724
|
+
failed.append(name)
|
|
725
|
+
for cmd in restore_commands(udid, dict(DEFAULTS)):
|
|
726
|
+
run(cmd)
|
|
727
|
+
time.sleep(5) # ReportCrash writes a few seconds after the process dies
|
|
728
|
+
crashes = pull_crashes(udid, [process, *data.get("extension_processes", [])], started, out / "crashes")
|
|
729
|
+
print(f"{len(screens) - len(failed)}/{len(screens)} screens survived, crashes {len(crashes)} artifacts {out}")
|
|
730
|
+
return 0 if not failed and not crashes else 1
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def cmd_crashes(args: argparse.Namespace) -> int:
|
|
734
|
+
_, data = load_app(args)
|
|
735
|
+
device = find_device(args.udid)
|
|
736
|
+
if not device:
|
|
737
|
+
print("no paired device")
|
|
738
|
+
return 1
|
|
739
|
+
since = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=args.since_minutes)
|
|
740
|
+
crashes = pull_crashes(device["udid"], [data["process_name"], *data.get("extension_processes", [])], since, Path(args.out))
|
|
741
|
+
print("\n".join(crashes) or "no crashes")
|
|
742
|
+
return 1 if crashes else 0
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
DEVICE_COMMANDS = {"test", "perf", "upgrade", "release", "traits", "devmode", "lock"} # these drive the phone: one session at a time
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def add_manifest_args(parser: argparse.ArgumentParser) -> None:
|
|
749
|
+
"""--manifest or --app (+ --apps-root); manifest_path_for raises when a command needs one and both are missing."""
|
|
750
|
+
parser.add_argument("--manifest", help="path to device_qa.json")
|
|
751
|
+
parser.add_argument("--app", help="app slug under --apps-root")
|
|
752
|
+
parser.add_argument("--apps-root", help="folder that holds <slug>/release/testing/device_qa.json (default $QAKIT_APPS_ROOT or <git top>/apps)")
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def cmd_lock(args: argparse.Namespace) -> int:
|
|
756
|
+
"""`status`: who holds the phone and who waits. `run`: hold it around a raw command (main() took the lock already)."""
|
|
757
|
+
if args.action == "run":
|
|
758
|
+
command = args.cmd[1:] if args.cmd[:1] == ["--"] else args.cmd
|
|
759
|
+
return subprocess.call(command) if command else 2
|
|
760
|
+
device = find_device(args.udid)
|
|
761
|
+
lock = DeviceLock(device["udid"] if device else (args.udid or "no-device"), "")
|
|
762
|
+
print("held by " + lock.describe(lock.holder()) if lock.holder() else "free")
|
|
763
|
+
for i, ticket in enumerate(lock.waiting(), 1):
|
|
764
|
+
print(f" {i}. waiting: {lock.describe(ticket)}")
|
|
765
|
+
return 0
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def main(argv: list[str] | None = None) -> int:
|
|
769
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
770
|
+
parser.add_argument("--udid", help="pin one device (default: the paired physical iPhone, wired first)")
|
|
771
|
+
parser.add_argument("--no-wait", action="store_true", help="fail (exit 75) instead of waiting when another session holds the phone")
|
|
772
|
+
parser.add_argument("--wait-timeout", type=float, default=4 * 3600, help="seconds to wait for the phone before giving up")
|
|
773
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
774
|
+
doctor = sub.add_parser("doctor")
|
|
775
|
+
add_manifest_args(doctor)
|
|
776
|
+
devmode = sub.add_parser("devmode")
|
|
777
|
+
devmode.add_argument("action", choices=["list", "reveal", "enable"])
|
|
778
|
+
test = sub.add_parser("test")
|
|
779
|
+
add_manifest_args(test)
|
|
780
|
+
test.add_argument("--suite", action="append", help="suite name from the manifest (repeatable; default ui)")
|
|
781
|
+
test.add_argument("--variant", action="append", help="variant name from the manifest (repeatable; default 'default')")
|
|
782
|
+
test.add_argument("--only", action="append", help="test identifier(s) to run instead of suites, e.g. PawdayUITests/PawdayUITests/testX")
|
|
783
|
+
test.add_argument("--out")
|
|
784
|
+
test.add_argument("--timeout", type=float, default=3600)
|
|
785
|
+
test.add_argument("--configuration", choices=["Debug", "Release"], default="Debug", help="Release runs the UI suites on the optimized build")
|
|
786
|
+
shots = sub.add_parser("shots")
|
|
787
|
+
shots.add_argument("xcresult")
|
|
788
|
+
shots.add_argument("--out", required=True)
|
|
789
|
+
perf = sub.add_parser("perf")
|
|
790
|
+
add_manifest_args(perf)
|
|
791
|
+
perf.add_argument("--seconds", type=int, default=40)
|
|
792
|
+
perf.add_argument("--arg", action="append", default=[], help="launch argument for the app (repeatable), e.g. --arg=-pawdayStart --arg=map")
|
|
793
|
+
perf.add_argument("--variant", help="manifest variant applied while recording, e.g. voiceover")
|
|
794
|
+
perf.add_argument("--out")
|
|
795
|
+
upgrade = sub.add_parser("upgrade")
|
|
796
|
+
add_manifest_args(upgrade)
|
|
797
|
+
upgrade.add_argument("--from", dest="from_ref", required=True, help="git ref of the previous release (tag or commit)")
|
|
798
|
+
upgrade.add_argument("--out")
|
|
799
|
+
upgrade.add_argument("--timeout", type=float, default=2400)
|
|
800
|
+
release = sub.add_parser("release")
|
|
801
|
+
add_manifest_args(release)
|
|
802
|
+
release.add_argument("--archive", required=True, help="the .xcarchive that is (or will be) uploaded")
|
|
803
|
+
release.add_argument("--cloudkit", choices=["Production", "Development"], default="Production")
|
|
804
|
+
release.add_argument("--launches", type=int, default=5)
|
|
805
|
+
release.add_argument("--wait", type=float, default=8)
|
|
806
|
+
release.add_argument("--out")
|
|
807
|
+
traits = sub.add_parser("traits")
|
|
808
|
+
add_manifest_args(traits)
|
|
809
|
+
traits.add_argument("--screen", action="append", help="screen name from the manifest's traits.screens (repeatable; default all)")
|
|
810
|
+
traits.add_argument("--wait", type=float, default=10, help="seconds after launch before the first change")
|
|
811
|
+
traits.add_argument("--step", type=float, default=4.5, help="seconds after each change")
|
|
812
|
+
traits.add_argument("--out")
|
|
813
|
+
crashes = sub.add_parser("crashes")
|
|
814
|
+
add_manifest_args(crashes)
|
|
815
|
+
crashes.add_argument("--since-minutes", type=int, default=60)
|
|
816
|
+
crashes.add_argument("--out", required=True)
|
|
817
|
+
lock = sub.add_parser("lock")
|
|
818
|
+
lock.add_argument("action", choices=["status", "run"])
|
|
819
|
+
lock.add_argument("--note", help="who holds the phone, shown to the sessions that wait")
|
|
820
|
+
lock.add_argument("cmd", nargs="*", help="run: the command to run while holding the phone, after `--`") # REMAINDER swallowed --note (found by the kit's tests, 0.2.0)
|
|
821
|
+
args = parser.parse_args(argv)
|
|
822
|
+
handler = {"doctor": cmd_doctor, "devmode": cmd_devmode, "test": cmd_test, "shots": cmd_shots, "crashes": cmd_crashes,
|
|
823
|
+
"perf": cmd_perf, "upgrade": cmd_upgrade, "release": cmd_release, "traits": cmd_traits, "lock": cmd_lock}[args.command]
|
|
824
|
+
if args.command not in DEVICE_COMMANDS or (args.command == "lock" and args.action == "status"):
|
|
825
|
+
return handler(args)
|
|
826
|
+
device = find_device(args.udid)
|
|
827
|
+
note = (getattr(args, "note", None) or os.environ.get("QAKIT_DEVICE_NOTE")
|
|
828
|
+
or f"{getattr(args, 'app', None) or 'phone'} {args.command} from {Path.cwd().name}")
|
|
829
|
+
try:
|
|
830
|
+
with DeviceLock(device["udid"] if device else (args.udid or "no-device"), note, wait=not args.no_wait, timeout=args.wait_timeout):
|
|
831
|
+
return handler(args)
|
|
832
|
+
except DeviceBusy as busy:
|
|
833
|
+
print(busy)
|
|
834
|
+
return 75 # EX_TEMPFAIL: try again later
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
if __name__ == "__main__":
|
|
838
|
+
sys.exit(main()) # mutation-ok: entry point, pytest calls main() directly
|