qa-agent-kit 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qa_agent_kit-0.6.0/.claude-plugin/marketplace.json +17 -0
- qa_agent_kit-0.6.0/.claude-plugin/plugin.json +21 -0
- qa_agent_kit-0.6.0/.gitignore +10 -0
- qa_agent_kit-0.6.0/AGENTS.md +48 -0
- qa_agent_kit-0.6.0/CHANGELOG.md +202 -0
- qa_agent_kit-0.6.0/CONTRIBUTING.md +32 -0
- qa_agent_kit-0.6.0/LICENSE +21 -0
- qa_agent_kit-0.6.0/PKG-INFO +157 -0
- qa_agent_kit-0.6.0/README.md +131 -0
- qa_agent_kit-0.6.0/RELEASING.md +34 -0
- qa_agent_kit-0.6.0/ROADMAP.md +48 -0
- qa_agent_kit-0.6.0/docs/README.md +14 -0
- qa_agent_kit-0.6.0/docs/agents.md +40 -0
- qa_agent_kit-0.6.0/docs/case-studies.md +146 -0
- qa_agent_kit-0.6.0/docs/device-qa-protocol.md +426 -0
- qa_agent_kit-0.6.0/docs/getting-started.md +100 -0
- qa_agent_kit-0.6.0/docs/ko/device-qa-protocol.md +433 -0
- qa_agent_kit-0.6.0/docs/manifest.md +96 -0
- qa_agent_kit-0.6.0/docs/mutation-testing.md +69 -0
- qa_agent_kit-0.6.0/docs/prose-qa.md +49 -0
- qa_agent_kit-0.6.0/docs/pypi.md +40 -0
- qa_agent_kit-0.6.0/docs/report.md +64 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/AboutView.swift +44 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/Assets.xcassets/AccentColor.colorset/Contents.json +1 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/Assets.xcassets/AppIcon.appiconset/Contents.json +1 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/Assets.xcassets/Contents.json +1 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/Info.plist +22 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/NoteStore.swift +42 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/NotesApp.swift +52 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/NotesListView.swift +122 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes/SettingsView.swift +48 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes.xcodeproj/project.pbxproj +362 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes.xcodeproj/project.xcworkspace/contents.xcworkspacedata +5 -0
- qa_agent_kit-0.6.0/examples/Notes/Notes.xcodeproj/xcshareddata/xcschemes/Notes.xcscheme +19 -0
- qa_agent_kit-0.6.0/examples/Notes/NotesCore/Package.swift +15 -0
- qa_agent_kit-0.6.0/examples/Notes/NotesCore/Sources/NotesCore/NoteList.swift +79 -0
- qa_agent_kit-0.6.0/examples/Notes/NotesCore/Tests/NotesCoreTests/NoteListTests.swift +58 -0
- qa_agent_kit-0.6.0/examples/Notes/NotesUITests/NotesQAWalkthroughTests.swift +187 -0
- qa_agent_kit-0.6.0/examples/Notes/release/testing/device_qa.json +34 -0
- qa_agent_kit-0.6.0/examples/Notes/release/testing/mutation.json +3 -0
- qa_agent_kit-0.6.0/examples/Notes/scripts/generate_pbxproj.py +203 -0
- qa_agent_kit-0.6.0/examples/README.md +102 -0
- qa_agent_kit-0.6.0/examples/device_qa.json +43 -0
- qa_agent_kit-0.6.0/mcp/README.md +62 -0
- qa_agent_kit-0.6.0/mcp/server.py +369 -0
- qa_agent_kit-0.6.0/mutation.json +4 -0
- qa_agent_kit-0.6.0/pyproject.toml +73 -0
- qa_agent_kit-0.6.0/skills/README.md +51 -0
- qa_agent_kit-0.6.0/skills/device-qa/SKILL.md +87 -0
- qa_agent_kit-0.6.0/skills/prose-qa/SKILL.md +49 -0
- qa_agent_kit-0.6.0/skills/qa-rigor/SKILL.md +105 -0
- qa_agent_kit-0.6.0/src/qakit/__init__.py +21 -0
- qa_agent_kit-0.6.0/src/qakit/_cli.py +46 -0
- qa_agent_kit-0.6.0/tests/test_device_qa.py +268 -0
- qa_agent_kit-0.6.0/tests/test_device_qa_commands.py +443 -0
- qa_agent_kit-0.6.0/tests/test_device_qa_flow.py +447 -0
- qa_agent_kit-0.6.0/tests/test_install_skills.py +100 -0
- qa_agent_kit-0.6.0/tests/test_mcp_server.py +231 -0
- qa_agent_kit-0.6.0/tests/test_mutate.py +309 -0
- qa_agent_kit-0.6.0/tests/test_prose_check.py +183 -0
- qa_agent_kit-0.6.0/tests/test_release_checks.py +123 -0
- qa_agent_kit-0.6.0/tests/test_release_checks_flow.py +194 -0
- qa_agent_kit-0.6.0/tests/test_shots_gif.py +60 -0
- qa_agent_kit-0.6.0/tools/device_lock.py +151 -0
- qa_agent_kit-0.6.0/tools/device_qa.py +838 -0
- qa_agent_kit-0.6.0/tools/install_skills.py +120 -0
- qa_agent_kit-0.6.0/tools/mutate.py +684 -0
- qa_agent_kit-0.6.0/tools/prose_check.py +291 -0
- qa_agent_kit-0.6.0/tools/recording_frames.swift +78 -0
- qa_agent_kit-0.6.0/tools/release_checks.py +287 -0
- qa_agent_kit-0.6.0/tools/screenshot_contrast.swift +42 -0
- qa_agent_kit-0.6.0/tools/shots_gif.py +81 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "younatics",
|
|
3
|
+
"description": "Plugins by younatics: QAKit, the QA agent kit",
|
|
4
|
+
"owner": {
|
|
5
|
+
"name": "younatics",
|
|
6
|
+
"url": "https://github.com/younatics"
|
|
7
|
+
},
|
|
8
|
+
"plugins": [
|
|
9
|
+
{
|
|
10
|
+
"name": "qakit",
|
|
11
|
+
"source": "./",
|
|
12
|
+
"description": "QA procedures for coding agents (qa-rigor, device-qa, prose-qa) and the kit's tools as an MCP server",
|
|
13
|
+
"category": "testing",
|
|
14
|
+
"tags": ["qa", "testing", "mutation-testing", "ios", "prose"]
|
|
15
|
+
}
|
|
16
|
+
]
|
|
17
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "qakit",
|
|
3
|
+
"displayName": "QAKit",
|
|
4
|
+
"version": "0.5.1",
|
|
5
|
+
"description": "The QA agent kit: qa-rigor, device-qa and prose-qa procedures, and the kit's tools (real-device runs, release checks, mutation testing, prose gates, verdict ledger) as MCP tools",
|
|
6
|
+
"author": {
|
|
7
|
+
"name": "younatics",
|
|
8
|
+
"url": "https://github.com/younatics"
|
|
9
|
+
},
|
|
10
|
+
"homepage": "https://github.com/younatics/QAKit",
|
|
11
|
+
"repository": "https://github.com/younatics/QAKit",
|
|
12
|
+
"license": "MIT",
|
|
13
|
+
"keywords": ["qa", "testing", "mutation-testing", "ios", "xcuitest", "app-store", "prose"],
|
|
14
|
+
"skills": ["./skills/"],
|
|
15
|
+
"mcpServers": {
|
|
16
|
+
"qakit": {
|
|
17
|
+
"command": "python3",
|
|
18
|
+
"args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.py"]
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# AGENTS.md: how an AI coding agent uses QAKit
|
|
2
|
+
|
|
3
|
+
This file is read by agents that follow the `AGENTS.md` convention (Codex, Cursor, Aider, OpenHands, Gemini CLI and
|
|
4
|
+
others). Claude Code reads it through `CLAUDE.md`. It says what the kit is and which procedure to load for which job.
|
|
5
|
+
Copy or link it into the project that uses the kit, or point that project's own AGENTS.md here.
|
|
6
|
+
|
|
7
|
+
## What this is
|
|
8
|
+
|
|
9
|
+
QAKit is a QA agent kit: procedures (`skills/*/SKILL.md`), deterministic tools (`tools/*.py`, `Sources/QAKit`), and
|
|
10
|
+
the evidence formats a verdict cites (`docs/report.md`). The agent is the QA team; the tools make its findings
|
|
11
|
+
reproducible and its verdicts citable.
|
|
12
|
+
|
|
13
|
+
## Pick the procedure
|
|
14
|
+
|
|
15
|
+
| The job | Load | Run |
|
|
16
|
+
|---|---|---|
|
|
17
|
+
| Any code change before merge or release | `skills/qa-rigor/SKILL.md` | `python3 tools/mutate.py --config <mutation.json> --changed-since origin/main` |
|
|
18
|
+
| An Apple app before TestFlight or App Review | `skills/device-qa/SKILL.md` and `docs/device-qa-protocol.md` | `python3 tools/device_qa.py doctor --manifest <device_qa.json>`, then `test`, then `python3 tools/release_checks.py` |
|
|
19
|
+
| A memo, README, design doc, paper or post | `skills/prose-qa/SKILL.md` | `python3 tools/prose_check.py <file.md> --claims claims.csv` |
|
|
20
|
+
|
|
21
|
+
Read the skill fully before running a tool. Each skill ends with the verdict format; produce it, including what you did
|
|
22
|
+
not run and why.
|
|
23
|
+
|
|
24
|
+
## Rules the kit assumes
|
|
25
|
+
|
|
26
|
+
- Expected values are written before the test, the fix or the edit. Three values at every boundary.
|
|
27
|
+
- Green tests are not a verdict. The verdict is the ledger: gates run, results, gates skipped with one line of risk each.
|
|
28
|
+
- A survivor in a mutation report gets a test or a `mutation-ok: <reason>` on the line. Never an exclusion to lift a score.
|
|
29
|
+
- On a phone, evidence goes into the result bundle (`QA.snap`, `QA.note`), never to a Mac path.
|
|
30
|
+
- Submission to review, a release request or a destructive action waits for the owner's explicit word.
|
|
31
|
+
- A gap in the kit (a tool that needs a workaround, a protocol row that is wrong) is fixed in the kit with a test and
|
|
32
|
+
released by tag (`RELEASING.md`), not patched in the consumer project.
|
|
33
|
+
|
|
34
|
+
## Install the skills into your agent
|
|
35
|
+
|
|
36
|
+
`python3 tools/install_skills.py --target claude --scope project` (or `codex`, `cursor`, `gemini`, `all`; `--scope user`
|
|
37
|
+
for your home folder). It links `skills/*` into the folder each tool reads and writes a one-line rule file where the
|
|
38
|
+
tool needs one. `skills/README.md` has the manual steps.
|
|
39
|
+
|
|
40
|
+
## Layout
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
skills/ qa-rigor · device-qa · prose-qa
|
|
44
|
+
tools/ mutate.py · device_qa.py · release_checks.py · prose_check.py · install_skills.py
|
|
45
|
+
Sources/QAKit Swift helpers for XCUITest targets (snap, audit, sweep, alerts, settle)
|
|
46
|
+
docs/ protocol (en, ko), manifest, report, getting started, mutation testing, prose QA
|
|
47
|
+
tests/ pytest; mutation.json measures the kit's own tests
|
|
48
|
+
```
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to QAKit. The format follows Keep a Changelog; versions follow SemVer (MAJOR: a tool flag, manifest
|
|
4
|
+
key, Swift API or skill step removed or renamed; MINOR: added; PATCH: fixed).
|
|
5
|
+
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
## [0.6.0] - 2026-10-29
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- On PyPI as `qa-agent-kit` (PyPI refused `qakit` as too similar to the existing `qa-kit`); the import name and the
|
|
13
|
+
`qakit-*` commands are unchanged.
|
|
14
|
+
|
|
15
|
+
- Swift: `QA.searchField(app, prompt:)` finds the search field wherever iOS 26 puts it: `searchFields`, the bottom bar's
|
|
16
|
+
text field, or behind the minimised Search button, which it taps. `QA.flip(toggle)` taps a SwiftUI Toggle's inner
|
|
17
|
+
switch, because a tap at the row's centre hits the label and changes nothing. `QA.tree(app, step, in:)` attaches the
|
|
18
|
+
element tree as text evidence when something cannot be found.
|
|
19
|
+
- Explained audit class "system bar button": a Cancel, Save or Done drawn by iOS in any navigation bar or toolbar (a sheet's
|
|
20
|
+
own bar included) is measured by the audit against the content behind it and reported as low contrast, fixed size
|
|
21
|
+
("Dynamic Type font sizes are partially unsupported") or clipped.
|
|
22
|
+
- Swift: `QA.count(query, in: app)` counts the rows of a lazy list that keeps only the visible ones in the tree (pages down
|
|
23
|
+
until a page adds nothing, then back to the top).
|
|
24
|
+
- `shots` index: Xcode 27.2's attachment manifest suffixes every readable name with `_<n>_<UUID>.<ext>`; the index shows
|
|
25
|
+
the step name alone again.
|
|
26
|
+
- `doctor` names two more reasons a device run cannot start: a project pinned to manual App Store profiles (and the
|
|
27
|
+
manifest `xcodebuild_args` override that fixes it) and a login keychain locked for the session (`codesign
|
|
28
|
+
errSecInternalComponent`: run the phone commands from a GUI-session LaunchAgent). Both found on Daynest.
|
|
29
|
+
|
|
30
|
+
### Fixed
|
|
31
|
+
|
|
32
|
+
- `QA.reveal`: a lift from under a floating bar never starts in the bottom 12% of the screen. A press there dragged up is
|
|
33
|
+
the Home / app-switcher gesture, and it put the phone in the app switcher mid-walk on Daynest. An element whose centre
|
|
34
|
+
is past the screen's side edge (a horizontal carousel) is dragged in sideways instead of paged vertically forever.
|
|
35
|
+
- `QA.closeInAppSafari` falls back to a live query for Safari's own Done button when the tree snapshot cannot be taken over a
|
|
36
|
+
heavy web page.
|
|
37
|
+
- `device_qa.py` puts Xcode's `usr/bin` first on PATH so xcodebuild's own `xcrun` finds `devicectl` (the diagnostics step
|
|
38
|
+
failed on a Mac whose shell PATH lacked it).
|
|
39
|
+
- `examples/Notes`: the walkthrough uses the helpers above; the stored-note count is a row (a section footer is reported
|
|
40
|
+
as fixed-size text), the About link is a plain row (a `Label` is reported as clippable text), the list body has no line
|
|
41
|
+
limit. Found on its first two device runs (iPhone 14 Pro, iOS 27.2).
|
|
42
|
+
|
|
43
|
+
## [0.5.2] - 2026-10-29
|
|
44
|
+
|
|
45
|
+
### Fixed
|
|
46
|
+
|
|
47
|
+
- `device_qa.py`: a tool that does not answer within its timeout (an `xcrun devicectl` that hangs when CoreSimulator is older
|
|
48
|
+
than Xcode) comes back as a failed step with a one-line reason instead of a traceback; `doctor` prints that reason and still
|
|
49
|
+
validates the manifest when no phone answers.
|
|
50
|
+
|
|
51
|
+
### Added
|
|
52
|
+
|
|
53
|
+
- `examples/Notes`: a small SwiftUI app (three tabs, 538 lines) with a complete `NotesQAWalkthroughTests`, a `device_qa.json`,
|
|
54
|
+
a `NotesCore` package with a mutation config, and two deliberate defects (a 2.7:1 caption, a dead Help button) the walk is
|
|
55
|
+
meant to find. `examples/README.md` walks a newcomer through the whole loop.
|
|
56
|
+
|
|
57
|
+
- `mcp/server.py`: a stdio MCP server with no third-party dependency (`device_doctor`, `device_test`, `device_shots`,
|
|
58
|
+
`device_crashes`, `device_traits`, `release_checks`, `mutation`, `prose_check`, `verdict`), and a Claude Code plugin
|
|
59
|
+
(`.claude-plugin/plugin.json`, `marketplace.json`: `claude plugin marketplace add younatics/QAKit`, then
|
|
60
|
+
`claude plugin install qakit@younatics`). `docs/agents.md` says how to attach the kit to each agent.
|
|
61
|
+
- `tools/shots_gif.py`: a run's screenshots as one animated GIF in `index.md` order (needs Pillow), for demos and bug reports.
|
|
62
|
+
|
|
63
|
+
- `pip install qakit`: `pyproject.toml` (hatchling, version from this file), `src/qakit/_cli.py` shims and five console
|
|
64
|
+
scripts (`qakit-mutate`, `qakit-prose`, `qakit-device`, `qakit-release-checks`, `qakit-install-skills`); the skills and docs
|
|
65
|
+
ship in the wheel so `qakit-install-skills` finds them (`docs/pypi.md`). The files landed in 0.5.1; not on PyPI yet.
|
|
66
|
+
|
|
67
|
+
## [0.5.1] - 2026-10-29
|
|
68
|
+
|
|
69
|
+
### Fixed
|
|
70
|
+
|
|
71
|
+
- `release_checks.py` SUB-03: the file-timestamp pattern no longer matches `record.modificationDate` or `asset.creationDate`
|
|
72
|
+
(CKRecord and PHAsset properties, not file APIs); it keeps URL resource keys, attribute dictionaries, `attributesOfItem`,
|
|
73
|
+
`fileModificationDate`, `getattrlist` and the `stat` family. Found on the second app (Daynest's notification service).
|
|
74
|
+
|
|
75
|
+
## [0.5.0] - 2026-10-29
|
|
76
|
+
|
|
77
|
+
### Added
|
|
78
|
+
|
|
79
|
+
- Protocol FEED-07: the production data path, with no fake-data flag, crosses its clock boundaries (midnight, the morning data job, cache expiry) with core data on screen at every instant. BUILD-01 now fails a cold launch that is alive but shows no core data. Both from Pawday's blank weather after midnight (2026-10-29).
|
|
80
|
+
|
|
81
|
+
## [0.4.1] - 2026-10-28
|
|
82
|
+
|
|
83
|
+
### Changed
|
|
84
|
+
|
|
85
|
+
- `mutate.py` makes no mutants on typing-only Python (`TYPE_CHECKING = False`, the body of `if TYPE_CHECKING:`, `@overload`
|
|
86
|
+
stubs, the test of `if sys.version_info ...`) and no drop-call on a C-family call that continues the previous line
|
|
87
|
+
(`let x =`, an open `(`, a trailing `,` or operator). Both classes came from `docs/case-studies.md`: every one was
|
|
88
|
+
equivalent or an invalid build.
|
|
89
|
+
|
|
90
|
+
## [0.4.0] - 2026-10-28
|
|
91
|
+
|
|
92
|
+
### Added
|
|
93
|
+
|
|
94
|
+
- Manifest `fingerprint_root`: the folder whose edits mark a round as mixed, for apps whose core sources live beside the project
|
|
95
|
+
folder (Elsewhen).
|
|
96
|
+
- `QA.reveal(clearOf:)` lifts a covered element by the exact distance, slowly, from the margin at its height to a third of the
|
|
97
|
+
screen; a page fling overshot a cover strip narrower than the fling (Elsewhen, large text).
|
|
98
|
+
|
|
99
|
+
### Fixed
|
|
100
|
+
|
|
101
|
+
- The source fingerprint excludes the run's own DerivedData folders, so an app whose build folder is not gitignored no longer
|
|
102
|
+
reports every round as `sources_changed_during_run` (Elsewhen).
|
|
103
|
+
|
|
104
|
+
## [0.3.1] - 2026-10-28
|
|
105
|
+
|
|
106
|
+
### Fixed
|
|
107
|
+
|
|
108
|
+
- `QA.settleAfterLaunch(app)` and `Use VoiceOver` / `VoiceOver 사용` in the allow labels: the first time `--variant voiceover`
|
|
109
|
+
switches VoiceOver on, iOS 27.2 shows a welcome prompt over the app and swallowed the smoke test's first tap (reported by the
|
|
110
|
+
Elsewhen session from a screen recording, 2026-10-28). The manifest and skill docs say where to call it.
|
|
111
|
+
|
|
112
|
+
## [0.3.0] - 2026-10-28
|
|
113
|
+
|
|
114
|
+
### Added
|
|
115
|
+
|
|
116
|
+
- `QA.reveal(_:in:hittable:clearOf:)`: floating covers (a bottom bar, the tab bar) to drag clear of. On iOS 27 an element
|
|
117
|
+
under a floating bar still reports hittable and the tap lands on the bar (proposed by the Elsewhen session after a
|
|
118
|
+
large-text device run, 2026-10-28).
|
|
119
|
+
|
|
120
|
+
### Fixed
|
|
121
|
+
|
|
122
|
+
- `tests/test_install_skills.py` sandboxes HOME and `Path.home()`: a mutated `--scope` check in a mutation run linked
|
|
123
|
+
skills into the real `~/.claude/skills` and `~/.agents/skills` (dangling links to the scratch copy, 2026-10-28).
|
|
124
|
+
|
|
125
|
+
## [0.2.0] - 2026-10-28
|
|
126
|
+
|
|
127
|
+
The consumer monorepo had moved on while 0.1.0 was being cut: four sessions QA-ing four apps on one phone had grown the
|
|
128
|
+
runner and the Swift helpers. This release ports all of it, generalized.
|
|
129
|
+
|
|
130
|
+
### Added
|
|
131
|
+
|
|
132
|
+
- `tools/device_qa.py` gained six commands and three options. `perf` runs with Time Profiler attached and reads
|
|
133
|
+
Instruments' potential hangs and hang risks. `upgrade` builds the previous release in a temporary worktree, seeds it,
|
|
134
|
+
installs this build over it and runs the check test with QA_UPGRADE=1. `release` exports the Release archive for the
|
|
135
|
+
phone and cold-launches it with no debugger. `traits` changes text size and dark mode on each screen, in front and in
|
|
136
|
+
the background, and the process must survive. `lock status` and `lock run` share the phone. `test --only` and
|
|
137
|
+
`--configuration Release` narrow a run, and a source-tree fingerprint fails a round whose sources changed mid-run
|
|
138
|
+
(`sources_changed_during_run`). Manifest keys `bundle_id`, `team_id`, `traits`, `upgrade`, `release.version`.
|
|
139
|
+
- `tools/device_lock.py`: one phone, many sessions. flock on `~/.qakit/device-qa/<udid>.lock` plus arrival-order tickets;
|
|
140
|
+
`QAKIT_DEVICE_NOTE` names the holder, `QAKIT_DEVICE_HELD` lets a holder's children share its turn, `--no-wait` exits 75.
|
|
141
|
+
- `tools/recording_frames.swift` (frames and a contact sheet from a UI-test screen recording, or screenshots tiled side by
|
|
142
|
+
side) and `tools/screenshot_contrast.swift` (the real WCAG contrast of text in a screenshot box).
|
|
143
|
+
- Swift: `QA.walk` (snap, audit, sweep, scroll until the bottom), `QA.back`, `QA.closeInAppSafari`, `QA.exitSearch`,
|
|
144
|
+
`QA.onScreen`, `QA.reveal`, `QA.visibleButtons`, `QA.tap(frame)`, `QA.waitForStill`, `QA.tapUntilGone`, `QA.replaceText`,
|
|
145
|
+
`QA.switches(labelled:)`, `QA.waitForValue`, `QA.explainedReason`. `QA.audit` retries once on an audit timeout, attaches
|
|
146
|
+
Apple's detailed description and counts the classes the protocol explains (off screen, under the tab bar, empty-label
|
|
147
|
+
text field, nearly-passed secondary text) instead of failing them. `QA.sweepButtons` reads one snapshot of the tree and
|
|
148
|
+
taps coordinates (live queries by index kept the main thread busy and went stale), with `idPrefix`.
|
|
149
|
+
`QA.answerSystemAlert` only asks hosts that are in front (a suspended BusinessActionSheet cost 31 s per ask).
|
|
150
|
+
- `tools/release_checks.py`: SUB-12 rejects markup in store text (App Store Connect refuses `<library>`), SUB-15 checks
|
|
151
|
+
the age-rating block carries `socialMedia`; `--ipa` without `codesign` on the machine is a WARN, not a crash.
|
|
152
|
+
- `docs/case-studies.md`: the mutation layer measured on three public projects (80 to 86 percent scores; 7 of 15 sampled
|
|
153
|
+
survivors were real test gaps, 6 equivalent, 2 engine noise), with the engine improvements it points at.
|
|
154
|
+
- Protocol: INT-06 (trait changes), SUITE-08 rewritten around mutation testing, BUILD-01, BUILD-04 and PERF-05 name the new
|
|
155
|
+
commands, step 6a and exit criterion 3a (the mutation gate), in both editions.
|
|
156
|
+
|
|
157
|
+
### Fixed
|
|
158
|
+
|
|
159
|
+
- `device_qa.py lock run --note <who> -- <command>`: `--note` was swallowed into the command (argparse REMAINDER) and the
|
|
160
|
+
holder showed up under the default name; found by the kit's own CLI test.
|
|
161
|
+
|
|
162
|
+
### Changed
|
|
163
|
+
|
|
164
|
+
- `QA.dismissTopmost(_:closers:rootTitle:onPushedPage:)` replaces the 0.1.0 `closeIdentifiers:` form and
|
|
165
|
+
`QA.settle` takes `closers:` and `rootTitle:`. `QA.sweepScrolling` is gone; `QA.walk` does its job.
|
|
166
|
+
- `validate_manifest` reports `<key> must be a non-empty string` (0.1.0 said `missing <key>`) and requires `schema_version`.
|
|
167
|
+
- `swift build` on macOS: the iOS-only `textClipped` audit type is guarded, so the package builds on a Mac host and in CI.
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
## [0.1.0] - 2026-10-28
|
|
171
|
+
|
|
172
|
+
First release, extracted from a private monorepo where the three layers had been in daily use.
|
|
173
|
+
|
|
174
|
+
### Added
|
|
175
|
+
|
|
176
|
+
- `skills/qa-rigor`: the nine principles, the mutation-testing procedure, the verdict ledger.
|
|
177
|
+
- `tools/mutate.py`: mutation testing for Python (tokenize and ast) and C-family syntax (Swift, JS/TS, Kotlin, Go, Java,
|
|
178
|
+
C#); throwaway copies, coverage-aware uncovered reporting, `--changed-since`, `--resume`, seeded samples.
|
|
179
|
+
- `skills/device-qa` and `docs/device-qa-protocol.md`: 24 areas, 180 checks with IDs, pass criteria, methods, severities
|
|
180
|
+
and sources; English and Korean editions.
|
|
181
|
+
- `tools/device_qa.py`: `doctor`, `devmode`, `test` under device-wide variants (appearance, text size, location,
|
|
182
|
+
VoiceOver, language) with automatic restore, `shots` contact sheets, `crashes`; `--manifest` or `--app` resolution;
|
|
183
|
+
App Store Connect key signing from a `signing` block or an `appstore_config` file; `summary.json`
|
|
184
|
+
(`qakit.device-qa.summary.v1`).
|
|
185
|
+
- `tools/release_checks.py`: store metadata limits and placeholders, first-release wording after 1.0, review note
|
|
186
|
+
length, localized purpose strings, required-reason API declarations per bundle, links, AASA on Apple's CDN, IPA
|
|
187
|
+
entitlements.
|
|
188
|
+
- Swift package `QAKit` (iOS 17+): `snap`, `note`, `assertInFront`, `audit`, `answerSystemAlert`, `dismissTopmost`,
|
|
189
|
+
`settle`, `sweepButtons`, `sweepScrolling`; labels, alert hosts and locales configurable through `QA_*` environment.
|
|
190
|
+
- `skills/prose-qa`, `docs/prose-qa.md` and `tools/prose_check.py`: outline, sentence length, duplicates, unsourced
|
|
191
|
+
claims, unexpanded acronyms, links, machine-writing tells; a claims ledger skeleton CSV.
|
|
192
|
+
- `AGENTS.md` for any agent that reads the convention, and `tools/install_skills.py` that links the skills into Claude
|
|
193
|
+
Code, Codex, Cursor and Gemini CLI (project or user scope, idempotent, uninstall).
|
|
194
|
+
- `examples/device_qa.json`, `docs/manifest.md`, `docs/report.md`, `docs/getting-started.md`, `docs/mutation-testing.md`.
|
|
195
|
+
- `tests/`: 150 tests over the five Python tools, the installer and the mutation engine, including the command layer of
|
|
196
|
+
`device_qa.py` driven with a fake `run`; `mutation.json` measures the kit's own tests (first run: 47% with the command
|
|
197
|
+
layer untested, which is why the flow tests exist).
|
|
198
|
+
|
|
199
|
+
### Fixed
|
|
200
|
+
|
|
201
|
+
- `device_qa.py`: an empty `--json-output` file from a failed `devicectl` call raised instead of returning `{}`.
|
|
202
|
+
- `release_checks.py --json`: the summary line went into stdout and broke JSON parsing; it goes to stderr now.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thank you for looking. QAKit is small on purpose: three Python tools, one Swift file, three skills and a protocol.
|
|
4
|
+
|
|
5
|
+
## Ground rules
|
|
6
|
+
|
|
7
|
+
- **Every check comes from a defect.** A new gate, operator or protocol row names the defect it would have caught, in its
|
|
8
|
+
docstring or its table row. Checks without a story get removed.
|
|
9
|
+
- **Deterministic tools, bare interpreters.** `tools/*.py` run on Python 3.11+ with the standard library; `pytest` is the
|
|
10
|
+
only test dependency. The Swift package depends on XCTest only.
|
|
11
|
+
- **Tests before merge.** A change to a tool comes with a test that fails without it. Run `python3 -m pytest -q tests`,
|
|
12
|
+
`python3 -m pyflakes tools tests`, `swift build`, and the mutation gate from `RELEASING.md`.
|
|
13
|
+
- **Docs in the same commit.** A tool flag, a manifest key or a Swift API appears in `docs/` and in the skill that uses it
|
|
14
|
+
in the commit that adds it. Both protocol editions (`docs/device-qa-protocol.md`, `docs/ko/device-qa-protocol.md`)
|
|
15
|
+
move together.
|
|
16
|
+
- **Plain prose.** Short sentences, no em dashes, no "Firstly". `python3 tools/prose_check.py` on what you wrote.
|
|
17
|
+
|
|
18
|
+
## Layout
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
Sources/QAKit/QAKit.swift Swift helpers for XCUITest targets
|
|
22
|
+
tools/ device_qa.py · release_checks.py · mutate.py · prose_check.py
|
|
23
|
+
tests/ pytest for the tools and the mutation engine
|
|
24
|
+
skills/ qa-rigor · device-qa · prose-qa (SKILL.md each)
|
|
25
|
+
docs/ protocol (en, ko), manifest, report, getting started, mutation testing, prose QA
|
|
26
|
+
examples/ a complete device_qa.json
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Reporting a gap
|
|
30
|
+
|
|
31
|
+
Open an issue with: the tool and version, the command, what happened, what you expected, and the evidence folder's
|
|
32
|
+
`summary.json` or the mutation report when it applies. A failing test is the best report.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SeungYoun Yi (younatics)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: qa-agent-kit
|
|
3
|
+
Version: 0.6.0
|
|
4
|
+
Summary: The QA agent kit: mutation testing for any repo, prose gates with a claims ledger, and real-device QA for Apple apps, with the procedures an AI coding agent follows.
|
|
5
|
+
Project-URL: Homepage, https://github.com/younatics/QAKit
|
|
6
|
+
Project-URL: Changelog, https://github.com/younatics/QAKit/blob/main/CHANGELOG.md
|
|
7
|
+
Project-URL: Documentation, https://github.com/younatics/QAKit/blob/main/docs/getting-started.md
|
|
8
|
+
Author: younatics
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,claude-code,codex,ios,mutation-testing,prose,qa,xcuitest
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: MacOS
|
|
16
|
+
Classifier: Operating System :: POSIX
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: pyflakes; extra == 'test'
|
|
24
|
+
Requires-Dist: pytest; extra == 'test'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
<p align="center">
|
|
28
|
+
<img src="https://raw.githubusercontent.com/younatics/QAKit/main/assets/banner.svg" alt="QAKit: the QA agent kit" width="100%">
|
|
29
|
+
</p>
|
|
30
|
+
|
|
31
|
+
# QAKit
|
|
32
|
+
|
|
33
|
+
**The QA agent kit.** Procedures an AI coding agent follows to test like a QA engineer, the tools that run the checks,
|
|
34
|
+
and the evidence format a release verdict cites. Three layers, one repo:
|
|
35
|
+
|
|
36
|
+
| Layer | For | Skill | Tools |
|
|
37
|
+
|---|---|---|---|
|
|
38
|
+
| Code | any repo with tests | `skills/qa-rigor` | `tools/mutate.py` (mutation testing, Python and C-family syntax) |
|
|
39
|
+
| Real device | Apple apps | `skills/device-qa` | `tools/device_qa.py` (test, traits, perf, upgrade, release, one lock per phone), `tools/release_checks.py`, `recording_frames.swift`, `screenshot_contrast.swift`, Swift package `QAKit` for XCUITest |
|
|
40
|
+
| Prose | memos, READMEs, papers | `skills/prose-qa` | `tools/prose_check.py` (gates and a claims ledger) |
|
|
41
|
+
|
|
42
|
+
QAKit grew out of shipping several App Store apps as a solo developer whose QA team is an agent. Every check in it was
|
|
43
|
+
added after a defect that green tests let through: a `>=` that became `>` at every grading boundary with 13 mutants
|
|
44
|
+
surviving, a review note over the 4,000-character limit, a clipped caption nobody looked at, a What's New that still
|
|
45
|
+
announced the first release in version 1.1.
|
|
46
|
+
|
|
47
|
+
## See it
|
|
48
|
+
|
|
49
|
+
<p align="center"><img src="https://raw.githubusercontent.com/younatics/QAKit/main/assets/demo-walk.gif" alt="An agent walking an app on a real iPhone: onboarding, the home screen at every scroll position, each button tapped once" width="300"></p>
|
|
50
|
+
|
|
51
|
+
`QA.walk` on a real iPhone: the onboarding pages, the home screen at every scroll position, every visible button tapped once
|
|
52
|
+
and screenshotted, the accessibility audit per screen. These 29 frames are one walkthrough test's attachments, assembled by
|
|
53
|
+
`tools/shots_gif.py` from the result bundle (the app is the author's, live on the App Store).
|
|
54
|
+
|
|
55
|
+
<p align="center"><img src="https://raw.githubusercontent.com/younatics/QAKit/main/assets/demo-variants.gif" alt="The Notes example app's list, settings and search screens in the default, dark and large-text variants, one phone, one command" width="300"></p>
|
|
56
|
+
|
|
57
|
+
The same walk in three variants: `device_qa.py test --variant default --variant dark --variant large-text` switches the
|
|
58
|
+
phone's appearance and text size between runs. These nine frames are the `examples/Notes` app's list, settings and
|
|
59
|
+
search screens from one such run.
|
|
60
|
+
|
|
61
|
+
## What makes it different
|
|
62
|
+
|
|
63
|
+
- **Systematic, not random.** `QA.sweepButtons` taps every visible button on every screen at every scroll position,
|
|
64
|
+
screenshots what opened, checks the app is still in front and goes back. Monkey testers tap at random; agents reviewing a
|
|
65
|
+
contact sheet need every control seen once.
|
|
66
|
+
- **The whole phone changes, not the app.** `device_qa.py test --variant dark` switches the device to dark mode, large
|
|
67
|
+
accessibility text, a simulated location or VoiceOver through Xcode 27's `devicectl`, runs the suite, and restores iOS
|
|
68
|
+
defaults. System alerts and other apps render the same way the user will see them.
|
|
69
|
+
- **Apple's accessibility audit on every screen,** collected into one report per screen instead of stopping at the first
|
|
70
|
+
issue, with an allow list for reviewed exceptions.
|
|
71
|
+
- **Test strength is measured.** Coverage says a line ran. Mutation testing says whether a test would notice if the
|
|
72
|
+
line were wrong. The gate is "every mutant on a changed line is dead or explained".
|
|
73
|
+
- **Prose gets the same treatment.** Deterministic gates, then a claims ledger where every number is traced to a quote,
|
|
74
|
+
then a cold read by a reader who did not write it.
|
|
75
|
+
- **Written for agents.** The skills are `SKILL.md` files a coding agent loads. The protocol has 180 checks with IDs,
|
|
76
|
+
pass criteria, methods and sources, so an agent can cite what it did and what it skipped.
|
|
77
|
+
|
|
78
|
+
## Quick start
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install qa-agent-kit # qakit-mutate, qakit-prose, qakit-device, qakit-release-checks, qakit-install-skills
|
|
82
|
+
# or from source
|
|
83
|
+
git clone https://github.com/younatics/QAKit.git
|
|
84
|
+
cd QAKit && python3 -m pytest -q tests # 150 tests, no dependency beyond pytest
|
|
85
|
+
|
|
86
|
+
# mutation testing on any project
|
|
87
|
+
python3 tools/mutate.py --root ../myproject --source 'src/**/*.py' --test '{orig}/.venv/bin/python -m pytest -x -q'
|
|
88
|
+
|
|
89
|
+
# prose
|
|
90
|
+
python3 tools/prose_check.py ../memo.md --claims claims.csv
|
|
91
|
+
|
|
92
|
+
# an Apple app on a USB-attached test iPhone (macOS, Xcode 27)
|
|
93
|
+
python3 tools/device_qa.py doctor --manifest ../myapp/release/testing/device_qa.json
|
|
94
|
+
python3 tools/device_qa.py test --manifest ../myapp/release/testing/device_qa.json --suite qa --variant dark
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Give the procedures to your agent, whichever one you use:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
claude plugin marketplace add younatics/QAKit && claude plugin install qakit@younatics # Claude Code: skills + MCP tools
|
|
101
|
+
python3 tools/install_skills.py --target claude --scope project # or codex, cursor, gemini, all; --scope user
|
|
102
|
+
claude mcp add qakit -- python3 /path/to/qakit/mcp/server.py # any MCP client: doctor, test, shots, crashes, release checks, mutation, prose, verdict
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Then read [docs/getting-started.md](docs/getting-started.md). [AGENTS.md](AGENTS.md) tells any agent which skill to load
|
|
106
|
+
for which job.
|
|
107
|
+
|
|
108
|
+
## Documentation
|
|
109
|
+
|
|
110
|
+
| Document | What it covers |
|
|
111
|
+
|---|---|
|
|
112
|
+
| [docs/getting-started.md](docs/getting-started.md) | Installing, linking the Swift package, the first walkthrough test, the first device run |
|
|
113
|
+
| [docs/device-qa-protocol.md](docs/device-qa-protocol.md) | The protocol: 24 areas, 180 checks with IDs, exit criteria, what only a human can do ([한국어판](docs/ko/device-qa-protocol.md)) |
|
|
114
|
+
| [docs/manifest.md](docs/manifest.md) | Every key of the `device_qa.json` manifest |
|
|
115
|
+
| [docs/report.md](docs/report.md) | What a run leaves behind: `summary.json`, contact sheets, crash reports |
|
|
116
|
+
| [docs/mutation-testing.md](docs/mutation-testing.md) | `mutate.py` operators, config, reports, how to read a score |
|
|
117
|
+
| [docs/prose-qa.md](docs/prose-qa.md) | The prose gates and the claims ledger |
|
|
118
|
+
| [docs/case-studies.md](docs/case-studies.md) | Does the mutation layer find real gaps? Measured on humanize, tenacity and swift-argument-parser |
|
|
119
|
+
| [docs/pypi.md](docs/pypi.md) | The pip package: the five commands, where the skills land, the release procedure |
|
|
120
|
+
| [examples/README.md](examples/README.md) | The Notes example app: build it, walk it, see the two planted defects reported |
|
|
121
|
+
| [docs/agents.md](docs/agents.md) | Attaching the kit to Claude Code (plugin), Codex, Cursor, Gemini CLI and any MCP client |
|
|
122
|
+
| [mcp/README.md](mcp/README.md) | The MCP server: nine tools, one stdio process, no dependencies |
|
|
123
|
+
| [skills/README.md](skills/README.md) | Installing the skills by hand |
|
|
124
|
+
| [ROADMAP.md](ROADMAP.md) | What comes next and the loop that keeps the kit honest |
|
|
125
|
+
| [RELEASING.md](RELEASING.md) | How a change becomes a tagged release |
|
|
126
|
+
|
|
127
|
+
## The Swift package
|
|
128
|
+
|
|
129
|
+
```swift
|
|
130
|
+
import XCTest
|
|
131
|
+
import QAKit
|
|
132
|
+
|
|
133
|
+
QA.snap("settings", in: self) // screenshot into the result bundle, named by variant
|
|
134
|
+
QA.audit(app, "settings", in: self) // Apple's accessibility audit, every issue in one report
|
|
135
|
+
QA.walk(app, "settings", in: self, deny: ["Delete"]) { QA.dismissTopmost(app) } // snap, audit, sweep, scroll
|
|
136
|
+
QA.answerSystemAlert() // permission prompts, in the phone's language
|
|
137
|
+
QA.settle(app, on: screen, in: self) { relaunch() } // get back to a known screen, relaunch as the last resort
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Requirements: iOS 17 or later for the accessibility audit, Xcode 15 or later to build, Xcode 27 or later for the
|
|
141
|
+
device variants in `device_qa.py`.
|
|
142
|
+
|
|
143
|
+
## Who runs it
|
|
144
|
+
|
|
145
|
+
- The author's app monorepo pins the kit as a submodule and runs every layer on a USB-attached test iPhone.
|
|
146
|
+
- [forge](https://github.com/younatics/forge), the multi-agent coding loop, runs the mutation and prose gates as merge
|
|
147
|
+
authority on every task and gives its reviewer the qa-rigor principles.
|
|
148
|
+
- Gaps either of them hits come back here as a fix with a test and a tag (`RELEASING.md`).
|
|
149
|
+
|
|
150
|
+
## Status
|
|
151
|
+
|
|
152
|
+
Private beta. The three layers are in daily use on the author's apps and documents; the API and the manifest keys may
|
|
153
|
+
still change between minor versions until 1.0. See [CHANGELOG.md](CHANGELOG.md).
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/younatics/QAKit/main/assets/banner.svg" alt="QAKit: the QA agent kit" width="100%">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
# QAKit
|
|
6
|
+
|
|
7
|
+
**The QA agent kit.** Procedures an AI coding agent follows to test like a QA engineer, the tools that run the checks,
|
|
8
|
+
and the evidence format a release verdict cites. Three layers, one repo:
|
|
9
|
+
|
|
10
|
+
| Layer | For | Skill | Tools |
|
|
11
|
+
|---|---|---|---|
|
|
12
|
+
| Code | any repo with tests | `skills/qa-rigor` | `tools/mutate.py` (mutation testing, Python and C-family syntax) |
|
|
13
|
+
| Real device | Apple apps | `skills/device-qa` | `tools/device_qa.py` (test, traits, perf, upgrade, release, one lock per phone), `tools/release_checks.py`, `recording_frames.swift`, `screenshot_contrast.swift`, Swift package `QAKit` for XCUITest |
|
|
14
|
+
| Prose | memos, READMEs, papers | `skills/prose-qa` | `tools/prose_check.py` (gates and a claims ledger) |
|
|
15
|
+
|
|
16
|
+
QAKit grew out of shipping several App Store apps as a solo developer whose QA team is an agent. Every check in it was
|
|
17
|
+
added after a defect that green tests let through: a `>=` that became `>` at every grading boundary with 13 mutants
|
|
18
|
+
surviving, a review note over the 4,000-character limit, a clipped caption nobody looked at, a What's New that still
|
|
19
|
+
announced the first release in version 1.1.
|
|
20
|
+
|
|
21
|
+
## See it
|
|
22
|
+
|
|
23
|
+
<p align="center"><img src="https://raw.githubusercontent.com/younatics/QAKit/main/assets/demo-walk.gif" alt="An agent walking an app on a real iPhone: onboarding, the home screen at every scroll position, each button tapped once" width="300"></p>
|
|
24
|
+
|
|
25
|
+
`QA.walk` on a real iPhone: the onboarding pages, the home screen at every scroll position, every visible button tapped once
|
|
26
|
+
and screenshotted, the accessibility audit per screen. These 29 frames are one walkthrough test's attachments, assembled by
|
|
27
|
+
`tools/shots_gif.py` from the result bundle (the app is the author's, live on the App Store).
|
|
28
|
+
|
|
29
|
+
<p align="center"><img src="https://raw.githubusercontent.com/younatics/QAKit/main/assets/demo-variants.gif" alt="The Notes example app's list, settings and search screens in the default, dark and large-text variants, one phone, one command" width="300"></p>
|
|
30
|
+
|
|
31
|
+
The same walk in three variants: `device_qa.py test --variant default --variant dark --variant large-text` switches the
|
|
32
|
+
phone's appearance and text size between runs. These nine frames are the `examples/Notes` app's list, settings and
|
|
33
|
+
search screens from one such run.
|
|
34
|
+
|
|
35
|
+
## What makes it different
|
|
36
|
+
|
|
37
|
+
- **Systematic, not random.** `QA.sweepButtons` taps every visible button on every screen at every scroll position,
|
|
38
|
+
screenshots what opened, checks the app is still in front and goes back. Monkey testers tap at random; agents reviewing a
|
|
39
|
+
contact sheet need every control seen once.
|
|
40
|
+
- **The whole phone changes, not the app.** `device_qa.py test --variant dark` switches the device to dark mode, large
|
|
41
|
+
accessibility text, a simulated location or VoiceOver through Xcode 27's `devicectl`, runs the suite, and restores iOS
|
|
42
|
+
defaults. System alerts and other apps render the same way the user will see them.
|
|
43
|
+
- **Apple's accessibility audit on every screen,** collected into one report per screen instead of stopping at the first
|
|
44
|
+
issue, with an allow list for reviewed exceptions.
|
|
45
|
+
- **Test strength is measured.** Coverage says a line ran. Mutation testing says whether a test would notice if the
|
|
46
|
+
line were wrong. The gate is "every mutant on a changed line is dead or explained".
|
|
47
|
+
- **Prose gets the same treatment.** Deterministic gates, then a claims ledger where every number is traced to a quote,
|
|
48
|
+
then a cold read by a reader who did not write it.
|
|
49
|
+
- **Written for agents.** The skills are `SKILL.md` files a coding agent loads. The protocol has 180 checks with IDs,
|
|
50
|
+
pass criteria, methods and sources, so an agent can cite what it did and what it skipped.
|
|
51
|
+
|
|
52
|
+
## Quick start
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install qa-agent-kit # qakit-mutate, qakit-prose, qakit-device, qakit-release-checks, qakit-install-skills
|
|
56
|
+
# or from source
|
|
57
|
+
git clone https://github.com/younatics/QAKit.git
|
|
58
|
+
cd QAKit && python3 -m pytest -q tests # 150 tests, no dependency beyond pytest
|
|
59
|
+
|
|
60
|
+
# mutation testing on any project
|
|
61
|
+
python3 tools/mutate.py --root ../myproject --source 'src/**/*.py' --test '{orig}/.venv/bin/python -m pytest -x -q'
|
|
62
|
+
|
|
63
|
+
# prose
|
|
64
|
+
python3 tools/prose_check.py ../memo.md --claims claims.csv
|
|
65
|
+
|
|
66
|
+
# an Apple app on a USB-attached test iPhone (macOS, Xcode 27)
|
|
67
|
+
python3 tools/device_qa.py doctor --manifest ../myapp/release/testing/device_qa.json
|
|
68
|
+
python3 tools/device_qa.py test --manifest ../myapp/release/testing/device_qa.json --suite qa --variant dark
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Give the procedures to your agent, whichever one you use:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
claude plugin marketplace add younatics/QAKit && claude plugin install qakit@younatics # Claude Code: skills + MCP tools
|
|
75
|
+
python3 tools/install_skills.py --target claude --scope project # or codex, cursor, gemini, all; --scope user
|
|
76
|
+
claude mcp add qakit -- python3 /path/to/qakit/mcp/server.py # any MCP client: doctor, test, shots, crashes, release checks, mutation, prose, verdict
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Then read [docs/getting-started.md](docs/getting-started.md). [AGENTS.md](AGENTS.md) tells any agent which skill to load
|
|
80
|
+
for which job.
|
|
81
|
+
|
|
82
|
+
## Documentation
|
|
83
|
+
|
|
84
|
+
| Document | What it covers |
|
|
85
|
+
|---|---|
|
|
86
|
+
| [docs/getting-started.md](docs/getting-started.md) | Installing, linking the Swift package, the first walkthrough test, the first device run |
|
|
87
|
+
| [docs/device-qa-protocol.md](docs/device-qa-protocol.md) | The protocol: 24 areas, 180 checks with IDs, exit criteria, what only a human can do ([한국어판](docs/ko/device-qa-protocol.md)) |
|
|
88
|
+
| [docs/manifest.md](docs/manifest.md) | Every key of the `device_qa.json` manifest |
|
|
89
|
+
| [docs/report.md](docs/report.md) | What a run leaves behind: `summary.json`, contact sheets, crash reports |
|
|
90
|
+
| [docs/mutation-testing.md](docs/mutation-testing.md) | `mutate.py` operators, config, reports, how to read a score |
|
|
91
|
+
| [docs/prose-qa.md](docs/prose-qa.md) | The prose gates and the claims ledger |
|
|
92
|
+
| [docs/case-studies.md](docs/case-studies.md) | Does the mutation layer find real gaps? Measured on humanize, tenacity and swift-argument-parser |
|
|
93
|
+
| [docs/pypi.md](docs/pypi.md) | The pip package: the five commands, where the skills land, the release procedure |
|
|
94
|
+
| [examples/README.md](examples/README.md) | The Notes example app: build it, walk it, see the two planted defects reported |
|
|
95
|
+
| [docs/agents.md](docs/agents.md) | Attaching the kit to Claude Code (plugin), Codex, Cursor, Gemini CLI and any MCP client |
|
|
96
|
+
| [mcp/README.md](mcp/README.md) | The MCP server: nine tools, one stdio process, no dependencies |
|
|
97
|
+
| [skills/README.md](skills/README.md) | Installing the skills by hand |
|
|
98
|
+
| [ROADMAP.md](ROADMAP.md) | What comes next and the loop that keeps the kit honest |
|
|
99
|
+
| [RELEASING.md](RELEASING.md) | How a change becomes a tagged release |
|
|
100
|
+
|
|
101
|
+
## The Swift package
|
|
102
|
+
|
|
103
|
+
```swift
|
|
104
|
+
import XCTest
|
|
105
|
+
import QAKit
|
|
106
|
+
|
|
107
|
+
QA.snap("settings", in: self) // screenshot into the result bundle, named by variant
|
|
108
|
+
QA.audit(app, "settings", in: self) // Apple's accessibility audit, every issue in one report
|
|
109
|
+
QA.walk(app, "settings", in: self, deny: ["Delete"]) { QA.dismissTopmost(app) } // snap, audit, sweep, scroll
|
|
110
|
+
QA.answerSystemAlert() // permission prompts, in the phone's language
|
|
111
|
+
QA.settle(app, on: screen, in: self) { relaunch() } // get back to a known screen, relaunch as the last resort
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Requirements: iOS 17 or later for the accessibility audit, Xcode 15 or later to build, Xcode 27 or later for the
|
|
115
|
+
device variants in `device_qa.py`.
|
|
116
|
+
|
|
117
|
+
## Who runs it
|
|
118
|
+
|
|
119
|
+
- The author's app monorepo pins the kit as a submodule and runs every layer on a USB-attached test iPhone.
|
|
120
|
+
- [forge](https://github.com/younatics/forge), the multi-agent coding loop, runs the mutation and prose gates as merge
|
|
121
|
+
authority on every task and gives its reviewer the qa-rigor principles.
|
|
122
|
+
- Gaps either of them hits come back here as a fix with a test and a tag (`RELEASING.md`).
|
|
123
|
+
|
|
124
|
+
## Status
|
|
125
|
+
|
|
126
|
+
Private beta. The three layers are in daily use on the author's apps and documents; the API and the manifest keys may
|
|
127
|
+
still change between minor versions until 1.0. See [CHANGELOG.md](CHANGELOG.md).
|
|
128
|
+
|
|
129
|
+
## License
|
|
130
|
+
|
|
131
|
+
MIT. See [LICENSE](LICENSE).
|