qa-agent 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qa_agent-0.2.0/qa_agent.egg-info → qa_agent-0.2.2}/PKG-INFO +53 -110
- {qa_agent-0.2.0 → qa_agent-0.2.2}/README.md +47 -106
- {qa_agent-0.2.0 → qa_agent-0.2.2}/pyproject.toml +6 -4
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/__init__.py +1 -1
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/accessibility.py +28 -21
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/forms.py +36 -24
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/keyboard.py +20 -14
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/mouse.py +30 -28
- {qa_agent-0.2.0 → qa_agent-0.2.2/qa_agent.egg-info}/PKG-INFO +53 -110
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent.egg-info/SOURCES.txt +1 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent.egg-info/requires.txt +4 -3
- qa_agent-0.2.2/tests/integration/test_target_coverage.py +80 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/LICENSE +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/__main__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/agent.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/ai_planner.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/cli.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/config.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/llm_client.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/models.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/plan_cache.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/reporters/__init__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/reporters/base.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/reporters/console.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/reporters/json_reporter.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/reporters/markdown.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/reporters/pdf.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/__init__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/base.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/custom.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/errors.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/testers/wcag_compliance.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/__init__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/server.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/static/app.js +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/static/style.css +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/templates/base.html +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/templates/index.html +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/templates/run.html +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/templates/session.html +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent/web/templates/sessions.html +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent.egg-info/dependency_links.txt +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent.egg-info/entry_points.txt +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/qa_agent.egg-info/top_level.txt +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/setup.cfg +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/__init__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/_cli_exit_helper.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/conftest.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/integration/__init__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/integration/test_smoke.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_agent.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_ai_planner.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_cli.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_config.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_llm_client.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_models.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_packaging.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_plan_cache.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_plan_warnings.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/test_reporters.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/__init__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/test_accessibility.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/test_base.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/test_custom.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/test_errors.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/test_forms.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/test_keyboard.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/testers/test_mouse.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/web/__init__.py +0 -0
- {qa_agent-0.2.0 → qa_agent-0.2.2}/tests/web/test_server.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: qa-agent
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Agentic exploratory QA testing for web applications
|
|
5
5
|
Author: Bill Richards
|
|
6
6
|
License: MIT License
|
|
@@ -27,6 +27,7 @@ License: MIT License
|
|
|
27
27
|
|
|
28
28
|
Project-URL: Homepage, https://github.com/billrichards/qa-agent
|
|
29
29
|
Project-URL: Repository, https://github.com/billrichards/qa-agent
|
|
30
|
+
Project-URL: Documentation, https://github.com/billrichards/qa-agent/blob/main/docs/README.md
|
|
30
31
|
Project-URL: Issues, https://github.com/billrichards/qa-agent/issues
|
|
31
32
|
Project-URL: Changelog, https://github.com/billrichards/qa-agent/blob/main/CHANGELOG.md
|
|
32
33
|
Keywords: qa,testing,playwright,accessibility,agentic,automation,exploratory-testing
|
|
@@ -51,12 +52,13 @@ Requires-Dist: flask>=3.0; extra == "web"
|
|
|
51
52
|
Requires-Dist: markdown>=3.5; extra == "web"
|
|
52
53
|
Requires-Dist: nh3>=0.2.15; extra == "web"
|
|
53
54
|
Provides-Extra: dev
|
|
54
|
-
Requires-Dist: pytest>=
|
|
55
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
55
56
|
Requires-Dist: pytest-playwright>=0.4.0; extra == "dev"
|
|
56
|
-
Requires-Dist: pytest-cov>=
|
|
57
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
|
|
57
58
|
Requires-Dist: pytest-mock>=3.10.0; extra == "dev"
|
|
59
|
+
Requires-Dist: pytest-xdist>=3.0.0; extra == "dev"
|
|
58
60
|
Requires-Dist: black>=23.0.0; extra == "dev"
|
|
59
|
-
Requires-Dist: ruff>=0.
|
|
61
|
+
Requires-Dist: ruff>=0.4.0; extra == "dev"
|
|
60
62
|
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
61
63
|
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
62
64
|
Provides-Extra: all
|
|
@@ -67,6 +69,10 @@ Dynamic: license-file
|
|
|
67
69
|
|
|
68
70
|
Automated exploratory QA testing for web applications — powered by Playwright and, optionally, LLMs (Claude or GPT-4o).
|
|
69
71
|
|
|
72
|
+
<p align="center">
|
|
73
|
+
<code>qa-agent example.com</code>
|
|
74
|
+
</p>
|
|
75
|
+
|
|
70
76
|
<p align="center">
|
|
71
77
|
<a href="https://github.com/billrichards/qa-agent/actions/workflows/test.yml"><img src="https://github.com/billrichards/qa-agent/actions/workflows/test.yml/badge.svg" alt="Tests"></a>
|
|
72
78
|
<a href="https://pypi.org/project/qa-agent/"><img src="https://img.shields.io/pypi/v/qa-agent" alt="PyPI version"></a>
|
|
@@ -89,18 +95,16 @@ Need targeted tests? Pass plain-English instructions and an LLM generates custom
|
|
|
89
95
|
- [Features](#features)
|
|
90
96
|
- [Installation](#installation)
|
|
91
97
|
- [Quick Start](#quick-start)
|
|
98
|
+
- [Programmatic Usage](#programmatic-usage)
|
|
92
99
|
- [Agentic Testing](#agentic-testing)
|
|
93
100
|
- [Web Interface & API](#web-interface--api)
|
|
94
101
|
- [CLI Reference](#cli-reference)
|
|
95
|
-
- [Programmatic Usage](#programmatic-usage)
|
|
96
102
|
- [Test Categories](#test-categories)
|
|
97
103
|
- [Output Formats](#output-formats)
|
|
98
104
|
- [CI/CD Integration](#cicd-integration)
|
|
99
|
-
- [Architecture](#architecture)
|
|
100
|
-
- [Development](#development)
|
|
101
|
-
- [Contributing](#contributing)
|
|
102
105
|
- [Exit Codes](#exit-codes)
|
|
103
106
|
- [Troubleshooting](#troubleshooting)
|
|
107
|
+
- [Further Documentation](#further-documentation)
|
|
104
108
|
- [License](#license)
|
|
105
109
|
|
|
106
110
|
---
|
|
@@ -169,6 +173,31 @@ python -m qa_agent https://example.com
|
|
|
169
173
|
|
|
170
174
|
---
|
|
171
175
|
|
|
176
|
+
## Programmatic Usage
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
from qa_agent import QAAgent, TestConfig, TestMode, OutputFormat
|
|
180
|
+
|
|
181
|
+
config = TestConfig(
|
|
182
|
+
urls=["https://example.com"],
|
|
183
|
+
mode=TestMode.EXPLORE,
|
|
184
|
+
output_formats=[OutputFormat.CONSOLE, OutputFormat.JSON],
|
|
185
|
+
max_depth=2,
|
|
186
|
+
max_pages=10,
|
|
187
|
+
instructions="Verify the password reset flow.", # optional
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
agent = QAAgent(config)
|
|
191
|
+
session = agent.run()
|
|
192
|
+
|
|
193
|
+
print(f"Pages tested: {len(session.pages_tested)}")
|
|
194
|
+
print(f"Total findings: {session.total_findings}")
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
→ [Full Python API Reference](https://github.com/billrichards/qa-agent/blob/main/docs/api-reference.md) — all classes, methods, and configuration options.
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
172
201
|
## Agentic Testing
|
|
173
202
|
|
|
174
203
|
Pass natural-language instructions and an LLM generates custom test steps that run alongside the standard suite. Supports **Anthropic** (Claude) and **OpenAI** (GPT-4o and others). No third-party AI packages are required — all API calls use Python's built-in `urllib`.
|
|
@@ -261,7 +290,7 @@ curl -N http://127.0.0.1:5000/api/stream/a1b2c3d4
|
|
|
261
290
|
curl http://127.0.0.1:5000/api/sessions?limit=10
|
|
262
291
|
```
|
|
263
292
|
|
|
264
|
-
→ [Full API reference](docs/web-api.md) — all endpoints, request body schema, and SSE event types.
|
|
293
|
+
→ [Full API reference](https://github.com/billrichards/qa-agent/blob/main/docs/web-api.md) — all endpoints, request body schema, and SSE event types.
|
|
265
294
|
|
|
266
295
|
---
|
|
267
296
|
|
|
@@ -366,40 +395,11 @@ qa-agent --wcag-compliance https://example.com
|
|
|
366
395
|
|
|
367
396
|
---
|
|
368
397
|
|
|
369
|
-
## Programmatic Usage
|
|
370
|
-
|
|
371
|
-
```python
|
|
372
|
-
from qa_agent import QAAgent, TestConfig, TestMode, OutputFormat
|
|
373
|
-
from qa_agent.llm_client import LLMProvider
|
|
374
|
-
|
|
375
|
-
config = TestConfig(
|
|
376
|
-
urls=["https://example.com"],
|
|
377
|
-
mode=TestMode.EXPLORE,
|
|
378
|
-
output_formats=[OutputFormat.CONSOLE, OutputFormat.JSON],
|
|
379
|
-
max_depth=2,
|
|
380
|
-
max_pages=10,
|
|
381
|
-
instructions="Verify the password reset flow.", # optional
|
|
382
|
-
llm_provider=LLMProvider.OPENAI, # optional, default: LLMProvider.ANTHROPIC
|
|
383
|
-
ai_model="gpt-4o-mini", # optional, default: None (uses provider default)
|
|
384
|
-
)
|
|
385
|
-
|
|
386
|
-
agent = QAAgent(config)
|
|
387
|
-
session = agent.run()
|
|
388
|
-
|
|
389
|
-
print(f"Pages tested: {len(session.pages_tested)}")
|
|
390
|
-
print(f"Total findings: {session.total_findings}")
|
|
391
|
-
|
|
392
|
-
for finding in session.get_all_findings():
|
|
393
|
-
print(f" [{finding.severity.value.upper()}] {finding.title}")
|
|
394
|
-
```
|
|
395
|
-
|
|
396
|
-
---
|
|
397
|
-
|
|
398
398
|
## Test Categories
|
|
399
399
|
|
|
400
400
|
Six built-in suites cover keyboard navigation, mouse interaction, form handling, accessibility (WCAG), runtime error detection, and an opt-in WCAG 2.1 AA compliance audit. Five run by default; enable the sixth with `--wcag-compliance`.
|
|
401
401
|
|
|
402
|
-
→ [Detailed test-by-test reference](docs/test-categories.md)
|
|
402
|
+
→ [Detailed test-by-test reference](https://github.com/billrichards/qa-agent/blob/main/docs/test-categories.md)
|
|
403
403
|
|
|
404
404
|
---
|
|
405
405
|
|
|
@@ -464,76 +464,6 @@ Exits with code `1` when critical or high severity issues are found, failing the
|
|
|
464
464
|
|
|
465
465
|
---
|
|
466
466
|
|
|
467
|
-
## Architecture
|
|
468
|
-
|
|
469
|
-
```
|
|
470
|
-
qa_agent/
|
|
471
|
-
├── cli.py # CLI entry point
|
|
472
|
-
├── agent.py # Core orchestrator
|
|
473
|
-
├── config.py # Configuration dataclasses
|
|
474
|
-
├── models.py # Finding, PageAnalysis, TestSession, TestPlan
|
|
475
|
-
├── llm_client.py # Anthropic & OpenAI clients via stdlib urllib
|
|
476
|
-
├── ai_planner.py # LLM-powered test plan generation
|
|
477
|
-
├── plan_cache.py # Filesystem cache for test plans
|
|
478
|
-
├── testers/
|
|
479
|
-
│ ├── base.py # BaseTester abstract class
|
|
480
|
-
│ ├── keyboard.py # Keyboard navigation
|
|
481
|
-
│ ├── mouse.py # Mouse interaction
|
|
482
|
-
│ ├── forms.py # Form handling
|
|
483
|
-
│ ├── accessibility.py # WCAG accessibility
|
|
484
|
-
│ ├── wcag_compliance.py # WCAG 2.1 AA compliance (opt-in)
|
|
485
|
-
│ ├── errors.py # Console & network errors
|
|
486
|
-
│ └── custom.py # AI-generated test steps
|
|
487
|
-
├── reporters/
|
|
488
|
-
│ ├── console.py # Colored terminal output
|
|
489
|
-
│ ├── markdown.py # Markdown report
|
|
490
|
-
│ ├── json_reporter.py # JSON report
|
|
491
|
-
│ └── pdf.py # PDF report (requires weasyprint)
|
|
492
|
-
└── web/
|
|
493
|
-
├── server.py # Flask app with SSE streaming
|
|
494
|
-
├── templates/ # Jinja2 templates
|
|
495
|
-
└── static/ # CSS and JavaScript
|
|
496
|
-
```
|
|
497
|
-
|
|
498
|
-
→ [Extending QA Agent — adding custom testers](docs/architecture.md)
|
|
499
|
-
|
|
500
|
-
---
|
|
501
|
-
|
|
502
|
-
## Development
|
|
503
|
-
|
|
504
|
-
```bash
|
|
505
|
-
git clone https://github.com/billrichards/qa-agent.git
|
|
506
|
-
cd qa-agent
|
|
507
|
-
pip install -e ".[dev,web,ai]"
|
|
508
|
-
playwright install chromium
|
|
509
|
-
|
|
510
|
-
# Unit tests (no browser needed)
|
|
511
|
-
pytest -v -m "not integration and not network"
|
|
512
|
-
|
|
513
|
-
# Integration tests (real Playwright)
|
|
514
|
-
pytest -v -m integration --no-cov
|
|
515
|
-
|
|
516
|
-
# Lint & type check
|
|
517
|
-
ruff check .
|
|
518
|
-
mypy qa_agent
|
|
519
|
-
```
|
|
520
|
-
|
|
521
|
-
CI runs unit tests across Python 3.10–3.12 on Ubuntu, macOS, and Windows. Integration tests run on Ubuntu with Playwright. See [`.github/workflows/test.yml`](.github/workflows/test.yml).
|
|
522
|
-
|
|
523
|
-
---
|
|
524
|
-
|
|
525
|
-
## Contributing
|
|
526
|
-
|
|
527
|
-
1. Fork the repository
|
|
528
|
-
2. Create a feature branch (`git checkout -b my-feature`)
|
|
529
|
-
3. Make your changes and add tests
|
|
530
|
-
4. Run `pytest -v -m "not integration and not network"`
|
|
531
|
-
5. Open a pull request against `main`
|
|
532
|
-
|
|
533
|
-
Code style: Ruff + Black, line length 100.
|
|
534
|
-
|
|
535
|
-
---
|
|
536
|
-
|
|
537
467
|
## Exit Codes
|
|
538
468
|
|
|
539
469
|
| Code | Meaning |
|
|
@@ -588,6 +518,19 @@ Requires **3.10+**. Check with `python --version`.
|
|
|
588
518
|
|
|
589
519
|
---
|
|
590
520
|
|
|
521
|
+
## Further Documentation
|
|
522
|
+
|
|
523
|
+
| Document | Description |
|
|
524
|
+
|---|---|
|
|
525
|
+
| [Architecture](https://github.com/billrichards/qa-agent/blob/main/docs/architecture.md) | System architecture and how to extend QA Agent |
|
|
526
|
+
| [Development Guide](https://github.com/billrichards/qa-agent/blob/main/docs/development.md) | Detailed development setup, testing, and build processes |
|
|
527
|
+
| [Python API Reference](https://github.com/billrichards/qa-agent/blob/main/docs/api-reference.md) | Programmatic usage for embedding QA Agent in Python code |
|
|
528
|
+
| [Test Categories](https://github.com/billrichards/qa-agent/blob/main/docs/test-categories.md) | Detailed test-by-test reference |
|
|
529
|
+
| [Web API](https://github.com/billrichards/qa-agent/blob/main/docs/web-api.md) | Full REST API documentation |
|
|
530
|
+
| [Contributing](https://github.com/billrichards/qa-agent/blob/main/CONTRIBUTING.md) | Contribution guidelines and pull request process |
|
|
531
|
+
|
|
532
|
+
---
|
|
533
|
+
|
|
591
534
|
## License
|
|
592
535
|
|
|
593
|
-
MIT — Copyright (c) 2026 Bill Richards. See [LICENSE](LICENSE).
|
|
536
|
+
MIT — Copyright (c) 2026 Bill Richards. See [LICENSE](https://github.com/billrichards/qa-agent/blob/main/LICENSE).
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
Automated exploratory QA testing for web applications — powered by Playwright and, optionally, LLMs (Claude or GPT-4o).
|
|
4
4
|
|
|
5
|
+
<p align="center">
|
|
6
|
+
<code>qa-agent example.com</code>
|
|
7
|
+
</p>
|
|
8
|
+
|
|
5
9
|
<p align="center">
|
|
6
10
|
<a href="https://github.com/billrichards/qa-agent/actions/workflows/test.yml"><img src="https://github.com/billrichards/qa-agent/actions/workflows/test.yml/badge.svg" alt="Tests"></a>
|
|
7
11
|
<a href="https://pypi.org/project/qa-agent/"><img src="https://img.shields.io/pypi/v/qa-agent" alt="PyPI version"></a>
|
|
@@ -24,18 +28,16 @@ Need targeted tests? Pass plain-English instructions and an LLM generates custom
|
|
|
24
28
|
- [Features](#features)
|
|
25
29
|
- [Installation](#installation)
|
|
26
30
|
- [Quick Start](#quick-start)
|
|
31
|
+
- [Programmatic Usage](#programmatic-usage)
|
|
27
32
|
- [Agentic Testing](#agentic-testing)
|
|
28
33
|
- [Web Interface & API](#web-interface--api)
|
|
29
34
|
- [CLI Reference](#cli-reference)
|
|
30
|
-
- [Programmatic Usage](#programmatic-usage)
|
|
31
35
|
- [Test Categories](#test-categories)
|
|
32
36
|
- [Output Formats](#output-formats)
|
|
33
37
|
- [CI/CD Integration](#cicd-integration)
|
|
34
|
-
- [Architecture](#architecture)
|
|
35
|
-
- [Development](#development)
|
|
36
|
-
- [Contributing](#contributing)
|
|
37
38
|
- [Exit Codes](#exit-codes)
|
|
38
39
|
- [Troubleshooting](#troubleshooting)
|
|
40
|
+
- [Further Documentation](#further-documentation)
|
|
39
41
|
- [License](#license)
|
|
40
42
|
|
|
41
43
|
---
|
|
@@ -104,6 +106,31 @@ python -m qa_agent https://example.com
|
|
|
104
106
|
|
|
105
107
|
---
|
|
106
108
|
|
|
109
|
+
## Programmatic Usage
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from qa_agent import QAAgent, TestConfig, TestMode, OutputFormat
|
|
113
|
+
|
|
114
|
+
config = TestConfig(
|
|
115
|
+
urls=["https://example.com"],
|
|
116
|
+
mode=TestMode.EXPLORE,
|
|
117
|
+
output_formats=[OutputFormat.CONSOLE, OutputFormat.JSON],
|
|
118
|
+
max_depth=2,
|
|
119
|
+
max_pages=10,
|
|
120
|
+
instructions="Verify the password reset flow.", # optional
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
agent = QAAgent(config)
|
|
124
|
+
session = agent.run()
|
|
125
|
+
|
|
126
|
+
print(f"Pages tested: {len(session.pages_tested)}")
|
|
127
|
+
print(f"Total findings: {session.total_findings}")
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
→ [Full Python API Reference](https://github.com/billrichards/qa-agent/blob/main/docs/api-reference.md) — all classes, methods, and configuration options.
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
107
134
|
## Agentic Testing
|
|
108
135
|
|
|
109
136
|
Pass natural-language instructions and an LLM generates custom test steps that run alongside the standard suite. Supports **Anthropic** (Claude) and **OpenAI** (GPT-4o and others). No third-party AI packages are required — all API calls use Python's built-in `urllib`.
|
|
@@ -196,7 +223,7 @@ curl -N http://127.0.0.1:5000/api/stream/a1b2c3d4
|
|
|
196
223
|
curl http://127.0.0.1:5000/api/sessions?limit=10
|
|
197
224
|
```
|
|
198
225
|
|
|
199
|
-
→ [Full API reference](docs/web-api.md) — all endpoints, request body schema, and SSE event types.
|
|
226
|
+
→ [Full API reference](https://github.com/billrichards/qa-agent/blob/main/docs/web-api.md) — all endpoints, request body schema, and SSE event types.
|
|
200
227
|
|
|
201
228
|
---
|
|
202
229
|
|
|
@@ -301,40 +328,11 @@ qa-agent --wcag-compliance https://example.com
|
|
|
301
328
|
|
|
302
329
|
---
|
|
303
330
|
|
|
304
|
-
## Programmatic Usage
|
|
305
|
-
|
|
306
|
-
```python
|
|
307
|
-
from qa_agent import QAAgent, TestConfig, TestMode, OutputFormat
|
|
308
|
-
from qa_agent.llm_client import LLMProvider
|
|
309
|
-
|
|
310
|
-
config = TestConfig(
|
|
311
|
-
urls=["https://example.com"],
|
|
312
|
-
mode=TestMode.EXPLORE,
|
|
313
|
-
output_formats=[OutputFormat.CONSOLE, OutputFormat.JSON],
|
|
314
|
-
max_depth=2,
|
|
315
|
-
max_pages=10,
|
|
316
|
-
instructions="Verify the password reset flow.", # optional
|
|
317
|
-
llm_provider=LLMProvider.OPENAI, # optional, default: LLMProvider.ANTHROPIC
|
|
318
|
-
ai_model="gpt-4o-mini", # optional, default: None (uses provider default)
|
|
319
|
-
)
|
|
320
|
-
|
|
321
|
-
agent = QAAgent(config)
|
|
322
|
-
session = agent.run()
|
|
323
|
-
|
|
324
|
-
print(f"Pages tested: {len(session.pages_tested)}")
|
|
325
|
-
print(f"Total findings: {session.total_findings}")
|
|
326
|
-
|
|
327
|
-
for finding in session.get_all_findings():
|
|
328
|
-
print(f" [{finding.severity.value.upper()}] {finding.title}")
|
|
329
|
-
```
|
|
330
|
-
|
|
331
|
-
---
|
|
332
|
-
|
|
333
331
|
## Test Categories
|
|
334
332
|
|
|
335
333
|
Six built-in suites cover keyboard navigation, mouse interaction, form handling, accessibility (WCAG), runtime error detection, and an opt-in WCAG 2.1 AA compliance audit. Five run by default; enable the sixth with `--wcag-compliance`.
|
|
336
334
|
|
|
337
|
-
→ [Detailed test-by-test reference](docs/test-categories.md)
|
|
335
|
+
→ [Detailed test-by-test reference](https://github.com/billrichards/qa-agent/blob/main/docs/test-categories.md)
|
|
338
336
|
|
|
339
337
|
---
|
|
340
338
|
|
|
@@ -399,76 +397,6 @@ Exits with code `1` when critical or high severity issues are found, failing the
|
|
|
399
397
|
|
|
400
398
|
---
|
|
401
399
|
|
|
402
|
-
## Architecture
|
|
403
|
-
|
|
404
|
-
```
|
|
405
|
-
qa_agent/
|
|
406
|
-
├── cli.py # CLI entry point
|
|
407
|
-
├── agent.py # Core orchestrator
|
|
408
|
-
├── config.py # Configuration dataclasses
|
|
409
|
-
├── models.py # Finding, PageAnalysis, TestSession, TestPlan
|
|
410
|
-
├── llm_client.py # Anthropic & OpenAI clients via stdlib urllib
|
|
411
|
-
├── ai_planner.py # LLM-powered test plan generation
|
|
412
|
-
├── plan_cache.py # Filesystem cache for test plans
|
|
413
|
-
├── testers/
|
|
414
|
-
│ ├── base.py # BaseTester abstract class
|
|
415
|
-
│ ├── keyboard.py # Keyboard navigation
|
|
416
|
-
│ ├── mouse.py # Mouse interaction
|
|
417
|
-
│ ├── forms.py # Form handling
|
|
418
|
-
│ ├── accessibility.py # WCAG accessibility
|
|
419
|
-
│ ├── wcag_compliance.py # WCAG 2.1 AA compliance (opt-in)
|
|
420
|
-
│ ├── errors.py # Console & network errors
|
|
421
|
-
│ └── custom.py # AI-generated test steps
|
|
422
|
-
├── reporters/
|
|
423
|
-
│ ├── console.py # Colored terminal output
|
|
424
|
-
│ ├── markdown.py # Markdown report
|
|
425
|
-
│ ├── json_reporter.py # JSON report
|
|
426
|
-
│ └── pdf.py # PDF report (requires weasyprint)
|
|
427
|
-
└── web/
|
|
428
|
-
├── server.py # Flask app with SSE streaming
|
|
429
|
-
├── templates/ # Jinja2 templates
|
|
430
|
-
└── static/ # CSS and JavaScript
|
|
431
|
-
```
|
|
432
|
-
|
|
433
|
-
→ [Extending QA Agent — adding custom testers](docs/architecture.md)
|
|
434
|
-
|
|
435
|
-
---
|
|
436
|
-
|
|
437
|
-
## Development
|
|
438
|
-
|
|
439
|
-
```bash
|
|
440
|
-
git clone https://github.com/billrichards/qa-agent.git
|
|
441
|
-
cd qa-agent
|
|
442
|
-
pip install -e ".[dev,web,ai]"
|
|
443
|
-
playwright install chromium
|
|
444
|
-
|
|
445
|
-
# Unit tests (no browser needed)
|
|
446
|
-
pytest -v -m "not integration and not network"
|
|
447
|
-
|
|
448
|
-
# Integration tests (real Playwright)
|
|
449
|
-
pytest -v -m integration --no-cov
|
|
450
|
-
|
|
451
|
-
# Lint & type check
|
|
452
|
-
ruff check .
|
|
453
|
-
mypy qa_agent
|
|
454
|
-
```
|
|
455
|
-
|
|
456
|
-
CI runs unit tests across Python 3.10–3.12 on Ubuntu, macOS, and Windows. Integration tests run on Ubuntu with Playwright. See [`.github/workflows/test.yml`](.github/workflows/test.yml).
|
|
457
|
-
|
|
458
|
-
---
|
|
459
|
-
|
|
460
|
-
## Contributing
|
|
461
|
-
|
|
462
|
-
1. Fork the repository
|
|
463
|
-
2. Create a feature branch (`git checkout -b my-feature`)
|
|
464
|
-
3. Make your changes and add tests
|
|
465
|
-
4. Run `pytest -v -m "not integration and not network"`
|
|
466
|
-
5. Open a pull request against `main`
|
|
467
|
-
|
|
468
|
-
Code style: Ruff + Black, line length 100.
|
|
469
|
-
|
|
470
|
-
---
|
|
471
|
-
|
|
472
400
|
## Exit Codes
|
|
473
401
|
|
|
474
402
|
| Code | Meaning |
|
|
@@ -523,6 +451,19 @@ Requires **3.10+**. Check with `python --version`.
|
|
|
523
451
|
|
|
524
452
|
---
|
|
525
453
|
|
|
454
|
+
## Further Documentation
|
|
455
|
+
|
|
456
|
+
| Document | Description |
|
|
457
|
+
|---|---|
|
|
458
|
+
| [Architecture](https://github.com/billrichards/qa-agent/blob/main/docs/architecture.md) | System architecture and how to extend QA Agent |
|
|
459
|
+
| [Development Guide](https://github.com/billrichards/qa-agent/blob/main/docs/development.md) | Detailed development setup, testing, and build processes |
|
|
460
|
+
| [Python API Reference](https://github.com/billrichards/qa-agent/blob/main/docs/api-reference.md) | Programmatic usage for embedding QA Agent in Python code |
|
|
461
|
+
| [Test Categories](https://github.com/billrichards/qa-agent/blob/main/docs/test-categories.md) | Detailed test-by-test reference |
|
|
462
|
+
| [Web API](https://github.com/billrichards/qa-agent/blob/main/docs/web-api.md) | Full REST API documentation |
|
|
463
|
+
| [Contributing](https://github.com/billrichards/qa-agent/blob/main/CONTRIBUTING.md) | Contribution guidelines and pull request process |
|
|
464
|
+
|
|
465
|
+
---
|
|
466
|
+
|
|
526
467
|
## License
|
|
527
468
|
|
|
528
|
-
MIT — Copyright (c) 2026 Bill Richards. See [LICENSE](LICENSE).
|
|
469
|
+
MIT — Copyright (c) 2026 Bill Richards. See [LICENSE](https://github.com/billrichards/qa-agent/blob/main/LICENSE).
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "qa-agent"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Agentic exploratory QA testing for web applications"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {file = "LICENSE"}
|
|
@@ -37,12 +37,13 @@ web = [
|
|
|
37
37
|
"nh3>=0.2.15",
|
|
38
38
|
]
|
|
39
39
|
dev = [
|
|
40
|
-
"pytest>=
|
|
40
|
+
"pytest>=8.0.0",
|
|
41
41
|
"pytest-playwright>=0.4.0",
|
|
42
|
-
"pytest-cov>=
|
|
42
|
+
"pytest-cov>=5.0.0",
|
|
43
43
|
"pytest-mock>=3.10.0",
|
|
44
|
+
"pytest-xdist>=3.0.0",
|
|
44
45
|
"black>=23.0.0",
|
|
45
|
-
"ruff>=0.
|
|
46
|
+
"ruff>=0.4.0",
|
|
46
47
|
"mypy>=1.0.0",
|
|
47
48
|
"build>=1.0.0",
|
|
48
49
|
]
|
|
@@ -57,6 +58,7 @@ qa-agent-web = "qa_agent.web:serve_web_cli"
|
|
|
57
58
|
[project.urls]
|
|
58
59
|
Homepage = "https://github.com/billrichards/qa-agent"
|
|
59
60
|
Repository = "https://github.com/billrichards/qa-agent"
|
|
61
|
+
Documentation = "https://github.com/billrichards/qa-agent/blob/main/docs/README.md"
|
|
60
62
|
Issues = "https://github.com/billrichards/qa-agent/issues"
|
|
61
63
|
Changelog = "https://github.com/billrichards/qa-agent/blob/main/CHANGELOG.md"
|
|
62
64
|
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
"""Accessibility testing module."""
|
|
2
2
|
|
|
3
|
+
import logging
|
|
4
|
+
|
|
3
5
|
from playwright.sync_api import Page
|
|
4
6
|
|
|
5
7
|
from ..config import TestConfig
|
|
6
8
|
from ..models import Finding, FindingCategory, Severity
|
|
7
9
|
from .base import BaseTester
|
|
8
10
|
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
9
13
|
|
|
10
14
|
class AccessibilityTester(BaseTester):
|
|
11
15
|
"""Tests accessibility issues and WCAG compliance."""
|
|
@@ -78,7 +82,8 @@ class AccessibilityTester(BaseTester):
|
|
|
78
82
|
if info['isInLink'] and info['alt'] in ['click here', 'read more', 'link', 'image']:
|
|
79
83
|
issues['suspicious_alt'].append({"alt": info['alt'], "src": info['src'], "issue": "link_image"})
|
|
80
84
|
|
|
81
|
-
except Exception:
|
|
85
|
+
except Exception as e:
|
|
86
|
+
logger.debug("%s: error evaluating image element: %s", self.__class__.__name__, e)
|
|
82
87
|
continue
|
|
83
88
|
|
|
84
89
|
if len(issues['missing_alt']) > 0:
|
|
@@ -105,8 +110,8 @@ class AccessibilityTester(BaseTester):
|
|
|
105
110
|
metadata={"images": issues['suspicious_alt'][:5]},
|
|
106
111
|
))
|
|
107
112
|
|
|
108
|
-
except Exception:
|
|
109
|
-
|
|
113
|
+
except Exception as e:
|
|
114
|
+
logger.debug("%s: _test_images_alt_text failed: %s", self.__class__.__name__, e)
|
|
110
115
|
|
|
111
116
|
def _test_headings_structure(self):
|
|
112
117
|
"""Test heading hierarchy (h1-h6)."""
|
|
@@ -171,8 +176,8 @@ class AccessibilityTester(BaseTester):
|
|
|
171
176
|
metadata={"skips": skipped_levels[:5]},
|
|
172
177
|
))
|
|
173
178
|
|
|
174
|
-
except Exception:
|
|
175
|
-
|
|
179
|
+
except Exception as e:
|
|
180
|
+
logger.debug("%s: _test_headings_structure failed: %s", self.__class__.__name__, e)
|
|
176
181
|
|
|
177
182
|
def _test_link_text(self):
|
|
178
183
|
"""Test that links have descriptive text."""
|
|
@@ -220,7 +225,8 @@ class AccessibilityTester(BaseTester):
|
|
|
220
225
|
"href": info['href']
|
|
221
226
|
})
|
|
222
227
|
|
|
223
|
-
except Exception:
|
|
228
|
+
except Exception as e:
|
|
229
|
+
logger.debug("%s: error evaluating link element: %s", self.__class__.__name__, e)
|
|
224
230
|
continue
|
|
225
231
|
|
|
226
232
|
empty_links = [lnk for lnk in bad_links if lnk['issue'] == 'empty']
|
|
@@ -250,8 +256,8 @@ class AccessibilityTester(BaseTester):
|
|
|
250
256
|
metadata={"links": generic_links[:5]},
|
|
251
257
|
))
|
|
252
258
|
|
|
253
|
-
except Exception:
|
|
254
|
-
|
|
259
|
+
except Exception as e:
|
|
260
|
+
logger.debug("%s: _test_link_text failed: %s", self.__class__.__name__, e)
|
|
255
261
|
|
|
256
262
|
def _test_color_contrast(self):
|
|
257
263
|
"""Test color contrast of text elements."""
|
|
@@ -315,7 +321,8 @@ class AccessibilityTester(BaseTester):
|
|
|
315
321
|
if contrast_info and not contrast_info.get('passes') and not contrast_info.get('isTransparentBg'):
|
|
316
322
|
low_contrast.append(contrast_info)
|
|
317
323
|
|
|
318
|
-
except Exception:
|
|
324
|
+
except Exception as e:
|
|
325
|
+
logger.debug("%s: error evaluating contrast for element: %s", self.__class__.__name__, e)
|
|
319
326
|
continue
|
|
320
327
|
|
|
321
328
|
if len(low_contrast) > 3:
|
|
@@ -330,8 +337,8 @@ class AccessibilityTester(BaseTester):
|
|
|
330
337
|
metadata={"elements": low_contrast[:5]},
|
|
331
338
|
))
|
|
332
339
|
|
|
333
|
-
except Exception:
|
|
334
|
-
|
|
340
|
+
except Exception as e:
|
|
341
|
+
logger.debug("%s: _test_color_contrast failed: %s", self.__class__.__name__, e)
|
|
335
342
|
|
|
336
343
|
def _test_aria_usage(self):
|
|
337
344
|
"""Test correct usage of ARIA attributes."""
|
|
@@ -439,8 +446,8 @@ class AccessibilityTester(BaseTester):
|
|
|
439
446
|
actual_behavior=f"Referenced ID '{issue['id']}' not found",
|
|
440
447
|
))
|
|
441
448
|
|
|
442
|
-
except Exception:
|
|
443
|
-
|
|
449
|
+
except Exception as e:
|
|
450
|
+
logger.debug("%s: _test_aria_usage failed: %s", self.__class__.__name__, e)
|
|
444
451
|
|
|
445
452
|
def _test_landmark_regions(self):
|
|
446
453
|
"""Test for proper landmark regions."""
|
|
@@ -477,8 +484,8 @@ class AccessibilityTester(BaseTester):
|
|
|
477
484
|
actual_behavior=f"Found {landmarks['main']} main landmarks",
|
|
478
485
|
))
|
|
479
486
|
|
|
480
|
-
except Exception:
|
|
481
|
-
|
|
487
|
+
except Exception as e:
|
|
488
|
+
logger.debug("%s: _test_landmark_regions failed: %s", self.__class__.__name__, e)
|
|
482
489
|
|
|
483
490
|
def _test_language_attribute(self):
|
|
484
491
|
"""Test for lang attribute on html element."""
|
|
@@ -510,8 +517,8 @@ class AccessibilityTester(BaseTester):
|
|
|
510
517
|
actual_behavior=f"Lang attribute value: '{lang_info.get('lang')}'",
|
|
511
518
|
))
|
|
512
519
|
|
|
513
|
-
except Exception:
|
|
514
|
-
|
|
520
|
+
except Exception as e:
|
|
521
|
+
logger.debug("%s: _test_language_attribute failed: %s", self.__class__.__name__, e)
|
|
515
522
|
|
|
516
523
|
def _test_skip_links(self):
|
|
517
524
|
"""Test for skip navigation links."""
|
|
@@ -579,8 +586,8 @@ class AccessibilityTester(BaseTester):
|
|
|
579
586
|
actual_behavior="Skip link target ID not found",
|
|
580
587
|
))
|
|
581
588
|
|
|
582
|
-
except Exception:
|
|
583
|
-
|
|
589
|
+
except Exception as e:
|
|
590
|
+
logger.debug("%s: _test_skip_links failed: %s", self.__class__.__name__, e)
|
|
584
591
|
|
|
585
592
|
def _test_motion_preferences(self):
|
|
586
593
|
"""Test respect for reduced motion preference."""
|
|
@@ -640,5 +647,5 @@ class AccessibilityTester(BaseTester):
|
|
|
640
647
|
actual_behavior="No prefers-reduced-motion media query found in stylesheets",
|
|
641
648
|
))
|
|
642
649
|
|
|
643
|
-
except Exception:
|
|
644
|
-
|
|
650
|
+
except Exception as e:
|
|
651
|
+
logger.debug("%s: _test_motion_preferences failed: %s", self.__class__.__name__, e)
|