recon-qa 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- recon_qa-0.1.0/PKG-INFO +322 -0
- recon_qa-0.1.0/README.md +288 -0
- recon_qa-0.1.0/pyproject.toml +61 -0
- recon_qa-0.1.0/recon/__init__.py +1 -0
- recon_qa-0.1.0/recon/analysis/__init__.py +1 -0
- recon_qa-0.1.0/recon/analysis/classifier.py +107 -0
- recon_qa-0.1.0/recon/analysis/rca_engine.py +175 -0
- recon_qa-0.1.0/recon/apps/__init__.py +1 -0
- recon_qa-0.1.0/recon/apps/api/__init__.py +1 -0
- recon_qa-0.1.0/recon/apps/api/server.py +59 -0
- recon_qa-0.1.0/recon/apps/cli/__init__.py +1 -0
- recon_qa-0.1.0/recon/apps/cli/config_cli.py +224 -0
- recon_qa-0.1.0/recon/apps/cli/doctor.py +90 -0
- recon_qa-0.1.0/recon/apps/cli/main.py +286 -0
- recon_qa-0.1.0/recon/common/__init__.py +1 -0
- recon_qa-0.1.0/recon/common/config.py +88 -0
- recon_qa-0.1.0/recon/common/exceptions.py +29 -0
- recon_qa-0.1.0/recon/common/logging.py +81 -0
- recon_qa-0.1.0/recon/common/models.py +216 -0
- recon_qa-0.1.0/recon/common/security.py +121 -0
- recon_qa-0.1.0/recon/demo_app/__init__.py +1 -0
- recon_qa-0.1.0/recon/demo_app/main.py +117 -0
- recon_qa-0.1.0/recon/discovery/__init__.py +1 -0
- recon_qa-0.1.0/recon/discovery/endpoint_detector.py +74 -0
- recon_qa-0.1.0/recon/discovery/models.py +74 -0
- recon_qa-0.1.0/recon/discovery/openapi.py +272 -0
- recon_qa-0.1.0/recon/discovery/web_crawler.py +280 -0
- recon_qa-0.1.0/recon/execution/__init__.py +1 -0
- recon_qa-0.1.0/recon/execution/api/__init__.py +1 -0
- recon_qa-0.1.0/recon/execution/api/assertions.py +197 -0
- recon_qa-0.1.0/recon/execution/api/runner.py +254 -0
- recon_qa-0.1.0/recon/execution/browser/__init__.py +1 -0
- recon_qa-0.1.0/recon/execution/browser/evidence.py +73 -0
- recon_qa-0.1.0/recon/execution/browser/runner.py +274 -0
- recon_qa-0.1.0/recon/llm/__init__.py +1 -0
- recon_qa-0.1.0/recon/llm/failure_analyzer.py +116 -0
- recon_qa-0.1.0/recon/llm/provider.py +296 -0
- recon_qa-0.1.0/recon/llm/test_generator.py +140 -0
- recon_qa-0.1.0/recon/orchestration/__init__.py +1 -0
- recon_qa-0.1.0/recon/orchestration/orchestrator.py +189 -0
- recon_qa-0.1.0/recon/orchestration/worker_pool.py +94 -0
- recon_qa-0.1.0/recon/persistence/__init__.py +1 -0
- recon_qa-0.1.0/recon/persistence/database.py +107 -0
- recon_qa-0.1.0/recon/persistence/models.py +43 -0
- recon_qa-0.1.0/recon/planning/__init__.py +1 -0
- recon_qa-0.1.0/recon/planning/generator.py +412 -0
- recon_qa-0.1.0/recon/planning/schema_fuzzer.py +176 -0
- recon_qa-0.1.0/recon/reporting/__init__.py +1 -0
- recon_qa-0.1.0/recon/reporting/html_reporter.py +346 -0
- recon_qa-0.1.0/recon/reporting/json_reporter.py +36 -0
- recon_qa-0.1.0/recon_qa.egg-info/PKG-INFO +322 -0
- recon_qa-0.1.0/recon_qa.egg-info/SOURCES.txt +55 -0
- recon_qa-0.1.0/recon_qa.egg-info/dependency_links.txt +1 -0
- recon_qa-0.1.0/recon_qa.egg-info/entry_points.txt +3 -0
- recon_qa-0.1.0/recon_qa.egg-info/requires.txt +27 -0
- recon_qa-0.1.0/recon_qa.egg-info/top_level.txt +1 -0
- recon_qa-0.1.0/setup.cfg +4 -0
recon_qa-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: recon-qa
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Recon: AI QA Agent — Autonomous Testing & Failure Analysis Platform
|
|
5
|
+
Author: Recon Engineering Team
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: qa,testing,automation,playwright,ai,failure-analysis,openapi
|
|
8
|
+
Requires-Python: >=3.12
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: typer>=0.12.0
|
|
11
|
+
Requires-Dist: rich>=13.7.0
|
|
12
|
+
Requires-Dist: pydantic>=2.7.0
|
|
13
|
+
Requires-Dist: pydantic-settings>=2.2.0
|
|
14
|
+
Requires-Dist: httpx>=0.27.0
|
|
15
|
+
Requires-Dist: fastapi>=0.111.0
|
|
16
|
+
Requires-Dist: uvicorn>=0.30.0
|
|
17
|
+
Requires-Dist: sqlalchemy>=2.0.30
|
|
18
|
+
Requires-Dist: aiosqlite>=0.20.0
|
|
19
|
+
Requires-Dist: jinja2>=3.1.4
|
|
20
|
+
Requires-Dist: jsonschema>=4.22.0
|
|
21
|
+
Requires-Dist: pytest>=8.2.0
|
|
22
|
+
Requires-Dist: pytest-asyncio>=0.23.0
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
24
|
+
Requires-Dist: pyyaml>=6.0.1
|
|
25
|
+
Provides-Extra: browser
|
|
26
|
+
Requires-Dist: playwright>=1.44.0; extra == "browser"
|
|
27
|
+
Provides-Extra: ai
|
|
28
|
+
Requires-Dist: google-genai>=1.0.0; extra == "ai"
|
|
29
|
+
Requires-Dist: openai>=1.30.0; extra == "ai"
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: ruff>=0.4.0; extra == "dev"
|
|
32
|
+
Requires-Dist: mypy>=1.10.0; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
|
|
34
|
+
|
|
35
|
+
# Recon: AI QA Agent — Autonomous Testing & Failure Analysis Platform
|
|
36
|
+
|
|
37
|
+
[](https://github.com/recon-qa/recon/actions/workflows/ci.yml)
|
|
38
|
+
[](https://www.python.org/downloads/)
|
|
39
|
+
[](https://opensource.org/licenses/MIT)
|
|
40
|
+
|
|
41
|
+
> **Recon** (`qa-agent`) is an autonomous, developer-first testing platform designed to inspect applications, plan multi-category test suites, execute HTTP & Playwright browser tests concurrently, deterministically classify failures, perform AI-assisted root-cause analysis (RCA), and generate actionable reports.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 1. Architecture Overview
|
|
46
|
+
|
|
47
|
+
```mermaid
|
|
48
|
+
flowchart TD
|
|
49
|
+
subgraph Discovery ["1. Application Discovery"]
|
|
50
|
+
Target["Target Application URL / OpenAPI Spec"] --> Engine["Discovery Engine"]
|
|
51
|
+
Engine --> OA["OpenAPI 3.x / Swagger Parser"]
|
|
52
|
+
Engine --> Crawl["Playwright Web Crawler"]
|
|
53
|
+
OA --> Assets["Endpoints, Parameters, Schemas"]
|
|
54
|
+
Crawl --> WebAssets["Forms, Inputs, Buttons, JS Errors"]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
subgraph Planning ["2. Test Planning & Generation"]
|
|
58
|
+
Assets & WebAssets --> Planner["Deterministic Test Planner"]
|
|
59
|
+
Planner --> Happy["Happy Path Cases"]
|
|
60
|
+
Planner --> Val["Validation & Schema Checks"]
|
|
61
|
+
Planner --> Boundary["Boundary & Edge Cases"]
|
|
62
|
+
Planner --> Negative["Type Violations & Negative Cases"]
|
|
63
|
+
Planner --> Auth["Auth / Security Cases"]
|
|
64
|
+
Planner --> AI_Gen["AI Exploratory Edge Cases"]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
subgraph Execution ["3. Orchestrated Concurrent Execution"]
|
|
68
|
+
WorkerPool["Worker Pool (Bounded Concurrency)"]
|
|
69
|
+
Happy & Val & Boundary & Negative & Auth & AI_Gen --> WorkerPool
|
|
70
|
+
WorkerPool --> APIRunner["API Runner (httpx, Schemas, Retries)"]
|
|
71
|
+
WorkerPool --> BrowserRunner["Browser Runner (Playwright, DOM, Screenshots)"]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
subgraph Analysis ["4. Failure Analysis & RCA"]
|
|
75
|
+
APIRunner & BrowserRunner --> Evidence["Evidence Collector (Traces, Screenshots, Logs)"]
|
|
76
|
+
Evidence --> Classifier["Deterministic Failure Classifier"]
|
|
77
|
+
Classifier --> AI_RCA["AI Root-Cause Analyzer (Fact vs Hypothesis vs Fix)"]
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
subgraph Reporting ["5. Output & Persistence"]
|
|
81
|
+
AI_RCA --> JSONRep["Machine-readable JSON"]
|
|
82
|
+
AI_RCA --> HTMLRep["Self-Contained Interactive HTML Report"]
|
|
83
|
+
AI_RCA --> DB["PostgreSQL / SQLite Persistence"]
|
|
84
|
+
end
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## 2. Core Capabilities
|
|
90
|
+
|
|
91
|
+
- **Deterministic Testing First**: AI is an analysis and exploratory proposal layer, not an unpredictable execution core. Tests execute against concrete assertions (Status codes, JSONPath, JSON Schema, latency, DOM visibility).
|
|
92
|
+
- **OpenAPI & Headless Browser Discovery**: Auto-detects and resolves OpenAPI 3.0, 3.1, and Swagger 2.0 schemas, or crawls dynamic Web applications using Playwright to extract forms, interactive buttons, and JavaScript console errors.
|
|
93
|
+
- **Multi-Category Test Suites**:
|
|
94
|
+
1. `HAPPY_PATH`: Valid payload matching schemas and expected 200/201 responses.
|
|
95
|
+
2. `VALIDATION`: Missing required property permutations (expected 400/422).
|
|
96
|
+
3. `BOUNDARY`: Empty strings, zero values, negative IDs, and oversized strings.
|
|
97
|
+
4. `NEGATIVE`: Type mismatches (e.g. strings for integer fields, malformed JSON).
|
|
98
|
+
5. `AUTHENTICATION`: Verifies secure endpoints reject unauthenticated requests.
|
|
99
|
+
6. `AUTHORIZATION`: Verifies role-restricted endpoints enforce access controls.
|
|
100
|
+
7. `ERROR_HANDLING`: Verifies non-existent resource IDs return clean 404s.
|
|
101
|
+
8. `EXPLORATORY`: AI-proposed edge cases with strict sandbox validation.
|
|
102
|
+
- **Deterministic Failure Taxonomy**:
|
|
103
|
+
- `ASSERTION_FAILURE`
|
|
104
|
+
- `APPLICATION_ERROR` (500 Internal Server Error, unhandled exceptions)
|
|
105
|
+
- `VALIDATION_FAILURE`
|
|
106
|
+
- `AUTHENTICATION_FAILURE`
|
|
107
|
+
- `AUTHORIZATION_FAILURE`
|
|
108
|
+
- `TIMEOUT`
|
|
109
|
+
- `NETWORK_ERROR`
|
|
110
|
+
- `BROWSER_ERROR`
|
|
111
|
+
- `HTTP_ERROR`
|
|
112
|
+
- `TEST_CONFIGURATION_ERROR`
|
|
113
|
+
- **AI Root Cause Analysis**: Distinguishes between **Observed Facts**, **Hypotheses** (with confidence score 0.0–1.0), and **Suggested Fixes**.
|
|
114
|
+
- **Enterprise Security**: Built-in SSRF protection (blocking cloud metadata IPs `169.254.169.254`, loopbacks unless permitted), response size limits, and automatic secret redaction (`Authorization`, `Bearer`, `Cookie`, passwords, API keys).
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## 3. Quick Start
|
|
119
|
+
|
|
120
|
+
### Installation
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
# Clone repository
|
|
124
|
+
git clone https://github.com/recon-qa/recon.git
|
|
125
|
+
cd recon
|
|
126
|
+
|
|
127
|
+
# Install in editable mode with browser and AI extras
|
|
128
|
+
pip install -e .[browser,ai]
|
|
129
|
+
|
|
130
|
+
# Install Playwright browser dependencies
|
|
131
|
+
python -m playwright install chromium
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Verify Environment (`doctor`)
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
recon doctor
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Output:
|
|
141
|
+
```
|
|
142
|
+
Recon QA Agent — System Diagnostics (Doctor)
|
|
143
|
+
Component Status Details
|
|
144
|
+
Python Version OK Python 3.12.10
|
|
145
|
+
Playwright & Chromium OK Headless Chromium ready
|
|
146
|
+
Docker CLI OK Found at /usr/bin/docker
|
|
147
|
+
Persistence (DB) OK sqlite+aiosqlite:///./recon.db
|
|
148
|
+
AI Provider (Gemini) CONFIGURED Key present: True (Model: gemini-2.5-flash)
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
### AI Configuration (Bring Your Own Key)
|
|
154
|
+
|
|
155
|
+
Recon is **100% Bring-Your-Own-Key (BYOK)**. You can configure and manage keys directly from your terminal or via `.env`:
|
|
156
|
+
|
|
157
|
+
#### Interactive Terminal Commands (Easiest):
|
|
158
|
+
```bash
|
|
159
|
+
# 1. View all supported providers and current active model:
|
|
160
|
+
recon providers
|
|
161
|
+
|
|
162
|
+
# 2. Interactively add or update your API key:
|
|
163
|
+
recon set-key
|
|
164
|
+
# (or specify directly: recon set-key gemini --key AIzaSy...)
|
|
165
|
+
|
|
166
|
+
# 3. Switch active provider at any time:
|
|
167
|
+
recon use mistral
|
|
168
|
+
recon use gemini
|
|
169
|
+
recon use ollama
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
#### Manual `.env` Configuration (Alternative):
|
|
175
|
+
Create a `.env` file in the root directory:
|
|
176
|
+
|
|
177
|
+
#### 1. Google Gemini (Fastest & Free Tier)
|
|
178
|
+
```ini
|
|
179
|
+
RECON_LLM_PROVIDER=gemini
|
|
180
|
+
GEMINI_API_KEY=your_gemini_api_key
|
|
181
|
+
RECON_GEMINI_MODEL=gemini-2.5-flash
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
#### 2. Mistral AI
|
|
185
|
+
```ini
|
|
186
|
+
RECON_LLM_PROVIDER=mistral
|
|
187
|
+
MISTRAL_API_KEY=your_mistral_api_key
|
|
188
|
+
RECON_MISTRAL_MODEL=mistral-small-latest
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
#### 3. OpenAI
|
|
192
|
+
```ini
|
|
193
|
+
RECON_LLM_PROVIDER=openai
|
|
194
|
+
OPENAI_API_KEY=your_openai_api_key
|
|
195
|
+
RECON_OPENAI_MODEL=gpt-4o-mini
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
#### 4. Anthropic Claude
|
|
199
|
+
```ini
|
|
200
|
+
RECON_LLM_PROVIDER=anthropic
|
|
201
|
+
ANTHROPIC_API_KEY=your_anthropic_api_key
|
|
202
|
+
RECON_ANTHROPIC_MODEL=claude-3-5-haiku-20241022
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
#### 5. Local Models (Ollama) & Any OpenAI-Compatible API (Groq, DeepSeek, OpenRouter)
|
|
206
|
+
```ini
|
|
207
|
+
RECON_LLM_PROVIDER=ollama
|
|
208
|
+
RECON_LLM_BASE_URL=http://localhost:11434/v1
|
|
209
|
+
RECON_LLM_MODEL=llama3.2
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
#### 6. Offline Mock (Default — No API Key Required)
|
|
213
|
+
If no key is configured, Recon defaults to `RECON_LLM_PROVIDER=mock` using deterministic rule-based analysis with zero network calls and zero cost.
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## 4. CLI Reference
|
|
218
|
+
|
|
219
|
+
Both `recon` and `qa-agent` are available as entrypoints.
|
|
220
|
+
|
|
221
|
+
### 1. Scan Target Application
|
|
222
|
+
```bash
|
|
223
|
+
recon scan http://localhost:8000
|
|
224
|
+
# or with explicit OpenAPI spec
|
|
225
|
+
recon scan http://localhost:8000 --spec http://localhost:8000/openapi.json --browser
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
### 2. Generate Test Plan
|
|
229
|
+
```bash
|
|
230
|
+
recon generate http://localhost:8000 --output tests.json
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
### 3. Run Autonomous QA Tests
|
|
234
|
+
```bash
|
|
235
|
+
recon test http://localhost:8000 --browser --concurrency 4
|
|
236
|
+
```
|
|
237
|
+
**Exit Codes:**
|
|
238
|
+
- `0`: All tests passed cleanly.
|
|
239
|
+
- `1`: Test failures or intentional defects detected.
|
|
240
|
+
- `2`: Configuration or unreachable target error.
|
|
241
|
+
|
|
242
|
+
### 4. Analyze Past Test Results
|
|
243
|
+
```bash
|
|
244
|
+
recon analyze ./reports/latest.json
|
|
245
|
+
# or by run ID
|
|
246
|
+
recon analyze 20260828-011000-a1b2c3
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
### 5. View Interactive HTML Report
|
|
250
|
+
```bash
|
|
251
|
+
recon report latest
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
### 6. Launch Built-in Demo Target
|
|
255
|
+
```bash
|
|
256
|
+
recon serve-demo --port 8000
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
|
|
261
|
+
## 5. Built-in Demo Target Application
|
|
262
|
+
|
|
263
|
+
Recon includes a deliberately defective target application inside the repository (`recon/demo_app/`):
|
|
264
|
+
|
|
265
|
+
1. **`POST /api/orders`**: Fails with HTTP 500 (`NullReferenceException`) when the optional `currency` parameter is omitted.
|
|
266
|
+
2. **`POST /api/users`**: Flawed validation regex rejects valid emails containing numbers.
|
|
267
|
+
3. **`GET /api/admin/secrets`**: Missing authorization check exposes sensitive keys without authentication.
|
|
268
|
+
4. **`GET /api/slow-analytics`**: Delayed execution (2.0s) triggering latency warnings.
|
|
269
|
+
5. **Web UI Login**: Form click produces `Uncaught TypeError: Cannot read properties of undefined (reading 'token')` and fails navigation.
|
|
270
|
+
|
|
271
|
+
To run Recon against the demo target:
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
# Terminal 1: Start demo app
|
|
275
|
+
recon serve-demo --port 8000
|
|
276
|
+
|
|
277
|
+
# Terminal 2: Run Recon with browser automation
|
|
278
|
+
recon test http://localhost:8000 --browser --concurrency 4
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
---
|
|
282
|
+
|
|
283
|
+
## 6. Docker & Docker Compose
|
|
284
|
+
|
|
285
|
+
Run the entire suite and demo app in Docker:
|
|
286
|
+
|
|
287
|
+
```bash
|
|
288
|
+
docker compose up
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
This starts:
|
|
292
|
+
1. `recon-demo-app` on port `8000`
|
|
293
|
+
2. `recon-agent` which runs full discovery, test execution, failure classification, and generates HTML/JSON reports in `./reports`.
|
|
294
|
+
|
|
295
|
+
---
|
|
296
|
+
|
|
297
|
+
## 7. CI/CD Integration
|
|
298
|
+
|
|
299
|
+
Example GitHub Actions workflow:
|
|
300
|
+
|
|
301
|
+
```yaml
|
|
302
|
+
- name: Start Target Application
|
|
303
|
+
run: python -m uvicorn recon.demo_app.main:app --port 8000 &
|
|
304
|
+
|
|
305
|
+
- name: Run Recon QA Agent
|
|
306
|
+
run: recon test http://127.0.0.1:8000 --browser --concurrency 4
|
|
307
|
+
|
|
308
|
+
- name: Upload Test Report
|
|
309
|
+
if: always()
|
|
310
|
+
uses: actions/upload-artifact@v4
|
|
311
|
+
with:
|
|
312
|
+
name: recon-qa-report
|
|
313
|
+
path: reports/
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
---
|
|
317
|
+
|
|
318
|
+
## 8. Limitations & Scope
|
|
319
|
+
|
|
320
|
+
- **Arbitrary Dynamic Endpoints**: Non-standard API endpoints without OpenAPI documentation or HTML links cannot be guessed with 100% certainty. Recon probes standard paths (`/openapi.json`, `/swagger.json`, `/docs`).
|
|
321
|
+
- **Complex Multi-Step State**: Endpoints requiring complex state transitions (e.g. 2FA SMS tokens) require pre-configured authentication headers.
|
|
322
|
+
- **Heuristic Boundaries**: High-dimensional schemas are fuzz-tested at key boundaries (lengths, zero, null, type mismatch) rather than combinatorial explosion.
|
recon_qa-0.1.0/README.md
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
# Recon: AI QA Agent — Autonomous Testing & Failure Analysis Platform
|
|
2
|
+
|
|
3
|
+
[](https://github.com/recon-qa/recon/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/downloads/)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
|
|
7
|
+
> **Recon** (`qa-agent`) is an autonomous, developer-first testing platform designed to inspect applications, plan multi-category test suites, execute HTTP & Playwright browser tests concurrently, deterministically classify failures, perform AI-assisted root-cause analysis (RCA), and generate actionable reports.
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## 1. Architecture Overview
|
|
12
|
+
|
|
13
|
+
```mermaid
|
|
14
|
+
flowchart TD
|
|
15
|
+
subgraph Discovery ["1. Application Discovery"]
|
|
16
|
+
Target["Target Application URL / OpenAPI Spec"] --> Engine["Discovery Engine"]
|
|
17
|
+
Engine --> OA["OpenAPI 3.x / Swagger Parser"]
|
|
18
|
+
Engine --> Crawl["Playwright Web Crawler"]
|
|
19
|
+
OA --> Assets["Endpoints, Parameters, Schemas"]
|
|
20
|
+
Crawl --> WebAssets["Forms, Inputs, Buttons, JS Errors"]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
subgraph Planning ["2. Test Planning & Generation"]
|
|
24
|
+
Assets & WebAssets --> Planner["Deterministic Test Planner"]
|
|
25
|
+
Planner --> Happy["Happy Path Cases"]
|
|
26
|
+
Planner --> Val["Validation & Schema Checks"]
|
|
27
|
+
Planner --> Boundary["Boundary & Edge Cases"]
|
|
28
|
+
Planner --> Negative["Type Violations & Negative Cases"]
|
|
29
|
+
Planner --> Auth["Auth / Security Cases"]
|
|
30
|
+
Planner --> AI_Gen["AI Exploratory Edge Cases"]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
subgraph Execution ["3. Orchestrated Concurrent Execution"]
|
|
34
|
+
WorkerPool["Worker Pool (Bounded Concurrency)"]
|
|
35
|
+
Happy & Val & Boundary & Negative & Auth & AI_Gen --> WorkerPool
|
|
36
|
+
WorkerPool --> APIRunner["API Runner (httpx, Schemas, Retries)"]
|
|
37
|
+
WorkerPool --> BrowserRunner["Browser Runner (Playwright, DOM, Screenshots)"]
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
subgraph Analysis ["4. Failure Analysis & RCA"]
|
|
41
|
+
APIRunner & BrowserRunner --> Evidence["Evidence Collector (Traces, Screenshots, Logs)"]
|
|
42
|
+
Evidence --> Classifier["Deterministic Failure Classifier"]
|
|
43
|
+
Classifier --> AI_RCA["AI Root-Cause Analyzer (Fact vs Hypothesis vs Fix)"]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
subgraph Reporting ["5. Output & Persistence"]
|
|
47
|
+
AI_RCA --> JSONRep["Machine-readable JSON"]
|
|
48
|
+
AI_RCA --> HTMLRep["Self-Contained Interactive HTML Report"]
|
|
49
|
+
AI_RCA --> DB["PostgreSQL / SQLite Persistence"]
|
|
50
|
+
end
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 2. Core Capabilities
|
|
56
|
+
|
|
57
|
+
- **Deterministic Testing First**: AI is an analysis and exploratory proposal layer, not an unpredictable execution core. Tests execute against concrete assertions (Status codes, JSONPath, JSON Schema, latency, DOM visibility).
|
|
58
|
+
- **OpenAPI & Headless Browser Discovery**: Auto-detects and resolves OpenAPI 3.0, 3.1, and Swagger 2.0 schemas, or crawls dynamic Web applications using Playwright to extract forms, interactive buttons, and JavaScript console errors.
|
|
59
|
+
- **Multi-Category Test Suites**:
|
|
60
|
+
1. `HAPPY_PATH`: Valid payload matching schemas and expected 200/201 responses.
|
|
61
|
+
2. `VALIDATION`: Missing required property permutations (expected 400/422).
|
|
62
|
+
3. `BOUNDARY`: Empty strings, zero values, negative IDs, and oversized strings.
|
|
63
|
+
4. `NEGATIVE`: Type mismatches (e.g. strings for integer fields, malformed JSON).
|
|
64
|
+
5. `AUTHENTICATION`: Verifies secure endpoints reject unauthenticated requests.
|
|
65
|
+
6. `AUTHORIZATION`: Verifies role-restricted endpoints enforce access controls.
|
|
66
|
+
7. `ERROR_HANDLING`: Verifies non-existent resource IDs return clean 404s.
|
|
67
|
+
8. `EXPLORATORY`: AI-proposed edge cases with strict sandbox validation.
|
|
68
|
+
- **Deterministic Failure Taxonomy**:
|
|
69
|
+
- `ASSERTION_FAILURE`
|
|
70
|
+
- `APPLICATION_ERROR` (500 Internal Server Error, unhandled exceptions)
|
|
71
|
+
- `VALIDATION_FAILURE`
|
|
72
|
+
- `AUTHENTICATION_FAILURE`
|
|
73
|
+
- `AUTHORIZATION_FAILURE`
|
|
74
|
+
- `TIMEOUT`
|
|
75
|
+
- `NETWORK_ERROR`
|
|
76
|
+
- `BROWSER_ERROR`
|
|
77
|
+
- `HTTP_ERROR`
|
|
78
|
+
- `TEST_CONFIGURATION_ERROR`
|
|
79
|
+
- **AI Root Cause Analysis**: Distinguishes between **Observed Facts**, **Hypotheses** (with confidence score 0.0–1.0), and **Suggested Fixes**.
|
|
80
|
+
- **Enterprise Security**: Built-in SSRF protection (blocking cloud metadata IPs `169.254.169.254`, loopbacks unless permitted), response size limits, and automatic secret redaction (`Authorization`, `Bearer`, `Cookie`, passwords, API keys).
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## 3. Quick Start
|
|
85
|
+
|
|
86
|
+
### Installation
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
# Clone repository
|
|
90
|
+
git clone https://github.com/recon-qa/recon.git
|
|
91
|
+
cd recon
|
|
92
|
+
|
|
93
|
+
# Install in editable mode with browser and AI extras
|
|
94
|
+
pip install -e .[browser,ai]
|
|
95
|
+
|
|
96
|
+
# Install Playwright browser dependencies
|
|
97
|
+
python -m playwright install chromium
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### Verify Environment (`doctor`)
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
recon doctor
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Output:
|
|
107
|
+
```
|
|
108
|
+
Recon QA Agent — System Diagnostics (Doctor)
|
|
109
|
+
Component Status Details
|
|
110
|
+
Python Version OK Python 3.12.10
|
|
111
|
+
Playwright & Chromium OK Headless Chromium ready
|
|
112
|
+
Docker CLI OK Found at /usr/bin/docker
|
|
113
|
+
Persistence (DB) OK sqlite+aiosqlite:///./recon.db
|
|
114
|
+
AI Provider (Gemini) CONFIGURED Key present: True (Model: gemini-2.5-flash)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
---
|
|
118
|
+
|
|
119
|
+
### AI Configuration (Bring Your Own Key)
|
|
120
|
+
|
|
121
|
+
Recon is **100% Bring-Your-Own-Key (BYOK)**. You can configure and manage keys directly from your terminal or via `.env`:
|
|
122
|
+
|
|
123
|
+
#### Interactive Terminal Commands (Easiest):
|
|
124
|
+
```bash
|
|
125
|
+
# 1. View all supported providers and current active model:
|
|
126
|
+
recon providers
|
|
127
|
+
|
|
128
|
+
# 2. Interactively add or update your API key:
|
|
129
|
+
recon set-key
|
|
130
|
+
# (or specify directly: recon set-key gemini --key AIzaSy...)
|
|
131
|
+
|
|
132
|
+
# 3. Switch active provider at any time:
|
|
133
|
+
recon use mistral
|
|
134
|
+
recon use gemini
|
|
135
|
+
recon use ollama
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
#### Manual `.env` Configuration (Alternative):
|
|
141
|
+
Create a `.env` file in the root directory:
|
|
142
|
+
|
|
143
|
+
#### 1. Google Gemini (Fastest & Free Tier)
|
|
144
|
+
```ini
|
|
145
|
+
RECON_LLM_PROVIDER=gemini
|
|
146
|
+
GEMINI_API_KEY=your_gemini_api_key
|
|
147
|
+
RECON_GEMINI_MODEL=gemini-2.5-flash
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
#### 2. Mistral AI
|
|
151
|
+
```ini
|
|
152
|
+
RECON_LLM_PROVIDER=mistral
|
|
153
|
+
MISTRAL_API_KEY=your_mistral_api_key
|
|
154
|
+
RECON_MISTRAL_MODEL=mistral-small-latest
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
#### 3. OpenAI
|
|
158
|
+
```ini
|
|
159
|
+
RECON_LLM_PROVIDER=openai
|
|
160
|
+
OPENAI_API_KEY=your_openai_api_key
|
|
161
|
+
RECON_OPENAI_MODEL=gpt-4o-mini
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
#### 4. Anthropic Claude
|
|
165
|
+
```ini
|
|
166
|
+
RECON_LLM_PROVIDER=anthropic
|
|
167
|
+
ANTHROPIC_API_KEY=your_anthropic_api_key
|
|
168
|
+
RECON_ANTHROPIC_MODEL=claude-3-5-haiku-20241022
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
#### 5. Local Models (Ollama) & Any OpenAI-Compatible API (Groq, DeepSeek, OpenRouter)
|
|
172
|
+
```ini
|
|
173
|
+
RECON_LLM_PROVIDER=ollama
|
|
174
|
+
RECON_LLM_BASE_URL=http://localhost:11434/v1
|
|
175
|
+
RECON_LLM_MODEL=llama3.2
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
#### 6. Offline Mock (Default — No API Key Required)
|
|
179
|
+
If no key is configured, Recon defaults to `RECON_LLM_PROVIDER=mock` using deterministic rule-based analysis with zero network calls and zero cost.
|
|
180
|
+
|
|
181
|
+
---
|
|
182
|
+
|
|
183
|
+
## 4. CLI Reference
|
|
184
|
+
|
|
185
|
+
Both `recon` and `qa-agent` are available as entrypoints.
|
|
186
|
+
|
|
187
|
+
### 1. Scan Target Application
|
|
188
|
+
```bash
|
|
189
|
+
recon scan http://localhost:8000
|
|
190
|
+
# or with explicit OpenAPI spec
|
|
191
|
+
recon scan http://localhost:8000 --spec http://localhost:8000/openapi.json --browser
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### 2. Generate Test Plan
|
|
195
|
+
```bash
|
|
196
|
+
recon generate http://localhost:8000 --output tests.json
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### 3. Run Autonomous QA Tests
|
|
200
|
+
```bash
|
|
201
|
+
recon test http://localhost:8000 --browser --concurrency 4
|
|
202
|
+
```
|
|
203
|
+
**Exit Codes:**
|
|
204
|
+
- `0`: All tests passed cleanly.
|
|
205
|
+
- `1`: Test failures or intentional defects detected.
|
|
206
|
+
- `2`: Configuration or unreachable target error.
|
|
207
|
+
|
|
208
|
+
### 4. Analyze Past Test Results
|
|
209
|
+
```bash
|
|
210
|
+
recon analyze ./reports/latest.json
|
|
211
|
+
# or by run ID
|
|
212
|
+
recon analyze 20260828-011000-a1b2c3
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### 5. View Interactive HTML Report
|
|
216
|
+
```bash
|
|
217
|
+
recon report latest
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
### 6. Launch Built-in Demo Target
|
|
221
|
+
```bash
|
|
222
|
+
recon serve-demo --port 8000
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
|
|
227
|
+
## 5. Built-in Demo Target Application
|
|
228
|
+
|
|
229
|
+
Recon includes a deliberately defective target application inside the repository (`recon/demo_app/`):
|
|
230
|
+
|
|
231
|
+
1. **`POST /api/orders`**: Fails with HTTP 500 (`NullReferenceException`) when the optional `currency` parameter is omitted.
|
|
232
|
+
2. **`POST /api/users`**: Flawed validation regex rejects valid emails containing numbers.
|
|
233
|
+
3. **`GET /api/admin/secrets`**: Missing authorization check exposes sensitive keys without authentication.
|
|
234
|
+
4. **`GET /api/slow-analytics`**: Delayed execution (2.0s) triggering latency warnings.
|
|
235
|
+
5. **Web UI Login**: Form click produces `Uncaught TypeError: Cannot read properties of undefined (reading 'token')` and fails navigation.
|
|
236
|
+
|
|
237
|
+
To run Recon against the demo target:
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
# Terminal 1: Start demo app
|
|
241
|
+
recon serve-demo --port 8000
|
|
242
|
+
|
|
243
|
+
# Terminal 2: Run Recon with browser automation
|
|
244
|
+
recon test http://localhost:8000 --browser --concurrency 4
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
---
|
|
248
|
+
|
|
249
|
+
## 6. Docker & Docker Compose
|
|
250
|
+
|
|
251
|
+
Run the entire suite and demo app in Docker:
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
docker compose up
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
This starts:
|
|
258
|
+
1. `recon-demo-app` on port `8000`
|
|
259
|
+
2. `recon-agent` which runs full discovery, test execution, failure classification, and generates HTML/JSON reports in `./reports`.
|
|
260
|
+
|
|
261
|
+
---
|
|
262
|
+
|
|
263
|
+
## 7. CI/CD Integration
|
|
264
|
+
|
|
265
|
+
Example GitHub Actions workflow:
|
|
266
|
+
|
|
267
|
+
```yaml
|
|
268
|
+
- name: Start Target Application
|
|
269
|
+
run: python -m uvicorn recon.demo_app.main:app --port 8000 &
|
|
270
|
+
|
|
271
|
+
- name: Run Recon QA Agent
|
|
272
|
+
run: recon test http://127.0.0.1:8000 --browser --concurrency 4
|
|
273
|
+
|
|
274
|
+
- name: Upload Test Report
|
|
275
|
+
if: always()
|
|
276
|
+
uses: actions/upload-artifact@v4
|
|
277
|
+
with:
|
|
278
|
+
name: recon-qa-report
|
|
279
|
+
path: reports/
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
---
|
|
283
|
+
|
|
284
|
+
## 8. Limitations & Scope
|
|
285
|
+
|
|
286
|
+
- **Arbitrary Dynamic Endpoints**: Non-standard API endpoints without OpenAPI documentation or HTML links cannot be guessed with 100% certainty. Recon probes standard paths (`/openapi.json`, `/swagger.json`, `/docs`).
|
|
287
|
+
- **Complex Multi-Step State**: Endpoints requiring complex state transitions (e.g. 2FA SMS tokens) require pre-configured authentication headers.
|
|
288
|
+
- **Heuristic Boundaries**: High-dimensional schemas are fuzz-tested at key boundaries (lengths, zero, null, type mismatch) rather than combinatorial explosion.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "recon-qa"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Recon: AI QA Agent — Autonomous Testing & Failure Analysis Platform"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Recon Engineering Team" }]
|
|
13
|
+
keywords = ["qa", "testing", "automation", "playwright", "ai", "failure-analysis", "openapi"]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"typer>=0.12.0",
|
|
16
|
+
"rich>=13.7.0",
|
|
17
|
+
"pydantic>=2.7.0",
|
|
18
|
+
"pydantic-settings>=2.2.0",
|
|
19
|
+
"httpx>=0.27.0",
|
|
20
|
+
"fastapi>=0.111.0",
|
|
21
|
+
"uvicorn>=0.30.0",
|
|
22
|
+
"sqlalchemy>=2.0.30",
|
|
23
|
+
"aiosqlite>=0.20.0",
|
|
24
|
+
"jinja2>=3.1.4",
|
|
25
|
+
"jsonschema>=4.22.0",
|
|
26
|
+
"pytest>=8.2.0",
|
|
27
|
+
"pytest-asyncio>=0.23.0",
|
|
28
|
+
"beautifulsoup4>=4.12.0",
|
|
29
|
+
"pyyaml>=6.0.1",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
browser = [
|
|
34
|
+
"playwright>=1.44.0",
|
|
35
|
+
]
|
|
36
|
+
ai = [
|
|
37
|
+
"google-genai>=1.0.0",
|
|
38
|
+
"openai>=1.30.0",
|
|
39
|
+
]
|
|
40
|
+
dev = [
|
|
41
|
+
"ruff>=0.4.0",
|
|
42
|
+
"mypy>=1.10.0",
|
|
43
|
+
"pytest-cov>=5.0.0",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.scripts]
|
|
47
|
+
recon = "recon.apps.cli.main:app"
|
|
48
|
+
qa-agent = "recon.apps.cli.main:app"
|
|
49
|
+
|
|
50
|
+
[tool.setuptools.packages.find]
|
|
51
|
+
where = ["."]
|
|
52
|
+
include = ["recon*"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
asyncio_mode = "auto"
|
|
56
|
+
testpaths = ["tests"]
|
|
57
|
+
pythonpath = ["."]
|
|
58
|
+
|
|
59
|
+
[tool.ruff]
|
|
60
|
+
line-length = 100
|
|
61
|
+
target-version = "py312"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Package marker
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Package marker
|