trunft 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trunft-0.1.0/LICENSE +21 -0
- trunft-0.1.0/PKG-INFO +237 -0
- trunft-0.1.0/README.md +195 -0
- trunft-0.1.0/pyproject.toml +65 -0
- trunft-0.1.0/setup.cfg +4 -0
- trunft-0.1.0/src/trunft/__init__.py +3 -0
- trunft-0.1.0/src/trunft/api.py +197 -0
- trunft-0.1.0/src/trunft/cli.py +219 -0
- trunft-0.1.0/src/trunft/core/__init__.py +0 -0
- trunft-0.1.0/src/trunft/core/config.py +97 -0
- trunft-0.1.0/src/trunft/core/disruption.py +96 -0
- trunft-0.1.0/src/trunft/core/models.py +57 -0
- trunft-0.1.0/src/trunft/core/pillar.py +127 -0
- trunft-0.1.0/src/trunft/core/runner.py +79 -0
- trunft-0.1.0/src/trunft/core/thresholds.py +35 -0
- trunft-0.1.0/src/trunft/core/tooling.py +125 -0
- trunft-0.1.0/src/trunft/mcp_server.py +165 -0
- trunft-0.1.0/src/trunft/pillars/__init__.py +1 -0
- trunft-0.1.0/src/trunft/pillars/accessibility.py +78 -0
- trunft-0.1.0/src/trunft/pillars/chaos.py +93 -0
- trunft-0.1.0/src/trunft/pillars/compatibility.py +73 -0
- trunft-0.1.0/src/trunft/pillars/compliance.py +99 -0
- trunft-0.1.0/src/trunft/pillars/installability.py +85 -0
- trunft-0.1.0/src/trunft/pillars/performance.py +88 -0
- trunft-0.1.0/src/trunft/pillars/reliability.py +57 -0
- trunft-0.1.0/src/trunft/pillars/security.py +96 -0
- trunft-0.1.0/src/trunft/reporting/__init__.py +91 -0
- trunft-0.1.0/src/trunft/scaffold.py +42 -0
- trunft-0.1.0/src/trunft/setup_tools.py +115 -0
- trunft-0.1.0/src/trunft/templates/__init__.py +0 -0
- trunft-0.1.0/src/trunft/templates/project/chaos/experiments/api-pod-kill.json.example +31 -0
- trunft-0.1.0/src/trunft/templates/project/config/environments/dev.yaml +23 -0
- trunft-0.1.0/src/trunft/templates/project/config/environments/staging.yaml +19 -0
- trunft-0.1.0/src/trunft/templates/project/config/trunft.yaml +161 -0
- trunft-0.1.0/src/trunft/templates/project/docker/docker-compose.yml +23 -0
- trunft-0.1.0/src/trunft/templates/project/reports/.gitkeep +0 -0
- trunft-0.1.0/src/trunft/templates/project/workloads/k6/scenario.js +42 -0
- trunft-0.1.0/src/trunft.egg-info/PKG-INFO +237 -0
- trunft-0.1.0/src/trunft.egg-info/SOURCES.txt +44 -0
- trunft-0.1.0/src/trunft.egg-info/dependency_links.txt +1 -0
- trunft-0.1.0/src/trunft.egg-info/entry_points.txt +3 -0
- trunft-0.1.0/src/trunft.egg-info/requires.txt +21 -0
- trunft-0.1.0/src/trunft.egg-info/top_level.txt +1 -0
- trunft-0.1.0/tests/test_framework.py +132 -0
- trunft-0.1.0/tests/test_interfaces.py +134 -0
- trunft-0.1.0/tests/test_packaging.py +37 -0
trunft-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shan Konduru
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
trunft-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: trunft
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: TruNFT - unified Non-Functional Testing platform (CLI + MCP server) for Web, Mobile Native, Mobile Browser and Mobile Hybrid apps
|
|
5
|
+
Author-email: Shan Konduru <ShanKonduru@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ShanKonduru/TruNFT
|
|
8
|
+
Project-URL: Source, https://github.com/ShanKonduru/TruNFT
|
|
9
|
+
Project-URL: Issues, https://github.com/ShanKonduru/TruNFT/issues
|
|
10
|
+
Keywords: non-functional-testing,performance-testing,load-testing,security-testing,chaos-engineering,accessibility,compatibility-testing,mobile-testing,k6,mcp
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
22
|
+
Classifier: Topic :: System :: Benchmark
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: PyYAML>=6.0
|
|
27
|
+
Provides-Extra: web
|
|
28
|
+
Requires-Dist: playwright>=1.45; extra == "web"
|
|
29
|
+
Provides-Extra: mobile
|
|
30
|
+
Requires-Dist: Appium-Python-Client>=4.0; extra == "mobile"
|
|
31
|
+
Provides-Extra: chaos
|
|
32
|
+
Requires-Dist: chaostoolkit>=1.19; extra == "chaos"
|
|
33
|
+
Provides-Extra: mcp
|
|
34
|
+
Requires-Dist: mcp<3,>=2.3; extra == "mcp"
|
|
35
|
+
Provides-Extra: all
|
|
36
|
+
Requires-Dist: trunft[chaos,mcp,mobile,web]; extra == "all"
|
|
37
|
+
Provides-Extra: dev
|
|
38
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
39
|
+
Requires-Dist: anyio>=4; extra == "dev"
|
|
40
|
+
Requires-Dist: mcp<3,>=2.3; extra == "dev"
|
|
41
|
+
Dynamic: license-file
|
|
42
|
+
|
|
43
|
+
# TruNFT — Unified Non-Functional Testing Platform
|
|
44
|
+
|
|
45
|
+
One framework, one CLI, one config and one report for **non-functional testing** across:
|
|
46
|
+
|
|
47
|
+
| Platform | Key | What is exercised |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| Web UI | `web` | Site URLs in desktop browsers |
|
|
50
|
+
| Mobile browser | `mobile-browser` | Site URLs with mobile form factor / device emulation |
|
|
51
|
+
| Mobile native | `mobile-native` | App binary (APK/IPA) + the backend APIs it calls |
|
|
52
|
+
| Mobile hybrid | `mobile-hybrid` | App binary + embedded web content + backend APIs |
|
|
53
|
+
|
|
54
|
+
TruNFT is an **orchestrator**. It sends each kind of test to an established tool (k6, OWASP ZAP, Toxiproxy, Chaos Toolkit, Lighthouse, Playwright, adb/xcrun), applies the same SLO thresholds to the results, and writes JSON, JUnit and HTML reports.
|
|
55
|
+
|
|
56
|
+
## Coverage
|
|
57
|
+
|
|
58
|
+
| Pillar (`--pillar`) | Test types (`-t`) | Engine |
|
|
59
|
+
|---|---|---|
|
|
60
|
+
| **Performance** `performance` | load, stress, spike, endurance (soak), scalability, capacity | k6 (one parameterised script, profiles in config) |
|
|
61
|
+
| **Security** `security` | vulnerability, penetration, access_control | OWASP ZAP baseline/full (Docker); built-in role × endpoint authZ matrix |
|
|
62
|
+
| **Reliability & availability** `reliability` | stability, failover, recovery, disaster_recovery | Background availability probe + configurable trigger/restore hooks (RTO, outage) |
|
|
63
|
+
| **Resilience & chaos** `chaos` | network_latency, service_failure, resource_exhaustion, experiment | Toxiproxy, shell hooks (kubectl / docker / AWS FIS), Chaos Toolkit |
|
|
64
|
+
| **Usability & accessibility** `accessibility` | accessibility, usability | Lighthouse (axe-core rules, Core Web Vitals); mobile vs desktop form factor |
|
|
65
|
+
| **Compatibility** `compatibility` | browser, device, os, configuration | Playwright (Chromium/Firefox/WebKit, device emulation, locale/dark mode/…) |
|
|
66
|
+
| **Installability & portability** `installability` | install, upgrade, uninstall, portability | adb (Android), xcrun simctl (iOS); multi-host health checks |
|
|
67
|
+
| **Compliance & localization** `compliance` | security_headers, privacy_cookies, localization | Built-in HTTP checks (OWASP headers, cookie flags, per-locale `lang` and untranslated strings) |
|
|
68
|
+
|
|
69
|
+
Run `trunft list` for the full platform × test-type matrix.
|
|
70
|
+
|
|
71
|
+
## Two interfaces, one engine
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
trunft (CLI) trunft-mcp (MCP server: stdio / streamable-http)
|
|
75
|
+
\ /
|
|
76
|
+
+---- trunft.api ----+ capabilities, doctor, plan, run, reports
|
|
77
|
+
|
|
|
78
|
+
runner -> pillars -> external tools (k6, ZAP, Lighthouse, ...)
|
|
79
|
+
|
|
|
80
|
+
reports/<run>/results.json, junit.xml, report.html
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Both interfaces read the same config and write the same reports, so a run started from one can be inspected from the other.
|
|
84
|
+
|
|
85
|
+
## Quick start (CLI)
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
python -m venv .venv && .venv/Scripts/activate # source .venv/bin/activate on Linux/macOS
|
|
89
|
+
pip install -e ".[dev]" # add [web] for Playwright, [mobile] for Appium client
|
|
90
|
+
trunft doctor # shows which external tools are available
|
|
91
|
+
trunft setup # installs missing ones (winget/brew/npm/pip); --dry-run to preview
|
|
92
|
+
trunft plan -e staging --pillar performance # preview, runs nothing
|
|
93
|
+
trunft run --dry-run # plan everything, execute nothing
|
|
94
|
+
trunft run -e dev -p web --pillar compliance
|
|
95
|
+
trunft run -e staging --pillar performance -t load -t spike
|
|
96
|
+
trunft reports # recent runs
|
|
97
|
+
trunft report <run> --failures # what broke and which thresholds were breached
|
|
98
|
+
pytest # framework self-tests
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`list`, `doctor`, `plan`, `reports` and `report` accept `--json`.
|
|
102
|
+
Exit code: `0` = no failures, `1` = at least one FAILED/ERROR, `2` = bad config or arguments.
|
|
103
|
+
|
|
104
|
+
## MCP server (AI agents / IDEs)
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
pip install -e ".[mcp]"
|
|
108
|
+
trunft-mcp --root C:/myprojects/TruNFT # stdio
|
|
109
|
+
trunft-mcp --transport streamable-http --port 8765 # http://127.0.0.1:8765/mcp
|
|
110
|
+
trunft mcp # same server via the main CLI
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Register it with Claude Code:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
claude mcp add trunft --env TRUNFT_ROOT=C:/myprojects/TruNFT -- C:/myprojects/TruNFT/.venv/Scripts/trunft-mcp.exe
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Or in Claude Desktop / any MCP client config:
|
|
120
|
+
|
|
121
|
+
```json
|
|
122
|
+
{ "mcpServers": { "trunft": {
|
|
123
|
+
"command": "C:/myprojects/TruNFT/.venv/Scripts/trunft-mcp.exe",
|
|
124
|
+
"env": { "TRUNFT_ROOT": "C:/myprojects/TruNFT" } } } }
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
| Tool | Purpose |
|
|
128
|
+
|---|---|
|
|
129
|
+
| `list_capabilities` | Pillars, test types, required tools, and which platforms each applies to |
|
|
130
|
+
| `check_tools` | Which external tools are installed |
|
|
131
|
+
| `list_environments` / `get_config` | Environments and the merged config (secrets redacted) |
|
|
132
|
+
| `plan_tests` | Preview the platform × pillar × test-type combinations |
|
|
133
|
+
| `start_test_run` | Start a background run and return a `run_id` at once. **`dry_run` defaults to `true`** |
|
|
134
|
+
| `get_test_run` / `cancel_test_run` | Poll progress and results, or stop the remaining tests |
|
|
135
|
+
| `list_test_reports` / `get_test_report` | Past runs from CLI or MCP; `only_failures` filter |
|
|
136
|
+
|
|
137
|
+
Runs are asynchronous because soak tests can last for hours. The agent starts a run, polls it, then reads the report. Tools that only read are marked read-only. `start_test_run` is marked destructive and open-world because real runs generate load, run security scans and can inject faults.
|
|
138
|
+
|
|
139
|
+
### Result statuses
|
|
140
|
+
| Status | Meaning |
|
|
141
|
+
|---|---|
|
|
142
|
+
| `passed` | Ran, and every threshold was met |
|
|
143
|
+
| `failed` | Ran, but breached a threshold or assertion |
|
|
144
|
+
| `error` | Couldn't complete (tool crashed, target unreachable) |
|
|
145
|
+
| `skipped` | Dry-run, not applicable to the platform, or a required tool isn't installed |
|
|
146
|
+
| `not_implemented` | Placeholder: needs config (e.g. trigger hook) or code; the message says what |
|
|
147
|
+
|
|
148
|
+
## Layout
|
|
149
|
+
|
|
150
|
+
```
|
|
151
|
+
config/trunft.yaml master config: targets, thresholds (SLOs), pillar settings
|
|
152
|
+
config/environments/*.yaml per-environment overrides (dev, staging, ...); ${ENV_VAR:-default} supported
|
|
153
|
+
src/trunft/api.py service layer shared by CLI and MCP server
|
|
154
|
+
src/trunft/cli.py `trunft` command line
|
|
155
|
+
src/trunft/mcp_server.py `trunft-mcp` MCP server (stdio / streamable-http)
|
|
156
|
+
src/trunft/core/ config, models, pillar base + registry, runner, thresholds, probes
|
|
157
|
+
src/trunft/pillars/*.py one module per NFT pillar (auto-discovered)
|
|
158
|
+
src/trunft/reporting/ results.json, junit.xml, report.html
|
|
159
|
+
workloads/k6/scenario.js parameterised k6 workload - put real user journeys here
|
|
160
|
+
chaos/experiments/ Chaos Toolkit experiments (*.json)
|
|
161
|
+
docker/docker-compose.yml toxiproxy, influxdb, grafana for local runs
|
|
162
|
+
apps/android, apps/ios app binaries under test (git-ignored)
|
|
163
|
+
.github/workflows/ci.yml unit tests on push/PR (Linux + Windows); no live tests in CI
|
|
164
|
+
.github/workflows/publish.yml gated PyPI release on v* tags
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Thresholds (SLO gates)
|
|
168
|
+
|
|
169
|
+
Every pillar returns metrics. A threshold `max_<metric>` or `min_<metric>` under `thresholds.<pillar>.default`, or under a per-test-type block, decides pass or fail:
|
|
170
|
+
|
|
171
|
+
```yaml
|
|
172
|
+
thresholds:
|
|
173
|
+
performance:
|
|
174
|
+
default: {max_p95_ms: 800, max_error_rate: 0.01}
|
|
175
|
+
stress: {max_p95_ms: 2000}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## Installing from PyPI
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
pip install trunft # core; extras: trunft[web], trunft[mcp], trunft[chaos], trunft[all]
|
|
182
|
+
trunft init my-nft-project # creates config/, workloads/k6, chaos/, docker/ from bundled templates
|
|
183
|
+
cd my-nft-project && trunft setup && trunft run --dry-run
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Releasing
|
|
187
|
+
|
|
188
|
+
Publishing uses PyPI **Trusted Publishing** (`.github/workflows/publish.yml`, no API tokens):
|
|
189
|
+
|
|
190
|
+
1. Bump `__version__` in `src/trunft/__init__.py` (the single source of the version) and commit.
|
|
191
|
+
2. Tag that commit and push the tag: `git tag v0.2.0 && git push origin v0.2.0`
|
|
192
|
+
|
|
193
|
+
The workflow uploads only if **all** of these gates pass:
|
|
194
|
+
|
|
195
|
+
| Gate | Check |
|
|
196
|
+
|---|---|
|
|
197
|
+
| Version | Tag `v<x.y.z>` == `__version__` == built sdist/wheel versions |
|
|
198
|
+
| Unit | `pytest` on Linux + Windows, Python 3.10 and 3.13 |
|
|
199
|
+
| Security | Sources compile; Bandit SAST with zero findings; `pip-audit --strict` on the runtime dependency set |
|
|
200
|
+
| Build | `twine check --strict`; the wheel is installed into a clean venv, then `trunft init` and a dry run are executed |
|
|
201
|
+
|
|
202
|
+
Then it publishes to PyPI (GitHub environment `pypi`).
|
|
203
|
+
Run the version gate locally with `python scripts/check_release_version.py v0.2.0`.
|
|
204
|
+
|
|
205
|
+
If you edit files under `config/`, `workloads/`, `chaos/` or `docker/`, copy them to `src/trunft/templates/project/`. A test fails if the two copies drift apart.
|
|
206
|
+
|
|
207
|
+
## Extending
|
|
208
|
+
|
|
209
|
+
Add a module under `src/trunft/pillars/`:
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
from trunft.core.pillar import Pillar, RunContext, register
|
|
213
|
+
from trunft.core.models import Status
|
|
214
|
+
|
|
215
|
+
@register
|
|
216
|
+
class Energy(Pillar):
|
|
217
|
+
name, title = "energy", "Battery & Energy"
|
|
218
|
+
test_types = {"drain": "Battery drain over a 30-min journey"}
|
|
219
|
+
type_platforms = {"drain": Platform.mobile_apps()}
|
|
220
|
+
tools = {"drain": ["adb"]}
|
|
221
|
+
|
|
222
|
+
def run(self, ctx: RunContext, test_type: str):
|
|
223
|
+
...
|
|
224
|
+
return self.result(ctx, test_type, Status.PASSED, metrics={"mah_per_hour": 120})
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## Roadmap: open placeholders (`not_implemented`)
|
|
228
|
+
|
|
229
|
+
- **Mobile client performance**: startup time, memory, CPU and frame rate (adb `dumpsys`, Xcode Instruments)
|
|
230
|
+
- **Mobile security**: MobSF static/dynamic analysis of APK/IPA
|
|
231
|
+
- **Native accessibility**: Espresso Accessibility Test Framework, XCUITest `performAccessibilityAudit`
|
|
232
|
+
- **Device-cloud compatibility**: Appium or Selenium grid (BrowserStack, Sauce Labs, LambdaTest) for the OS and native matrix
|
|
233
|
+
- **Upgrade data retention**, **DR RPO verification**, **client-side network conditioning** (offline/3G)
|
|
234
|
+
- **Native localization**: resource-bundle key completeness plus per-locale screenshots
|
|
235
|
+
- **Compliance evidence packs**: GDPR, PCI-DSS, HIPAA, WCAG conformance statement
|
|
236
|
+
|
|
237
|
+
> ⚠️ Run active security scans (`penetration`) and chaos/DR hooks only against environments you are authorised to test. `allow_active_scan` is `false` by default.
|
trunft-0.1.0/README.md
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# TruNFT — Unified Non-Functional Testing Platform
|
|
2
|
+
|
|
3
|
+
One framework, one CLI, one config and one report for **non-functional testing** across:
|
|
4
|
+
|
|
5
|
+
| Platform | Key | What is exercised |
|
|
6
|
+
|---|---|---|
|
|
7
|
+
| Web UI | `web` | Site URLs in desktop browsers |
|
|
8
|
+
| Mobile browser | `mobile-browser` | Site URLs with mobile form factor / device emulation |
|
|
9
|
+
| Mobile native | `mobile-native` | App binary (APK/IPA) + the backend APIs it calls |
|
|
10
|
+
| Mobile hybrid | `mobile-hybrid` | App binary + embedded web content + backend APIs |
|
|
11
|
+
|
|
12
|
+
TruNFT is an **orchestrator**. It sends each kind of test to an established tool (k6, OWASP ZAP, Toxiproxy, Chaos Toolkit, Lighthouse, Playwright, adb/xcrun), applies the same SLO thresholds to the results, and writes JSON, JUnit and HTML reports.
|
|
13
|
+
|
|
14
|
+
## Coverage
|
|
15
|
+
|
|
16
|
+
| Pillar (`--pillar`) | Test types (`-t`) | Engine |
|
|
17
|
+
|---|---|---|
|
|
18
|
+
| **Performance** `performance` | load, stress, spike, endurance (soak), scalability, capacity | k6 (one parameterised script, profiles in config) |
|
|
19
|
+
| **Security** `security` | vulnerability, penetration, access_control | OWASP ZAP baseline/full (Docker); built-in role × endpoint authZ matrix |
|
|
20
|
+
| **Reliability & availability** `reliability` | stability, failover, recovery, disaster_recovery | Background availability probe + configurable trigger/restore hooks (RTO, outage) |
|
|
21
|
+
| **Resilience & chaos** `chaos` | network_latency, service_failure, resource_exhaustion, experiment | Toxiproxy, shell hooks (kubectl / docker / AWS FIS), Chaos Toolkit |
|
|
22
|
+
| **Usability & accessibility** `accessibility` | accessibility, usability | Lighthouse (axe-core rules, Core Web Vitals); mobile vs desktop form factor |
|
|
23
|
+
| **Compatibility** `compatibility` | browser, device, os, configuration | Playwright (Chromium/Firefox/WebKit, device emulation, locale/dark mode/…) |
|
|
24
|
+
| **Installability & portability** `installability` | install, upgrade, uninstall, portability | adb (Android), xcrun simctl (iOS); multi-host health checks |
|
|
25
|
+
| **Compliance & localization** `compliance` | security_headers, privacy_cookies, localization | Built-in HTTP checks (OWASP headers, cookie flags, per-locale `lang` and untranslated strings) |
|
|
26
|
+
|
|
27
|
+
Run `trunft list` for the full platform × test-type matrix.
|
|
28
|
+
|
|
29
|
+
## Two interfaces, one engine
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
trunft (CLI) trunft-mcp (MCP server: stdio / streamable-http)
|
|
33
|
+
\ /
|
|
34
|
+
+---- trunft.api ----+ capabilities, doctor, plan, run, reports
|
|
35
|
+
|
|
|
36
|
+
runner -> pillars -> external tools (k6, ZAP, Lighthouse, ...)
|
|
37
|
+
|
|
|
38
|
+
reports/<run>/results.json, junit.xml, report.html
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Both interfaces read the same config and write the same reports, so a run started from one can be inspected from the other.
|
|
42
|
+
|
|
43
|
+
## Quick start (CLI)
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
python -m venv .venv && .venv/Scripts/activate # source .venv/bin/activate on Linux/macOS
|
|
47
|
+
pip install -e ".[dev]" # add [web] for Playwright, [mobile] for Appium client
|
|
48
|
+
trunft doctor # shows which external tools are available
|
|
49
|
+
trunft setup # installs missing ones (winget/brew/npm/pip); --dry-run to preview
|
|
50
|
+
trunft plan -e staging --pillar performance # preview, runs nothing
|
|
51
|
+
trunft run --dry-run # plan everything, execute nothing
|
|
52
|
+
trunft run -e dev -p web --pillar compliance
|
|
53
|
+
trunft run -e staging --pillar performance -t load -t spike
|
|
54
|
+
trunft reports # recent runs
|
|
55
|
+
trunft report <run> --failures # what broke and which thresholds were breached
|
|
56
|
+
pytest # framework self-tests
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
`list`, `doctor`, `plan`, `reports` and `report` accept `--json`.
|
|
60
|
+
Exit code: `0` = no failures, `1` = at least one FAILED/ERROR, `2` = bad config or arguments.
|
|
61
|
+
|
|
62
|
+
## MCP server (AI agents / IDEs)
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install -e ".[mcp]"
|
|
66
|
+
trunft-mcp --root C:/myprojects/TruNFT # stdio
|
|
67
|
+
trunft-mcp --transport streamable-http --port 8765 # http://127.0.0.1:8765/mcp
|
|
68
|
+
trunft mcp # same server via the main CLI
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Register it with Claude Code:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
claude mcp add trunft --env TRUNFT_ROOT=C:/myprojects/TruNFT -- C:/myprojects/TruNFT/.venv/Scripts/trunft-mcp.exe
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Or in Claude Desktop / any MCP client config:
|
|
78
|
+
|
|
79
|
+
```json
|
|
80
|
+
{ "mcpServers": { "trunft": {
|
|
81
|
+
"command": "C:/myprojects/TruNFT/.venv/Scripts/trunft-mcp.exe",
|
|
82
|
+
"env": { "TRUNFT_ROOT": "C:/myprojects/TruNFT" } } } }
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
| Tool | Purpose |
|
|
86
|
+
|---|---|
|
|
87
|
+
| `list_capabilities` | Pillars, test types, required tools, and which platforms each applies to |
|
|
88
|
+
| `check_tools` | Which external tools are installed |
|
|
89
|
+
| `list_environments` / `get_config` | Environments and the merged config (secrets redacted) |
|
|
90
|
+
| `plan_tests` | Preview the platform × pillar × test-type combinations |
|
|
91
|
+
| `start_test_run` | Start a background run and return a `run_id` at once. **`dry_run` defaults to `true`** |
|
|
92
|
+
| `get_test_run` / `cancel_test_run` | Poll progress and results, or stop the remaining tests |
|
|
93
|
+
| `list_test_reports` / `get_test_report` | Past runs from CLI or MCP; `only_failures` filter |
|
|
94
|
+
|
|
95
|
+
Runs are asynchronous because soak tests can last for hours. The agent starts a run, polls it, then reads the report. Tools that only read are marked read-only. `start_test_run` is marked destructive and open-world because real runs generate load, run security scans and can inject faults.
|
|
96
|
+
|
|
97
|
+
### Result statuses
|
|
98
|
+
| Status | Meaning |
|
|
99
|
+
|---|---|
|
|
100
|
+
| `passed` | Ran, and every threshold was met |
|
|
101
|
+
| `failed` | Ran, but breached a threshold or assertion |
|
|
102
|
+
| `error` | Couldn't complete (tool crashed, target unreachable) |
|
|
103
|
+
| `skipped` | Dry-run, not applicable to the platform, or a required tool isn't installed |
|
|
104
|
+
| `not_implemented` | Placeholder: needs config (e.g. trigger hook) or code; the message says what |
|
|
105
|
+
|
|
106
|
+
## Layout
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
config/trunft.yaml master config: targets, thresholds (SLOs), pillar settings
|
|
110
|
+
config/environments/*.yaml per-environment overrides (dev, staging, ...); ${ENV_VAR:-default} supported
|
|
111
|
+
src/trunft/api.py service layer shared by CLI and MCP server
|
|
112
|
+
src/trunft/cli.py `trunft` command line
|
|
113
|
+
src/trunft/mcp_server.py `trunft-mcp` MCP server (stdio / streamable-http)
|
|
114
|
+
src/trunft/core/ config, models, pillar base + registry, runner, thresholds, probes
|
|
115
|
+
src/trunft/pillars/*.py one module per NFT pillar (auto-discovered)
|
|
116
|
+
src/trunft/reporting/ results.json, junit.xml, report.html
|
|
117
|
+
workloads/k6/scenario.js parameterised k6 workload - put real user journeys here
|
|
118
|
+
chaos/experiments/ Chaos Toolkit experiments (*.json)
|
|
119
|
+
docker/docker-compose.yml toxiproxy, influxdb, grafana for local runs
|
|
120
|
+
apps/android, apps/ios app binaries under test (git-ignored)
|
|
121
|
+
.github/workflows/ci.yml unit tests on push/PR (Linux + Windows); no live tests in CI
|
|
122
|
+
.github/workflows/publish.yml gated PyPI release on v* tags
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Thresholds (SLO gates)
|
|
126
|
+
|
|
127
|
+
Every pillar returns metrics. A threshold `max_<metric>` or `min_<metric>` under `thresholds.<pillar>.default`, or under a per-test-type block, decides pass or fail:
|
|
128
|
+
|
|
129
|
+
```yaml
|
|
130
|
+
thresholds:
|
|
131
|
+
performance:
|
|
132
|
+
default: {max_p95_ms: 800, max_error_rate: 0.01}
|
|
133
|
+
stress: {max_p95_ms: 2000}
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## Installing from PyPI
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install trunft # core; extras: trunft[web], trunft[mcp], trunft[chaos], trunft[all]
|
|
140
|
+
trunft init my-nft-project # creates config/, workloads/k6, chaos/, docker/ from bundled templates
|
|
141
|
+
cd my-nft-project && trunft setup && trunft run --dry-run
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
## Releasing
|
|
145
|
+
|
|
146
|
+
Publishing uses PyPI **Trusted Publishing** (`.github/workflows/publish.yml`, no API tokens):
|
|
147
|
+
|
|
148
|
+
1. Bump `__version__` in `src/trunft/__init__.py` (the single source of the version) and commit.
|
|
149
|
+
2. Tag that commit and push the tag: `git tag v0.2.0 && git push origin v0.2.0`
|
|
150
|
+
|
|
151
|
+
The workflow uploads only if **all** of these gates pass:
|
|
152
|
+
|
|
153
|
+
| Gate | Check |
|
|
154
|
+
|---|---|
|
|
155
|
+
| Version | Tag `v<x.y.z>` == `__version__` == built sdist/wheel versions |
|
|
156
|
+
| Unit | `pytest` on Linux + Windows, Python 3.10 and 3.13 |
|
|
157
|
+
| Security | Sources compile; Bandit SAST with zero findings; `pip-audit --strict` on the runtime dependency set |
|
|
158
|
+
| Build | `twine check --strict`; the wheel is installed into a clean venv, then `trunft init` and a dry run are executed |
|
|
159
|
+
|
|
160
|
+
Then it publishes to PyPI (GitHub environment `pypi`).
|
|
161
|
+
Run the version gate locally with `python scripts/check_release_version.py v0.2.0`.
|
|
162
|
+
|
|
163
|
+
If you edit files under `config/`, `workloads/`, `chaos/` or `docker/`, copy them to `src/trunft/templates/project/`. A test fails if the two copies drift apart.
|
|
164
|
+
|
|
165
|
+
## Extending
|
|
166
|
+
|
|
167
|
+
Add a module under `src/trunft/pillars/`:
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from trunft.core.pillar import Pillar, RunContext, register
|
|
171
|
+
from trunft.core.models import Status
|
|
172
|
+
|
|
173
|
+
@register
|
|
174
|
+
class Energy(Pillar):
|
|
175
|
+
name, title = "energy", "Battery & Energy"
|
|
176
|
+
test_types = {"drain": "Battery drain over a 30-min journey"}
|
|
177
|
+
type_platforms = {"drain": Platform.mobile_apps()}
|
|
178
|
+
tools = {"drain": ["adb"]}
|
|
179
|
+
|
|
180
|
+
def run(self, ctx: RunContext, test_type: str):
|
|
181
|
+
...
|
|
182
|
+
return self.result(ctx, test_type, Status.PASSED, metrics={"mah_per_hour": 120})
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Roadmap: open placeholders (`not_implemented`)
|
|
186
|
+
|
|
187
|
+
- **Mobile client performance**: startup time, memory, CPU and frame rate (adb `dumpsys`, Xcode Instruments)
|
|
188
|
+
- **Mobile security**: MobSF static/dynamic analysis of APK/IPA
|
|
189
|
+
- **Native accessibility**: Espresso Accessibility Test Framework, XCUITest `performAccessibilityAudit`
|
|
190
|
+
- **Device-cloud compatibility**: Appium or Selenium grid (BrowserStack, Sauce Labs, LambdaTest) for the OS and native matrix
|
|
191
|
+
- **Upgrade data retention**, **DR RPO verification**, **client-side network conditioning** (offline/3G)
|
|
192
|
+
- **Native localization**: resource-bundle key completeness plus per-locale screenshots
|
|
193
|
+
- **Compliance evidence packs**: GDPR, PCI-DSS, HIPAA, WCAG conformance statement
|
|
194
|
+
|
|
195
|
+
> ⚠️ Run active security scans (`penetration`) and chaos/DR hooks only against environments you are authorised to test. `allow_active_scan` is `false` by default.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "trunft"
|
|
7
|
+
dynamic = ["version"] # single source of truth: trunft/__init__.py (__version__)
|
|
8
|
+
description = "TruNFT - unified Non-Functional Testing platform (CLI + MCP server) for Web, Mobile Native, Mobile Browser and Mobile Hybrid apps"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Shan Konduru", email = "ShanKonduru@gmail.com" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"non-functional-testing", "performance-testing", "load-testing", "security-testing",
|
|
16
|
+
"chaos-engineering", "accessibility", "compatibility-testing", "mobile-testing", "k6", "mcp",
|
|
17
|
+
]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Environment :: Console",
|
|
21
|
+
"Intended Audience :: Developers",
|
|
22
|
+
"Operating System :: OS Independent",
|
|
23
|
+
"Programming Language :: Python :: 3",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
"Topic :: Software Development :: Testing",
|
|
29
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
30
|
+
"Topic :: System :: Benchmark",
|
|
31
|
+
]
|
|
32
|
+
dependencies = ["PyYAML>=6.0"]
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
web = ["playwright>=1.45"]
|
|
36
|
+
mobile = ["Appium-Python-Client>=4.0"]
|
|
37
|
+
chaos = ["chaostoolkit>=1.19"]
|
|
38
|
+
mcp = ["mcp>=2.3,<3"]
|
|
39
|
+
all = ["trunft[web,mobile,chaos,mcp]"]
|
|
40
|
+
dev = ["pytest>=8.0", "anyio>=4", "mcp>=2.3,<3"]
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/ShanKonduru/TruNFT"
|
|
44
|
+
Source = "https://github.com/ShanKonduru/TruNFT"
|
|
45
|
+
Issues = "https://github.com/ShanKonduru/TruNFT/issues"
|
|
46
|
+
|
|
47
|
+
[project.scripts]
|
|
48
|
+
trunft = "trunft.cli:main"
|
|
49
|
+
trunft-mcp = "trunft.mcp_server:main"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.dynamic]
|
|
52
|
+
version = { attr = "trunft.__version__" }
|
|
53
|
+
|
|
54
|
+
[tool.setuptools.packages.find]
|
|
55
|
+
where = ["src"]
|
|
56
|
+
|
|
57
|
+
[tool.setuptools.package-data]
|
|
58
|
+
trunft = ["templates/project/**/*", "templates/project/**/.gitkeep"]
|
|
59
|
+
|
|
60
|
+
[tool.pytest.ini_options]
|
|
61
|
+
testpaths = ["tests"]
|
|
62
|
+
pythonpath = ["src"]
|
|
63
|
+
|
|
64
|
+
[tool.bandit]
|
|
65
|
+
exclude_dirs = ["tests", ".venv"]
|
trunft-0.1.0/setup.cfg
ADDED