test-reporting 4.4.0__tar.gz → 4.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {test_reporting-4.4.0 → test_reporting-4.4.2}/PKG-INFO +4 -34
- {test_reporting-4.4.0 → test_reporting-4.4.2}/README.md +3 -33
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/cli.py +4 -6
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/config.py +0 -9
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/exporters/pptx_exporter.py +0 -28
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/exporters/xlsx_exporter.py +0 -20
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/monthly_report.py +31 -177
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/monthly_ui.py +11 -3
- {test_reporting-4.4.0 → test_reporting-4.4.2}/setup.py +1 -1
- {test_reporting-4.4.0 → test_reporting-4.4.2}/test_reporting.egg-info/PKG-INFO +4 -34
- {test_reporting-4.4.0 → test_reporting-4.4.2}/tests/test_published_monthly_report.py +16 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/LICENSE +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/__init__.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/classifier.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/exporters/__init__.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/exporters/pdf_exporter.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/pdf_analysis.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/plugin.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/publisher.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/report_insights.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/run_source.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/storage.py +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/templates/index.html +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/templates/project.html +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/reporting/templates/run.html +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/setup.cfg +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/test_reporting.egg-info/SOURCES.txt +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/test_reporting.egg-info/dependency_links.txt +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/test_reporting.egg-info/entry_points.txt +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/test_reporting.egg-info/requires.txt +0 -0
- {test_reporting-4.4.0 → test_reporting-4.4.2}/test_reporting.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: test-reporting
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.2
|
|
4
4
|
Summary: Multi-project test reporting dashboard — collect results locally or push to S3
|
|
5
5
|
Home-page: https://github.com/amahdy77/test-reporting.git
|
|
6
6
|
Author: Ashfaqur Mahdy
|
|
@@ -260,7 +260,7 @@ dashboard_max_runs_per_suite = 50 # Run history per suite (default: 50)
|
|
|
260
260
|
| `test-report suites` | Show suite statistics (reads local SQLite) |
|
|
261
261
|
| `test-report cleanup` | Delete old runs from local SQLite |
|
|
262
262
|
| `test-report cleanup-artifacts` | Delete old screenshots and traces (preserves data) |
|
|
263
|
-
| `test-report monthly-report` | Generate
|
|
263
|
+
| `test-report monthly-report` | Generate a project report (PDF default; PPTX/XLSX optional) |
|
|
264
264
|
| `test-report monthly-ui` | Open local project and date picker for reports |
|
|
265
265
|
|
|
266
266
|
### Cleaning up old data
|
|
@@ -320,39 +320,9 @@ retention_days = 90 # Keep 90 days of artifacts and data
|
|
|
320
320
|
|
|
321
321
|
## Monthly Reports
|
|
322
322
|
|
|
323
|
-
Generate a report for one project from its published run history.
|
|
323
|
+
Generate a report for one project from its published S3 or local run history. PDF is the default stakeholder report; PowerPoint and Excel are optional exports. The local UI offers rolling date presets, all published history, and custom dates. Reports are downloaded locally and do not change the publishing pipeline.
|
|
324
324
|
|
|
325
|
-
For
|
|
326
|
-
|
|
327
|
-
### Quick Start
|
|
328
|
-
|
|
329
|
-
```bash
|
|
330
|
-
# Install with report dependencies and S3 support
|
|
331
|
-
pip install test-reporting[s3,reports]
|
|
332
|
-
|
|
333
|
-
# Open the local project and date picker
|
|
334
|
-
test-report monthly-ui --config reporting.ini
|
|
335
|
-
|
|
336
|
-
# Generate without the UI (defaults to the last 30 days)
|
|
337
|
-
test-report monthly-report --project "My Project" --format pptx,xlsx,pdf
|
|
338
|
-
|
|
339
|
-
# Custom date range
|
|
340
|
-
test-report monthly-report --project "My Project" --start 2026-08-01 --end 2026-08-31 --format xlsx
|
|
341
|
-
|
|
342
|
-
# Specific project
|
|
343
|
-
test-report monthly-report --project "My Project" --format pptx
|
|
344
|
-
```
|
|
345
|
-
|
|
346
|
-
### Features
|
|
347
|
-
|
|
348
|
-
- 📊 Executive summary with health score
|
|
349
|
-
- 📈 Pass rate trends and test volume charts
|
|
350
|
-
- 🔥 Failure hotspots and flaky test detection
|
|
351
|
-
- 🐌 Slowest tests and performance regressions
|
|
352
|
-
- 📁 File health report
|
|
353
|
-
- 💡 Actionable recommendations
|
|
354
|
-
|
|
355
|
-
**See [REPORT_GENERATOR.md](REPORT_GENERATOR.md) for setup and metric definitions.**
|
|
325
|
+
For AWS SSO setup, commands, date selection, and metric definitions, see [MONTHLY_REPORTS.md](MONTHLY_REPORTS.md).
|
|
356
326
|
|
|
357
327
|
---
|
|
358
328
|
|
|
@@ -219,7 +219,7 @@ dashboard_max_runs_per_suite = 50 # Run history per suite (default: 50)
|
|
|
219
219
|
| `test-report suites` | Show suite statistics (reads local SQLite) |
|
|
220
220
|
| `test-report cleanup` | Delete old runs from local SQLite |
|
|
221
221
|
| `test-report cleanup-artifacts` | Delete old screenshots and traces (preserves data) |
|
|
222
|
-
| `test-report monthly-report` | Generate
|
|
222
|
+
| `test-report monthly-report` | Generate a project report (PDF default; PPTX/XLSX optional) |
|
|
223
223
|
| `test-report monthly-ui` | Open local project and date picker for reports |
|
|
224
224
|
|
|
225
225
|
### Cleaning up old data
|
|
@@ -279,39 +279,9 @@ retention_days = 90 # Keep 90 days of artifacts and data
|
|
|
279
279
|
|
|
280
280
|
## Monthly Reports
|
|
281
281
|
|
|
282
|
-
Generate a report for one project from its published run history.
|
|
282
|
+
Generate a report for one project from its published S3 or local run history. PDF is the default stakeholder report; PowerPoint and Excel are optional exports. The local UI offers rolling date presets, all published history, and custom dates. Reports are downloaded locally and do not change the publishing pipeline.
|
|
283
283
|
|
|
284
|
-
For
|
|
285
|
-
|
|
286
|
-
### Quick Start
|
|
287
|
-
|
|
288
|
-
```bash
|
|
289
|
-
# Install with report dependencies and S3 support
|
|
290
|
-
pip install test-reporting[s3,reports]
|
|
291
|
-
|
|
292
|
-
# Open the local project and date picker
|
|
293
|
-
test-report monthly-ui --config reporting.ini
|
|
294
|
-
|
|
295
|
-
# Generate without the UI (defaults to the last 30 days)
|
|
296
|
-
test-report monthly-report --project "My Project" --format pptx,xlsx,pdf
|
|
297
|
-
|
|
298
|
-
# Custom date range
|
|
299
|
-
test-report monthly-report --project "My Project" --start 2026-08-01 --end 2026-08-31 --format xlsx
|
|
300
|
-
|
|
301
|
-
# Specific project
|
|
302
|
-
test-report monthly-report --project "My Project" --format pptx
|
|
303
|
-
```
|
|
304
|
-
|
|
305
|
-
### Features
|
|
306
|
-
|
|
307
|
-
- 📊 Executive summary with health score
|
|
308
|
-
- 📈 Pass rate trends and test volume charts
|
|
309
|
-
- 🔥 Failure hotspots and flaky test detection
|
|
310
|
-
- 🐌 Slowest tests and performance regressions
|
|
311
|
-
- 📁 File health report
|
|
312
|
-
- 💡 Actionable recommendations
|
|
313
|
-
|
|
314
|
-
**See [REPORT_GENERATOR.md](REPORT_GENERATOR.md) for setup and metric definitions.**
|
|
284
|
+
For AWS SSO setup, commands, date selection, and metric definitions, see [MONTHLY_REPORTS.md](MONTHLY_REPORTS.md).
|
|
315
285
|
|
|
316
286
|
---
|
|
317
287
|
|
|
@@ -362,10 +362,8 @@ def monthly_report():
|
|
|
362
362
|
parser.add_argument('--start', help='Start date (YYYY-MM-DD)')
|
|
363
363
|
parser.add_argument('--end', help='End date (YYYY-MM-DD)')
|
|
364
364
|
parser.add_argument('--project', required=True, help='One published project name')
|
|
365
|
-
parser.add_argument('--
|
|
366
|
-
|
|
367
|
-
parser.add_argument('--format', '--formats', dest='formats', default='pptx',
|
|
368
|
-
help='Output format(s): pptx, xlsx, pdf (comma-separated)')
|
|
365
|
+
parser.add_argument('--format', '--formats', dest='formats', default='pdf',
|
|
366
|
+
help='Output format(s): pdf, pptx, xlsx (comma-separated; default: pdf)')
|
|
369
367
|
parser.add_argument('--output', help='Output directory (default: ./reports)')
|
|
370
368
|
parser.add_argument('--top-n', type=int, default=10, help='Number of items in top N lists')
|
|
371
369
|
parser.add_argument('--config', default='reporting.ini', help='Config file path')
|
|
@@ -402,7 +400,7 @@ def monthly_report():
|
|
|
402
400
|
print(f"[Monthly Report] Formats: {', '.join(formats)}")
|
|
403
401
|
|
|
404
402
|
# Generate report
|
|
405
|
-
generator = MonthlyReportGenerator(config
|
|
403
|
+
generator = MonthlyReportGenerator(config)
|
|
406
404
|
files = generator.generate(
|
|
407
405
|
start_date=start_date,
|
|
408
406
|
end_date=end_date,
|
|
@@ -442,7 +440,7 @@ def main():
|
|
|
442
440
|
'suites': (show_suites, 'Show statistics for all test suites'),
|
|
443
441
|
'cleanup': (cleanup_old_data, 'Clean up old data from local SQLite'),
|
|
444
442
|
'cleanup-artifacts': (cleanup_artifacts, 'Clean up old screenshots and traces'),
|
|
445
|
-
'monthly-report': (monthly_report, 'Generate
|
|
443
|
+
'monthly-report': (monthly_report, 'Generate a project report (PDF default; PPTX/XLSX optional)'),
|
|
446
444
|
'monthly-ui': (monthly_ui, 'Open local project and date picker for reports'),
|
|
447
445
|
}
|
|
448
446
|
|
|
@@ -55,11 +55,6 @@ class ReportingConfig:
|
|
|
55
55
|
or None
|
|
56
56
|
)
|
|
57
57
|
|
|
58
|
-
# Monthly report settings
|
|
59
|
-
monthly_section = self.config['monthly_report'] if 'monthly_report' in self.config else {}
|
|
60
|
-
self.REPORT_PERIOD_DAYS = int(monthly_section.get('report_period_days', 30))
|
|
61
|
-
self.REPORT_TOP_N = int(monthly_section.get('top_n_items', 10))
|
|
62
|
-
|
|
63
58
|
def _load_from_defaults(self):
|
|
64
59
|
self.PROJECT_NAME = os.getenv('PROJECT_NAME', 'test-automation-project')
|
|
65
60
|
|
|
@@ -86,10 +81,6 @@ class ReportingConfig:
|
|
|
86
81
|
|
|
87
82
|
self.REGRESSION_TAG = os.getenv('REGRESSION_TAG') or None
|
|
88
83
|
|
|
89
|
-
# Monthly report defaults
|
|
90
|
-
self.REPORT_PERIOD_DAYS = 30
|
|
91
|
-
self.REPORT_TOP_N = 10
|
|
92
|
-
|
|
93
84
|
# Timeout thresholds (ms)
|
|
94
85
|
TIMEOUT_UI_WARNING = 30000
|
|
95
86
|
TIMEOUT_UI_MAX = 60000
|
|
@@ -86,10 +86,6 @@ class PowerPointExporter:
|
|
|
86
86
|
if sections.get('file_health', True) and metrics.get('file_health'):
|
|
87
87
|
self._add_file_health_slide(prs, metrics['file_health'][:10])
|
|
88
88
|
|
|
89
|
-
# Slide 11: Recommendations
|
|
90
|
-
if sections.get('recommendations', True) and metrics.get('recommendations'):
|
|
91
|
-
self._add_recommendations_slide(prs, metrics['recommendations'])
|
|
92
|
-
|
|
93
89
|
self._polish_deck(prs, report_meta)
|
|
94
90
|
prs.save(str(output_path))
|
|
95
91
|
|
|
@@ -500,30 +496,6 @@ class PowerPointExporter:
|
|
|
500
496
|
for j in range(3):
|
|
501
497
|
table.cell(i, j).text_frame.paragraphs[0].font.size = Pt(10)
|
|
502
498
|
|
|
503
|
-
def _add_recommendations_slide(self, prs, recommendations):
|
|
504
|
-
"""Add recommendations slide."""
|
|
505
|
-
slide = prs.slides.add_slide(prs.slide_layouts[5])
|
|
506
|
-
|
|
507
|
-
title = slide.shapes.title
|
|
508
|
-
title.text = "Recommendations"
|
|
509
|
-
|
|
510
|
-
# Add recommendations as bullet points
|
|
511
|
-
left = Inches(1)
|
|
512
|
-
top = Inches(2)
|
|
513
|
-
width = Inches(8)
|
|
514
|
-
height = Inches(4.5)
|
|
515
|
-
|
|
516
|
-
text_box = slide.shapes.add_textbox(left, top, width, height)
|
|
517
|
-
text_frame = text_box.text_frame
|
|
518
|
-
text_frame.word_wrap = True
|
|
519
|
-
|
|
520
|
-
for rec in recommendations:
|
|
521
|
-
p = text_frame.add_paragraph()
|
|
522
|
-
p.text = rec
|
|
523
|
-
p.font.size = Pt(14)
|
|
524
|
-
p.level = 0
|
|
525
|
-
p.space_after = Pt(12)
|
|
526
|
-
|
|
527
499
|
def _get_health_color(self, score):
|
|
528
500
|
"""Get color based on health score."""
|
|
529
501
|
if score >= 95:
|
|
@@ -75,10 +75,6 @@ class ExcelExporter:
|
|
|
75
75
|
if sections.get('file_health', True) and metrics.get('file_health'):
|
|
76
76
|
self._create_file_health_sheet(wb, metrics['file_health'])
|
|
77
77
|
|
|
78
|
-
# Sheet 9: Recommendations
|
|
79
|
-
if sections.get('recommendations', True) and metrics.get('recommendations'):
|
|
80
|
-
self._create_recommendations_sheet(wb, metrics['recommendations'])
|
|
81
|
-
|
|
82
78
|
self._polish_workbook(wb, metrics, report_meta)
|
|
83
79
|
wb.save(str(output_path))
|
|
84
80
|
|
|
@@ -464,22 +460,6 @@ class ExcelExporter:
|
|
|
464
460
|
for col in ['C', 'D', 'E', 'F']:
|
|
465
461
|
ws.column_dimensions[col].width = 15
|
|
466
462
|
|
|
467
|
-
def _create_recommendations_sheet(self, wb, recommendations):
|
|
468
|
-
"""Create recommendations sheet."""
|
|
469
|
-
ws = wb.create_sheet("Recommendations")
|
|
470
|
-
|
|
471
|
-
# Title
|
|
472
|
-
ws['A1'] = "Recommendations"
|
|
473
|
-
ws['A1'].font = Font(size=16, bold=True)
|
|
474
|
-
|
|
475
|
-
# Recommendations
|
|
476
|
-
for row, rec in enumerate(recommendations, 3):
|
|
477
|
-
ws[f'A{row}'] = f"{row - 2}. {rec}"
|
|
478
|
-
ws[f'A{row}'].alignment = Alignment(wrap_text=True, vertical='top')
|
|
479
|
-
|
|
480
|
-
# Format
|
|
481
|
-
ws.column_dimensions['A'].width = 100
|
|
482
|
-
|
|
483
463
|
def _get_health_fill(self, score):
|
|
484
464
|
"""Get fill color based on health score."""
|
|
485
465
|
if score >= 95:
|
|
@@ -2,14 +2,12 @@
|
|
|
2
2
|
Monthly Report Generator
|
|
3
3
|
Generates comprehensive monthly test reports in multiple formats (PPTX, XLSX, PDF).
|
|
4
4
|
"""
|
|
5
|
-
import json
|
|
6
5
|
from datetime import date, datetime, timedelta
|
|
7
6
|
from pathlib import Path
|
|
8
7
|
from typing import Dict, Any, List, Optional
|
|
9
8
|
from collections import defaultdict
|
|
10
9
|
|
|
11
10
|
from .config import ReportingConfig
|
|
12
|
-
from .storage import TestResultStorage
|
|
13
11
|
from .run_source import PublishedRunSource
|
|
14
12
|
from .report_insights import build_highlights
|
|
15
13
|
|
|
@@ -17,11 +15,9 @@ from .report_insights import build_highlights
|
|
|
17
15
|
class MonthlyReportGenerator:
|
|
18
16
|
"""Generates monthly test reports with metrics and visualizations."""
|
|
19
17
|
|
|
20
|
-
def __init__(self, config: Optional[ReportingConfig] = None,
|
|
18
|
+
def __init__(self, config: Optional[ReportingConfig] = None, run_source=None):
|
|
21
19
|
self.config = config or ReportingConfig()
|
|
22
|
-
self.source = source
|
|
23
20
|
self.run_source = run_source or PublishedRunSource(self.config)
|
|
24
|
-
self.storage = TestResultStorage(config=self.config) if source == 'sqlite' else None
|
|
25
21
|
|
|
26
22
|
def generate(
|
|
27
23
|
self,
|
|
@@ -31,7 +27,6 @@ class MonthlyReportGenerator:
|
|
|
31
27
|
formats: List[str] = None,
|
|
32
28
|
sections: Dict[str, bool] = None,
|
|
33
29
|
top_n: int = 10,
|
|
34
|
-
chart_theme: str = 'light',
|
|
35
30
|
output_dir: Optional[Path] = None
|
|
36
31
|
) -> List[Path]:
|
|
37
32
|
"""
|
|
@@ -44,21 +39,18 @@ class MonthlyReportGenerator:
|
|
|
44
39
|
formats: List of formats to generate ['pptx', 'xlsx', 'pdf']
|
|
45
40
|
sections: Dict of section names to include/exclude
|
|
46
41
|
top_n: Number of items to show in top N lists
|
|
47
|
-
chart_theme: Chart theme ('light' or 'dark')
|
|
48
42
|
output_dir: Output directory (defaults to temp)
|
|
49
43
|
|
|
50
44
|
Returns:
|
|
51
45
|
List of generated file paths
|
|
52
46
|
"""
|
|
53
|
-
if self.source not in ('published', 'sqlite'):
|
|
54
|
-
raise ValueError(f'Unknown report source: {self.source}')
|
|
55
47
|
start = date.fromisoformat(start_date)
|
|
56
48
|
end = date.fromisoformat(end_date)
|
|
57
49
|
if start > end:
|
|
58
50
|
raise ValueError('Start date must be on or before end date')
|
|
59
51
|
if not project:
|
|
60
52
|
raise ValueError('Select one project for the report')
|
|
61
|
-
formats = formats or ['
|
|
53
|
+
formats = formats or ['pdf']
|
|
62
54
|
if not set(formats) <= {'pptx', 'xlsx', 'pdf'}:
|
|
63
55
|
raise ValueError('Formats must be pptx, xlsx, or pdf')
|
|
64
56
|
sections = sections or {}
|
|
@@ -100,7 +92,6 @@ class MonthlyReportGenerator:
|
|
|
100
92
|
'project': project or 'All Projects',
|
|
101
93
|
'generated_at': datetime.now().isoformat(),
|
|
102
94
|
'top_n': top_n,
|
|
103
|
-
'chart_theme': chart_theme
|
|
104
95
|
}
|
|
105
96
|
|
|
106
97
|
# Generate each format
|
|
@@ -144,93 +135,36 @@ class MonthlyReportGenerator:
|
|
|
144
135
|
return f"monthly-report_{project_part}{start_date}_to_{end_date}.{ext}"
|
|
145
136
|
|
|
146
137
|
def collect_data(self, start_date: str, end_date: str, project: Optional[str] = None) -> Dict[str, Any]:
|
|
147
|
-
"""Collect test results from published run files
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
for t in run.get('failed_tests', [])
|
|
156
|
-
}
|
|
157
|
-
tests = run.get('all_tests')
|
|
158
|
-
if tests is None:
|
|
159
|
-
tests = run.get('passed_tests', []) + run.get('failed_tests', [])
|
|
160
|
-
for test in tests:
|
|
161
|
-
status = test.get('status')
|
|
162
|
-
if not status:
|
|
163
|
-
status = 'failed' if (test.get('test_name'), test.get('file_name')) in failed_details else 'passed'
|
|
164
|
-
detail = failed_details.get((test.get('test_name'), test.get('file_name')), {})
|
|
165
|
-
results.append({
|
|
166
|
-
'run_id': index,
|
|
167
|
-
'test_name': test.get('test_name'),
|
|
168
|
-
'file_name': test.get('file_name'),
|
|
169
|
-
'status': status,
|
|
170
|
-
'duration_seconds': test.get('duration_seconds') or 0,
|
|
171
|
-
'failure_type': test.get('failure_type') or detail.get('failure_type'),
|
|
172
|
-
'is_soft_fail': detail.get('is_soft_fail', False),
|
|
173
|
-
'created_at': run.get('timestamp', ''),
|
|
174
|
-
})
|
|
175
|
-
runs.sort(key=lambda r: r.get('timestamp', ''))
|
|
176
|
-
return {'runs': runs, 'results': results, 'start_date': start_date,
|
|
177
|
-
'end_date': end_date, 'project': project}
|
|
178
|
-
return self._collect_sqlite_data(start_date, end_date, project)
|
|
179
|
-
|
|
180
|
-
def _collect_sqlite_data(self, start_date, end_date, project):
|
|
181
|
-
"""Read the legacy local database when explicitly requested."""
|
|
182
|
-
import sqlite3
|
|
183
|
-
|
|
184
|
-
conn = sqlite3.connect(self.storage.db_path)
|
|
185
|
-
conn.row_factory = sqlite3.Row
|
|
186
|
-
|
|
187
|
-
try:
|
|
188
|
-
# Build WHERE clause
|
|
189
|
-
where_clauses = [
|
|
190
|
-
"timestamp >= ?",
|
|
191
|
-
"timestamp <= ?"
|
|
192
|
-
]
|
|
193
|
-
params = [start_date, end_date + 'T23:59:59']
|
|
194
|
-
|
|
195
|
-
if project:
|
|
196
|
-
where_clauses.append("project_name = ?")
|
|
197
|
-
params.append(project)
|
|
198
|
-
|
|
199
|
-
where_clause = " AND ".join(where_clauses)
|
|
200
|
-
|
|
201
|
-
# Get all runs in date range
|
|
202
|
-
runs_query = f"""
|
|
203
|
-
SELECT * FROM test_runs
|
|
204
|
-
WHERE {where_clause}
|
|
205
|
-
ORDER BY timestamp ASC
|
|
206
|
-
"""
|
|
207
|
-
runs_cursor = conn.execute(runs_query, params)
|
|
208
|
-
runs = [dict(row) for row in runs_cursor.fetchall()]
|
|
209
|
-
|
|
210
|
-
# Get all test results for these runs
|
|
211
|
-
if runs:
|
|
212
|
-
run_ids = [r['id'] for r in runs]
|
|
213
|
-
placeholders = ','.join('?' * len(run_ids))
|
|
214
|
-
results_query = f"""
|
|
215
|
-
SELECT * FROM test_results
|
|
216
|
-
WHERE run_id IN ({placeholders})
|
|
217
|
-
ORDER BY run_id, id
|
|
218
|
-
"""
|
|
219
|
-
results_cursor = conn.execute(results_query, run_ids)
|
|
220
|
-
results = [dict(row) for row in results_cursor.fetchall()]
|
|
221
|
-
else:
|
|
222
|
-
results = []
|
|
223
|
-
|
|
224
|
-
return {
|
|
225
|
-
'runs': runs,
|
|
226
|
-
'results': results,
|
|
227
|
-
'start_date': start_date,
|
|
228
|
-
'end_date': end_date,
|
|
229
|
-
'project': project
|
|
138
|
+
"""Collect test results from published run files."""
|
|
139
|
+
runs = []
|
|
140
|
+
results = []
|
|
141
|
+
for index, run in enumerate(self.run_source.runs(project, start_date, end_date), 1):
|
|
142
|
+
runs.append({**run, 'id': index})
|
|
143
|
+
failed_details = {
|
|
144
|
+
(t.get('test_name'), t.get('file_name')): t
|
|
145
|
+
for t in run.get('failed_tests', [])
|
|
230
146
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
147
|
+
tests = run.get('all_tests')
|
|
148
|
+
if tests is None:
|
|
149
|
+
tests = run.get('passed_tests', []) + run.get('failed_tests', [])
|
|
150
|
+
for test in tests:
|
|
151
|
+
status = test.get('status')
|
|
152
|
+
if not status:
|
|
153
|
+
status = 'failed' if (test.get('test_name'), test.get('file_name')) in failed_details else 'passed'
|
|
154
|
+
detail = failed_details.get((test.get('test_name'), test.get('file_name')), {})
|
|
155
|
+
results.append({
|
|
156
|
+
'run_id': index,
|
|
157
|
+
'test_name': test.get('test_name'),
|
|
158
|
+
'file_name': test.get('file_name'),
|
|
159
|
+
'status': status,
|
|
160
|
+
'duration_seconds': test.get('duration_seconds') or 0,
|
|
161
|
+
'failure_type': test.get('failure_type') or detail.get('failure_type'),
|
|
162
|
+
'is_soft_fail': detail.get('is_soft_fail', False),
|
|
163
|
+
'created_at': run.get('timestamp', ''),
|
|
164
|
+
})
|
|
165
|
+
runs.sort(key=lambda r: r.get('timestamp', ''))
|
|
166
|
+
return {'runs': runs, 'results': results, 'start_date': start_date,
|
|
167
|
+
'end_date': end_date, 'project': project}
|
|
234
168
|
|
|
235
169
|
def calculate_metrics(self, data: Dict[str, Any], top_n: int = 10) -> Dict[str, Any]:
|
|
236
170
|
"""Calculate all metrics from collected data."""
|
|
@@ -240,11 +174,6 @@ class MonthlyReportGenerator:
|
|
|
240
174
|
if not runs:
|
|
241
175
|
return self._empty_metrics()
|
|
242
176
|
|
|
243
|
-
# Group results by run_id
|
|
244
|
-
results_by_run = defaultdict(list)
|
|
245
|
-
for result in results:
|
|
246
|
-
results_by_run[result['run_id']].append(result)
|
|
247
|
-
|
|
248
177
|
# Calculate summary metrics
|
|
249
178
|
summary = self._calculate_summary(runs, results)
|
|
250
179
|
|
|
@@ -267,11 +196,6 @@ class MonthlyReportGenerator:
|
|
|
267
196
|
# Calculate file health
|
|
268
197
|
file_health = self._calculate_file_health(results)
|
|
269
198
|
|
|
270
|
-
# Generate recommendations
|
|
271
|
-
recommendations = self._generate_recommendations(
|
|
272
|
-
summary, failure_hotspots, flaky_tests, slowest_tests
|
|
273
|
-
)
|
|
274
|
-
|
|
275
199
|
# Older published files may only contain selected test details. Never
|
|
276
200
|
# present test-level rates or trends as complete when records are missing.
|
|
277
201
|
if not summary['detail_complete']:
|
|
@@ -279,8 +203,6 @@ class MonthlyReportGenerator:
|
|
|
279
203
|
slowest_tests = []
|
|
280
204
|
performance_regressions = []
|
|
281
205
|
file_health = []
|
|
282
|
-
recommendations = self._generate_recommendations(summary, failure_hotspots, [], [])
|
|
283
|
-
recommendations.insert(0, 'Detailed test history is incomplete for this period; test-level analysis is omitted.')
|
|
284
206
|
|
|
285
207
|
return {
|
|
286
208
|
'summary': summary,
|
|
@@ -291,7 +213,6 @@ class MonthlyReportGenerator:
|
|
|
291
213
|
'slowest_tests': slowest_tests,
|
|
292
214
|
'performance_regressions': performance_regressions,
|
|
293
215
|
'file_health': file_health,
|
|
294
|
-
'recommendations': recommendations
|
|
295
216
|
}
|
|
296
217
|
|
|
297
218
|
def _empty_metrics(self) -> Dict[str, Any]:
|
|
@@ -311,7 +232,6 @@ class MonthlyReportGenerator:
|
|
|
311
232
|
'slowest_tests': [],
|
|
312
233
|
'performance_regressions': [],
|
|
313
234
|
'file_health': [],
|
|
314
|
-
'recommendations': []
|
|
315
235
|
}
|
|
316
236
|
|
|
317
237
|
def _calculate_summary(self, runs: List[Dict], results: List[Dict]) -> Dict[str, Any]:
|
|
@@ -599,69 +519,3 @@ class MonthlyReportGenerator:
|
|
|
599
519
|
|
|
600
520
|
file_health.sort(key=lambda x: x['pass_rate']) # Worst first
|
|
601
521
|
return file_health
|
|
602
|
-
|
|
603
|
-
def _generate_recommendations(
|
|
604
|
-
self,
|
|
605
|
-
summary: Dict,
|
|
606
|
-
failure_hotspots: List[Dict],
|
|
607
|
-
flaky_tests: List[Dict],
|
|
608
|
-
slowest_tests: List[Dict]
|
|
609
|
-
) -> List[str]:
|
|
610
|
-
"""Generate actionable recommendations based on metrics."""
|
|
611
|
-
recommendations = []
|
|
612
|
-
|
|
613
|
-
# Health-based recommendations
|
|
614
|
-
health_score = summary.get('health_score', 0)
|
|
615
|
-
if health_score < 80:
|
|
616
|
-
recommendations.append(
|
|
617
|
-
f"Critical: Test health score is {health_score}%. "
|
|
618
|
-
"Review the most frequent failures and confirm their causes."
|
|
619
|
-
)
|
|
620
|
-
elif health_score < 95:
|
|
621
|
-
recommendations.append(
|
|
622
|
-
f"Watch: Test health score is {health_score}%. "
|
|
623
|
-
"Focus on reducing failures to reach 95%+ target."
|
|
624
|
-
)
|
|
625
|
-
else:
|
|
626
|
-
recommendations.append(
|
|
627
|
-
f"Strong result: Test health score is {health_score}%. "
|
|
628
|
-
"Continue monitoring the pass rate and failure mix."
|
|
629
|
-
)
|
|
630
|
-
|
|
631
|
-
# Failure hotspot recommendations
|
|
632
|
-
if len(failure_hotspots) > 0:
|
|
633
|
-
top_3 = failure_hotspots[:3]
|
|
634
|
-
recommendations.append(
|
|
635
|
-
f"Investigate the top {len(top_3)} recorded failure hotspots: " +
|
|
636
|
-
", ".join(f"{h['test_name']} ({h['failure_count']} failures)" for h in top_3)
|
|
637
|
-
)
|
|
638
|
-
|
|
639
|
-
# Flaky test recommendations
|
|
640
|
-
if len(flaky_tests) > 0:
|
|
641
|
-
recommendations.append(
|
|
642
|
-
f"Investigate {len(flaky_tests)} tests that both passed and failed across runs. "
|
|
643
|
-
"Check whether failures come from the product, environment, or test code."
|
|
644
|
-
)
|
|
645
|
-
|
|
646
|
-
# Performance recommendations
|
|
647
|
-
if len(slowest_tests) > 0:
|
|
648
|
-
top_slow = slowest_tests[0]
|
|
649
|
-
recommendations.append(
|
|
650
|
-
f"Review the slowest test: {top_slow['test_name']} "
|
|
651
|
-
f"(avg {top_slow['avg_duration']}s) for avoidable delays."
|
|
652
|
-
)
|
|
653
|
-
|
|
654
|
-
# Trend-based recommendations
|
|
655
|
-
trend = summary.get('trend', '0%')
|
|
656
|
-
if trend.startswith('-'):
|
|
657
|
-
recommendations.append(
|
|
658
|
-
f"Pass rate is declining ({trend}). "
|
|
659
|
-
"Review recent failures and changes to the test environment or product."
|
|
660
|
-
)
|
|
661
|
-
elif trend.startswith('+') and float(trend.replace('%', '').replace('+', '')) > 2:
|
|
662
|
-
recommendations.append(
|
|
663
|
-
f"Pass rate is improving ({trend}). "
|
|
664
|
-
"Continue current practices and share learnings with team."
|
|
665
|
-
)
|
|
666
|
-
|
|
667
|
-
return recommendations
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import html
|
|
4
4
|
import io
|
|
5
5
|
import json
|
|
6
|
+
import re
|
|
6
7
|
import tempfile
|
|
7
8
|
import webbrowser
|
|
8
9
|
import zipfile
|
|
@@ -56,8 +57,8 @@ button:disabled {{opacity:.55;cursor:wait}}
|
|
|
56
57
|
<p id="period-status" role="status"><small>Change either date to use a custom range.</small></p>
|
|
57
58
|
<label>Formats</label><div class="formats">
|
|
58
59
|
<label><input type="checkbox" name="format" value="pdf" checked> PDF</label>
|
|
59
|
-
<label><input type="checkbox" name="format" value="pptx"
|
|
60
|
-
<label><input type="checkbox" name="format" value="xlsx"
|
|
60
|
+
<label><input type="checkbox" name="format" value="pptx"> PowerPoint</label>
|
|
61
|
+
<label><input type="checkbox" name="format" value="xlsx"> Excel</label></div>
|
|
61
62
|
<button id="generate" type="submit" onclick="this.textContent='Generating…';">Generate and download</button></form>
|
|
62
63
|
<p><small>Runs stay in S3. The ZIP downloads when generation finishes. Watch the PowerShell window for progress; large date ranges may take a while.</small></p>
|
|
63
64
|
<script>
|
|
@@ -119,6 +120,12 @@ endInput.addEventListener('change', chooseCustom);
|
|
|
119
120
|
</main></body></html>'''.encode('utf-8')
|
|
120
121
|
|
|
121
122
|
|
|
123
|
+
def _download_name(project, start, end):
|
|
124
|
+
"""Build a safe, recognizable ZIP name for the browser download."""
|
|
125
|
+
project_part = re.sub(r'[^A-Za-z0-9._-]+', '-', project).strip('.-')[:60] or 'project'
|
|
126
|
+
return f'Test_Report_{project_part}_{start}_to_{end}.zip'
|
|
127
|
+
|
|
128
|
+
|
|
122
129
|
def make_handler(config, run_source=None):
|
|
123
130
|
source = run_source or PublishedRunSource(config)
|
|
124
131
|
|
|
@@ -179,7 +186,8 @@ def make_handler(config, run_source=None):
|
|
|
179
186
|
body = output.getvalue()
|
|
180
187
|
self.send_response(200)
|
|
181
188
|
self.send_header('Content-Type', 'application/zip')
|
|
182
|
-
self.send_header('Content-Disposition',
|
|
189
|
+
self.send_header('Content-Disposition',
|
|
190
|
+
f'attachment; filename="{_download_name(project, start, end)}"')
|
|
183
191
|
self.send_header('Content-Length', str(len(body)))
|
|
184
192
|
self.send_header('X-Content-Type-Options', 'nosniff')
|
|
185
193
|
self.end_headers()
|
|
@@ -9,7 +9,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
|
9
9
|
|
|
10
10
|
setup(
|
|
11
11
|
name='test-reporting',
|
|
12
|
-
version='4.4.
|
|
12
|
+
version='4.4.2',
|
|
13
13
|
description='Multi-project test reporting dashboard — collect results locally or push to S3',
|
|
14
14
|
long_description=long_description,
|
|
15
15
|
long_description_content_type="text/markdown",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: test-reporting
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.2
|
|
4
4
|
Summary: Multi-project test reporting dashboard — collect results locally or push to S3
|
|
5
5
|
Home-page: https://github.com/amahdy77/test-reporting.git
|
|
6
6
|
Author: Ashfaqur Mahdy
|
|
@@ -260,7 +260,7 @@ dashboard_max_runs_per_suite = 50 # Run history per suite (default: 50)
|
|
|
260
260
|
| `test-report suites` | Show suite statistics (reads local SQLite) |
|
|
261
261
|
| `test-report cleanup` | Delete old runs from local SQLite |
|
|
262
262
|
| `test-report cleanup-artifacts` | Delete old screenshots and traces (preserves data) |
|
|
263
|
-
| `test-report monthly-report` | Generate
|
|
263
|
+
| `test-report monthly-report` | Generate a project report (PDF default; PPTX/XLSX optional) |
|
|
264
264
|
| `test-report monthly-ui` | Open local project and date picker for reports |
|
|
265
265
|
|
|
266
266
|
### Cleaning up old data
|
|
@@ -320,39 +320,9 @@ retention_days = 90 # Keep 90 days of artifacts and data
|
|
|
320
320
|
|
|
321
321
|
## Monthly Reports
|
|
322
322
|
|
|
323
|
-
Generate a report for one project from its published run history.
|
|
323
|
+
Generate a report for one project from its published S3 or local run history. PDF is the default stakeholder report; PowerPoint and Excel are optional exports. The local UI offers rolling date presets, all published history, and custom dates. Reports are downloaded locally and do not change the publishing pipeline.
|
|
324
324
|
|
|
325
|
-
For
|
|
326
|
-
|
|
327
|
-
### Quick Start
|
|
328
|
-
|
|
329
|
-
```bash
|
|
330
|
-
# Install with report dependencies and S3 support
|
|
331
|
-
pip install test-reporting[s3,reports]
|
|
332
|
-
|
|
333
|
-
# Open the local project and date picker
|
|
334
|
-
test-report monthly-ui --config reporting.ini
|
|
335
|
-
|
|
336
|
-
# Generate without the UI (defaults to the last 30 days)
|
|
337
|
-
test-report monthly-report --project "My Project" --format pptx,xlsx,pdf
|
|
338
|
-
|
|
339
|
-
# Custom date range
|
|
340
|
-
test-report monthly-report --project "My Project" --start 2026-08-01 --end 2026-08-31 --format xlsx
|
|
341
|
-
|
|
342
|
-
# Specific project
|
|
343
|
-
test-report monthly-report --project "My Project" --format pptx
|
|
344
|
-
```
|
|
345
|
-
|
|
346
|
-
### Features
|
|
347
|
-
|
|
348
|
-
- 📊 Executive summary with health score
|
|
349
|
-
- 📈 Pass rate trends and test volume charts
|
|
350
|
-
- 🔥 Failure hotspots and flaky test detection
|
|
351
|
-
- 🐌 Slowest tests and performance regressions
|
|
352
|
-
- 📁 File health report
|
|
353
|
-
- 💡 Actionable recommendations
|
|
354
|
-
|
|
355
|
-
**See [REPORT_GENERATOR.md](REPORT_GENERATOR.md) for setup and metric definitions.**
|
|
325
|
+
For AWS SSO setup, commands, date selection, and metric definitions, see [MONTHLY_REPORTS.md](MONTHLY_REPORTS.md).
|
|
356
326
|
|
|
357
327
|
---
|
|
358
328
|
|
|
@@ -156,6 +156,9 @@ class PublishedMonthlyReportTests(unittest.TestCase):
|
|
|
156
156
|
self.assertIn(b'Project A', page)
|
|
157
157
|
self.assertIn(b'Last 3 months', page)
|
|
158
158
|
self.assertIn(b'All data', page)
|
|
159
|
+
self.assertIn(b'value="pdf" checked', page)
|
|
160
|
+
self.assertIn(b'value="pptx"> PowerPoint', page)
|
|
161
|
+
self.assertIn(b'value="xlsx"> Excel', page)
|
|
159
162
|
with urllib.request.urlopen(address + '/date-bounds?project=Project%20A') as response:
|
|
160
163
|
self.assertEqual(json.loads(response.read()),
|
|
161
164
|
{'start': '2026-09-30', 'end': '2026-10-01'})
|
|
@@ -170,6 +173,9 @@ class PublishedMonthlyReportTests(unittest.TestCase):
|
|
|
170
173
|
'end': '2026-10-01', 'format': 'xlsx'}).encode()
|
|
171
174
|
with urllib.request.urlopen(address + '/generate', data=good_form) as response:
|
|
172
175
|
self.assertEqual(response.headers.get_content_type(), 'application/zip')
|
|
176
|
+
self.assertEqual(
|
|
177
|
+
response.headers['Content-Disposition'],
|
|
178
|
+
'attachment; filename="Test_Report_Project-A_2026-10-01_to_2026-10-01.zip"')
|
|
173
179
|
with zipfile.ZipFile(io.BytesIO(response.read())) as archive:
|
|
174
180
|
self.assertEqual(len(archive.namelist()), 1)
|
|
175
181
|
self.assertTrue(archive.namelist()[0].endswith('.xlsx'))
|
|
@@ -186,14 +192,24 @@ class PublishedMonthlyReportTests(unittest.TestCase):
|
|
|
186
192
|
from openpyxl import load_workbook
|
|
187
193
|
from pptx import Presentation
|
|
188
194
|
workbook = load_workbook(next(p for p in files if p.suffix == '.xlsx'))
|
|
195
|
+
self.assertNotIn('Recommendations', workbook.sheetnames)
|
|
189
196
|
self.assertEqual(workbook['Summary']['A24'].value, 'KEY FINDINGS')
|
|
190
197
|
self.assertIn('100.0% pass rate', workbook['Summary']['A25'].value)
|
|
191
198
|
self.assertEqual(workbook['Pass Rate Trend'].freeze_panes, 'B2')
|
|
192
199
|
deck = Presentation(next(p for p in files if p.suffix == '.pptx'))
|
|
200
|
+
self.assertNotIn('Recommendations',
|
|
201
|
+
[slide.shapes.title.text for slide in deck.slides if slide.shapes.title])
|
|
193
202
|
summary_text = ' '.join(shape.text for shape in deck.slides[1].shapes if shape.has_text_frame)
|
|
194
203
|
self.assertIn('KEY FINDINGS', summary_text)
|
|
195
204
|
self.assertIn('100.0% pass rate', summary_text)
|
|
196
205
|
|
|
206
|
+
@unittest.skipUnless(importlib.util.find_spec('reportlab'), 'Install test-reporting[reports]')
|
|
207
|
+
def test_default_report_format_is_pdf(self):
|
|
208
|
+
files = MonthlyReportGenerator(self.config).generate(
|
|
209
|
+
'2026-10-01', '2026-10-01', project='Project A',
|
|
210
|
+
output_dir=self.config.STORE_PATH / 'exports')
|
|
211
|
+
self.assertEqual([path.suffix for path in files], ['.pdf'])
|
|
212
|
+
|
|
197
213
|
|
|
198
214
|
if __name__ == '__main__':
|
|
199
215
|
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|