test-reporting 4.3.0__tar.gz → 4.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {test_reporting-4.3.0 → test_reporting-4.4.1}/PKG-INFO +3 -1
- {test_reporting-4.3.0 → test_reporting-4.4.1}/README.md +2 -0
- test_reporting-4.4.1/reporting/exporters/pdf_exporter.py +337 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/monthly_report.py +2 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/monthly_ui.py +94 -4
- test_reporting-4.4.1/reporting/pdf_analysis.py +58 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/run_source.py +41 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/setup.py +1 -1
- {test_reporting-4.3.0 → test_reporting-4.4.1}/test_reporting.egg-info/PKG-INFO +3 -1
- {test_reporting-4.3.0 → test_reporting-4.4.1}/test_reporting.egg-info/SOURCES.txt +1 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/tests/test_published_monthly_report.py +47 -1
- test_reporting-4.3.0/reporting/exporters/pdf_exporter.py +0 -542
- {test_reporting-4.3.0 → test_reporting-4.4.1}/LICENSE +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/__init__.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/classifier.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/cli.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/config.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/exporters/__init__.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/exporters/pptx_exporter.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/exporters/xlsx_exporter.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/plugin.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/publisher.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/report_insights.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/storage.py +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/templates/index.html +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/templates/project.html +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/reporting/templates/run.html +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/setup.cfg +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/test_reporting.egg-info/dependency_links.txt +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/test_reporting.egg-info/entry_points.txt +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/test_reporting.egg-info/requires.txt +0 -0
- {test_reporting-4.3.0 → test_reporting-4.4.1}/test_reporting.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: test-reporting
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.4.1
|
|
4
4
|
Summary: Multi-project test reporting dashboard — collect results locally or push to S3
|
|
5
5
|
Home-page: https://github.com/amahdy77/test-reporting.git
|
|
6
6
|
Author: Ashfaqur Mahdy
|
|
@@ -322,6 +322,8 @@ retention_days = 90 # Keep 90 days of artifacts and data
|
|
|
322
322
|
|
|
323
323
|
Generate a report for one project from its published run history. The report reader supports the same local or S3 store used by `test-report publish`; it does not need a copied SQLite database.
|
|
324
324
|
|
|
325
|
+
For Windows AWS SSO sign-in, S3 configuration, and local UI or CLI steps, see [LOCAL_REPORT_GENERATION.md](LOCAL_REPORT_GENERATION.md).
|
|
326
|
+
|
|
325
327
|
### Quick Start
|
|
326
328
|
|
|
327
329
|
```bash
|
|
@@ -281,6 +281,8 @@ retention_days = 90 # Keep 90 days of artifacts and data
|
|
|
281
281
|
|
|
282
282
|
Generate a report for one project from its published run history. The report reader supports the same local or S3 store used by `test-report publish`; it does not need a copied SQLite database.
|
|
283
283
|
|
|
284
|
+
For Windows AWS SSO sign-in, S3 configuration, and local UI or CLI steps, see [LOCAL_REPORT_GENERATION.md](LOCAL_REPORT_GENERATION.md).
|
|
285
|
+
|
|
284
286
|
### Quick Start
|
|
285
287
|
|
|
286
288
|
```bash
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"""Stakeholder PDF built from published run summaries and test records."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Dict
|
|
5
|
+
from xml.sax.saxutils import escape
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from reportlab.graphics.shapes import Circle, Drawing, Line, Rect, String
|
|
9
|
+
from reportlab.lib import colors
|
|
10
|
+
from reportlab.lib.enums import TA_CENTER, TA_LEFT
|
|
11
|
+
from reportlab.lib.pagesizes import A4
|
|
12
|
+
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
|
13
|
+
from reportlab.lib.units import inch
|
|
14
|
+
from reportlab.platypus import (
|
|
15
|
+
KeepTogether, LongTable, PageBreak, Paragraph, SimpleDocTemplate,
|
|
16
|
+
Spacer, Table, TableStyle,
|
|
17
|
+
)
|
|
18
|
+
except ImportError as exc:
|
|
19
|
+
raise ImportError(
|
|
20
|
+
"reportlab is required for PDF export. Install with: pip install test-reporting[reports]"
|
|
21
|
+
) from exc
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class PDFExporter:
|
|
25
|
+
"""Present period measures, charts, and a file-to-test breakdown."""
|
|
26
|
+
|
|
27
|
+
def __init__(self):
|
|
28
|
+
self.navy = colors.HexColor('#142B49')
|
|
29
|
+
self.teal = colors.HexColor('#087E83')
|
|
30
|
+
self.red = colors.HexColor('#C0444B')
|
|
31
|
+
self.gold = colors.HexColor('#C58A24')
|
|
32
|
+
self.ink = colors.HexColor('#26374A')
|
|
33
|
+
self.muted = colors.HexColor('#64748B')
|
|
34
|
+
self.light = colors.HexColor('#F3F7FA')
|
|
35
|
+
self.border = colors.HexColor('#DCE5ED')
|
|
36
|
+
styles = getSampleStyleSheet()
|
|
37
|
+
styles['Normal'].fontName = 'Helvetica'
|
|
38
|
+
styles['Normal'].fontSize = 9.5
|
|
39
|
+
styles['Normal'].leading = 14
|
|
40
|
+
styles['Normal'].textColor = self.ink
|
|
41
|
+
self.styles = styles
|
|
42
|
+
self.styles.add(ParagraphStyle(
|
|
43
|
+
name='ReportTitle', parent=styles['Normal'], fontName='Helvetica-Bold',
|
|
44
|
+
fontSize=29, leading=36, textColor=self.navy, spaceAfter=15))
|
|
45
|
+
self.styles.add(ParagraphStyle(
|
|
46
|
+
name='SectionTitle', parent=styles['Normal'], fontName='Helvetica-Bold',
|
|
47
|
+
fontSize=16, leading=20, textColor=self.navy, spaceBefore=10, spaceAfter=9))
|
|
48
|
+
self.styles.add(ParagraphStyle(
|
|
49
|
+
name='SubTitle', parent=styles['Normal'], fontName='Helvetica-Bold',
|
|
50
|
+
fontSize=11, leading=15, textColor=self.teal, spaceBefore=10, spaceAfter=6))
|
|
51
|
+
self.styles.add(ParagraphStyle(
|
|
52
|
+
name='Small', parent=styles['Normal'], fontSize=8, leading=11, textColor=self.muted))
|
|
53
|
+
self.styles.add(ParagraphStyle(
|
|
54
|
+
name='Cell', parent=styles['Normal'], fontSize=8, leading=11, splitLongWords=True))
|
|
55
|
+
self.styles.add(ParagraphStyle(
|
|
56
|
+
name='CellHead', parent=styles['Cell'], fontName='Helvetica-Bold',
|
|
57
|
+
textColor=colors.white))
|
|
58
|
+
self.styles.add(ParagraphStyle(
|
|
59
|
+
name='Center', parent=styles['Normal'], alignment=TA_CENTER))
|
|
60
|
+
|
|
61
|
+
def export(self, metrics: Dict[str, Any], sections: Dict[str, bool],
|
|
62
|
+
report_meta: Dict[str, Any], output_path: Path):
|
|
63
|
+
analysis = metrics['pdf_analysis']
|
|
64
|
+
summary = metrics['summary']
|
|
65
|
+
doc = SimpleDocTemplate(
|
|
66
|
+
str(output_path), pagesize=A4, leftMargin=49, rightMargin=49,
|
|
67
|
+
topMargin=57, bottomMargin=48,
|
|
68
|
+
title=f"Test Execution Report - {report_meta['project']}",
|
|
69
|
+
author='QA Reporting')
|
|
70
|
+
story = self._cover(report_meta, summary, analysis)
|
|
71
|
+
story.append(PageBreak())
|
|
72
|
+
if sections.get('summary', True):
|
|
73
|
+
story.extend(self._overview(metrics, analysis))
|
|
74
|
+
if sections.get('trends', True) and metrics.get('daily_trends'):
|
|
75
|
+
story.append(PageBreak())
|
|
76
|
+
story.extend(self._activity(metrics['daily_trends'], summary, analysis))
|
|
77
|
+
if sections.get('failures', True):
|
|
78
|
+
story.append(PageBreak())
|
|
79
|
+
story.extend(self._failure_mix(metrics, analysis))
|
|
80
|
+
if summary.get('detail_complete') and (
|
|
81
|
+
sections.get('file_health', True) or sections.get('failures', True)):
|
|
82
|
+
story.append(PageBreak())
|
|
83
|
+
story.extend(self._file_breakdown(analysis, report_meta['top_n']))
|
|
84
|
+
self._project = report_meta['project']
|
|
85
|
+
self._period = f"{report_meta['start_date']} to {report_meta['end_date']}"
|
|
86
|
+
doc.build(story, onFirstPage=self._page, onLaterPages=self._page)
|
|
87
|
+
|
|
88
|
+
def _page(self, canvas, doc):
|
|
89
|
+
width, height = A4
|
|
90
|
+
canvas.saveState()
|
|
91
|
+
canvas.setFillColor(self.navy)
|
|
92
|
+
canvas.rect(0, height - 12, width, 12, fill=1, stroke=0)
|
|
93
|
+
if doc.page > 1:
|
|
94
|
+
canvas.setFont('Helvetica-Bold', 8)
|
|
95
|
+
canvas.setFillColor(self.navy)
|
|
96
|
+
canvas.drawString(49, height - 34, 'TEST EXECUTION REPORT')
|
|
97
|
+
canvas.setStrokeColor(self.border)
|
|
98
|
+
canvas.line(49, 35, width - 49, 35)
|
|
99
|
+
canvas.setFont('Helvetica', 7.5)
|
|
100
|
+
canvas.setFillColor(self.muted)
|
|
101
|
+
canvas.drawString(49, 22, f'{self._project} | {self._period}'[:95])
|
|
102
|
+
canvas.drawRightString(width - 49, 22, str(doc.page))
|
|
103
|
+
canvas.restoreState()
|
|
104
|
+
|
|
105
|
+
def _p(self, value, style='Normal'):
|
|
106
|
+
return Paragraph(escape(str(value)), self.styles[style])
|
|
107
|
+
|
|
108
|
+
def _heading(self, title, description):
|
|
109
|
+
return [self._p(title, 'SectionTitle'), self._p(description), Spacer(1, 10)]
|
|
110
|
+
|
|
111
|
+
def _table(self, headings, rows, widths):
|
|
112
|
+
values = [[self._p(h, 'CellHead') for h in headings]]
|
|
113
|
+
values.extend([[self._p(cell, 'Cell') for cell in row] for row in rows])
|
|
114
|
+
table = LongTable(values, colWidths=widths, repeatRows=1, hAlign='LEFT')
|
|
115
|
+
table.setStyle(TableStyle([
|
|
116
|
+
('BACKGROUND', (0, 0), (-1, 0), self.navy),
|
|
117
|
+
('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, self.light]),
|
|
118
|
+
('LINEBELOW', (0, 0), (-1, 0), 1, self.teal),
|
|
119
|
+
('LINEBELOW', (0, 1), (-1, -1), 0.3, self.border),
|
|
120
|
+
('TOPPADDING', (0, 0), (-1, -1), 7),
|
|
121
|
+
('BOTTOMPADDING', (0, 0), (-1, -1), 7),
|
|
122
|
+
('LEFTPADDING', (0, 0), (-1, -1), 8),
|
|
123
|
+
('RIGHTPADDING', (0, 0), (-1, -1), 8),
|
|
124
|
+
('VALIGN', (0, 0), (-1, -1), 'TOP'),
|
|
125
|
+
]))
|
|
126
|
+
return table
|
|
127
|
+
|
|
128
|
+
def _cover(self, meta, summary, analysis):
|
|
129
|
+
story = [Spacer(1, 1.15 * inch), self._p('QUALITY INTELLIGENCE / PERIOD REVIEW', 'SubTitle'),
|
|
130
|
+
Spacer(1, 12), self._p('Test Execution Report', 'ReportTitle'),
|
|
131
|
+
self._p(meta['project'], 'SubTitle'),
|
|
132
|
+
self._p(f"{meta['start_date']} to {meta['end_date']}"),
|
|
133
|
+
Spacer(1, 0.65 * inch)]
|
|
134
|
+
cards = [
|
|
135
|
+
('PASS RATE', f"{summary['pass_rate']}%"),
|
|
136
|
+
('TEST EXECUTIONS', f"{analysis['total']:,}"),
|
|
137
|
+
('RUNS', f"{summary['total_runs']:,}"),
|
|
138
|
+
]
|
|
139
|
+
card_data = [[self._p(label, 'Small') for label, _ in cards],
|
|
140
|
+
[self._p(value, 'ReportTitle') for _, value in cards]]
|
|
141
|
+
cards_table = Table(card_data, colWidths=[165, 165, 165], hAlign='LEFT')
|
|
142
|
+
cards_table.setStyle(TableStyle([
|
|
143
|
+
('BACKGROUND', (0, 0), (-1, -1), self.light),
|
|
144
|
+
('BOX', (0, 0), (-1, -1), 0.5, self.border),
|
|
145
|
+
('TOPPADDING', (0, 0), (-1, 0), 13),
|
|
146
|
+
('BOTTOMPADDING', (0, -1), (-1, -1), 10),
|
|
147
|
+
('LEFTPADDING', (0, 0), (-1, -1), 12),
|
|
148
|
+
]))
|
|
149
|
+
story.extend([cards_table, Spacer(1, 0.45 * inch),
|
|
150
|
+
self._p('What this report measures', 'SubTitle'),
|
|
151
|
+
self._p('Results from published automated test runs for one project and the selected period. '
|
|
152
|
+
'Pass rate is passed executions divided by all executions, including skipped tests. '
|
|
153
|
+
'These measures describe observed test execution, not product coverage or defect severity.'),
|
|
154
|
+
Spacer(1, 12),
|
|
155
|
+
self._p('Generated from run history; individual test measures appear only when detailed '
|
|
156
|
+
'records cover all passed and failed executions.', 'Small')])
|
|
157
|
+
return story
|
|
158
|
+
|
|
159
|
+
def _overview(self, metrics, analysis):
|
|
160
|
+
summary = metrics['summary']
|
|
161
|
+
story = self._heading('At a glance',
|
|
162
|
+
'The period totals below use published run summaries. A test execution is one recorded test result in one run.')
|
|
163
|
+
for insight in metrics.get('highlights', []):
|
|
164
|
+
story.extend([self._p(insight), Spacer(1, 6)])
|
|
165
|
+
story.append(Spacer(1, 8))
|
|
166
|
+
previous = (f"{summary['previous_pass_rate']}% ({summary['pass_rate_change']:+.1f} points)"
|
|
167
|
+
if summary.get('previous_pass_rate') is not None else 'No earlier runs')
|
|
168
|
+
rows = [
|
|
169
|
+
('Passed executions', f"{analysis['passed']:,}", 'Completed successfully.'),
|
|
170
|
+
('Failed executions', f"{analysis['failed']:,}", 'Recorded as failed by the test runner.'),
|
|
171
|
+
('Skipped executions', f"{analysis['skipped']:,}", 'Did not run to a pass or fail result.'),
|
|
172
|
+
('Failure rate', f"{analysis['failure_rate']}%", 'Failed executions / all test executions.'),
|
|
173
|
+
('Skip rate', f"{analysis['skip_rate']}%", 'Skipped executions / all test executions.'),
|
|
174
|
+
('Runs without failures', f"{analysis['clean_runs']:,} / {summary['total_runs']:,}",
|
|
175
|
+
'Runs with zero recorded failures; skipped tests may still be present.'),
|
|
176
|
+
('Runs without failures rate', f"{analysis['clean_run_rate']}%",
|
|
177
|
+
'Runs without failures / all runs.'),
|
|
178
|
+
('Previous period pass rate', previous,
|
|
179
|
+
'Compared with the immediately preceding period of equal length.'),
|
|
180
|
+
]
|
|
181
|
+
if analysis['unclassified']:
|
|
182
|
+
rows.insert(3, ('Unclassified executions', f"{analysis['unclassified']:,}",
|
|
183
|
+
'Difference between run total and recorded outcome counts.'))
|
|
184
|
+
story.append(self._table(['Measure', 'Value', 'How to read it'], rows, [165, 110, 220]))
|
|
185
|
+
story.extend([Spacer(1, 16), self._p('Execution outcome mix', 'SubTitle'),
|
|
186
|
+
self._p('Share of all test executions by outcome. The denominator includes skipped tests.', 'Small'),
|
|
187
|
+
Spacer(1, 6), self._outcome_chart(analysis)])
|
|
188
|
+
if not summary.get('detail_complete'):
|
|
189
|
+
story.extend([Spacer(1, 12), self._p('Data completeness', 'SubTitle'),
|
|
190
|
+
self._p('Some published runs do not contain full test-level detail. File and test case '
|
|
191
|
+
'rates are omitted; run-level totals remain available.')])
|
|
192
|
+
return story
|
|
193
|
+
|
|
194
|
+
def _outcome_chart(self, analysis):
|
|
195
|
+
chart = Drawing(495, 65)
|
|
196
|
+
total = max(analysis['total'], analysis['passed'] + analysis['failed'] +
|
|
197
|
+
analysis['skipped'], 1)
|
|
198
|
+
x = 0
|
|
199
|
+
for value, color in ((analysis['passed'], self.teal),
|
|
200
|
+
(analysis['failed'], self.red),
|
|
201
|
+
(analysis['skipped'], self.gold),
|
|
202
|
+
(analysis['unclassified'], self.muted)):
|
|
203
|
+
width = 495 * value / total
|
|
204
|
+
if width:
|
|
205
|
+
chart.add(Rect(x, 30, width, 22, fillColor=color, strokeColor=None))
|
|
206
|
+
x += width
|
|
207
|
+
labels = [('Passed', self.teal), ('Failed', self.red), ('Skipped', self.gold)]
|
|
208
|
+
if analysis['unclassified']:
|
|
209
|
+
labels.append(('Unclassified', self.muted))
|
|
210
|
+
for index, (label, color) in enumerate(labels):
|
|
211
|
+
lx = index * (120 if len(labels) == 4 else 145)
|
|
212
|
+
chart.add(Rect(lx, 8, 8, 8, fillColor=color, strokeColor=None))
|
|
213
|
+
chart.add(String(lx + 13, 8, label, fontName='Helvetica', fontSize=8,
|
|
214
|
+
fillColor=self.ink))
|
|
215
|
+
return chart
|
|
216
|
+
|
|
217
|
+
def _activity(self, daily, summary, analysis):
|
|
218
|
+
story = self._heading('Activity and change',
|
|
219
|
+
'Daily charts show active days with published runs. Values are weighted by the number of test executions.')
|
|
220
|
+
rows = [
|
|
221
|
+
('Active days', analysis['active_days'], 'Calendar days with at least one published run.'),
|
|
222
|
+
('Runs per active day', analysis['runs_per_active_day'], 'Total runs / active days.'),
|
|
223
|
+
('Executions per run', analysis['executions_per_run'], 'Total test executions / runs.'),
|
|
224
|
+
('Average run duration', f"{summary['avg_duration']}s", 'Mean recorded duration across runs.'),
|
|
225
|
+
]
|
|
226
|
+
story.append(self._table(['Measure', 'Value', 'How to read it'], rows, [165, 110, 220]))
|
|
227
|
+
shown = daily[-31:]
|
|
228
|
+
if len(daily) > len(shown):
|
|
229
|
+
story.extend([Spacer(1, 8), self._p('Charts show the most recent 31 active days; '
|
|
230
|
+
'period totals above include all selected runs.', 'Small')])
|
|
231
|
+
story.extend([Spacer(1, 16), self._p('Daily pass rate', 'SubTitle'),
|
|
232
|
+
self._p('Passed executions / all executions on each active day.', 'Small'),
|
|
233
|
+
self._series_chart(shown, 'pass_rate', 100, self.teal, '%'),
|
|
234
|
+
Spacer(1, 12), self._p('Daily execution volume', 'SubTitle'),
|
|
235
|
+
self._p('Number of test executions recorded on each active day.', 'Small'),
|
|
236
|
+
self._series_chart(shown, 'total_tests',
|
|
237
|
+
max(row['total_tests'] for row in shown), self.navy, '')])
|
|
238
|
+
return story
|
|
239
|
+
|
|
240
|
+
def _series_chart(self, daily, key, maximum, color, suffix):
|
|
241
|
+
chart = Drawing(495, 160)
|
|
242
|
+
x0, x1, y0, y1 = 39, 490, 25, 145
|
|
243
|
+
maximum = max(maximum, 1)
|
|
244
|
+
for fraction in (0, 0.5, 1):
|
|
245
|
+
y = y0 + (y1 - y0) * fraction
|
|
246
|
+
chart.add(Line(x0, y, x1, y, strokeColor=self.border, strokeWidth=0.6))
|
|
247
|
+
chart.add(String(2, y - 3, f'{maximum * fraction:g}{suffix}',
|
|
248
|
+
fontName='Helvetica', fontSize=7, fillColor=self.muted))
|
|
249
|
+
points = []
|
|
250
|
+
for i, item in enumerate(daily):
|
|
251
|
+
x = x0 + (x1 - x0) * i / max(len(daily) - 1, 1)
|
|
252
|
+
y = y0 + (y1 - y0) * item[key] / maximum
|
|
253
|
+
points.append((x, y))
|
|
254
|
+
if i and key == 'pass_rate':
|
|
255
|
+
chart.add(Line(*points[i - 1], x, y, strokeColor=color, strokeWidth=1.6))
|
|
256
|
+
elif key != 'pass_rate':
|
|
257
|
+
width = min(10, (x1 - x0) / max(len(daily), 1) * 0.65)
|
|
258
|
+
chart.add(Rect(x - width / 2, y0, width, y - y0,
|
|
259
|
+
fillColor=color, strokeColor=None))
|
|
260
|
+
if i == 0 or i == len(daily) - 1 or i % max(1, len(daily) // 6) == 0:
|
|
261
|
+
chart.add(String(x - 12, 8, item['date'][5:], fontName='Helvetica',
|
|
262
|
+
fontSize=6.8, fillColor=self.muted))
|
|
263
|
+
if key == 'pass_rate':
|
|
264
|
+
for x, y in points:
|
|
265
|
+
chart.add(Circle(x, y, 2.3, fillColor=color, strokeColor=None))
|
|
266
|
+
return chart
|
|
267
|
+
|
|
268
|
+
def _failure_mix(self, metrics, analysis):
|
|
269
|
+
story = self._heading('Failures and their distribution',
|
|
270
|
+
'Failure categories come from recorded test-level details. They are shown only when those '
|
|
271
|
+
'details cover all passed and failed executions.')
|
|
272
|
+
if not metrics['summary'].get('detail_complete'):
|
|
273
|
+
story.append(self._p('Detailed test history is incomplete, so failure categories and '
|
|
274
|
+
'test-level comparisons are not reported.'))
|
|
275
|
+
return story
|
|
276
|
+
failure_types = metrics.get('failure_types', [])
|
|
277
|
+
if not failure_types:
|
|
278
|
+
story.append(self._p('No failed test executions were recorded in this period.'))
|
|
279
|
+
return story
|
|
280
|
+
story.append(self._p('Failure types', 'SubTitle'))
|
|
281
|
+
story.append(self._p('Count of failed test executions in each recorded category.', 'Small'))
|
|
282
|
+
story.append(Spacer(1, 8))
|
|
283
|
+
story.append(self._category_chart(failure_types[:8]))
|
|
284
|
+
story.append(Spacer(1, 8))
|
|
285
|
+
story.append(self._table(['Failure type', 'Failed executions', 'Share of classified failures'],
|
|
286
|
+
[(item['type'], item['count'], f"{item['percentage']}%")
|
|
287
|
+
for item in failure_types], [230, 125, 140]))
|
|
288
|
+
story.extend([Spacer(1, 14), self._p('Where failures occurred', 'SubTitle'),
|
|
289
|
+
self._p('Files below are ranked by failed executions. Full file and test case names '
|
|
290
|
+
'appear in the next section.', 'Small')])
|
|
291
|
+
top_files = analysis['files'][:10]
|
|
292
|
+
story.append(self._table(['Test file', 'Failed', 'Executions', 'Pass rate'],
|
|
293
|
+
[(row['name'], row['failed'], row['executions'],
|
|
294
|
+
f"{row['pass_rate']}%") for row in top_files],
|
|
295
|
+
[300, 55, 70, 70]))
|
|
296
|
+
return story
|
|
297
|
+
|
|
298
|
+
def _category_chart(self, failure_types):
|
|
299
|
+
height = max(70, 26 * len(failure_types) + 10)
|
|
300
|
+
chart = Drawing(495, height)
|
|
301
|
+
maximum = max(item['count'] for item in failure_types) or 1
|
|
302
|
+
for index, item in enumerate(failure_types):
|
|
303
|
+
y = height - 25 - index * 26
|
|
304
|
+
label = str(item['type'])
|
|
305
|
+
if len(label) > 24:
|
|
306
|
+
label = label[:21] + '...'
|
|
307
|
+
chart.add(String(0, y + 5, label, fontName='Helvetica',
|
|
308
|
+
fontSize=8, fillColor=self.ink))
|
|
309
|
+
chart.add(Rect(160, y, 275 * item['count'] / maximum, 13,
|
|
310
|
+
fillColor=self.red, strokeColor=None))
|
|
311
|
+
chart.add(String(445, y + 3, str(item['count']), fontName='Helvetica-Bold',
|
|
312
|
+
fontSize=8, fillColor=self.ink))
|
|
313
|
+
return chart
|
|
314
|
+
|
|
315
|
+
def _file_breakdown(self, analysis, top_n):
|
|
316
|
+
limit = max(1, top_n)
|
|
317
|
+
files = analysis['files'][:limit]
|
|
318
|
+
story = self._heading('Test files and test cases',
|
|
319
|
+
f'Files are ranked by failed executions. Showing {len(files)} of {len(analysis["files"])} '
|
|
320
|
+
'files, with up to 8 cases per file. Cases are ranked by failed executions. '
|
|
321
|
+
'Every displayed name is complete and may wrap onto multiple lines.')
|
|
322
|
+
if not files:
|
|
323
|
+
story.append(self._p('No named test-level records are available.'))
|
|
324
|
+
return story
|
|
325
|
+
for file in files:
|
|
326
|
+
title = self._p(file['name'], 'SubTitle')
|
|
327
|
+
summary = self._p(
|
|
328
|
+
f"{file['case_count']} distinct cases | {file['executions']} executions | "
|
|
329
|
+
f"{file['passed']} passed | {file['failed']} failed | "
|
|
330
|
+
f"{file['skipped']} skipped | {file['pass_rate']}% pass rate", 'Small')
|
|
331
|
+
cases = file['cases'][:8]
|
|
332
|
+
table = self._table(['Test case', 'Passed', 'Failed', 'Skipped', 'Pass rate'],
|
|
333
|
+
[(case['name'], case['passed'], case['failed'],
|
|
334
|
+
case['skipped'], f"{case['pass_rate']}%") for case in cases],
|
|
335
|
+
[285, 50, 50, 55, 55])
|
|
336
|
+
story.extend([KeepTogether([title, summary, Spacer(1, 6)]), table, Spacer(1, 12)])
|
|
337
|
+
return story
|
|
@@ -126,9 +126,11 @@ class MonthlyReportGenerator:
|
|
|
126
126
|
|
|
127
127
|
if 'pdf' in formats:
|
|
128
128
|
from .exporters.pdf_exporter import PDFExporter
|
|
129
|
+
from .pdf_analysis import build_pdf_analysis
|
|
129
130
|
exporter = PDFExporter()
|
|
130
131
|
filename = self._make_filename(start_date, end_date, project, 'pdf')
|
|
131
132
|
output_path = output_dir / filename
|
|
133
|
+
metrics['pdf_analysis'] = build_pdf_analysis(data)
|
|
132
134
|
exporter.export(metrics, sections, report_meta, output_path)
|
|
133
135
|
generated_files.append(output_path)
|
|
134
136
|
print(f"[Monthly Report] Generated: {output_path}")
|
|
@@ -2,13 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
import html
|
|
4
4
|
import io
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
5
7
|
import tempfile
|
|
6
8
|
import webbrowser
|
|
7
9
|
import zipfile
|
|
8
10
|
from datetime import date, timedelta
|
|
9
11
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
10
12
|
from pathlib import Path
|
|
11
|
-
from urllib.parse import parse_qs
|
|
13
|
+
from urllib.parse import parse_qs, urlsplit
|
|
12
14
|
|
|
13
15
|
from .monthly_report import MonthlyReportGenerator
|
|
14
16
|
from .run_source import PublishedRunSource
|
|
@@ -31,24 +33,99 @@ h1 {{margin-top:0}} label {{display:block;font-weight:600;margin-top:1rem}}
|
|
|
31
33
|
input,select,button {{font:inherit;padding:.7rem;border:1px solid #aab6c6;border-radius:7px;box-sizing:border-box}}
|
|
32
34
|
input,select {{width:100%}} .dates {{display:grid;grid-template-columns:1fr 1fr;gap:1rem}}
|
|
33
35
|
.formats label {{display:inline-block;font-weight:400;margin-right:1rem}} .formats input {{width:auto}}
|
|
36
|
+
.presets {{display:flex;flex-wrap:wrap;gap:.45rem;margin-top:.6rem}}
|
|
37
|
+
.presets label {{margin:0;padding:.5rem .7rem;border:1px solid #cbd5e1;border-radius:8px;font-weight:500;cursor:pointer}}
|
|
38
|
+
.presets label:has(input:checked) {{background:#eaf2ff;border-color:#1d4ed8;color:#1d4ed8}}
|
|
39
|
+
.presets input {{width:auto;margin-right:.35rem}}
|
|
34
40
|
button {{margin-top:1.5rem;background:#1d4ed8;color:white;border:0;cursor:pointer}}
|
|
41
|
+
button:disabled {{opacity:.55;cursor:wait}}
|
|
35
42
|
.error {{color:#b91c1c}} small {{color:#64748b}}
|
|
36
43
|
</style></head><body><main><h1>Test report generator</h1>
|
|
37
44
|
<p>Select one project and a date range. Reports are generated from its published run history.</p>
|
|
38
45
|
{notice}
|
|
39
46
|
<form method="post" action="/generate">
|
|
40
47
|
<label for="project">Project</label><select id="project" name="project" required>{options}</select>
|
|
48
|
+
<label>Reporting period</label><div class="presets" role="group" aria-label="Reporting period">
|
|
49
|
+
<label><input type="radio" name="period" value="month" checked>Last month</label>
|
|
50
|
+
<label><input type="radio" name="period" value="quarter">Last 3 months</label>
|
|
51
|
+
<label><input type="radio" name="period" value="year">Last year</label>
|
|
52
|
+
<label><input type="radio" name="period" value="all">All data</label>
|
|
53
|
+
<label><input type="radio" name="period" value="custom">Custom</label></div>
|
|
54
|
+
<p><small>Rolling periods end today: 30, 90, or 365 days. All data uses the selected project's first and last published runs.</small></p>
|
|
41
55
|
<div class="dates"><div><label for="start">Start date</label><input id="start" name="start" type="date" value="{start.isoformat()}" required></div>
|
|
42
56
|
<div><label for="end">End date</label><input id="end" name="end" type="date" value="{today.isoformat()}" required></div></div>
|
|
57
|
+
<p id="period-status" role="status"><small>Change either date to use a custom range.</small></p>
|
|
43
58
|
<label>Formats</label><div class="formats">
|
|
44
59
|
<label><input type="checkbox" name="format" value="pdf" checked> PDF</label>
|
|
45
60
|
<label><input type="checkbox" name="format" value="pptx" checked> PowerPoint</label>
|
|
46
61
|
<label><input type="checkbox" name="format" value="xlsx" checked> Excel</label></div>
|
|
47
|
-
<button type="submit" onclick="this.textContent='Generating…';">Generate and download</button></form>
|
|
62
|
+
<button id="generate" type="submit" onclick="this.textContent='Generating…';">Generate and download</button></form>
|
|
48
63
|
<p><small>Runs stay in S3. The ZIP downloads when generation finishes. Watch the PowerShell window for progress; large date ranges may take a while.</small></p>
|
|
64
|
+
<script>
|
|
65
|
+
const projectInput = document.getElementById('project');
|
|
66
|
+
const startInput = document.getElementById('start');
|
|
67
|
+
const endInput = document.getElementById('end');
|
|
68
|
+
const generateButton = document.getElementById('generate');
|
|
69
|
+
const periodStatus = document.getElementById('period-status');
|
|
70
|
+
let boundsRequest = 0;
|
|
71
|
+
function selectedPeriod() {{ return document.querySelector('input[name="period"]:checked').value; }}
|
|
72
|
+
function formatDate(day) {{
|
|
73
|
+
return [day.getFullYear(), String(day.getMonth() + 1).padStart(2, '0'),
|
|
74
|
+
String(day.getDate()).padStart(2, '0')].join('-');
|
|
75
|
+
}}
|
|
76
|
+
async function applyPeriod() {{
|
|
77
|
+
const period = selectedPeriod();
|
|
78
|
+
const request = ++boundsRequest;
|
|
79
|
+
if (period !== 'all') generateButton.disabled = false;
|
|
80
|
+
if (period === 'custom') return;
|
|
81
|
+
if (period === 'all') {{
|
|
82
|
+
generateButton.disabled = true;
|
|
83
|
+
periodStatus.textContent = 'Finding the first and last published runs…';
|
|
84
|
+
const requestedProject = projectInput.value;
|
|
85
|
+
try {{
|
|
86
|
+
const response = await fetch('/date-bounds?project=' + encodeURIComponent(requestedProject));
|
|
87
|
+
const bounds = await response.json();
|
|
88
|
+
if (!response.ok) throw new Error(bounds.error || 'Unable to find published dates');
|
|
89
|
+
if (request !== boundsRequest || selectedPeriod() !== 'all') return;
|
|
90
|
+
startInput.value = bounds.start;
|
|
91
|
+
endInput.value = bounds.end;
|
|
92
|
+
periodStatus.textContent = 'All published runs for this project are selected.';
|
|
93
|
+
}} catch (error) {{
|
|
94
|
+
if (request !== boundsRequest) return;
|
|
95
|
+
document.querySelector('input[value="custom"]').checked = true;
|
|
96
|
+
periodStatus.textContent = error.message;
|
|
97
|
+
}} finally {{ if (request === boundsRequest) generateButton.disabled = false; }}
|
|
98
|
+
return;
|
|
99
|
+
}}
|
|
100
|
+
const days = {{month: 30, quarter: 90, year: 365}}[period];
|
|
101
|
+
const today = new Date();
|
|
102
|
+
const first = new Date(today.getFullYear(), today.getMonth(), today.getDate());
|
|
103
|
+
first.setDate(first.getDate() - days + 1);
|
|
104
|
+
startInput.value = formatDate(first);
|
|
105
|
+
endInput.value = formatDate(today);
|
|
106
|
+
periodStatus.textContent = `Selected ${{days}} days ending today.`;
|
|
107
|
+
}}
|
|
108
|
+
document.querySelectorAll('input[name="period"]').forEach(
|
|
109
|
+
radio => radio.addEventListener('change', applyPeriod));
|
|
110
|
+
projectInput.addEventListener('change', () => {{ if (selectedPeriod() === 'all') applyPeriod(); }});
|
|
111
|
+
function chooseCustom() {{
|
|
112
|
+
boundsRequest++;
|
|
113
|
+
generateButton.disabled = false;
|
|
114
|
+
document.querySelector('input[value="custom"]').checked = true;
|
|
115
|
+
periodStatus.textContent = 'Custom date range selected.';
|
|
116
|
+
}}
|
|
117
|
+
startInput.addEventListener('change', chooseCustom);
|
|
118
|
+
endInput.addEventListener('change', chooseCustom);
|
|
119
|
+
</script>
|
|
49
120
|
</main></body></html>'''.encode('utf-8')
|
|
50
121
|
|
|
51
122
|
|
|
123
|
+
def _download_name(project, start, end):
|
|
124
|
+
"""Build a safe, recognizable ZIP name for the browser download."""
|
|
125
|
+
project_part = re.sub(r'[^A-Za-z0-9._-]+', '-', project).strip('.-')[:60] or 'project'
|
|
126
|
+
return f'Test_Report_{project_part}_{start}_to_{end}.zip'
|
|
127
|
+
|
|
128
|
+
|
|
52
129
|
def make_handler(config, run_source=None):
|
|
53
130
|
source = run_source or PublishedRunSource(config)
|
|
54
131
|
|
|
@@ -62,7 +139,19 @@ def make_handler(config, run_source=None):
|
|
|
62
139
|
self.wfile.write(body)
|
|
63
140
|
|
|
64
141
|
def do_GET(self):
|
|
65
|
-
|
|
142
|
+
parsed = urlsplit(self.path)
|
|
143
|
+
if parsed.path == '/date-bounds':
|
|
144
|
+
try:
|
|
145
|
+
project = parse_qs(parsed.query).get('project', [''])[0]
|
|
146
|
+
first, last = source.date_bounds(project)
|
|
147
|
+
body = json.dumps({'start': first.isoformat(), 'end': last.isoformat()}).encode('utf-8')
|
|
148
|
+
self._send(200, body, 'application/json; charset=utf-8')
|
|
149
|
+
except Exception as exc:
|
|
150
|
+
status = 400 if isinstance(exc, ValueError) else 500
|
|
151
|
+
body = json.dumps({'error': str(exc)}).encode('utf-8')
|
|
152
|
+
self._send(status, body, 'application/json; charset=utf-8')
|
|
153
|
+
return
|
|
154
|
+
if parsed.path != '/' or parsed.query:
|
|
66
155
|
self._send(404, b'Not found', 'text/plain; charset=utf-8')
|
|
67
156
|
return
|
|
68
157
|
try:
|
|
@@ -97,7 +186,8 @@ def make_handler(config, run_source=None):
|
|
|
97
186
|
body = output.getvalue()
|
|
98
187
|
self.send_response(200)
|
|
99
188
|
self.send_header('Content-Type', 'application/zip')
|
|
100
|
-
self.send_header('Content-Disposition',
|
|
189
|
+
self.send_header('Content-Disposition',
|
|
190
|
+
f'attachment; filename="{_download_name(project, start, end)}"')
|
|
101
191
|
self.send_header('Content-Length', str(len(body)))
|
|
102
192
|
self.send_header('X-Content-Type-Options', 'nosniff')
|
|
103
193
|
self.end_headers()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Measures derived from published runs for the stakeholder PDF."""
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def build_pdf_analysis(data):
|
|
7
|
+
runs = data['runs']
|
|
8
|
+
results = data['results']
|
|
9
|
+
total = sum(run.get('total_tests', 0) for run in runs)
|
|
10
|
+
passed = sum(run.get('passed', 0) for run in runs)
|
|
11
|
+
failed = sum(run.get('failed', 0) for run in runs)
|
|
12
|
+
skipped = sum(run.get('skipped', 0) for run in runs)
|
|
13
|
+
unclassified = max(0, total - passed - failed - skipped)
|
|
14
|
+
clean_runs = sum(run.get('failed', 0) == 0 for run in runs)
|
|
15
|
+
active_days = len({run.get('timestamp', '')[:10] for run in runs if run.get('timestamp')})
|
|
16
|
+
|
|
17
|
+
files = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'cases': defaultdict(
|
|
18
|
+
lambda: {'passed': 0, 'failed': 0, 'skipped': 0})})
|
|
19
|
+
for result in results:
|
|
20
|
+
file_name = result.get('file_name') or '(file not recorded)'
|
|
21
|
+
test_name = result.get('test_name') or '(test not recorded)'
|
|
22
|
+
status = result.get('status')
|
|
23
|
+
if status not in ('passed', 'failed', 'skipped'):
|
|
24
|
+
continue
|
|
25
|
+
files[file_name][status] += 1
|
|
26
|
+
files[file_name]['cases'][test_name][status] += 1
|
|
27
|
+
|
|
28
|
+
file_rows = []
|
|
29
|
+
for name, counts in files.items():
|
|
30
|
+
cases = []
|
|
31
|
+
for test_name, values in counts['cases'].items():
|
|
32
|
+
executions = sum(values.values())
|
|
33
|
+
cases.append({'name': test_name, **values, 'executions': executions,
|
|
34
|
+
'pass_rate': round(100 * values['passed'] / executions, 1)})
|
|
35
|
+
cases.sort(key=lambda row: (-row['failed'], -row['executions'], row['name']))
|
|
36
|
+
executions = counts['passed'] + counts['failed'] + counts['skipped']
|
|
37
|
+
file_rows.append({'name': name, 'passed': counts['passed'],
|
|
38
|
+
'failed': counts['failed'], 'skipped': counts['skipped'],
|
|
39
|
+
'executions': executions, 'case_count': len(cases),
|
|
40
|
+
'pass_rate': round(100 * counts['passed'] / executions, 1),
|
|
41
|
+
'cases': cases})
|
|
42
|
+
file_rows.sort(key=lambda row: (-row['failed'], row['pass_rate'], row['name']))
|
|
43
|
+
|
|
44
|
+
return {
|
|
45
|
+
'total': total,
|
|
46
|
+
'passed': passed,
|
|
47
|
+
'failed': failed,
|
|
48
|
+
'skipped': skipped,
|
|
49
|
+
'unclassified': unclassified,
|
|
50
|
+
'clean_runs': clean_runs,
|
|
51
|
+
'clean_run_rate': round(100 * clean_runs / len(runs), 1) if runs else 0,
|
|
52
|
+
'active_days': active_days,
|
|
53
|
+
'runs_per_active_day': round(len(runs) / active_days, 1) if active_days else 0,
|
|
54
|
+
'executions_per_run': round(total / len(runs), 1) if runs else 0,
|
|
55
|
+
'failure_rate': round(100 * failed / total, 1) if total else 0,
|
|
56
|
+
'skip_rate': round(100 * skipped / total, 1) if total else 0,
|
|
57
|
+
'files': file_rows,
|
|
58
|
+
}
|
|
@@ -79,6 +79,47 @@ class PublishedRunSource:
|
|
|
79
79
|
if self._selected(run, project, start, end):
|
|
80
80
|
yield run
|
|
81
81
|
|
|
82
|
+
def date_bounds(self, project):
|
|
83
|
+
"""Earliest and latest published run dates for one project."""
|
|
84
|
+
if project not in self.projects():
|
|
85
|
+
raise ValueError(f"Published project not found: {project}")
|
|
86
|
+
dates = []
|
|
87
|
+
if self.config.STORE_TYPE == 's3':
|
|
88
|
+
prefix = self._key('runs', project) + '/'
|
|
89
|
+
pages = self._client().get_paginator('list_objects_v2').paginate(
|
|
90
|
+
Bucket=self.config.S3_BUCKET, Prefix=prefix)
|
|
91
|
+
for page in pages:
|
|
92
|
+
for entry in page.get('Contents', []):
|
|
93
|
+
key = entry['Key']
|
|
94
|
+
if not key.endswith('.json'):
|
|
95
|
+
continue
|
|
96
|
+
run_date = self._date_from_key(key)
|
|
97
|
+
if run_date is None:
|
|
98
|
+
body = self._client().get_object(Bucket=self.config.S3_BUCKET, Key=key)['Body'].read()
|
|
99
|
+
run_date = self._date_from_run(json.loads(body), project)
|
|
100
|
+
if run_date is not None:
|
|
101
|
+
dates.append(run_date)
|
|
102
|
+
else:
|
|
103
|
+
for path in (self.config.STORE_PATH / 'runs' / project).glob('*.json'):
|
|
104
|
+
run_date = self._date_from_key(path.name)
|
|
105
|
+
if run_date is None:
|
|
106
|
+
run_date = self._date_from_run(
|
|
107
|
+
json.loads(path.read_text(encoding='utf-8')), project)
|
|
108
|
+
if run_date is not None:
|
|
109
|
+
dates.append(run_date)
|
|
110
|
+
if not dates:
|
|
111
|
+
raise ValueError(f'No published runs found for {project}')
|
|
112
|
+
return min(dates), max(dates)
|
|
113
|
+
|
|
114
|
+
@staticmethod
|
|
115
|
+
def _date_from_run(run, project):
|
|
116
|
+
if run.get('project_name') != project:
|
|
117
|
+
return None
|
|
118
|
+
try:
|
|
119
|
+
return date.fromisoformat(run['timestamp'][:10])
|
|
120
|
+
except (KeyError, TypeError, ValueError):
|
|
121
|
+
return None
|
|
122
|
+
|
|
82
123
|
@staticmethod
|
|
83
124
|
def _date_from_key(key):
|
|
84
125
|
"""Use publisher run IDs to skip out-of-range objects before download."""
|
|
@@ -9,7 +9,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
|
9
9
|
|
|
10
10
|
setup(
|
|
11
11
|
name='test-reporting',
|
|
12
|
-
version='4.
|
|
12
|
+
version='4.4.1',
|
|
13
13
|
description='Multi-project test reporting dashboard — collect results locally or push to S3',
|
|
14
14
|
long_description=long_description,
|
|
15
15
|
long_description_content_type="text/markdown",
|