qa360 1.4.5 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/README.md +1 -1
  2. package/dist/commands/ai.d.ts +41 -0
  3. package/dist/commands/ai.js +499 -0
  4. package/dist/commands/ask.js +12 -12
  5. package/dist/commands/coverage.d.ts +8 -0
  6. package/dist/commands/coverage.js +252 -0
  7. package/dist/commands/explain.d.ts +27 -0
  8. package/dist/commands/explain.js +630 -0
  9. package/dist/commands/flakiness.d.ts +73 -0
  10. package/dist/commands/flakiness.js +435 -0
  11. package/dist/commands/generate.d.ts +66 -0
  12. package/dist/commands/generate.js +438 -0
  13. package/dist/commands/init.d.ts +56 -9
  14. package/dist/commands/init.js +217 -10
  15. package/dist/commands/monitor.d.ts +27 -0
  16. package/dist/commands/monitor.js +225 -0
  17. package/dist/commands/ollama.d.ts +40 -0
  18. package/dist/commands/ollama.js +301 -0
  19. package/dist/commands/pack.d.ts +37 -9
  20. package/dist/commands/pack.js +240 -141
  21. package/dist/commands/regression.d.ts +8 -0
  22. package/dist/commands/regression.js +340 -0
  23. package/dist/commands/repair.d.ts +26 -0
  24. package/dist/commands/repair.js +307 -0
  25. package/dist/commands/retry.d.ts +43 -0
  26. package/dist/commands/retry.js +275 -0
  27. package/dist/commands/run.d.ts +8 -3
  28. package/dist/commands/run.js +45 -31
  29. package/dist/commands/slo.d.ts +8 -0
  30. package/dist/commands/slo.js +327 -0
  31. package/dist/core/adapters/playwright-native-api.d.ts +183 -0
  32. package/dist/core/adapters/playwright-native-api.js +461 -0
  33. package/dist/core/adapters/playwright-ui.d.ts +7 -0
  34. package/dist/core/adapters/playwright-ui.js +29 -1
  35. package/dist/core/ai/anthropic-provider.d.ts +50 -0
  36. package/dist/core/ai/anthropic-provider.js +211 -0
  37. package/dist/core/ai/deepseek-provider.d.ts +81 -0
  38. package/dist/core/ai/deepseek-provider.js +254 -0
  39. package/dist/core/ai/index.d.ts +60 -0
  40. package/dist/core/ai/index.js +18 -0
  41. package/dist/core/ai/llm-client.d.ts +45 -0
  42. package/dist/core/ai/llm-client.js +7 -0
  43. package/dist/core/ai/mock-provider.d.ts +49 -0
  44. package/dist/core/ai/mock-provider.js +121 -0
  45. package/dist/core/ai/ollama-provider.d.ts +78 -0
  46. package/dist/core/ai/ollama-provider.js +192 -0
  47. package/dist/core/ai/openai-provider.d.ts +48 -0
  48. package/dist/core/ai/openai-provider.js +188 -0
  49. package/dist/core/ai/provider-factory.d.ts +160 -0
  50. package/dist/core/ai/provider-factory.js +269 -0
  51. package/dist/core/auth/api-key-provider.d.ts +16 -0
  52. package/dist/core/auth/api-key-provider.js +63 -0
  53. package/dist/core/auth/aws-iam-provider.d.ts +35 -0
  54. package/dist/core/auth/aws-iam-provider.js +177 -0
  55. package/dist/core/auth/azure-ad-provider.d.ts +15 -0
  56. package/dist/core/auth/azure-ad-provider.js +99 -0
  57. package/dist/core/auth/basic-auth-provider.d.ts +26 -0
  58. package/dist/core/auth/basic-auth-provider.js +111 -0
  59. package/dist/core/auth/gcp-adc-provider.d.ts +27 -0
  60. package/dist/core/auth/gcp-adc-provider.js +126 -0
  61. package/dist/core/auth/index.d.ts +238 -0
  62. package/dist/core/auth/index.js +82 -0
  63. package/dist/core/auth/jwt-provider.d.ts +19 -0
  64. package/dist/core/auth/jwt-provider.js +160 -0
  65. package/dist/core/auth/manager.d.ts +84 -0
  66. package/dist/core/auth/manager.js +230 -0
  67. package/dist/core/auth/oauth2-provider.d.ts +17 -0
  68. package/dist/core/auth/oauth2-provider.js +114 -0
  69. package/dist/core/auth/totp-provider.d.ts +31 -0
  70. package/dist/core/auth/totp-provider.js +134 -0
  71. package/dist/core/auth/ui-login-provider.d.ts +26 -0
  72. package/dist/core/auth/ui-login-provider.js +198 -0
  73. package/dist/core/cache/index.d.ts +7 -0
  74. package/dist/core/cache/index.js +6 -0
  75. package/dist/core/cache/lru-cache.d.ts +203 -0
  76. package/dist/core/cache/lru-cache.js +397 -0
  77. package/dist/core/coverage/analyzer.d.ts +101 -0
  78. package/dist/core/coverage/analyzer.js +415 -0
  79. package/dist/core/coverage/collector.d.ts +74 -0
  80. package/dist/core/coverage/collector.js +459 -0
  81. package/dist/core/coverage/config.d.ts +37 -0
  82. package/dist/core/coverage/config.js +156 -0
  83. package/dist/core/coverage/index.d.ts +11 -0
  84. package/dist/core/coverage/index.js +15 -0
  85. package/dist/core/coverage/types.d.ts +267 -0
  86. package/dist/core/coverage/types.js +6 -0
  87. package/dist/core/coverage/vault.d.ts +95 -0
  88. package/dist/core/coverage/vault.js +405 -0
  89. package/dist/core/dashboard/assets.d.ts +6 -0
  90. package/dist/core/dashboard/assets.js +690 -0
  91. package/dist/core/dashboard/index.d.ts +6 -0
  92. package/dist/core/dashboard/index.js +5 -0
  93. package/dist/core/dashboard/server.d.ts +72 -0
  94. package/dist/core/dashboard/server.js +354 -0
  95. package/dist/core/dashboard/types.d.ts +70 -0
  96. package/dist/core/dashboard/types.js +5 -0
  97. package/dist/core/discoverer/index.d.ts +115 -0
  98. package/dist/core/discoverer/index.js +250 -0
  99. package/dist/core/flakiness/index.d.ts +228 -0
  100. package/dist/core/flakiness/index.js +384 -0
  101. package/dist/core/generation/code-formatter.d.ts +111 -0
  102. package/dist/core/generation/code-formatter.js +307 -0
  103. package/dist/core/generation/code-generator.d.ts +144 -0
  104. package/dist/core/generation/code-generator.js +293 -0
  105. package/dist/core/generation/generator.d.ts +40 -0
  106. package/dist/core/generation/generator.js +76 -0
  107. package/dist/core/generation/index.d.ts +30 -0
  108. package/dist/core/generation/index.js +28 -0
  109. package/dist/core/generation/pack-generator.d.ts +107 -0
  110. package/dist/core/generation/pack-generator.js +416 -0
  111. package/dist/core/generation/prompt-builder.d.ts +132 -0
  112. package/dist/core/generation/prompt-builder.js +672 -0
  113. package/dist/core/generation/source-analyzer.d.ts +213 -0
  114. package/dist/core/generation/source-analyzer.js +657 -0
  115. package/dist/core/generation/test-optimizer.d.ts +117 -0
  116. package/dist/core/generation/test-optimizer.js +328 -0
  117. package/dist/core/generation/types.d.ts +214 -0
  118. package/dist/core/generation/types.js +4 -0
  119. package/dist/core/index.d.ts +23 -1
  120. package/dist/core/index.js +39 -0
  121. package/dist/core/pack/validator.js +31 -1
  122. package/dist/core/pack-v2/index.d.ts +9 -0
  123. package/dist/core/pack-v2/index.js +8 -0
  124. package/dist/core/pack-v2/loader.d.ts +62 -0
  125. package/dist/core/pack-v2/loader.js +231 -0
  126. package/dist/core/pack-v2/migrator.d.ts +56 -0
  127. package/dist/core/pack-v2/migrator.js +455 -0
  128. package/dist/core/pack-v2/validator.d.ts +61 -0
  129. package/dist/core/pack-v2/validator.js +577 -0
  130. package/dist/core/regression/detector.d.ts +107 -0
  131. package/dist/core/regression/detector.js +497 -0
  132. package/dist/core/regression/index.d.ts +9 -0
  133. package/dist/core/regression/index.js +11 -0
  134. package/dist/core/regression/trend-analyzer.d.ts +102 -0
  135. package/dist/core/regression/trend-analyzer.js +345 -0
  136. package/dist/core/regression/types.d.ts +222 -0
  137. package/dist/core/regression/types.js +7 -0
  138. package/dist/core/regression/vault.d.ts +87 -0
  139. package/dist/core/regression/vault.js +289 -0
  140. package/dist/core/repair/engine/fixer.d.ts +24 -0
  141. package/dist/core/repair/engine/fixer.js +226 -0
  142. package/dist/core/repair/engine/suggestion-engine.d.ts +18 -0
  143. package/dist/core/repair/engine/suggestion-engine.js +187 -0
  144. package/dist/core/repair/index.d.ts +10 -0
  145. package/dist/core/repair/index.js +13 -0
  146. package/dist/core/repair/repairer.d.ts +90 -0
  147. package/dist/core/repair/repairer.js +284 -0
  148. package/dist/core/repair/types.d.ts +91 -0
  149. package/dist/core/repair/types.js +6 -0
  150. package/dist/core/repair/utils/error-analyzer.d.ts +28 -0
  151. package/dist/core/repair/utils/error-analyzer.js +264 -0
  152. package/dist/core/retry/flakiness-integration.d.ts +60 -0
  153. package/dist/core/retry/flakiness-integration.js +228 -0
  154. package/dist/core/retry/index.d.ts +14 -0
  155. package/dist/core/retry/index.js +16 -0
  156. package/dist/core/retry/retry-engine.d.ts +80 -0
  157. package/dist/core/retry/retry-engine.js +296 -0
  158. package/dist/core/retry/types.d.ts +178 -0
  159. package/dist/core/retry/types.js +52 -0
  160. package/dist/core/retry/vault.d.ts +77 -0
  161. package/dist/core/retry/vault.js +304 -0
  162. package/dist/core/runner/e2e-helpers.d.ts +102 -0
  163. package/dist/core/runner/e2e-helpers.js +153 -0
  164. package/dist/core/runner/phase3-runner.d.ts +101 -2
  165. package/dist/core/runner/phase3-runner.js +559 -24
  166. package/dist/core/self-healing/assertion-healer.d.ts +97 -0
  167. package/dist/core/self-healing/assertion-healer.js +371 -0
  168. package/dist/core/self-healing/engine.d.ts +122 -0
  169. package/dist/core/self-healing/engine.js +538 -0
  170. package/dist/core/self-healing/index.d.ts +10 -0
  171. package/dist/core/self-healing/index.js +11 -0
  172. package/dist/core/self-healing/selector-healer.d.ts +103 -0
  173. package/dist/core/self-healing/selector-healer.js +372 -0
  174. package/dist/core/self-healing/types.d.ts +152 -0
  175. package/dist/core/self-healing/types.js +6 -0
  176. package/dist/core/slo/config.d.ts +107 -0
  177. package/dist/core/slo/config.js +360 -0
  178. package/dist/core/slo/index.d.ts +11 -0
  179. package/dist/core/slo/index.js +15 -0
  180. package/dist/core/slo/sli-calculator.d.ts +92 -0
  181. package/dist/core/slo/sli-calculator.js +364 -0
  182. package/dist/core/slo/slo-tracker.d.ts +148 -0
  183. package/dist/core/slo/slo-tracker.js +379 -0
  184. package/dist/core/slo/types.d.ts +281 -0
  185. package/dist/core/slo/types.js +7 -0
  186. package/dist/core/slo/vault.d.ts +102 -0
  187. package/dist/core/slo/vault.js +427 -0
  188. package/dist/core/tui/index.d.ts +7 -0
  189. package/dist/core/tui/index.js +6 -0
  190. package/dist/core/tui/monitor.d.ts +92 -0
  191. package/dist/core/tui/monitor.js +271 -0
  192. package/dist/core/tui/renderer.d.ts +33 -0
  193. package/dist/core/tui/renderer.js +218 -0
  194. package/dist/core/tui/types.d.ts +63 -0
  195. package/dist/core/tui/types.js +5 -0
  196. package/dist/core/types/pack-v2.d.ts +425 -0
  197. package/dist/core/types/pack-v2.js +8 -0
  198. package/dist/core/vault/index.d.ts +116 -0
  199. package/dist/core/vault/index.js +400 -5
  200. package/dist/core/watch/index.d.ts +7 -0
  201. package/dist/core/watch/index.js +6 -0
  202. package/dist/core/watch/watch-mode.d.ts +213 -0
  203. package/dist/core/watch/watch-mode.js +389 -0
  204. package/dist/index.js +68 -68
  205. package/dist/utils/config.d.ts +5 -0
  206. package/dist/utils/config.js +136 -0
  207. package/package.json +5 -1
  208. package/dist/core/adapters/playwright-api.d.ts +0 -82
  209. package/dist/core/adapters/playwright-api.js +0 -264
@@ -124,6 +124,201 @@ export class EvidenceVault {
124
124
  console.log(`[VAULT] FINDING_RECORDED: ${runId}/${finding.gate} (severity: ${finding.severity}, rule: ${finding.rule})`);
125
125
  return findingId;
126
126
  }
127
+ /**
128
+ * Record flakiness analysis result
129
+ */
130
+ async recordFlakiness(runId, flakiness) {
131
+ const flakinessRecord = {
132
+ run_id: runId,
133
+ created_at: Date.now(),
134
+ ...flakiness
135
+ };
136
+ const flakinessId = await this.insertFlakiness(flakinessRecord);
137
+ console.log(`[VAULT] FLAKINESS_RECORDED: ${runId}/${flakiness.test_name} (score: ${flakiness.score}, category: ${flakiness.category})`);
138
+ return flakinessId;
139
+ }
140
+ /**
141
+ * Record detected flakiness pattern
142
+ */
143
+ async recordFlakinessPattern(pattern) {
144
+ const now = Date.now();
145
+ // Check if pattern already exists for this test
146
+ const existing = await this.getFlakinessPattern(pattern.test_id, pattern.pattern_type);
147
+ if (existing) {
148
+ // Update existing pattern
149
+ await this.updateFlakinessPattern(existing.id, {
150
+ last_detected: now,
151
+ detection_count: (existing.detection_count || 0) + 1,
152
+ updated_at: now
153
+ });
154
+ console.log(`[VAULT] FLAKY_PATTERN_UPDATED: ${pattern.test_name} (${pattern.pattern_type})`);
155
+ return existing.id;
156
+ }
157
+ // Insert new pattern
158
+ const patternRecord = {
159
+ first_detected: now,
160
+ last_detected: now,
161
+ detection_count: 1,
162
+ created_at: now,
163
+ updated_at: now,
164
+ ...pattern
165
+ };
166
+ const patternId = await this.insertFlakinessPattern(patternRecord);
167
+ console.log(`[VAULT] FLAKY_PATTERN_DETECTED: ${pattern.test_name} (${pattern.pattern_type})`);
168
+ return patternId;
169
+ }
170
+ /**
171
+ * Add test to quarantine
172
+ */
173
+ async addToQuarantine(quarantine) {
174
+ const now = Date.now();
175
+ // Check if already in quarantine
176
+ const existing = await this.getQuarantine(quarantine.test_id);
177
+ if (existing && !existing.resolved_at) {
178
+ console.log(`[VAULT] ALREADY_IN_QUARANTINE: ${quarantine.test_name}`);
179
+ return existing.id;
180
+ }
181
+ const quarantineRecord = {
182
+ created_at: now,
183
+ updated_at: now,
184
+ ...quarantine
185
+ };
186
+ const quarantineId = await this.insertQuarantine(quarantineRecord);
187
+ console.log(`[VAULT] QUARANTINED: ${quarantine.test_name} (${quarantine.category})`);
188
+ return quarantineId;
189
+ }
190
+ /**
191
+ * Remove test from quarantine
192
+ */
193
+ async removeFromQuarantine(testId, resolvedBy, notes) {
194
+ await this.updateQuarantine(testId, {
195
+ resolved_at: Date.now(),
196
+ resolved_by: resolvedBy,
197
+ notes,
198
+ updated_at: Date.now()
199
+ });
200
+ console.log(`[VAULT] QUARANTINE_REMOVED: ${testId} (by: ${resolvedBy})`);
201
+ }
202
+ /**
203
+ * Get flakiness history for a test
204
+ */
205
+ async getFlakinessHistory(testId, limit = 50) {
206
+ return new Promise((resolve, reject) => {
207
+ this.db.all(`SELECT * FROM flakiness_history WHERE test_id = ? ORDER BY created_at DESC LIMIT ?`, [testId, limit], (err, rows) => {
208
+ if (err)
209
+ reject(err);
210
+ else
211
+ resolve(rows);
212
+ });
213
+ });
214
+ }
215
+ /**
216
+ * Get flakiness history for a run
217
+ */
218
+ async getRunFlakiness(runId) {
219
+ return new Promise((resolve, reject) => {
220
+ this.db.all(`SELECT * FROM flakiness_history WHERE run_id = ? ORDER BY score ASC`, [runId], (err, rows) => {
221
+ if (err)
222
+ reject(err);
223
+ else
224
+ resolve(rows);
225
+ });
226
+ });
227
+ }
228
+ /**
229
+ * Get all quarantined tests
230
+ */
231
+ async getQuarantinedTests(includeResolved = false) {
232
+ let query = 'SELECT * FROM flakiness_quarantine WHERE 1=1';
233
+ const params = [];
234
+ if (!includeResolved) {
235
+ query += ' AND resolved_at IS NULL';
236
+ }
237
+ query += ' ORDER BY quarantined_at DESC';
238
+ return new Promise((resolve, reject) => {
239
+ this.db.all(query, params, (err, rows) => {
240
+ if (err)
241
+ reject(err);
242
+ else
243
+ resolve(rows);
244
+ });
245
+ });
246
+ }
247
+ /**
248
+ * Get quarantine record for a test
249
+ */
250
+ async getQuarantine(testId) {
251
+ return new Promise((resolve, reject) => {
252
+ this.db.get('SELECT * FROM flakiness_quarantine WHERE test_id = ? ORDER BY created_at DESC LIMIT 1', [testId], (err, row) => {
253
+ if (err)
254
+ reject(err);
255
+ else
256
+ resolve(row || null);
257
+ });
258
+ });
259
+ }
260
+ /**
261
+ * Get flakiness pattern for a test
262
+ */
263
+ async getFlakinessPattern(testId, patternType) {
264
+ return new Promise((resolve, reject) => {
265
+ this.db.get('SELECT * FROM flakiness_patterns WHERE test_id = ? AND pattern_type = ?', [testId, patternType], (err, row) => {
266
+ if (err)
267
+ reject(err);
268
+ else
269
+ resolve(row || null);
270
+ });
271
+ });
272
+ }
273
+ /**
274
+ * Get flakiness trends for a test
275
+ */
276
+ async getFlakinessTrends(testId, days = 30) {
277
+ const cutoff = Date.now() - (days * 24 * 60 * 60 * 1000);
278
+ return new Promise((resolve, reject) => {
279
+ this.db.all(`SELECT created_at, score FROM flakiness_history WHERE test_id = ? AND created_at >= ? ORDER BY created_at ASC`, [testId, cutoff], (err, rows) => {
280
+ if (err) {
281
+ reject(err);
282
+ }
283
+ else {
284
+ const rawPoints = rows;
285
+ // Convert to expected format with 'date' property
286
+ const dataPoints = rawPoints.map(p => ({ date: p.created_at, score: p.score }));
287
+ if (dataPoints.length === 0) {
288
+ resolve({
289
+ testId,
290
+ currentScore: 100,
291
+ averageScore: 100,
292
+ trend: 'stable',
293
+ dataPoints: []
294
+ });
295
+ return;
296
+ }
297
+ const currentScore = dataPoints[dataPoints.length - 1].score;
298
+ const averageScore = Math.round(dataPoints.reduce((sum, p) => sum + p.score, 0) / dataPoints.length);
299
+ // Calculate trend
300
+ let trend = 'stable';
301
+ if (dataPoints.length >= 3) {
302
+ const recent = dataPoints.slice(-3);
303
+ const older = dataPoints.slice(0, -3);
304
+ const recentAvg = recent.reduce((sum, p) => sum + p.score, 0) / recent.length;
305
+ const olderAvg = older.reduce((sum, p) => sum + p.score, 0) / older.length;
306
+ if (recentAvg > olderAvg + 5)
307
+ trend = 'improving';
308
+ else if (recentAvg < olderAvg - 5)
309
+ trend = 'degrading';
310
+ }
311
+ resolve({
312
+ testId,
313
+ currentScore,
314
+ averageScore,
315
+ trend,
316
+ dataPoints
317
+ });
318
+ }
319
+ });
320
+ });
321
+ }
127
322
  /**
128
323
  * Store artifact in CAS and link to run
129
324
  */
@@ -433,6 +628,69 @@ export class EvidenceVault {
433
628
  });
434
629
  });
435
630
  }
631
+ async insertFlakiness(flakiness) {
632
+ const fields = Object.keys(flakiness).join(', ');
633
+ const placeholders = Object.keys(flakiness).map(() => '?').join(', ');
634
+ const values = Object.values(flakiness);
635
+ return new Promise((resolve, reject) => {
636
+ this.db.run(`INSERT INTO flakiness_history (${fields}) VALUES (${placeholders})`, values, function (err) {
637
+ if (err)
638
+ reject(err);
639
+ else
640
+ resolve(this.lastID);
641
+ });
642
+ });
643
+ }
644
+ async insertFlakinessPattern(pattern) {
645
+ const fields = Object.keys(pattern).join(', ');
646
+ const placeholders = Object.keys(pattern).map(() => '?').join(', ');
647
+ const values = Object.values(pattern);
648
+ return new Promise((resolve, reject) => {
649
+ this.db.run(`INSERT INTO flakiness_patterns (${fields}) VALUES (${placeholders})`, values, function (err) {
650
+ if (err)
651
+ reject(err);
652
+ else
653
+ resolve(this.lastID);
654
+ });
655
+ });
656
+ }
657
+ async updateFlakinessPattern(id, updates) {
658
+ const fields = Object.keys(updates).map(key => `${key} = ?`).join(', ');
659
+ const values = [...Object.values(updates), id];
660
+ return new Promise((resolve, reject) => {
661
+ this.db.run(`UPDATE flakiness_patterns SET ${fields} WHERE id = ?`, values, (err) => {
662
+ if (err)
663
+ reject(err);
664
+ else
665
+ resolve();
666
+ });
667
+ });
668
+ }
669
+ async insertQuarantine(quarantine) {
670
+ const fields = Object.keys(quarantine).join(', ');
671
+ const placeholders = Object.keys(quarantine).map(() => '?').join(', ');
672
+ const values = Object.values(quarantine);
673
+ return new Promise((resolve, reject) => {
674
+ this.db.run(`INSERT INTO flakiness_quarantine (${fields}) VALUES (${placeholders})`, values, function (err) {
675
+ if (err)
676
+ reject(err);
677
+ else
678
+ resolve(this.lastID);
679
+ });
680
+ });
681
+ }
682
+ async updateQuarantine(testId, updates) {
683
+ const fields = Object.keys(updates).map(key => `${key} = ?`).join(', ');
684
+ const values = [...Object.values(updates), testId];
685
+ return new Promise((resolve, reject) => {
686
+ this.db.run(`UPDATE flakiness_quarantine SET ${fields} WHERE test_id = ?`, values, (err) => {
687
+ if (err)
688
+ reject(err);
689
+ else
690
+ resolve();
691
+ });
692
+ });
693
+ }
436
694
  generateFindingFingerprint(finding) {
437
695
  const content = `${finding.gate}:${finding.rule}:${finding.location || ''}:${finding.message}`;
438
696
  return createHash('sha256').update(content).digest('hex').substring(0, 16);
@@ -525,6 +783,66 @@ CREATE TABLE IF NOT EXISTS run_artifacts (
525
783
  PRIMARY KEY (run_id, sha256, label)
526
784
  );
527
785
 
786
+ -- Flakiness history table: tracks test reliability over time
787
+ CREATE TABLE IF NOT EXISTS flakiness_history (
788
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
789
+ run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
790
+ test_id TEXT NOT NULL,
791
+ test_name TEXT NOT NULL,
792
+ gate TEXT NOT NULL,
793
+ file_path TEXT NOT NULL,
794
+ score INTEGER NOT NULL CHECK (score >= 0 AND score <= 100),
795
+ category TEXT NOT NULL CHECK (category IN ('legendary', 'solid', 'good', 'shaky', 'unstable')),
796
+ total_runs INTEGER NOT NULL,
797
+ successful_runs INTEGER NOT NULL,
798
+ avg_duration_ms INTEGER NOT NULL,
799
+ pattern_type TEXT,
800
+ suggested_fix TEXT,
801
+ confidence REAL,
802
+ first_seen INTEGER NOT NULL,
803
+ last_seen INTEGER NOT NULL,
804
+ current_streak INTEGER,
805
+ streak_type TEXT CHECK (streak_type IN ('pass', 'fail')),
806
+ created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
807
+ );
808
+
809
+ -- Flakiness patterns table: detected patterns for flaky tests
810
+ CREATE TABLE IF NOT EXISTS flakiness_patterns (
811
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
812
+ test_id TEXT NOT NULL,
813
+ test_name TEXT NOT NULL,
814
+ pattern_type TEXT NOT NULL CHECK (pattern_type IN ('timing', 'race_condition', 'external_dependency', 'environment_specific', 'selector_issue', 'unknown')),
815
+ description TEXT NOT NULL,
816
+ suggested_fix TEXT NOT NULL,
817
+ confidence REAL NOT NULL CHECK (confidence >= 0 AND confidence <= 1),
818
+ auto_fix_possible INTEGER NOT NULL DEFAULT 0,
819
+ first_detected INTEGER NOT NULL,
820
+ last_detected INTEGER NOT NULL,
821
+ detection_count INTEGER NOT NULL DEFAULT 1,
822
+ created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
823
+ updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
824
+ UNIQUE(test_id, pattern_type)
825
+ );
826
+
827
+ -- Flakiness quarantine table: tests quarantined for flakiness
828
+ CREATE TABLE IF NOT EXISTS flakiness_quarantine (
829
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
830
+ test_id TEXT NOT NULL UNIQUE,
831
+ test_name TEXT NOT NULL,
832
+ gate TEXT NOT NULL,
833
+ file_path TEXT NOT NULL,
834
+ reason TEXT NOT NULL,
835
+ score INTEGER NOT NULL CHECK (score >= 0 AND score <= 100),
836
+ category TEXT NOT NULL CHECK (category IN ('shaky', 'unstable')),
837
+ quarantined_at INTEGER NOT NULL,
838
+ quarantined_by TEXT NOT NULL,
839
+ resolved_at INTEGER,
840
+ resolved_by TEXT,
841
+ notes TEXT,
842
+ created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
843
+ updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
844
+ );
845
+
528
846
  -- Vault metadata table: schema version and configuration
529
847
  CREATE TABLE IF NOT EXISTS vault_metadata (
530
848
  key TEXT PRIMARY KEY,
@@ -559,14 +877,30 @@ CREATE INDEX IF NOT EXISTS idx_run_artifacts_run_id ON run_artifacts(run_id);
559
877
  CREATE INDEX IF NOT EXISTS idx_run_artifacts_sha256 ON run_artifacts(sha256);
560
878
  CREATE INDEX IF NOT EXISTS idx_run_artifacts_label ON run_artifacts(label);
561
879
 
880
+ -- Flakiness indexes
881
+ CREATE INDEX IF NOT EXISTS idx_flakiness_run_id ON flakiness_history(run_id);
882
+ CREATE INDEX IF NOT EXISTS idx_flakiness_test_id ON flakiness_history(test_id);
883
+ CREATE INDEX IF NOT EXISTS idx_flakiness_score ON flakiness_history(score);
884
+ CREATE INDEX IF NOT EXISTS idx_flakiness_category ON flakiness_history(category);
885
+ CREATE INDEX IF NOT EXISTS idx_flakiness_created_at ON flakiness_history(created_at DESC);
886
+
887
+ CREATE INDEX IF NOT EXISTS idx_flaky_patterns_test_id ON flakiness_patterns(test_id);
888
+ CREATE INDEX IF NOT EXISTS idx_flaky_patterns_pattern_type ON flakiness_patterns(pattern_type);
889
+ CREATE INDEX IF NOT EXISTS idx_flaky_patterns_detection_count ON flakiness_patterns(detection_count DESC);
890
+
891
+ CREATE INDEX IF NOT EXISTS idx_quarantine_test_id ON flakiness_quarantine(test_id);
892
+ CREATE INDEX IF NOT EXISTS idx_quarantine_quarantined_at ON flakiness_quarantine(quarantined_at DESC);
893
+ CREATE INDEX IF NOT EXISTS idx_quarantine_resolved_at ON flakiness_quarantine(resolved_at);
894
+ CREATE INDEX IF NOT EXISTS idx_quarantine_category ON flakiness_quarantine(category);
895
+
562
896
  -- Insert initial metadata
563
- INSERT OR IGNORE INTO vault_metadata (key, value) VALUES
564
- ('schema_version', '1.0.0'),
897
+ INSERT OR IGNORE INTO vault_metadata (key, value) VALUES
898
+ ('schema_version', '2.0.0'),
565
899
  ('created_at', strftime('%s', 'now') * 1000),
566
- ('vault_format', 'qa360-evidence-vault-v1');
900
+ ('vault_format', 'qa360-evidence-vault-v2');
567
901
 
568
902
  -- Triggers for updated_at timestamps
569
- CREATE TRIGGER IF NOT EXISTS trigger_runs_updated_at
903
+ CREATE TRIGGER IF NOT EXISTS trigger_runs_updated_at
570
904
  AFTER UPDATE ON runs
571
905
  BEGIN
572
906
  UPDATE runs SET updated_at = strftime('%s', 'now') * 1000 WHERE id = NEW.id;
@@ -578,6 +912,18 @@ CREATE TRIGGER IF NOT EXISTS trigger_artifacts_last_accessed
578
912
  UPDATE artifacts SET last_accessed = strftime('%s', 'now') * 1000 WHERE sha256 = NEW.sha256;
579
913
  END;
580
914
 
915
+ CREATE TRIGGER IF NOT EXISTS trigger_flakiness_patterns_updated_at
916
+ AFTER UPDATE ON flakiness_patterns
917
+ BEGIN
918
+ UPDATE flakiness_patterns SET updated_at = strftime('%s', 'now') * 1000 WHERE id = NEW.id;
919
+ END;
920
+
921
+ CREATE TRIGGER IF NOT EXISTS trigger_quarantine_updated_at
922
+ AFTER UPDATE ON flakiness_quarantine
923
+ BEGIN
924
+ UPDATE flakiness_quarantine SET updated_at = strftime('%s', 'now') * 1000 WHERE id = NEW.id;
925
+ END;
926
+
581
927
  -- Views for common queries
582
928
  CREATE VIEW IF NOT EXISTS v_recent_runs AS
583
929
  SELECT
@@ -600,7 +946,7 @@ GROUP BY r.id
600
946
  ORDER BY r.started_at DESC;
601
947
 
602
948
  CREATE VIEW IF NOT EXISTS v_gate_trends AS
603
- SELECT
949
+ SELECT
604
950
  g.name,
605
951
  g.status,
606
952
  r.started_at,
@@ -611,6 +957,55 @@ FROM gates g
611
957
  JOIN runs r ON g.run_id = r.id
612
958
  WHERE r.status IN ('passed', 'failed')
613
959
  ORDER BY r.started_at DESC;
960
+
961
+ -- Flakiness summary view: aggregate flakiness stats by test
962
+ CREATE VIEW IF NOT EXISTS v_flakiness_summary AS
963
+ SELECT
964
+ test_id,
965
+ test_name,
966
+ gate,
967
+ file_path,
968
+ MAX(score) as latest_score,
969
+ MIN(score) as worst_score,
970
+ AVG(score) as avg_score,
971
+ COUNT(*) as total_analyses,
972
+ MAX(category) as latest_category,
973
+ MAX(last_seen) as last_analyzed,
974
+ MAX(first_seen) as first_seen
975
+ FROM flakiness_history
976
+ GROUP BY test_id, test_name, gate, file_path
977
+ ORDER BY latest_score ASC;
978
+
979
+ -- Flakiness trends view: time series analysis
980
+ CREATE VIEW IF NOT EXISTS v_flakiness_trends AS
981
+ SELECT
982
+ DATE(created_at / 1000, 'unixepoch') as analysis_date,
983
+ category,
984
+ COUNT(*) as test_count,
985
+ AVG(score) as avg_score
986
+ FROM flakiness_history
987
+ WHERE created_at >= strftime('%s', 'now', '-30 days') * 1000
988
+ GROUP BY analysis_date, category
989
+ ORDER BY analysis_date DESC, category DESC;
990
+
991
+ -- Quarantined tests view
992
+ CREATE VIEW IF NOT EXISTS v_quarantined_tests AS
993
+ SELECT
994
+ q.*,
995
+ fh.score as current_score,
996
+ fh.category as current_category
997
+ FROM flakiness_quarantine q
998
+ LEFT JOIN (
999
+ SELECT test_id, score, category
1000
+ FROM flakiness_history fh1
1001
+ WHERE fh1.created_at = (
1002
+ SELECT MAX(fh2.created_at)
1003
+ FROM flakiness_history fh2
1004
+ WHERE fh2.test_id = fh1.test_id
1005
+ )
1006
+ ) fh ON q.test_id = fh.test_id
1007
+ WHERE q.resolved_at IS NULL
1008
+ ORDER BY q.quarantined_at DESC;
614
1009
  `;
615
1010
  return new Promise((resolve, reject) => {
616
1011
  this.db.exec(schema, (err) => {
@@ -0,0 +1,7 @@
1
+ /**
2
+ * QA360 Watch Module
3
+ *
4
+ * Continuous testing and benchmarking capabilities.
5
+ */
6
+ export { WatchMode, createWatchMode } from './watch-mode.js';
7
+ export type { WatchModeOptions, WatchRunResult, WatchStats, BenchmarkConfig, BenchmarkResult } from './watch-mode.js';
@@ -0,0 +1,6 @@
1
+ /**
2
+ * QA360 Watch Module
3
+ *
4
+ * Continuous testing and benchmarking capabilities.
5
+ */
6
+ export { WatchMode, createWatchMode } from './watch-mode.js';
@@ -0,0 +1,213 @@
1
+ /**
2
+ * QA360 Watch Mode
3
+ *
4
+ * Continuous testing mode that monitors file changes and re-runs tests.
5
+ * Features:
6
+ * - File watching with debounce
7
+ * - Smart test selection based on changed files
8
+ * - Benchmark mode for performance comparison
9
+ * - Statistics aggregation across runs
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const watcher = new WatchMode({
14
+ * workingDir: process.cwd(),
15
+ * debounceMs: 300,
16
+ * onRunStart: () => console.log('Running tests...'),
17
+ * onRunComplete: (result) => console.log('Done:', result)
18
+ * });
19
+ *
20
+ * await watcher.start();
21
+ * ```
22
+ */
23
+ import { type Phase3RunResult } from '../runner/phase3-runner.js';
24
+ /**
25
+ * Watch mode configuration options
26
+ */
27
+ export interface WatchModeOptions {
28
+ /** Working directory for tests */
29
+ workingDir: string;
30
+ /** Path to pack.yml file */
31
+ packPath?: string;
32
+ /** Debounce delay in milliseconds (default: 300) */
33
+ debounceMs?: number;
34
+ /** Patterns to ignore (default: node_modules, .git, dist) */
35
+ ignore?: string[];
36
+ /** Whether to run initial test on start (default: true) */
37
+ runOnStart?: boolean;
38
+ /** Whether to clear screen between runs (default: true) */
39
+ clearScreen?: boolean;
40
+ /** Callback when a run starts */
41
+ onRunStart?: (runNumber: number) => void;
42
+ /** Callback when a run completes */
43
+ onRunComplete?: (result: WatchRunResult) => void;
44
+ /** Callback when a file changes */
45
+ onFileChange?: (path: string) => void;
46
+ /** Callback on error */
47
+ onError?: (error: Error) => void;
48
+ }
49
+ /**
50
+ * Result of a single watch run
51
+ */
52
+ export interface WatchRunResult {
53
+ /** Run number (incrementing) */
54
+ runNumber: number;
55
+ /** Timestamp when run started */
56
+ startedAt: Date;
57
+ /** Timestamp when run completed */
58
+ completedAt: Date;
59
+ /** Duration in milliseconds */
60
+ duration: number;
61
+ /** Phase3Runner result */
62
+ result: Phase3RunResult;
63
+ /** Files that triggered this run */
64
+ changedFiles: string[];
65
+ /** Whether this run was successful */
66
+ success: boolean;
67
+ }
68
+ /**
69
+ * Aggregated statistics across watch runs
70
+ */
71
+ export interface WatchStats {
72
+ /** Total number of runs */
73
+ totalRuns: number;
74
+ /** Number of successful runs */
75
+ successfulRuns: number;
76
+ /** Number of failed runs */
77
+ failedRuns: number;
78
+ /** Average duration in milliseconds */
79
+ avgDuration: number;
80
+ /** Fastest run duration */
81
+ fastestRun: number;
82
+ /** Slowest run duration */
83
+ slowestRun: number;
84
+ /** Total duration of all runs */
85
+ totalDuration: number;
86
+ /** Trust score average */
87
+ avgTrustScore: number;
88
+ /** Total tests executed */
89
+ totalTests: number;
90
+ /** Total passed tests */
91
+ totalPassed: number;
92
+ /** Total failed tests */
93
+ totalFailed: number;
94
+ }
95
+ /**
96
+ * Benchmark configuration
97
+ */
98
+ export interface BenchmarkConfig {
99
+ /** Number of iterations to run */
100
+ iterations: number;
101
+ /** Warm-up iterations (not counted in stats) */
102
+ warmupIterations: number;
103
+ /** Whether to run with cache enabled */
104
+ withCache: boolean;
105
+ /** Whether to run in parallel */
106
+ parallel?: boolean;
107
+ }
108
+ /**
109
+ * Benchmark result
110
+ */
111
+ export interface BenchmarkResult {
112
+ /** Average duration across iterations */
113
+ avgDuration: number;
114
+ /** Minimum duration */
115
+ minDuration: number;
116
+ /** Maximum duration */
117
+ maxDuration: number;
118
+ /** Standard deviation */
119
+ stdDev: number;
120
+ /** Median duration */
121
+ median: number;
122
+ /** P95 duration */
123
+ p95: number;
124
+ /** P99 duration */
125
+ p99: number;
126
+ /** Number of successful runs */
127
+ successfulRuns: number;
128
+ /** Total iterations */
129
+ totalIterations: number;
130
+ /** Throughput (runs per second) */
131
+ throughput: number;
132
+ }
133
+ /**
134
+ * Watch Mode Manager
135
+ *
136
+ * Monitors file changes and runs tests automatically.
137
+ */
138
+ export declare class WatchMode {
139
+ private options;
140
+ private watchers;
141
+ private runNumber;
142
+ private isRunning;
143
+ private debounceTimer?;
144
+ private pendingChanges;
145
+ private runResults;
146
+ private stats;
147
+ private packConfig?;
148
+ constructor(options: WatchModeOptions);
149
+ /**
150
+ * Start watching files and running tests
151
+ */
152
+ start(): Promise<void>;
153
+ /**
154
+ * Stop watching files
155
+ */
156
+ stop(): Promise<void>;
157
+ /**
158
+ * Get current statistics
159
+ */
160
+ getStats(): WatchStats;
161
+ /**
162
+ * Get run history
163
+ */
164
+ getRunHistory(): WatchRunResult[];
165
+ /**
166
+ * Run benchmark tests
167
+ */
168
+ benchmark(config: BenchmarkConfig): Promise<BenchmarkResult>;
169
+ /**
170
+ * Load pack configuration
171
+ */
172
+ private loadPackConfig;
173
+ /**
174
+ * Setup file watcher using native Node.js fs.watch
175
+ */
176
+ private setupWatcher;
177
+ /**
178
+ * Check if a path should be ignored
179
+ */
180
+ private shouldIgnore;
181
+ /**
182
+ * Handle file change with debouncing
183
+ */
184
+ private handleFileChange;
185
+ /**
186
+ * Run tests and update statistics
187
+ */
188
+ private runTests;
189
+ /**
190
+ * Update aggregated statistics
191
+ */
192
+ private updateStats;
193
+ /**
194
+ * Calculate benchmark statistics
195
+ */
196
+ private calculateBenchmarkStats;
197
+ /**
198
+ * Log formatted run result
199
+ */
200
+ private logRunResult;
201
+ /**
202
+ * Format statistics for display
203
+ */
204
+ private formatStats;
205
+ /**
206
+ * Log message
207
+ */
208
+ private log;
209
+ }
210
+ /**
211
+ * Factory function to create a watch mode instance
212
+ */
213
+ export declare function createWatchMode(options: WatchModeOptions): WatchMode;