squally-mcp 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@
3
3
  "jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
4
4
  "info": {
5
5
  "title": "Squally Read API",
6
- "version": "1.0.0-beta",
6
+ "version": "1.0.0-beta.2",
7
7
  "description": "Read access to a Squally organization's test results, for scripts, CI jobs and AI agents.\n\n**Authentication.** Every request carries an organization read key as `Authorization: Bearer sqly_ro_…`. A read key is organization-wide (every project), created by an organization admin in Settings → API keys, and expires after 30, 60, 90, 180 days as chosen at creation. There is no renewal: create a new key and revoke the old one. Project ingest keys (`sqly_…`) are refused with `wrong_key_type`.\n\n**Expiry warning.** From 7 days before a key expires, every authenticated response carries `Squally-Key-Expires-At` with the expiry instant (ISO-8601).\n\n**Rate limit.** Currently 60 requests per 60 seconds per key, counted per `Authorization` header - a read key has its own budget, shared with nothing else. Over the limit the edge answers 429 without a JSON body.\n\n**Retries.** 429: wait 60 seconds, then retry. The edge counts requests per key in 60-second windows and sends no Retry-After header. 5xx: retry with exponential backoff, starting at 1 second and capped at 30 seconds. 400, 401, 403, 404: never retry - the same request gets the same answer.\n\n**Plans.** The read API is part of the Standard and Pro plans. A key of an organization on Free is answered 403 `tier_too_low`; keys are not revoked on a downgrade and work again after an upgrade.\n\n**Errors.** Every error is `{\"error\": \"<sentence>\", \"code\": \"<slug>\"}`. The sentence may be reworded; the slug may not. An id that does not exist and an id that belongs to another organization get the same 404, byte for byte.\n\n**Test names in paths.** Test names are Playwright title paths and often contain `/`: URL-encode them, including `/` as `%2F`.\n\n**Sharded runs** are returned as one logical run: /runs lists one row per logical run, and the run routes answer for all shards whichever shard's id you pass.\n\n**Stability.** v1, beta. The API may change until the beta ends; every change is listed in the changelog (docs/openapi/CHANGELOG.md)."
8
8
  },
9
9
  "servers": [
@@ -27,11 +27,7 @@
27
27
  },
28
28
  {
29
29
  "name": "Tests",
30
- "description": "One test by name: its stored flakiness verdict and its history across runs."
31
- },
32
- {
33
- "name": "Flakiness",
34
- "description": "Flaky and broken tests as the flakiness engine judges them."
30
+ "description": "Every test's metrics over a period, and one test by name: its metrics and its history across runs."
35
31
  },
36
32
  {
37
33
  "name": "Errors",
@@ -61,7 +57,7 @@
61
57
  "code": "invalid_period",
62
58
  "status": 400,
63
59
  "when": "days=90 is beyond this plan's 60-day retention. Use 60 or less.",
64
- "action": "Use days=7, 30 or 90, within the plan's retention; do not retry.",
60
+ "action": "Use days=7, 14, 30 or 90, within the plan's retention; do not retry.",
65
61
  "hasBody": true
66
62
  },
67
63
  {
@@ -364,6 +360,7 @@
364
360
  "type": "integer",
365
361
  "enum": [
366
362
  7,
363
+ 14,
367
364
  30,
368
365
  90
369
366
  ],
@@ -482,10 +479,10 @@
482
479
  "durationMs": 515000,
483
480
  "counts": {
484
481
  "total": 412,
485
- "passed": 409,
482
+ "passed": 408,
486
483
  "failed": 0,
487
484
  "skipped": 3,
488
- "recovered": 1
485
+ "flaky": 1
489
486
  },
490
487
  "shards": {
491
488
  "received": 2,
@@ -508,7 +505,7 @@
508
505
  "passed": 401,
509
506
  "failed": 2,
510
507
  "skipped": 2,
511
- "recovered": 0
508
+ "flaky": 0
512
509
  },
513
510
  "shards": null
514
511
  }
@@ -827,10 +824,10 @@
827
824
  },
828
825
  "counts": {
829
826
  "total": 412,
830
- "passed": 409,
827
+ "passed": 408,
831
828
  "failed": 0,
832
829
  "skipped": 3,
833
- "recovered": 1
830
+ "flaky": 1
834
831
  },
835
832
  "tests": [
836
833
  {
@@ -1148,7 +1145,7 @@
1148
1145
  }
1149
1146
  }
1150
1147
  ],
1151
- "copyPrompt": "You have the repository open. Find the test below, explain the failure and propose a fix.\n\nFailing test: chromium > checkout.spec.ts > Checkout > applies a coupon code\nFile: tests/checkout.spec.ts\nBrowser: chromium\nRun branch: main\nAttempt: first try\nOutcome: a later retry of this test passed (the test is flaky rather than consistently broken).\n\nError message:\nError: expect(locator).toHaveText(expected) failed\n\nLocator: getByTestId('order-total')\nExpected: \"€ 45.00\"\nReceived: \"€ 50.00\"\nTimeout: 5000ms\nStack (top frames):\nat /home/runner/work/storefront-web/tests/checkout.spec.ts:48:52\nAttempt duration: 6210 ms\nFailure category: Assertion\nStatus from the flakiness engine: flaky (signals: Retry rate). Treat this as evidence to weigh, not as a verdict.",
1148
+ "copyPrompt": "You have the repository open. Find the test below, explain the failure and propose a fix.\n\nFailing test: chromium > checkout.spec.ts > Checkout > applies a coupon code\nFile: tests/checkout.spec.ts\nBrowser: chromium\nRun branch: main\nAttempt: first try\nOutcome: a later retry of this test passed (the test is flaky rather than consistently broken).\n\nError message:\nError: expect(locator).toHaveText(expected) failed\n\nLocator: getByTestId('order-total')\nExpected: \"€ 45.00\"\nReceived: \"€ 50.00\"\nTimeout: 5000ms\nStack (top frames):\nat /home/runner/work/storefront-web/tests/checkout.spec.ts:48:52\nAttempt duration: 6210 ms\nFailure category: Assertion\nLast 14 days on CI: flaky rate 50% (4/8), failure rate 0% (0/8)",
1152
1149
  "shard": {
1153
1150
  "index": 1,
1154
1151
  "runId": "b1e0c6d2-7a43-4f19-8d5e-0c3a9e6f2b71"
@@ -1634,13 +1631,13 @@
1634
1631
  }
1635
1632
  }
1636
1633
  },
1637
- "/projects/{projectId}/flaky": {
1634
+ "/projects/{projectId}/tests": {
1638
1635
  "get": {
1639
- "operationId": "listFlakyTests",
1640
- "summary": "List flaky tests",
1641
- "description": "List flaky and broken tests (expensive). Flaky and broken tests as the flakiness engine judges them over its fixed 14-day window, ranked by time lost. THE EXPENSIVE ENDPOINT: it runs the engine live on every call - call it once per question, not in a loop, and use the test status route for a single test.",
1636
+ "operationId": "listTests",
1637
+ "summary": "List tests",
1638
+ "description": "Every test with a CI run in the period and its metrics - stability, flaky rate, failure rate, runs, time lost, its last 20 runs, the branches it ran on and where it fails most - ranked by the chosen sort. Counts completed CI runs only, without mass-failure runs, on every branch unless branch names one (see population). Moderate: two queries over the period's CI results, paged in the database; use the metrics route for one test.",
1642
1639
  "tags": [
1643
- "Flakiness"
1640
+ "Tests"
1644
1641
  ],
1645
1642
  "parameters": [
1646
1643
  {
@@ -1653,44 +1650,71 @@
1653
1650
  }
1654
1651
  },
1655
1652
  {
1656
- "name": "status",
1653
+ "name": "days",
1657
1654
  "in": "query",
1658
1655
  "required": false,
1659
- "description": "Only tests with this verdict.",
1656
+ "description": "The period, in days, ending now. Must not exceed the plan's retention (Standard 60, Pro 90 days); a longer period is refused with 400 invalid_period rather than answered with less data.",
1660
1657
  "schema": {
1661
- "type": "string",
1658
+ "type": "integer",
1662
1659
  "enum": [
1663
- "flaky",
1664
- "broken"
1665
- ]
1660
+ 7,
1661
+ 14,
1662
+ 30,
1663
+ 90
1664
+ ],
1665
+ "default": 14
1666
1666
  }
1667
1667
  },
1668
1668
  {
1669
- "name": "sort",
1669
+ "name": "search",
1670
1670
  "in": "query",
1671
1671
  "required": false,
1672
- "description": "Ranking: time lost (default), flake rate, or impact on the stable branch.",
1672
+ "description": "Case-insensitive substring of the test name or its file.",
1673
1673
  "schema": {
1674
1674
  "type": "string",
1675
- "enum": [
1676
- "timeLost",
1677
- "rate",
1678
- "impact"
1679
- ],
1680
- "default": "timeLost"
1675
+ "minLength": 1,
1676
+ "maxLength": 200
1681
1677
  }
1682
1678
  },
1683
1679
  {
1684
- "name": "search",
1680
+ "name": "branch",
1685
1681
  "in": "query",
1686
1682
  "required": false,
1687
- "description": "Case-insensitive substring of the test name or its file.",
1683
+ "description": "Only runs on this branch, the name exactly. Every branch when absent.",
1684
+ "schema": {
1685
+ "type": "string",
1686
+ "minLength": 1,
1687
+ "maxLength": 200
1688
+ }
1689
+ },
1690
+ {
1691
+ "name": "browser",
1692
+ "in": "query",
1693
+ "required": false,
1694
+ "description": "Only tests of this Playwright project - the items' browser field, exactly (e.g. chromium). Every test when absent.",
1688
1695
  "schema": {
1689
1696
  "type": "string",
1690
1697
  "minLength": 1,
1691
1698
  "maxLength": 200
1692
1699
  }
1693
1700
  },
1701
+ {
1702
+ "name": "sort",
1703
+ "in": "query",
1704
+ "required": false,
1705
+ "description": "Ranking, the worst first: stability (the default) lowest first; flakyRate, failureRate, runs and timeLost highest first. Ties go to the most runs (for runs: the lowest stability), then to the test name and file. A rate with nothing to divide by (null) sorts after every number.",
1706
+ "schema": {
1707
+ "type": "string",
1708
+ "enum": [
1709
+ "stability",
1710
+ "flakyRate",
1711
+ "failureRate",
1712
+ "runs",
1713
+ "timeLost"
1714
+ ],
1715
+ "default": "stability"
1716
+ }
1717
+ },
1694
1718
  {
1695
1719
  "name": "page",
1696
1720
  "in": "query",
@@ -1706,15 +1730,12 @@
1706
1730
  "name": "perPage",
1707
1731
  "in": "query",
1708
1732
  "required": false,
1709
- "description": "Rows per page.",
1733
+ "description": "Rows per page, 1 to 100.",
1710
1734
  "schema": {
1711
1735
  "type": "integer",
1712
- "enum": [
1713
- 10,
1714
- 25,
1715
- 50
1716
- ],
1717
- "default": 10
1736
+ "minimum": 1,
1737
+ "maximum": 100,
1738
+ "default": 50
1718
1739
  }
1719
1740
  }
1720
1741
  ],
@@ -1729,74 +1750,272 @@
1729
1750
  "content": {
1730
1751
  "application/json": {
1731
1752
  "schema": {
1732
- "$ref": "#/components/schemas/FlakyResponse"
1753
+ "$ref": "#/components/schemas/TestsResponse"
1733
1754
  },
1734
1755
  "example": {
1735
1756
  "project": {
1736
1757
  "id": "3f7c2a91-5b8e-4d0a-9c61-2e4b7f0d8a15",
1737
1758
  "name": "storefront-web"
1738
1759
  },
1739
- "windowDays": 14,
1740
- "sort": "timeLost",
1741
- "page": {
1742
- "page": 1,
1743
- "perPage": 10,
1744
- "total": 3,
1745
- "pageCount": 1
1746
- },
1747
- "tests": [
1760
+ "population": "ci_completed_not_excluded",
1761
+ "days": 14,
1762
+ "since": "2026-09-10T08:00:00.000Z",
1763
+ "until": "2026-09-24T08:00:00.000Z",
1764
+ "branch": null,
1765
+ "browser": null,
1766
+ "sort": "stability",
1767
+ "page": 1,
1768
+ "perPage": 3,
1769
+ "total": 409,
1770
+ "pageCount": 137,
1771
+ "items": [
1748
1772
  {
1749
1773
  "testName": "chromium > checkout.spec.ts > Checkout > applies a coupon code",
1750
1774
  "filePath": "tests/checkout.spec.ts",
1751
- "status": "flaky",
1752
- "statusReason": "retry_rate",
1753
- "alsoActive": [
1754
- "same_sha_flip"
1775
+ "browser": "chromium",
1776
+ "runs": 8,
1777
+ "stableRuns": 4,
1778
+ "flakyRuns": 4,
1779
+ "failedRuns": 0,
1780
+ "stability": 0.5,
1781
+ "flakyRate": 0.5,
1782
+ "failureRate": 0,
1783
+ "timeLostMs": 7361,
1784
+ "recentResults": [
1785
+ {
1786
+ "runId": "5c8e2a17-9d4b-4f60-b3a1-7e2d9c0f4b58",
1787
+ "runNumber": 1261,
1788
+ "startedAt": "2026-09-12T07:30:41.000Z",
1789
+ "branch": "main",
1790
+ "result": "stable"
1791
+ },
1792
+ {
1793
+ "runId": "c2e7a940-5d1b-4f8e-9a36-0b4d7e1c5f92",
1794
+ "runNumber": 1264,
1795
+ "startedAt": "2026-09-14T15:12:09.000Z",
1796
+ "branch": "feature/saved-carts",
1797
+ "result": "flaky"
1798
+ },
1799
+ {
1800
+ "runId": "a3f19c62-0e7d-4b85-9c24-6d1e8b3f7a09",
1801
+ "runNumber": 1266,
1802
+ "startedAt": "2026-09-15T07:31:03.000Z",
1803
+ "branch": "main",
1804
+ "result": "flaky"
1805
+ },
1806
+ {
1807
+ "runId": "d07b4e95-6a2c-4e13-8f5d-2b9c1a7e6f30",
1808
+ "runNumber": 1270,
1809
+ "startedAt": "2026-09-17T07:29:58.000Z",
1810
+ "branch": "main",
1811
+ "result": "flaky"
1812
+ },
1813
+ {
1814
+ "runId": "6b1f8d23-9e4a-4c07-b5d2-3a8e0f6c1d74",
1815
+ "runNumber": 1273,
1816
+ "startedAt": "2026-09-18T11:47:35.000Z",
1817
+ "branch": "feature/saved-carts",
1818
+ "result": "stable"
1819
+ },
1820
+ {
1821
+ "runId": "8f6a3d21-4c9e-4a7b-b052-e1d7c3a9f864",
1822
+ "runNumber": 1277,
1823
+ "startedAt": "2026-09-20T07:32:16.000Z",
1824
+ "branch": "main",
1825
+ "result": "stable"
1826
+ },
1827
+ {
1828
+ "runId": "7d2f4b19-3c8e-4a61-9e07-5b1d8c2a6f34",
1829
+ "runNumber": 1283,
1830
+ "startedAt": "2026-09-23T16:05:40.000Z",
1831
+ "branch": "feature/saved-carts",
1832
+ "result": "stable"
1833
+ },
1834
+ {
1835
+ "runId": "b1e0c6d2-7a43-4f19-8d5e-0c3a9e6f2b71",
1836
+ "runNumber": 1284,
1837
+ "startedAt": "2026-09-24T07:31:12.000Z",
1838
+ "branch": "main",
1839
+ "result": "flaky"
1840
+ }
1755
1841
  ],
1756
- "timeLostMs": 1968000,
1757
- "flakeScore": 0.42,
1758
- "flipRate": 0.31,
1759
- "failureRate": 0.18,
1760
- "runsConsidered": 38,
1761
- "failureStreak": 0,
1762
- "lastSeenAt": "2026-09-24T07:34:05.000Z",
1763
- "lastRun": {
1764
- "runId": "b1e0c6d2-7a43-4f19-8d5e-0c3a9e6f2b71",
1765
- "runNumber": 1284,
1766
- "status": "passed",
1767
- "startedAt": "2026-09-24T07:31:12.000Z"
1842
+ "branches": [
1843
+ "feature/saved-carts",
1844
+ "main"
1845
+ ],
1846
+ "topBranch": {
1847
+ "branch": "main",
1848
+ "kind": "retries",
1849
+ "count": 3
1768
1850
  }
1769
1851
  },
1770
1852
  {
1771
1853
  "testName": "chromium > search.spec.ts > Search > filters by date",
1772
1854
  "filePath": "tests/search.spec.ts",
1773
- "status": "broken",
1774
- "statusReason": "consecutive_failures",
1775
- "alsoActive": [
1776
- "failure_rate_broken"
1855
+ "browser": "chromium",
1856
+ "runs": 8,
1857
+ "stableRuns": 5,
1858
+ "flakyRuns": 0,
1859
+ "failedRuns": 3,
1860
+ "stability": 0.625,
1861
+ "flakyRate": 0,
1862
+ "failureRate": 0.375,
1863
+ "timeLostMs": 14308,
1864
+ "recentResults": [
1865
+ {
1866
+ "runId": "5c8e2a17-9d4b-4f60-b3a1-7e2d9c0f4b58",
1867
+ "runNumber": 1261,
1868
+ "startedAt": "2026-09-12T07:30:41.000Z",
1869
+ "branch": "main",
1870
+ "result": "stable"
1871
+ },
1872
+ {
1873
+ "runId": "c2e7a940-5d1b-4f8e-9a36-0b4d7e1c5f92",
1874
+ "runNumber": 1264,
1875
+ "startedAt": "2026-09-14T15:12:09.000Z",
1876
+ "branch": "feature/saved-carts",
1877
+ "result": "stable"
1878
+ },
1879
+ {
1880
+ "runId": "a3f19c62-0e7d-4b85-9c24-6d1e8b3f7a09",
1881
+ "runNumber": 1266,
1882
+ "startedAt": "2026-09-15T07:31:03.000Z",
1883
+ "branch": "main",
1884
+ "result": "stable"
1885
+ },
1886
+ {
1887
+ "runId": "d07b4e95-6a2c-4e13-8f5d-2b9c1a7e6f30",
1888
+ "runNumber": 1270,
1889
+ "startedAt": "2026-09-17T07:29:58.000Z",
1890
+ "branch": "main",
1891
+ "result": "failed"
1892
+ },
1893
+ {
1894
+ "runId": "6b1f8d23-9e4a-4c07-b5d2-3a8e0f6c1d74",
1895
+ "runNumber": 1273,
1896
+ "startedAt": "2026-09-18T11:47:35.000Z",
1897
+ "branch": "feature/saved-carts",
1898
+ "result": "stable"
1899
+ },
1900
+ {
1901
+ "runId": "8f6a3d21-4c9e-4a7b-b052-e1d7c3a9f864",
1902
+ "runNumber": 1277,
1903
+ "startedAt": "2026-09-20T07:32:16.000Z",
1904
+ "branch": "main",
1905
+ "result": "failed"
1906
+ },
1907
+ {
1908
+ "runId": "7d2f4b19-3c8e-4a61-9e07-5b1d8c2a6f34",
1909
+ "runNumber": 1283,
1910
+ "startedAt": "2026-09-23T16:05:40.000Z",
1911
+ "branch": "feature/saved-carts",
1912
+ "result": "failed"
1913
+ },
1914
+ {
1915
+ "runId": "b1e0c6d2-7a43-4f19-8d5e-0c3a9e6f2b71",
1916
+ "runNumber": 1284,
1917
+ "startedAt": "2026-09-24T07:31:12.000Z",
1918
+ "branch": "main",
1919
+ "result": "stable"
1920
+ }
1921
+ ],
1922
+ "branches": [
1923
+ "feature/saved-carts",
1924
+ "main"
1925
+ ],
1926
+ "topBranch": {
1927
+ "branch": "main",
1928
+ "kind": "failures",
1929
+ "count": 2
1930
+ }
1931
+ },
1932
+ {
1933
+ "testName": "chromium > auth.spec.ts > Login > remembers the device",
1934
+ "filePath": "tests/auth.spec.ts",
1935
+ "browser": "chromium",
1936
+ "runs": 8,
1937
+ "stableRuns": 7,
1938
+ "flakyRuns": 1,
1939
+ "failedRuns": 0,
1940
+ "stability": 0.875,
1941
+ "flakyRate": 0.125,
1942
+ "failureRate": 0,
1943
+ "timeLostMs": 3071,
1944
+ "recentResults": [
1945
+ {
1946
+ "runId": "5c8e2a17-9d4b-4f60-b3a1-7e2d9c0f4b58",
1947
+ "runNumber": 1261,
1948
+ "startedAt": "2026-09-12T07:30:41.000Z",
1949
+ "branch": "main",
1950
+ "result": "stable"
1951
+ },
1952
+ {
1953
+ "runId": "c2e7a940-5d1b-4f8e-9a36-0b4d7e1c5f92",
1954
+ "runNumber": 1264,
1955
+ "startedAt": "2026-09-14T15:12:09.000Z",
1956
+ "branch": "feature/saved-carts",
1957
+ "result": "flaky"
1958
+ },
1959
+ {
1960
+ "runId": "a3f19c62-0e7d-4b85-9c24-6d1e8b3f7a09",
1961
+ "runNumber": 1266,
1962
+ "startedAt": "2026-09-15T07:31:03.000Z",
1963
+ "branch": "main",
1964
+ "result": "stable"
1965
+ },
1966
+ {
1967
+ "runId": "d07b4e95-6a2c-4e13-8f5d-2b9c1a7e6f30",
1968
+ "runNumber": 1270,
1969
+ "startedAt": "2026-09-17T07:29:58.000Z",
1970
+ "branch": "main",
1971
+ "result": "stable"
1972
+ },
1973
+ {
1974
+ "runId": "6b1f8d23-9e4a-4c07-b5d2-3a8e0f6c1d74",
1975
+ "runNumber": 1273,
1976
+ "startedAt": "2026-09-18T11:47:35.000Z",
1977
+ "branch": "feature/saved-carts",
1978
+ "result": "stable"
1979
+ },
1980
+ {
1981
+ "runId": "8f6a3d21-4c9e-4a7b-b052-e1d7c3a9f864",
1982
+ "runNumber": 1277,
1983
+ "startedAt": "2026-09-20T07:32:16.000Z",
1984
+ "branch": "main",
1985
+ "result": "stable"
1986
+ },
1987
+ {
1988
+ "runId": "7d2f4b19-3c8e-4a61-9e07-5b1d8c2a6f34",
1989
+ "runNumber": 1283,
1990
+ "startedAt": "2026-09-23T16:05:40.000Z",
1991
+ "branch": "feature/saved-carts",
1992
+ "result": "stable"
1993
+ },
1994
+ {
1995
+ "runId": "b1e0c6d2-7a43-4f19-8d5e-0c3a9e6f2b71",
1996
+ "runNumber": 1284,
1997
+ "startedAt": "2026-09-24T07:31:12.000Z",
1998
+ "branch": "main",
1999
+ "result": "stable"
2000
+ }
1777
2001
  ],
1778
- "timeLostMs": 540000,
1779
- "flakeScore": 0.9,
1780
- "flipRate": 0.05,
1781
- "failureRate": 1,
1782
- "runsConsidered": 12,
1783
- "failureStreak": 12,
1784
- "lastSeenAt": "2026-09-23T16:09:31.000Z",
1785
- "lastRun": {
1786
- "runId": "7d2f4b19-3c8e-4a61-9e07-5b1d8c2a6f34",
1787
- "runNumber": 1283,
1788
- "status": "failed",
1789
- "startedAt": "2026-09-23T16:05:40.000Z"
2002
+ "branches": [
2003
+ "feature/saved-carts",
2004
+ "main"
2005
+ ],
2006
+ "topBranch": {
2007
+ "branch": "feature/saved-carts",
2008
+ "kind": "retries",
2009
+ "count": 1
1790
2010
  }
1791
2011
  }
1792
- ],
1793
- "gatheringDataCount": 6
2012
+ ]
1794
2013
  }
1795
2014
  }
1796
2015
  }
1797
2016
  },
1798
2017
  "400": {
1799
- "description": "Error: invalid_parameter",
2018
+ "description": "Error: invalid_parameter, invalid_period",
1800
2019
  "content": {
1801
2020
  "application/json": {
1802
2021
  "schema": {
@@ -1809,7 +2028,8 @@
1809
2028
  "code": {
1810
2029
  "type": "string",
1811
2030
  "enum": [
1812
- "invalid_parameter"
2031
+ "invalid_parameter",
2032
+ "invalid_period"
1813
2033
  ],
1814
2034
  "description": "Stable slug - match on this, not on the sentence."
1815
2035
  }
@@ -1826,6 +2046,13 @@
1826
2046
  "error": "sha must be 4 to 40 hexadecimal characters (a commit SHA or a prefix of one).",
1827
2047
  "code": "invalid_parameter"
1828
2048
  }
2049
+ },
2050
+ "invalid_period": {
2051
+ "summary": "invalid_period",
2052
+ "value": {
2053
+ "error": "days=90 is beyond this plan's 60-day retention. Use 60 or less.",
2054
+ "code": "invalid_period"
2055
+ }
1829
2056
  }
1830
2057
  }
1831
2058
  }
@@ -2013,11 +2240,11 @@
2013
2240
  }
2014
2241
  }
2015
2242
  },
2016
- "/projects/{projectId}/tests/{testName}/status": {
2243
+ "/projects/{projectId}/tests/{testName}/metrics": {
2017
2244
  "get": {
2018
- "operationId": "getTestStatus",
2019
- "summary": "Get test status",
2020
- "description": "Get one test's flakiness verdict (cheap). The stored verdict for one test - status, active signals, labels - as of the last ingest. THE CHEAP ONE: a single lookup, no engine pass; prefer it to /flaky when the question is about one test.",
2245
+ "operationId": "getTestMetrics",
2246
+ "summary": "Get test metrics",
2247
+ "description": "One test's metrics over the period: the fields of its /tests row, flat, beside the same period. A test with results but no CI run in the period answers zeros and null rates rather than 404. Cheap: the list's queries, narrowed to one test; prefer it to /tests when the question is about one test.",
2021
2248
  "tags": [
2022
2249
  "Tests"
2023
2250
  ],
@@ -2048,6 +2275,33 @@
2048
2275
  "schema": {
2049
2276
  "type": "string"
2050
2277
  }
2278
+ },
2279
+ {
2280
+ "name": "days",
2281
+ "in": "query",
2282
+ "required": false,
2283
+ "description": "The period, in days, ending now. Must not exceed the plan's retention (Standard 60, Pro 90 days); a longer period is refused with 400 invalid_period rather than answered with less data.",
2284
+ "schema": {
2285
+ "type": "integer",
2286
+ "enum": [
2287
+ 7,
2288
+ 14,
2289
+ 30,
2290
+ 90
2291
+ ],
2292
+ "default": 14
2293
+ }
2294
+ },
2295
+ {
2296
+ "name": "branch",
2297
+ "in": "query",
2298
+ "required": false,
2299
+ "description": "Only runs on this branch, the name exactly. Every branch when absent.",
2300
+ "schema": {
2301
+ "type": "string",
2302
+ "minLength": 1,
2303
+ "maxLength": 200
2304
+ }
2051
2305
  }
2052
2306
  ],
2053
2307
  "responses": {
@@ -2061,38 +2315,102 @@
2061
2315
  "content": {
2062
2316
  "application/json": {
2063
2317
  "schema": {
2064
- "$ref": "#/components/schemas/V4StatusResponse"
2318
+ "$ref": "#/components/schemas/TestMetricsResponse"
2065
2319
  },
2066
2320
  "example": {
2067
2321
  "project": {
2068
2322
  "id": "3f7c2a91-5b8e-4d0a-9c61-2e4b7f0d8a15",
2069
2323
  "name": "storefront-web"
2070
2324
  },
2071
- "test": {
2072
- "name": "chromium > checkout.spec.ts > Checkout > applies a coupon code",
2073
- "filePath": "tests/checkout.spec.ts"
2074
- },
2075
- "status": "flaky",
2076
- "awaitingFirstWrite": false,
2077
- "activeSignals": [
2325
+ "population": "ci_completed_not_excluded",
2326
+ "days": 14,
2327
+ "since": "2026-09-10T08:00:00.000Z",
2328
+ "until": "2026-09-24T08:00:00.000Z",
2329
+ "branch": null,
2330
+ "testName": "chromium > checkout.spec.ts > Checkout > applies a coupon code",
2331
+ "filePath": "tests/checkout.spec.ts",
2332
+ "browser": "chromium",
2333
+ "runs": 8,
2334
+ "stableRuns": 4,
2335
+ "flakyRuns": 4,
2336
+ "failedRuns": 0,
2337
+ "stability": 0.5,
2338
+ "flakyRate": 0.5,
2339
+ "failureRate": 0,
2340
+ "timeLostMs": 7361,
2341
+ "recentResults": [
2342
+ {
2343
+ "runId": "5c8e2a17-9d4b-4f60-b3a1-7e2d9c0f4b58",
2344
+ "runNumber": 1261,
2345
+ "startedAt": "2026-09-12T07:30:41.000Z",
2346
+ "branch": "main",
2347
+ "result": "stable"
2348
+ },
2078
2349
  {
2079
- "signal": "retry_rate",
2080
- "label": "Retry rate"
2350
+ "runId": "c2e7a940-5d1b-4f8e-9a36-0b4d7e1c5f92",
2351
+ "runNumber": 1264,
2352
+ "startedAt": "2026-09-14T15:12:09.000Z",
2353
+ "branch": "feature/saved-carts",
2354
+ "result": "flaky"
2355
+ },
2356
+ {
2357
+ "runId": "a3f19c62-0e7d-4b85-9c24-6d1e8b3f7a09",
2358
+ "runNumber": 1266,
2359
+ "startedAt": "2026-09-15T07:31:03.000Z",
2360
+ "branch": "main",
2361
+ "result": "flaky"
2362
+ },
2363
+ {
2364
+ "runId": "d07b4e95-6a2c-4e13-8f5d-2b9c1a7e6f30",
2365
+ "runNumber": 1270,
2366
+ "startedAt": "2026-09-17T07:29:58.000Z",
2367
+ "branch": "main",
2368
+ "result": "flaky"
2369
+ },
2370
+ {
2371
+ "runId": "6b1f8d23-9e4a-4c07-b5d2-3a8e0f6c1d74",
2372
+ "runNumber": 1273,
2373
+ "startedAt": "2026-09-18T11:47:35.000Z",
2374
+ "branch": "feature/saved-carts",
2375
+ "result": "stable"
2376
+ },
2377
+ {
2378
+ "runId": "8f6a3d21-4c9e-4a7b-b052-e1d7c3a9f864",
2379
+ "runNumber": 1277,
2380
+ "startedAt": "2026-09-20T07:32:16.000Z",
2381
+ "branch": "main",
2382
+ "result": "stable"
2383
+ },
2384
+ {
2385
+ "runId": "7d2f4b19-3c8e-4a61-9e07-5b1d8c2a6f34",
2386
+ "runNumber": 1283,
2387
+ "startedAt": "2026-09-23T16:05:40.000Z",
2388
+ "branch": "feature/saved-carts",
2389
+ "result": "stable"
2081
2390
  },
2082
2391
  {
2083
- "signal": "same_sha_flip",
2084
- "label": "Same-code flip"
2392
+ "runId": "b1e0c6d2-7a43-4f19-8d5e-0c3a9e6f2b71",
2393
+ "runNumber": 1284,
2394
+ "startedAt": "2026-09-24T07:31:12.000Z",
2395
+ "branch": "main",
2396
+ "result": "flaky"
2085
2397
  }
2086
2398
  ],
2087
- "labels": [
2088
- "retries"
2089
- ]
2399
+ "branches": [
2400
+ "feature/saved-carts",
2401
+ "main"
2402
+ ],
2403
+ "topBranch": {
2404
+ "branch": "main",
2405
+ "kind": "retries",
2406
+ "count": 3
2407
+ }
2090
2408
  }
2091
2409
  }
2092
2410
  }
2093
2411
  },
2094
2412
  "400": {
2095
- "description": "Error: ambiguous_test",
2413
+ "description": "Error: ambiguous_test, invalid_parameter, invalid_period",
2096
2414
  "content": {
2097
2415
  "application/json": {
2098
2416
  "schema": {
@@ -2105,7 +2423,9 @@
2105
2423
  "code": {
2106
2424
  "type": "string",
2107
2425
  "enum": [
2108
- "ambiguous_test"
2426
+ "ambiguous_test",
2427
+ "invalid_parameter",
2428
+ "invalid_period"
2109
2429
  ],
2110
2430
  "description": "Stable slug - match on this, not on the sentence."
2111
2431
  }
@@ -2122,6 +2442,20 @@
2122
2442
  "error": "Several tests have this name. Pass ?filePath= with one of: tests/a.spec.ts, tests/b.spec.ts.",
2123
2443
  "code": "ambiguous_test"
2124
2444
  }
2445
+ },
2446
+ "invalid_parameter": {
2447
+ "summary": "invalid_parameter",
2448
+ "value": {
2449
+ "error": "sha must be 4 to 40 hexadecimal characters (a commit SHA or a prefix of one).",
2450
+ "code": "invalid_parameter"
2451
+ }
2452
+ },
2453
+ "invalid_period": {
2454
+ "summary": "invalid_period",
2455
+ "value": {
2456
+ "error": "days=90 is beyond this plan's 60-day retention. Use 60 or less.",
2457
+ "code": "invalid_period"
2458
+ }
2125
2459
  }
2126
2460
  }
2127
2461
  }
@@ -2362,6 +2696,7 @@
2362
2696
  "type": "integer",
2363
2697
  "enum": [
2364
2698
  7,
2699
+ 14,
2365
2700
  30,
2366
2701
  90
2367
2702
  ],
@@ -2685,6 +3020,7 @@
2685
3020
  "type": "integer",
2686
3021
  "enum": [
2687
3022
  7,
3023
+ 14,
2688
3024
  30,
2689
3025
  90
2690
3026
  ],
@@ -3034,6 +3370,7 @@
3034
3370
  "type": "integer",
3035
3371
  "enum": [
3036
3372
  7,
3373
+ 14,
3037
3374
  30,
3038
3375
  90
3039
3376
  ],
@@ -3336,7 +3673,7 @@
3336
3673
  "get": {
3337
3674
  "operationId": "getOverview",
3338
3675
  "summary": "Get dashboard figures",
3339
- "description": "The dashboard for a period: last run, stability, time lost, slowest tests, time sinks, recent runs. Moderately expensive: its flaky count rides on one engine pass.",
3676
+ "description": "The dashboard for a period: last run, suite stability, time lost, the two test-metric tiles (testMetrics), slowest tests, time sinks, recent runs. Suite stability and testMetrics count completed CI runs only. Moderate: one pass over the period's runs and one count over its CI results.",
3340
3677
  "tags": [
3341
3678
  "Project"
3342
3679
  ],
@@ -3359,6 +3696,7 @@
3359
3696
  "type": "integer",
3360
3697
  "enum": [
3361
3698
  7,
3699
+ 14,
3362
3700
  30,
3363
3701
  90
3364
3702
  ],
@@ -3413,13 +3751,13 @@
3413
3751
  "startedAt": "2026-09-24T07:31:12.000Z",
3414
3752
  "finishedAt": "2026-09-24T07:39:47.000Z",
3415
3753
  "total": 412,
3416
- "passed": 409,
3754
+ "passed": 408,
3417
3755
  "failed": 0,
3418
3756
  "isFallbackBranch": false
3419
3757
  },
3420
- "runsInWindow": 57,
3758
+ "runsInWindow": 11,
3421
3759
  "suiteSize": 412,
3422
- "windowStability": 91.2,
3760
+ "windowStability": 99,
3423
3761
  "timeLost": {
3424
3762
  "lostMs": 5112000,
3425
3763
  "lostPercent": 2.4,
@@ -3428,11 +3766,15 @@
3428
3766
  },
3429
3767
  "failingTests": {
3430
3768
  "failing": 1,
3431
- "withVerdict": 3,
3432
3769
  "testsRun": 412
3433
3770
  },
3434
- "flakyOrBrokenCount": 3,
3435
- "flakyWindowDays": 14,
3771
+ "testMetrics": {
3772
+ "population": "ci_completed_not_excluded",
3773
+ "days": 30,
3774
+ "testsWithRetries": 7,
3775
+ "failingTests": 2,
3776
+ "ciTests": 409
3777
+ },
3436
3778
  "slowestTests": [
3437
3779
  {
3438
3780
  "testName": "chromium > checkout.spec.ts > Checkout > pays with a saved card",
@@ -3450,14 +3792,14 @@
3450
3792
  ],
3451
3793
  "runsSeries": [
3452
3794
  {
3453
- "runNumber": 1282,
3454
- "passed": 410,
3455
- "failed": 0,
3795
+ "runNumber": 1277,
3796
+ "passed": 408,
3797
+ "failed": 1,
3456
3798
  "durationMs": 498000
3457
3799
  },
3458
3800
  {
3459
3801
  "runNumber": 1284,
3460
- "passed": 409,
3802
+ "passed": 408,
3461
3803
  "failed": 0,
3462
3804
  "durationMs": 515000
3463
3805
  }
@@ -3735,8 +4077,7 @@
3735
4077
  "slack": {
3736
4078
  "configured": true,
3737
4079
  "events": [
3738
- "flaky_detected",
3739
- "broken_detected"
4080
+ "run_summary"
3740
4081
  ]
3741
4082
  },
3742
4083
  "github": {
@@ -3928,6 +4269,57 @@
3928
4269
  }
3929
4270
  }
3930
4271
  }
4272
+ },
4273
+ "/projects/{projectId}/flaky": {
4274
+ "get": {
4275
+ "operationId": "listFlakyTests",
4276
+ "summary": "List flaky tests (deprecated)",
4277
+ "description": "Deprecated in 1.0.0-beta.2 and removed in the next release: call GET /projects/{projectId}/tests. Answers 301 Moved Permanently to it, carrying the query over - sort=rate becomes sort=flakyRate, sort=timeLost stays, sort=impact and status are dropped, everything else passes through unchanged. The redirect checks no key; the list it points to does.",
4278
+ "tags": [
4279
+ "Tests"
4280
+ ],
4281
+ "deprecated": true,
4282
+ "parameters": [
4283
+ {
4284
+ "name": "projectId",
4285
+ "in": "path",
4286
+ "required": true,
4287
+ "description": "A project id from GET /projects.",
4288
+ "schema": {
4289
+ "type": "string"
4290
+ }
4291
+ }
4292
+ ],
4293
+ "responses": {
4294
+ "301": {
4295
+ "description": "Moved permanently to /projects/{projectId}/tests. No body.",
4296
+ "headers": {
4297
+ "Location": {
4298
+ "description": "The absolute URL of /projects/{projectId}/tests, with the query carried over.",
4299
+ "schema": {
4300
+ "type": "string",
4301
+ "format": "uri"
4302
+ }
4303
+ },
4304
+ "Deprecation": {
4305
+ "description": "When the deprecation took effect, as @ and a Unix time (RFC 9745).",
4306
+ "schema": {
4307
+ "type": "string"
4308
+ }
4309
+ },
4310
+ "Link": {
4311
+ "description": "/projects/{projectId}/tests as rel=\"successor-version\".",
4312
+ "schema": {
4313
+ "type": "string"
4314
+ }
4315
+ }
4316
+ }
4317
+ },
4318
+ "429": {
4319
+ "description": "Rate limited by the edge firewall, no JSON body. 429: wait 60 seconds, then retry. The edge counts requests per key in 60-second windows and sends no Retry-After header."
4320
+ }
4321
+ }
4322
+ }
3931
4323
  }
3932
4324
  },
3933
4325
  "components": {
@@ -4127,8 +4519,9 @@
4127
4519
  "skipped": {
4128
4520
  "type": "integer"
4129
4521
  },
4130
- "recovered": {
4131
- "type": "integer"
4522
+ "flaky": {
4523
+ "type": "integer",
4524
+ "description": "Tests whose final attempt passed after at least one retry in this run. Not counted in passed: total = passed + failed + skipped + flaky."
4132
4525
  }
4133
4526
  },
4134
4527
  "required": [
@@ -4136,7 +4529,7 @@
4136
4529
  "passed",
4137
4530
  "failed",
4138
4531
  "skipped",
4139
- "recovered"
4532
+ "flaky"
4140
4533
  ]
4141
4534
  },
4142
4535
  "shards": {
@@ -4456,8 +4849,9 @@
4456
4849
  "skipped": {
4457
4850
  "type": "integer"
4458
4851
  },
4459
- "recovered": {
4460
- "type": "integer"
4852
+ "flaky": {
4853
+ "type": "integer",
4854
+ "description": "Tests whose final attempt passed after at least one retry in this run. Not counted in passed: total = passed + failed + skipped + flaky."
4461
4855
  }
4462
4856
  },
4463
4857
  "required": [
@@ -4465,7 +4859,7 @@
4465
4859
  "passed",
4466
4860
  "failed",
4467
4861
  "skipped",
4468
- "recovered"
4862
+ "flaky"
4469
4863
  ]
4470
4864
  },
4471
4865
  "tests": {
@@ -4858,7 +5252,7 @@
4858
5252
  "branches"
4859
5253
  ]
4860
5254
  },
4861
- "FlakyResponse": {
5255
+ "TestsResponse": {
4862
5256
  "type": "object",
4863
5257
  "properties": {
4864
5258
  "project": {
@@ -4876,43 +5270,78 @@
4876
5270
  "name"
4877
5271
  ]
4878
5272
  },
4879
- "windowDays": {
5273
+ "population": {
5274
+ "type": "string",
5275
+ "const": "ci_completed_not_excluded",
5276
+ "description": "Which runs every number counts: completed CI runs (reported with a CI provider) started inside the period, without the runs excluded as a mass failure (at least 20 tests, over 80% of them failed). Every branch unless the branch parameter names one. Local runs never count."
5277
+ },
5278
+ "days": {
4880
5279
  "type": "integer",
4881
- "description": "The engine's fixed window in days. The flaky list takes no other period."
5280
+ "description": "The period in days, ending at until."
4882
5281
  },
4883
- "sort": {
4884
- "type": "string"
5282
+ "since": {
5283
+ "type": "string",
5284
+ "format": "date-time",
5285
+ "description": "Start of the period: until minus days. ISO-8601."
4885
5286
  },
4886
- "page": {
4887
- "type": "object",
4888
- "properties": {
4889
- "page": {
4890
- "type": "integer"
4891
- },
4892
- "perPage": {
4893
- "type": "integer"
5287
+ "until": {
5288
+ "type": "string",
5289
+ "format": "date-time",
5290
+ "description": "End of the period: when the numbers were computed. ISO-8601."
5291
+ },
5292
+ "branch": {
5293
+ "anyOf": [
5294
+ {
5295
+ "type": "string"
4894
5296
  },
4895
- "total": {
4896
- "type": "integer"
5297
+ {
5298
+ "type": "null"
5299
+ }
5300
+ ],
5301
+ "description": "The branch filter applied; null for every branch."
5302
+ },
5303
+ "browser": {
5304
+ "anyOf": [
5305
+ {
5306
+ "type": "string"
4897
5307
  },
4898
- "pageCount": {
4899
- "type": "integer"
5308
+ {
5309
+ "type": "null"
4900
5310
  }
4901
- },
4902
- "required": [
4903
- "page",
4904
- "perPage",
4905
- "total",
4906
- "pageCount"
5311
+ ],
5312
+ "description": "The browser filter applied; null for every test."
5313
+ },
5314
+ "sort": {
5315
+ "type": "string",
5316
+ "enum": [
5317
+ "stability",
5318
+ "flakyRate",
5319
+ "failureRate",
5320
+ "runs",
5321
+ "timeLost"
4907
5322
  ]
4908
5323
  },
4909
- "tests": {
5324
+ "page": {
5325
+ "type": "integer"
5326
+ },
5327
+ "perPage": {
5328
+ "type": "integer"
5329
+ },
5330
+ "total": {
5331
+ "type": "integer",
5332
+ "description": "Tests matching the query across all pages: every test with at least one CI run in the period, narrowed by search, branch and browser."
5333
+ },
5334
+ "pageCount": {
5335
+ "type": "integer"
5336
+ },
5337
+ "items": {
4910
5338
  "type": "array",
4911
5339
  "items": {
4912
5340
  "type": "object",
4913
5341
  "properties": {
4914
5342
  "testName": {
4915
- "type": "string"
5343
+ "type": "string",
5344
+ "description": "The stored name: Playwright's title path, the Playwright project first when there is one. The identity other test routes take."
4916
5345
  },
4917
5346
  "filePath": {
4918
5347
  "anyOf": [
@@ -4924,19 +5353,34 @@
4924
5353
  }
4925
5354
  ]
4926
5355
  },
4927
- "status": {
4928
- "type": "string"
5356
+ "browser": {
5357
+ "anyOf": [
5358
+ {
5359
+ "type": "string"
5360
+ },
5361
+ {
5362
+ "type": "null"
5363
+ }
5364
+ ],
5365
+ "description": "The Playwright project the name starts with (chromium in \"chromium > cart.spec.ts > adds an item\") - null when the name has none, i.e. when its second part is not the test's own file."
4929
5366
  },
4930
- "statusReason": {
4931
- "type": "string"
5367
+ "runs": {
5368
+ "type": "integer",
5369
+ "description": "CI runs in the period in which the test has a final attempt that did not skip: stableRuns + flakyRuns + failedRuns."
4932
5370
  },
4933
- "alsoActive": {
4934
- "type": "array",
4935
- "items": {
4936
- "type": "string"
4937
- }
5371
+ "stableRuns": {
5372
+ "type": "integer",
5373
+ "description": "Of runs, those whose final attempt passed on the first try."
4938
5374
  },
4939
- "timeLostMs": {
5375
+ "flakyRuns": {
5376
+ "type": "integer",
5377
+ "description": "Of runs, those whose final attempt passed after at least one retry."
5378
+ },
5379
+ "failedRuns": {
5380
+ "type": "integer",
5381
+ "description": "Of runs, those whose final attempt failed."
5382
+ },
5383
+ "stability": {
4940
5384
  "anyOf": [
4941
5385
  {
4942
5386
  "type": "number"
@@ -4945,13 +5389,18 @@
4945
5389
  "type": "null"
4946
5390
  }
4947
5391
  ],
4948
- "description": "Time lost to failed and retried attempts in the window, ms (approximation: wasted attempts x the final attempt duration). A failed run wastes every attempt it made, a run rescued by a retry wastes the attempts before it, and a run that passed first time wastes none - so 0 is a real measurement. Computed for every listed test; null is a guard for a test with no occurrence in the window and should not occur here."
5392
+ "description": "stableRuns / runs, a fraction from 0 to 1. null only when runs is 0."
4949
5393
  },
4950
- "flakeScore": {
4951
- "type": "number"
4952
- },
4953
- "flipRate": {
4954
- "type": "number"
5394
+ "flakyRate": {
5395
+ "anyOf": [
5396
+ {
5397
+ "type": "number"
5398
+ },
5399
+ {
5400
+ "type": "null"
5401
+ }
5402
+ ],
5403
+ "description": "flakyRuns / runs, a fraction from 0 to 1. null only when runs is 0."
4955
5404
  },
4956
5405
  "failureRate": {
4957
5406
  "anyOf": [
@@ -4961,74 +5410,148 @@
4961
5410
  {
4962
5411
  "type": "null"
4963
5412
  }
4964
- ]
5413
+ ],
5414
+ "description": "failedRuns / runs, a fraction from 0 to 1. null only when runs is 0."
4965
5415
  },
4966
- "runsConsidered": {
4967
- "type": "integer"
5416
+ "timeLostMs": {
5417
+ "type": "integer",
5418
+ "description": "Time spent on attempts that did not count, in ms, summed over runs: a failed run loses every attempt it made, a run rescued by a retry the attempts before the pass, each at the final attempt's duration. 0 when every run passed on the first try."
4968
5419
  },
4969
- "failureStreak": {
4970
- "type": "integer"
5420
+ "recentResults": {
5421
+ "type": "array",
5422
+ "items": {
5423
+ "type": "object",
5424
+ "properties": {
5425
+ "runId": {
5426
+ "type": "string"
5427
+ },
5428
+ "runNumber": {
5429
+ "type": "integer"
5430
+ },
5431
+ "startedAt": {
5432
+ "type": "string",
5433
+ "format": "date-time"
5434
+ },
5435
+ "branch": {
5436
+ "anyOf": [
5437
+ {
5438
+ "type": "string"
5439
+ },
5440
+ {
5441
+ "type": "null"
5442
+ }
5443
+ ],
5444
+ "description": "The run's branch; null when its CI sent none."
5445
+ },
5446
+ "result": {
5447
+ "type": "string",
5448
+ "enum": [
5449
+ "stable",
5450
+ "flaky",
5451
+ "failed"
5452
+ ],
5453
+ "description": "The run's final attempt: stable (passed on the first try), flaky (passed after at least one retry) or failed."
5454
+ }
5455
+ },
5456
+ "required": [
5457
+ "runId",
5458
+ "runNumber",
5459
+ "startedAt",
5460
+ "branch",
5461
+ "result"
5462
+ ]
5463
+ },
5464
+ "description": "The test's last 20 runs in the period, oldest first. Empty when it has none."
4971
5465
  },
4972
- "lastSeenAt": {
4973
- "type": "string",
4974
- "format": "date-time"
5466
+ "branches": {
5467
+ "type": "array",
5468
+ "items": {
5469
+ "anyOf": [
5470
+ {
5471
+ "type": "string"
5472
+ },
5473
+ {
5474
+ "type": "null"
5475
+ }
5476
+ ]
5477
+ },
5478
+ "description": "Every branch the test ran on in the period, code-point order, a null branch (a run whose CI sent none) last."
4975
5479
  },
4976
- "lastRun": {
4977
- "type": "object",
4978
- "properties": {
4979
- "runId": {
4980
- "type": "string"
4981
- },
4982
- "runNumber": {
4983
- "type": "integer"
4984
- },
4985
- "status": {
4986
- "type": "string"
5480
+ "topBranch": {
5481
+ "anyOf": [
5482
+ {
5483
+ "type": "object",
5484
+ "properties": {
5485
+ "branch": {
5486
+ "anyOf": [
5487
+ {
5488
+ "type": "string"
5489
+ },
5490
+ {
5491
+ "type": "null"
5492
+ }
5493
+ ]
5494
+ },
5495
+ "kind": {
5496
+ "type": "string",
5497
+ "enum": [
5498
+ "failures",
5499
+ "retries"
5500
+ ]
5501
+ },
5502
+ "count": {
5503
+ "type": "integer"
5504
+ }
5505
+ },
5506
+ "required": [
5507
+ "branch",
5508
+ "kind",
5509
+ "count"
5510
+ ]
4987
5511
  },
4988
- "startedAt": {
4989
- "type": "string",
4990
- "format": "date-time"
5512
+ {
5513
+ "type": "null"
4991
5514
  }
4992
- },
4993
- "required": [
4994
- "runId",
4995
- "runNumber",
4996
- "status",
4997
- "startedAt"
4998
- ]
5515
+ ],
5516
+ "description": "The branch with the most failed runs (kind failures, count = those runs); when no run failed, the branch with the most flaky runs (kind retries, count = those runs); null when neither. Ties go to the branch name in code-point order, a null branch last."
4999
5517
  }
5000
5518
  },
5001
5519
  "required": [
5002
5520
  "testName",
5003
5521
  "filePath",
5004
- "status",
5005
- "statusReason",
5006
- "alsoActive",
5007
- "timeLostMs",
5008
- "flakeScore",
5009
- "flipRate",
5522
+ "browser",
5523
+ "runs",
5524
+ "stableRuns",
5525
+ "flakyRuns",
5526
+ "failedRuns",
5527
+ "stability",
5528
+ "flakyRate",
5010
5529
  "failureRate",
5011
- "runsConsidered",
5012
- "failureStreak",
5013
- "lastSeenAt",
5014
- "lastRun"
5530
+ "timeLostMs",
5531
+ "recentResults",
5532
+ "branches",
5533
+ "topBranch"
5015
5534
  ]
5016
5535
  }
5017
- },
5018
- "gatheringDataCount": {
5019
- "type": "integer"
5020
5536
  }
5021
5537
  },
5022
5538
  "required": [
5023
5539
  "project",
5024
- "windowDays",
5540
+ "population",
5541
+ "days",
5542
+ "since",
5543
+ "until",
5544
+ "branch",
5545
+ "browser",
5025
5546
  "sort",
5026
5547
  "page",
5027
- "tests",
5028
- "gatheringDataCount"
5548
+ "perPage",
5549
+ "total",
5550
+ "pageCount",
5551
+ "items"
5029
5552
  ]
5030
5553
  },
5031
- "V4StatusResponse": {
5554
+ "TestMetricsResponse": {
5032
5555
  "type": "object",
5033
5556
  "properties": {
5034
5557
  "project": {
@@ -5046,77 +5569,234 @@
5046
5569
  "name"
5047
5570
  ]
5048
5571
  },
5049
- "test": {
5050
- "type": "object",
5051
- "properties": {
5052
- "name": {
5572
+ "population": {
5573
+ "type": "string",
5574
+ "const": "ci_completed_not_excluded",
5575
+ "description": "Which runs every number counts: completed CI runs (reported with a CI provider) started inside the period, without the runs excluded as a mass failure (at least 20 tests, over 80% of them failed). Every branch unless the branch parameter names one. Local runs never count."
5576
+ },
5577
+ "days": {
5578
+ "type": "integer",
5579
+ "description": "The period in days, ending at until."
5580
+ },
5581
+ "since": {
5582
+ "type": "string",
5583
+ "format": "date-time",
5584
+ "description": "Start of the period: until minus days. ISO-8601."
5585
+ },
5586
+ "until": {
5587
+ "type": "string",
5588
+ "format": "date-time",
5589
+ "description": "End of the period: when the numbers were computed. ISO-8601."
5590
+ },
5591
+ "branch": {
5592
+ "anyOf": [
5593
+ {
5053
5594
  "type": "string"
5054
5595
  },
5055
- "filePath": {
5056
- "anyOf": [
5057
- {
5058
- "type": "string"
5059
- },
5060
- {
5061
- "type": "null"
5062
- }
5063
- ]
5596
+ {
5597
+ "type": "null"
5064
5598
  }
5065
- },
5066
- "required": [
5067
- "name",
5068
- "filePath"
5069
- ]
5599
+ ],
5600
+ "description": "The branch filter applied; null for every branch."
5070
5601
  },
5071
- "status": {
5602
+ "testName": {
5072
5603
  "type": "string",
5073
- "enum": [
5074
- "insufficient_data",
5075
- "healthy",
5076
- "flaky",
5077
- "broken"
5604
+ "description": "The stored name: Playwright's title path, the Playwright project first when there is one. The identity other test routes take."
5605
+ },
5606
+ "filePath": {
5607
+ "anyOf": [
5608
+ {
5609
+ "type": "string"
5610
+ },
5611
+ {
5612
+ "type": "null"
5613
+ }
5078
5614
  ]
5079
5615
  },
5080
- "awaitingFirstWrite": {
5081
- "type": "boolean",
5082
- "description": "True when the test is known but no verdict has been stored for it yet; status then reads insufficient_data."
5616
+ "browser": {
5617
+ "anyOf": [
5618
+ {
5619
+ "type": "string"
5620
+ },
5621
+ {
5622
+ "type": "null"
5623
+ }
5624
+ ],
5625
+ "description": "The Playwright project the name starts with (chromium in \"chromium > cart.spec.ts > adds an item\") - null when the name has none, i.e. when its second part is not the test's own file."
5083
5626
  },
5084
- "activeSignals": {
5627
+ "runs": {
5628
+ "type": "integer",
5629
+ "description": "CI runs in the period in which the test has a final attempt that did not skip: stableRuns + flakyRuns + failedRuns."
5630
+ },
5631
+ "stableRuns": {
5632
+ "type": "integer",
5633
+ "description": "Of runs, those whose final attempt passed on the first try."
5634
+ },
5635
+ "flakyRuns": {
5636
+ "type": "integer",
5637
+ "description": "Of runs, those whose final attempt passed after at least one retry."
5638
+ },
5639
+ "failedRuns": {
5640
+ "type": "integer",
5641
+ "description": "Of runs, those whose final attempt failed."
5642
+ },
5643
+ "stability": {
5644
+ "anyOf": [
5645
+ {
5646
+ "type": "number"
5647
+ },
5648
+ {
5649
+ "type": "null"
5650
+ }
5651
+ ],
5652
+ "description": "stableRuns / runs, a fraction from 0 to 1. null only when runs is 0."
5653
+ },
5654
+ "flakyRate": {
5655
+ "anyOf": [
5656
+ {
5657
+ "type": "number"
5658
+ },
5659
+ {
5660
+ "type": "null"
5661
+ }
5662
+ ],
5663
+ "description": "flakyRuns / runs, a fraction from 0 to 1. null only when runs is 0."
5664
+ },
5665
+ "failureRate": {
5666
+ "anyOf": [
5667
+ {
5668
+ "type": "number"
5669
+ },
5670
+ {
5671
+ "type": "null"
5672
+ }
5673
+ ],
5674
+ "description": "failedRuns / runs, a fraction from 0 to 1. null only when runs is 0."
5675
+ },
5676
+ "timeLostMs": {
5677
+ "type": "integer",
5678
+ "description": "Time spent on attempts that did not count, in ms, summed over runs: a failed run loses every attempt it made, a run rescued by a retry the attempts before the pass, each at the final attempt's duration. 0 when every run passed on the first try."
5679
+ },
5680
+ "recentResults": {
5085
5681
  "type": "array",
5086
5682
  "items": {
5087
5683
  "type": "object",
5088
5684
  "properties": {
5089
- "signal": {
5685
+ "runId": {
5090
5686
  "type": "string"
5091
5687
  },
5092
- "label": {
5093
- "type": "string"
5688
+ "runNumber": {
5689
+ "type": "integer"
5690
+ },
5691
+ "startedAt": {
5692
+ "type": "string",
5693
+ "format": "date-time"
5694
+ },
5695
+ "branch": {
5696
+ "anyOf": [
5697
+ {
5698
+ "type": "string"
5699
+ },
5700
+ {
5701
+ "type": "null"
5702
+ }
5703
+ ],
5704
+ "description": "The run's branch; null when its CI sent none."
5705
+ },
5706
+ "result": {
5707
+ "type": "string",
5708
+ "enum": [
5709
+ "stable",
5710
+ "flaky",
5711
+ "failed"
5712
+ ],
5713
+ "description": "The run's final attempt: stable (passed on the first try), flaky (passed after at least one retry) or failed."
5094
5714
  }
5095
5715
  },
5096
5716
  "required": [
5097
- "signal",
5098
- "label"
5717
+ "runId",
5718
+ "runNumber",
5719
+ "startedAt",
5720
+ "branch",
5721
+ "result"
5099
5722
  ]
5100
- }
5723
+ },
5724
+ "description": "The test's last 20 runs in the period, oldest first. Empty when it has none."
5101
5725
  },
5102
- "labels": {
5726
+ "branches": {
5103
5727
  "type": "array",
5104
5728
  "items": {
5105
- "type": "string",
5106
- "enum": [
5107
- "flips",
5108
- "retries"
5729
+ "anyOf": [
5730
+ {
5731
+ "type": "string"
5732
+ },
5733
+ {
5734
+ "type": "null"
5735
+ }
5109
5736
  ]
5110
- }
5737
+ },
5738
+ "description": "Every branch the test ran on in the period, code-point order, a null branch (a run whose CI sent none) last."
5739
+ },
5740
+ "topBranch": {
5741
+ "anyOf": [
5742
+ {
5743
+ "type": "object",
5744
+ "properties": {
5745
+ "branch": {
5746
+ "anyOf": [
5747
+ {
5748
+ "type": "string"
5749
+ },
5750
+ {
5751
+ "type": "null"
5752
+ }
5753
+ ]
5754
+ },
5755
+ "kind": {
5756
+ "type": "string",
5757
+ "enum": [
5758
+ "failures",
5759
+ "retries"
5760
+ ]
5761
+ },
5762
+ "count": {
5763
+ "type": "integer"
5764
+ }
5765
+ },
5766
+ "required": [
5767
+ "branch",
5768
+ "kind",
5769
+ "count"
5770
+ ]
5771
+ },
5772
+ {
5773
+ "type": "null"
5774
+ }
5775
+ ],
5776
+ "description": "The branch with the most failed runs (kind failures, count = those runs); when no run failed, the branch with the most flaky runs (kind retries, count = those runs); null when neither. Ties go to the branch name in code-point order, a null branch last."
5111
5777
  }
5112
5778
  },
5113
5779
  "required": [
5114
5780
  "project",
5115
- "test",
5116
- "status",
5117
- "awaitingFirstWrite",
5118
- "activeSignals",
5119
- "labels"
5781
+ "population",
5782
+ "days",
5783
+ "since",
5784
+ "until",
5785
+ "branch",
5786
+ "testName",
5787
+ "filePath",
5788
+ "browser",
5789
+ "runs",
5790
+ "stableRuns",
5791
+ "flakyRuns",
5792
+ "failedRuns",
5793
+ "stability",
5794
+ "flakyRate",
5795
+ "failureRate",
5796
+ "timeLostMs",
5797
+ "recentResults",
5798
+ "branches",
5799
+ "topBranch"
5120
5800
  ]
5121
5801
  },
5122
5802
  "TestHistoryResponse": {
@@ -5747,7 +6427,8 @@
5747
6427
  ]
5748
6428
  },
5749
6429
  "runsInWindow": {
5750
- "type": "integer"
6430
+ "type": "integer",
6431
+ "description": "Completed CI runs in the window on branchScope, without mass-failure runs - the runs windowStability is counted over."
5751
6432
  },
5752
6433
  "suiteSize": {
5753
6434
  "type": "integer"
@@ -5760,7 +6441,8 @@
5760
6441
  {
5761
6442
  "type": "null"
5762
6443
  }
5763
- ]
6444
+ ],
6445
+ "description": "Suite stability, 0-100: of the final test results in the window's completed CI runs on branchScope (mass-failure runs and skipped results left out), the share that passed on the first try, rounded. A result that passed only after a retry counts against it. null when no such result exists."
5764
6446
  },
5765
6447
  "timeLost": {
5766
6448
  "type": "object",
@@ -5798,24 +6480,49 @@
5798
6480
  "failing": {
5799
6481
  "type": "integer"
5800
6482
  },
5801
- "withVerdict": {
5802
- "type": "integer"
5803
- },
5804
6483
  "testsRun": {
5805
6484
  "type": "integer"
5806
6485
  }
5807
6486
  },
5808
6487
  "required": [
5809
6488
  "failing",
5810
- "withVerdict",
5811
6489
  "testsRun"
5812
- ]
5813
- },
5814
- "flakyOrBrokenCount": {
5815
- "type": "integer"
6490
+ ],
6491
+ "description": "Tests that failed at least once in the window on branchScope, out of the tests those runs reported - every run, local ones included. Not the dashboard's Failing tests tile: that is testMetrics.failingTests, over CI runs on every branch."
5816
6492
  },
5817
- "flakyWindowDays": {
5818
- "type": "integer"
6493
+ "testMetrics": {
6494
+ "type": "object",
6495
+ "properties": {
6496
+ "population": {
6497
+ "type": "string",
6498
+ "const": "ci_completed_not_excluded",
6499
+ "description": "Which runs every number counts: completed CI runs (reported with a CI provider) started inside the period, without the runs excluded as a mass failure (at least 20 tests, over 80% of them failed). Every branch unless the branch parameter names one. Local runs never count."
6500
+ },
6501
+ "days": {
6502
+ "type": "integer",
6503
+ "description": "The period in days: the same as window.days."
6504
+ },
6505
+ "testsWithRetries": {
6506
+ "type": "integer",
6507
+ "description": "Of ciTests, the tests with at least one flaky run in the period (a final attempt that passed after a retry)."
6508
+ },
6509
+ "failingTests": {
6510
+ "type": "integer",
6511
+ "description": "Of ciTests, the tests with at least one failed run in the period."
6512
+ },
6513
+ "ciTests": {
6514
+ "type": "integer",
6515
+ "description": "Tests with at least one CI run in the period."
6516
+ }
6517
+ },
6518
+ "required": [
6519
+ "population",
6520
+ "days",
6521
+ "testsWithRetries",
6522
+ "failingTests",
6523
+ "ciTests"
6524
+ ],
6525
+ "description": "The dashboard's Tests with retries and Failing tests tiles and their denominator, counted like /tests: CI runs only, every branch - not narrowed by branchScope. The top-level failingTests counts differently: every run in the window on branchScope, local ones included."
5819
6526
  },
5820
6527
  "slowestTests": {
5821
6528
  "type": "array",
@@ -5924,8 +6631,7 @@
5924
6631
  "windowStability",
5925
6632
  "timeLost",
5926
6633
  "failingTests",
5927
- "flakyOrBrokenCount",
5928
- "flakyWindowDays",
6634
+ "testMetrics",
5929
6635
  "slowestTests",
5930
6636
  "timeSinks",
5931
6637
  "runsSeries"