vouchington-tooling 0.0.20 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/coverage-transport/control-v2.d.mts +42 -0
- package/dist/coverage-transport/control-v2.mjs +117 -0
- package/dist/coverage-transport/control.d.mts +2 -1
- package/dist/coverage-transport/control.mjs +7 -2
- package/dist/coverage-transport/discovery.d.mts +17 -0
- package/dist/coverage-transport/discovery.mjs +99 -0
- package/dist/coverage-transport/http.d.mts +1 -0
- package/dist/coverage-transport/http.mjs +12 -1
- package/dist/coverage-transport/index.d.mts +5 -1
- package/dist/coverage-transport/index.mjs +5 -1
- package/dist/coverage-transport/keys.d.mts +16 -0
- package/dist/coverage-transport/keys.mjs +44 -0
- package/dist/coverage-transport/lib.mjs +15 -0
- package/dist/coverage-transport/outcome.mjs +3 -1
- package/dist/coverage-transport/prefix-transfer.d.mts +8 -0
- package/dist/coverage-transport/prefix-transfer.mjs +69 -0
- package/dist/coverage-transport/prefix.d.mts +13 -0
- package/dist/coverage-transport/prefix.mjs +30 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -95,7 +95,11 @@ import { runAstGrepRule } from 'vouchington-tooling/ast-grep-rule'
|
|
|
95
95
|
import { parseReviewPayload, remapReviewComments } from 'vouchington-tooling/gha-review-payload'
|
|
96
96
|
import { runPostReview } from 'vouchington-tooling/gha-post-review'
|
|
97
97
|
import { nextPageUrlFromLinkHeader } from 'vouchington-tooling/http-link-pagination'
|
|
98
|
-
import {
|
|
98
|
+
import {
|
|
99
|
+
cmdUpload,
|
|
100
|
+
discoverDownloadControl,
|
|
101
|
+
mintPrefixUploadControl,
|
|
102
|
+
} from 'vouchington-tooling/coverage-transport'
|
|
99
103
|
import { pruneDeployedRuntimeDeps } from 'vouchington-tooling/pnpm-deploy'
|
|
100
104
|
import { parseDockerfileRuntimeImages } from 'vouchington-tooling/dockerfile-parse'
|
|
101
105
|
import { checkSccComplexity } from 'vouchington-tooling/scc-complexity'
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export interface PrefixPostTarget {
|
|
2
|
+
readonly url: string;
|
|
3
|
+
readonly fields: Readonly<Record<string, string>>;
|
|
4
|
+
readonly keyPrefix: string;
|
|
5
|
+
readonly maxObjectBytes: number;
|
|
6
|
+
}
|
|
7
|
+
export interface PrefixUploadTransportControl {
|
|
8
|
+
readonly version: 2;
|
|
9
|
+
readonly mode: 'prefix-upload';
|
|
10
|
+
readonly repository: string;
|
|
11
|
+
readonly revision: string;
|
|
12
|
+
readonly run: {
|
|
13
|
+
readonly id: string;
|
|
14
|
+
readonly controlAttempt: number;
|
|
15
|
+
};
|
|
16
|
+
readonly expiresAt: string;
|
|
17
|
+
readonly upload: PrefixPostTarget;
|
|
18
|
+
}
|
|
19
|
+
export interface DownloadedTransportObject {
|
|
20
|
+
readonly key: string;
|
|
21
|
+
readonly url: string;
|
|
22
|
+
readonly attempt: number;
|
|
23
|
+
readonly byteLength: number;
|
|
24
|
+
}
|
|
25
|
+
export interface DiscoveredDownloadTransportControl {
|
|
26
|
+
readonly version: 2;
|
|
27
|
+
readonly mode: 'discovered-download';
|
|
28
|
+
readonly repository: string;
|
|
29
|
+
readonly revision: string;
|
|
30
|
+
readonly run: {
|
|
31
|
+
readonly id: string;
|
|
32
|
+
readonly controlAttempt: number;
|
|
33
|
+
};
|
|
34
|
+
readonly expiresAt: string;
|
|
35
|
+
readonly coverage: Readonly<Record<string, {
|
|
36
|
+
readonly lcov: DownloadedTransportObject;
|
|
37
|
+
readonly manifest: DownloadedTransportObject;
|
|
38
|
+
}>>;
|
|
39
|
+
readonly blobs: Readonly<Record<string, DownloadedTransportObject>>;
|
|
40
|
+
}
|
|
41
|
+
export type TransportControlV2 = PrefixUploadTransportControl | DiscoveredDownloadTransportControl;
|
|
42
|
+
export declare function parseTransportControlV2(raw: Record<string, unknown>): TransportControlV2;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { VITEST_SUITE_PATTERN } from '../vitest-blob-manifest/index.mjs';
|
|
2
|
+
import { DEFAULT_MAX_BODY_BYTES } from './constants.mjs';
|
|
3
|
+
import { assertPrefixTransportIdentity, parseTransportObjectKey, transportPrefix, } from './keys.mjs';
|
|
4
|
+
function isRecord(value) {
|
|
5
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function exactKeys(value, keys) {
|
|
8
|
+
return Object.keys(value).toSorted().join('\0') === keys.toSorted().join('\0');
|
|
9
|
+
}
|
|
10
|
+
function validUrl(value) {
|
|
11
|
+
return typeof value === 'string' && URL.canParse(value) && /^https?:/i.test(value);
|
|
12
|
+
}
|
|
13
|
+
function identity(raw) {
|
|
14
|
+
if (!isRecord(raw.run) || !exactKeys(raw.run, ['id', 'controlAttempt']))
|
|
15
|
+
return null;
|
|
16
|
+
if (typeof raw.repository !== 'string' ||
|
|
17
|
+
typeof raw.revision !== 'string' ||
|
|
18
|
+
typeof raw.run.id !== 'string' ||
|
|
19
|
+
typeof raw.run.controlAttempt !== 'number')
|
|
20
|
+
return null;
|
|
21
|
+
const result = {
|
|
22
|
+
repository: raw.repository,
|
|
23
|
+
revision: raw.revision,
|
|
24
|
+
runId: raw.run.id,
|
|
25
|
+
controlAttempt: raw.run.controlAttempt,
|
|
26
|
+
};
|
|
27
|
+
try {
|
|
28
|
+
assertPrefixTransportIdentity(result);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
return result;
|
|
34
|
+
}
|
|
35
|
+
function parseObject(raw, expected, suite, kind) {
|
|
36
|
+
if (!isRecord(raw) ||
|
|
37
|
+
!exactKeys(raw, ['attempt', 'byteLength', 'key', 'url']) ||
|
|
38
|
+
!validUrl(raw.url) ||
|
|
39
|
+
typeof raw.key !== 'string' ||
|
|
40
|
+
typeof raw.attempt !== 'number' ||
|
|
41
|
+
typeof raw.byteLength !== 'number' ||
|
|
42
|
+
!Number.isSafeInteger(raw.attempt) ||
|
|
43
|
+
!Number.isSafeInteger(raw.byteLength) ||
|
|
44
|
+
raw.byteLength < 0 ||
|
|
45
|
+
raw.byteLength > DEFAULT_MAX_BODY_BYTES)
|
|
46
|
+
throw new Error('Discovered transport object is invalid');
|
|
47
|
+
const parsed = parseTransportObjectKey(raw.key, expected);
|
|
48
|
+
if (!parsed || parsed.suite !== suite || parsed.kind !== kind || parsed.attempt !== raw.attempt)
|
|
49
|
+
throw new Error('Discovered transport object key is invalid');
|
|
50
|
+
return raw;
|
|
51
|
+
}
|
|
52
|
+
function parseDownloadMap(raw, expected, kind) {
|
|
53
|
+
if (!isRecord(raw))
|
|
54
|
+
throw new Error('Discovered transport object map is invalid');
|
|
55
|
+
for (const [suite, value] of Object.entries(raw)) {
|
|
56
|
+
if (!VITEST_SUITE_PATTERN.test(suite))
|
|
57
|
+
throw new Error('Discovered transport suite is invalid');
|
|
58
|
+
if (kind === 'blob')
|
|
59
|
+
parseObject(value, expected, suite, 'blob');
|
|
60
|
+
else if (!isRecord(value) || !exactKeys(value, ['lcov', 'manifest']))
|
|
61
|
+
throw new Error('Discovered coverage pair is invalid');
|
|
62
|
+
else {
|
|
63
|
+
const lcov = parseObject(value.lcov, expected, suite, 'lcov');
|
|
64
|
+
const manifest = parseObject(value.manifest, expected, suite, 'manifest');
|
|
65
|
+
if (lcov.attempt !== manifest.attempt)
|
|
66
|
+
throw new Error('Discovered coverage pair attempts do not match');
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return raw;
|
|
70
|
+
}
|
|
71
|
+
export function parseTransportControlV2(raw) {
|
|
72
|
+
const expected = identity(raw);
|
|
73
|
+
if (!expected ||
|
|
74
|
+
raw.version !== 2 ||
|
|
75
|
+
typeof raw.expiresAt !== 'string' ||
|
|
76
|
+
!Number.isFinite(Date.parse(raw.expiresAt)))
|
|
77
|
+
throw new Error('Coverage transport control has invalid identity fields');
|
|
78
|
+
if (raw.mode === 'prefix-upload') {
|
|
79
|
+
if (!exactKeys(raw, [
|
|
80
|
+
'expiresAt',
|
|
81
|
+
'mode',
|
|
82
|
+
'repository',
|
|
83
|
+
'revision',
|
|
84
|
+
'run',
|
|
85
|
+
'upload',
|
|
86
|
+
'version',
|
|
87
|
+
]) ||
|
|
88
|
+
!isRecord(raw.upload) ||
|
|
89
|
+
!exactKeys(raw.upload, ['fields', 'keyPrefix', 'maxObjectBytes', 'url']) ||
|
|
90
|
+
!validUrl(raw.upload.url) ||
|
|
91
|
+
typeof raw.upload.keyPrefix !== 'string' ||
|
|
92
|
+
raw.upload.keyPrefix !== `${transportPrefix(expected)}/` ||
|
|
93
|
+
typeof raw.upload.maxObjectBytes !== 'number' ||
|
|
94
|
+
!Number.isSafeInteger(raw.upload.maxObjectBytes) ||
|
|
95
|
+
raw.upload.maxObjectBytes < 1 ||
|
|
96
|
+
raw.upload.maxObjectBytes > DEFAULT_MAX_BODY_BYTES ||
|
|
97
|
+
!isRecord(raw.upload.fields) ||
|
|
98
|
+
Object.entries(raw.upload.fields).some(([key, value]) => key === 'key' || !key || typeof value !== 'string'))
|
|
99
|
+
throw new Error('Prefix upload coverage transport control is invalid');
|
|
100
|
+
return raw;
|
|
101
|
+
}
|
|
102
|
+
if (raw.mode !== 'discovered-download' ||
|
|
103
|
+
!exactKeys(raw, [
|
|
104
|
+
'blobs',
|
|
105
|
+
'coverage',
|
|
106
|
+
'expiresAt',
|
|
107
|
+
'mode',
|
|
108
|
+
'repository',
|
|
109
|
+
'revision',
|
|
110
|
+
'run',
|
|
111
|
+
'version',
|
|
112
|
+
]))
|
|
113
|
+
throw new Error('Discovered download coverage transport control is invalid');
|
|
114
|
+
parseDownloadMap(raw.coverage, expected, 'coverage');
|
|
115
|
+
parseDownloadMap(raw.blobs, expected, 'blob');
|
|
116
|
+
return raw;
|
|
117
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type TransportControlV2 } from './control-v2.mts';
|
|
1
2
|
export interface PresignedCoverageUrls {
|
|
2
3
|
lcovPut: string;
|
|
3
4
|
lcovGet: string;
|
|
@@ -27,7 +28,7 @@ export interface FallbackOnlyTransportControl extends TransportControlBase {
|
|
|
27
28
|
readonly mode: 'fallback-only';
|
|
28
29
|
readonly reason: string;
|
|
29
30
|
}
|
|
30
|
-
export type TransportControl = PresignedTransportControl | FallbackOnlyTransportControl;
|
|
31
|
+
export type TransportControl = PresignedTransportControl | FallbackOnlyTransportControl | TransportControlV2;
|
|
31
32
|
export interface ExpectedTransportIdentity {
|
|
32
33
|
readonly repository: string;
|
|
33
34
|
readonly revision: string;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { chmodSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
3
3
|
import { VITEST_SUITE_PATTERN } from '../vitest-blob-manifest/index.mjs';
|
|
4
|
+
import { parseTransportControlV2 } from './control-v2.mjs';
|
|
4
5
|
function isRecord(value) {
|
|
5
6
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
6
7
|
}
|
|
@@ -25,9 +26,13 @@ function assertUrlMap(raw, fields, label) {
|
|
|
25
26
|
}
|
|
26
27
|
}
|
|
27
28
|
export function parseTransportControl(raw) {
|
|
28
|
-
if (!isRecord(raw)
|
|
29
|
+
if (!isRecord(raw)) {
|
|
29
30
|
throw new Error('Coverage transport control has an unsupported schema');
|
|
30
31
|
}
|
|
32
|
+
if (raw.version === 2)
|
|
33
|
+
return parseTransportControlV2(raw);
|
|
34
|
+
if (raw.version !== 1 || !isRecord(raw.run))
|
|
35
|
+
throw new Error('Coverage transport control has an unsupported schema');
|
|
31
36
|
if (typeof raw.repository !== 'string' ||
|
|
32
37
|
!raw.repository ||
|
|
33
38
|
typeof raw.revision !== 'string' ||
|
|
@@ -105,7 +110,7 @@ export function readTransportControl(path, expected) {
|
|
|
105
110
|
control.run.controlAttempt > expected.currentAttempt)) {
|
|
106
111
|
throw new Error('Coverage transport control identity does not match this run');
|
|
107
112
|
}
|
|
108
|
-
if (control.mode
|
|
113
|
+
if (control.mode !== 'fallback-only' && Date.parse(control.expiresAt) <= Date.now()) {
|
|
109
114
|
throw new Error('Coverage transport control has expired');
|
|
110
115
|
}
|
|
111
116
|
return control;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { DiscoveredDownloadTransportControl, PrefixUploadTransportControl } from './control-v2.mts';
|
|
2
|
+
import { type MintPrefixUploadOptions } from './prefix.mts';
|
|
3
|
+
export declare const MAX_DISCOVERED_TRANSPORT_OBJECTS = 1024;
|
|
4
|
+
export interface ObjectGetSigner {
|
|
5
|
+
signGet(key: string, ttlSeconds: number): Promise<string>;
|
|
6
|
+
}
|
|
7
|
+
export interface ListedTransportObject {
|
|
8
|
+
readonly key: string;
|
|
9
|
+
readonly byteLength: number;
|
|
10
|
+
}
|
|
11
|
+
export interface TransportObjectLister {
|
|
12
|
+
list(prefix: string, continuationToken?: string): Promise<{
|
|
13
|
+
readonly objects: readonly ListedTransportObject[];
|
|
14
|
+
readonly continuationToken?: string;
|
|
15
|
+
}>;
|
|
16
|
+
}
|
|
17
|
+
export declare function discoverDownloadControl(source: PrefixUploadTransportControl, lister: TransportObjectLister, signer: ObjectGetSigner, options?: MintPrefixUploadOptions): Promise<DiscoveredDownloadTransportControl>;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { DEFAULT_MAX_BODY_BYTES } from './constants.mjs';
|
|
2
|
+
import { parseTransportControl } from './control.mjs';
|
|
3
|
+
import { parseTransportObjectKey, } from './keys.mjs';
|
|
4
|
+
import { DEFAULT_TRANSPORT_TTL_SECONDS, transportExpiresAt, } from './prefix.mjs';
|
|
5
|
+
export const MAX_DISCOVERED_TRANSPORT_OBJECTS = 1024;
|
|
6
|
+
function identity(control) {
|
|
7
|
+
return {
|
|
8
|
+
repository: control.repository,
|
|
9
|
+
revision: control.revision,
|
|
10
|
+
runId: control.run.id,
|
|
11
|
+
controlAttempt: control.run.controlAttempt,
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
async function listCandidates(identity, lister) {
|
|
15
|
+
const result = [];
|
|
16
|
+
const tokens = new Set();
|
|
17
|
+
const keys = new Set();
|
|
18
|
+
let continuationToken;
|
|
19
|
+
do {
|
|
20
|
+
const page = await lister.list(`coverage-transport/${identity.repository}/${identity.runId}/${identity.revision}/`, continuationToken);
|
|
21
|
+
if (!Array.isArray(page.objects) ||
|
|
22
|
+
page.objects.length + result.length > MAX_DISCOVERED_TRANSPORT_OBJECTS)
|
|
23
|
+
throw new Error('Coverage transport discovery exceeds object limit');
|
|
24
|
+
for (const object of page.objects) {
|
|
25
|
+
if (!object ||
|
|
26
|
+
typeof object.key !== 'string' ||
|
|
27
|
+
!Number.isSafeInteger(object.byteLength) ||
|
|
28
|
+
object.byteLength < 0 ||
|
|
29
|
+
object.byteLength > DEFAULT_MAX_BODY_BYTES)
|
|
30
|
+
throw new Error('Coverage transport discovery object is invalid');
|
|
31
|
+
if (keys.has(object.key))
|
|
32
|
+
throw new Error('Coverage transport discovery has duplicate object keys');
|
|
33
|
+
keys.add(object.key);
|
|
34
|
+
const parsed = parseTransportObjectKey(object.key, identity);
|
|
35
|
+
if (!parsed)
|
|
36
|
+
throw new Error('Coverage transport discovery key is invalid');
|
|
37
|
+
result.push({ object, parsed });
|
|
38
|
+
}
|
|
39
|
+
if (page.continuationToken !== undefined && typeof page.continuationToken !== 'string')
|
|
40
|
+
throw new Error('Coverage transport discovery continuation token is invalid');
|
|
41
|
+
continuationToken = page.continuationToken;
|
|
42
|
+
if (continuationToken) {
|
|
43
|
+
if (tokens.has(continuationToken))
|
|
44
|
+
throw new Error('Coverage transport discovery pagination is cyclic');
|
|
45
|
+
tokens.add(continuationToken);
|
|
46
|
+
}
|
|
47
|
+
} while (continuationToken);
|
|
48
|
+
return result;
|
|
49
|
+
}
|
|
50
|
+
export async function discoverDownloadControl(source, lister, signer, options = {}) {
|
|
51
|
+
const value = identity(source);
|
|
52
|
+
const expiresAt = transportExpiresAt(options);
|
|
53
|
+
const candidates = await listCandidates(value, lister);
|
|
54
|
+
const suites = new Map();
|
|
55
|
+
for (const candidate of candidates) {
|
|
56
|
+
const entries = suites.get(candidate.parsed.suite);
|
|
57
|
+
if (entries)
|
|
58
|
+
entries.push(candidate);
|
|
59
|
+
else
|
|
60
|
+
suites.set(candidate.parsed.suite, [candidate]);
|
|
61
|
+
}
|
|
62
|
+
const coverage = {};
|
|
63
|
+
const blobs = {};
|
|
64
|
+
const object = async (candidate) => ({
|
|
65
|
+
key: candidate.object.key,
|
|
66
|
+
attempt: candidate.parsed.attempt,
|
|
67
|
+
byteLength: candidate.object.byteLength,
|
|
68
|
+
url: await signer.signGet(candidate.object.key, options.ttlSeconds ?? DEFAULT_TRANSPORT_TTL_SECONDS),
|
|
69
|
+
});
|
|
70
|
+
await Promise.all([...suites].map(async ([suite, entries]) => {
|
|
71
|
+
const attempts = [...new Set(entries.map((entry) => entry.parsed.attempt))].toSorted((a, b) => b - a);
|
|
72
|
+
const pair = attempts
|
|
73
|
+
.map((attempt) => entries.filter((entry) => entry.parsed.attempt === attempt))
|
|
74
|
+
.map((entries) => ({
|
|
75
|
+
lcov: entries.find((entry) => entry.parsed.kind === 'lcov'),
|
|
76
|
+
manifest: entries.find((entry) => entry.parsed.kind === 'manifest'),
|
|
77
|
+
}))
|
|
78
|
+
.find((pair) => pair.lcov && pair.manifest);
|
|
79
|
+
if (pair?.lcov && pair.manifest) {
|
|
80
|
+
const [lcov, manifest] = await Promise.all([object(pair.lcov), object(pair.manifest)]);
|
|
81
|
+
coverage[suite] = { lcov, manifest };
|
|
82
|
+
}
|
|
83
|
+
const blob = attempts
|
|
84
|
+
.map((attempt) => entries.find((entry) => entry.parsed.attempt === attempt && entry.parsed.kind === 'blob'))
|
|
85
|
+
.find(Boolean);
|
|
86
|
+
if (blob)
|
|
87
|
+
blobs[suite] = await object(blob);
|
|
88
|
+
}));
|
|
89
|
+
return parseTransportControl({
|
|
90
|
+
version: 2,
|
|
91
|
+
mode: 'discovered-download',
|
|
92
|
+
repository: value.repository,
|
|
93
|
+
revision: value.revision,
|
|
94
|
+
run: { id: value.runId, controlAttempt: value.controlAttempt },
|
|
95
|
+
expiresAt,
|
|
96
|
+
coverage,
|
|
97
|
+
blobs,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
@@ -4,3 +4,4 @@ export declare function coveragePresignFailureLog(error: unknown): string;
|
|
|
4
4
|
export declare function logTransport(options: RequestOptions, line: string): void;
|
|
5
5
|
export declare function fetchPut(url: string, body: Buffer, options?: RequestOptions): Promise<boolean>;
|
|
6
6
|
export declare function fetchGet(url: string, options?: RequestOptions): Promise<Buffer | null>;
|
|
7
|
+
export declare function fetchPost(url: string, fields: Readonly<Record<string, string>>, key: string, body: Buffer, options?: RequestOptions): Promise<boolean>;
|
|
@@ -57,7 +57,7 @@ async function request(method, url, body, options) {
|
|
|
57
57
|
try {
|
|
58
58
|
const response = await fetch(url, {
|
|
59
59
|
method,
|
|
60
|
-
...(body ? { body: new Uint8Array(body) } : {}),
|
|
60
|
+
...(body ? { body: typeof body === 'function' ? body() : new Uint8Array(body) } : {}),
|
|
61
61
|
signal: controller.signal,
|
|
62
62
|
});
|
|
63
63
|
// A missing presigned object is terminal. Let fetchGet yield null or fetchPut yield false
|
|
@@ -97,3 +97,14 @@ export async function fetchGet(url, options = {}) {
|
|
|
97
97
|
throw new Error('[coverage-transport] GET exhausted');
|
|
98
98
|
return response.body;
|
|
99
99
|
}
|
|
100
|
+
export async function fetchPost(url, fields, key, body, options = {}) {
|
|
101
|
+
const response = await request('POST', url, () => {
|
|
102
|
+
const form = new FormData();
|
|
103
|
+
for (const [name, value] of Object.entries(fields))
|
|
104
|
+
form.set(name, value);
|
|
105
|
+
form.set('key', key);
|
|
106
|
+
form.set('file', new Blob([new Uint8Array(body).slice().buffer]));
|
|
107
|
+
return form;
|
|
108
|
+
}, options);
|
|
109
|
+
return response?.ok === true;
|
|
110
|
+
}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
export { DEFAULT_COVERAGE_MANIFEST_FILENAME, cmdDownloadCoverage, cmdDownloadVitestBlobs, cmdUpload, } from './lib.mts';
|
|
2
2
|
export { parseTransportControl, readTransportControl, writeTransportControl, type ExpectedTransportIdentity, type FallbackOnlyTransportControl, type PresignedBlobUrls, type PresignedCoverageUrls, type PresignedTransportControl, type RequestOptions, type TransportControl, } from './control.mts';
|
|
3
|
-
export {
|
|
3
|
+
export { type DiscoveredDownloadTransportControl, type DownloadedTransportObject, type PrefixPostTarget, type PrefixUploadTransportControl, } from './control-v2.mts';
|
|
4
|
+
export { coveragePresignFailureLog, fetchGet, fetchPost, fetchPut, logTransport, redactTransportLog, } from './http.mts';
|
|
4
5
|
export { assertCoverageTransportBlobOutcome, assertCoverageTransportOutcome, isBlobPrimaryState, isStepOutcome, writeUploadOutcomeOutput, type AppendOutput, type BlobPrimaryState, type StepOutcome, } from './outcome.mts';
|
|
5
6
|
export { downloadVitestBlobBundles, packVitestBlobBundle } from './vitest-blob-transport.mts';
|
|
6
7
|
export { mintPresignedControl, transportObjectKeys, type MintPresignedControlOptions, type ObjectSigner, type PresignIdentity, } from './presign.mts';
|
|
8
|
+
export { discoverDownloadControl, MAX_DISCOVERED_TRANSPORT_OBJECTS, type ListedTransportObject, type ObjectGetSigner, type TransportObjectLister, } from './discovery.mts';
|
|
9
|
+
export { mintPrefixUploadControl, type MintPrefixUploadOptions, type PrefixPostSigner, } from './prefix.mts';
|
|
10
|
+
export { parseTransportObjectKey, transportObjectKeysV2, transportPrefix, type PrefixTransportIdentity, type TransportObjectKey, type TransportObjectKind, } from './keys.mts';
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
export { DEFAULT_COVERAGE_MANIFEST_FILENAME, cmdDownloadCoverage, cmdDownloadVitestBlobs, cmdUpload, } from './lib.mjs';
|
|
2
2
|
export { parseTransportControl, readTransportControl, writeTransportControl, } from './control.mjs';
|
|
3
|
-
export {
|
|
3
|
+
export {} from './control-v2.mjs';
|
|
4
|
+
export { coveragePresignFailureLog, fetchGet, fetchPost, fetchPut, logTransport, redactTransportLog, } from './http.mjs';
|
|
4
5
|
export { assertCoverageTransportBlobOutcome, assertCoverageTransportOutcome, isBlobPrimaryState, isStepOutcome, writeUploadOutcomeOutput, } from './outcome.mjs';
|
|
5
6
|
export { downloadVitestBlobBundles, packVitestBlobBundle } from './vitest-blob-transport.mjs';
|
|
6
7
|
export { mintPresignedControl, transportObjectKeys, } from './presign.mjs';
|
|
8
|
+
export { discoverDownloadControl, MAX_DISCOVERED_TRANSPORT_OBJECTS, } from './discovery.mjs';
|
|
9
|
+
export { mintPrefixUploadControl, } from './prefix.mjs';
|
|
10
|
+
export { parseTransportObjectKey, transportObjectKeysV2, transportPrefix, } from './keys.mjs';
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface PrefixTransportIdentity {
|
|
2
|
+
readonly repository: string;
|
|
3
|
+
readonly revision: string;
|
|
4
|
+
readonly runId: string;
|
|
5
|
+
readonly controlAttempt: number;
|
|
6
|
+
}
|
|
7
|
+
export type TransportObjectKind = 'lcov' | 'manifest' | 'blob';
|
|
8
|
+
export interface TransportObjectKey {
|
|
9
|
+
readonly attempt: number;
|
|
10
|
+
readonly suite: string;
|
|
11
|
+
readonly kind: TransportObjectKind;
|
|
12
|
+
}
|
|
13
|
+
export declare function assertPrefixTransportIdentity(identity: PrefixTransportIdentity): void;
|
|
14
|
+
export declare function transportPrefix(identity: PrefixTransportIdentity, attempt?: number): string;
|
|
15
|
+
export declare function transportObjectKeysV2(identity: PrefixTransportIdentity, suite: string, attempt?: number): Readonly<Record<TransportObjectKind, string>>;
|
|
16
|
+
export declare function parseTransportObjectKey(key: string, identity: PrefixTransportIdentity): TransportObjectKey | null;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { VITEST_SUITE_PATTERN } from '../vitest-blob-manifest/index.mjs';
|
|
2
|
+
export function assertPrefixTransportIdentity(identity) {
|
|
3
|
+
if (identity.repository.includes('..') ||
|
|
4
|
+
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(identity.repository) ||
|
|
5
|
+
!/^[0-9a-f]{40}$/.test(identity.revision) ||
|
|
6
|
+
!/^[1-9][0-9]*$/.test(identity.runId) ||
|
|
7
|
+
!Number.isSafeInteger(identity.controlAttempt) ||
|
|
8
|
+
identity.controlAttempt < 1) {
|
|
9
|
+
throw new Error('Coverage transport key identity is invalid');
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export function transportPrefix(identity, attempt = identity.controlAttempt) {
|
|
13
|
+
assertPrefixTransportIdentity(identity);
|
|
14
|
+
if (!Number.isSafeInteger(attempt) || attempt < 1 || attempt > identity.controlAttempt) {
|
|
15
|
+
throw new Error('Coverage transport attempt is invalid');
|
|
16
|
+
}
|
|
17
|
+
return `coverage-transport/${identity.repository}/${identity.runId}/${identity.revision}/attempt-${attempt}`;
|
|
18
|
+
}
|
|
19
|
+
export function transportObjectKeysV2(identity, suite, attempt = identity.controlAttempt) {
|
|
20
|
+
if (!VITEST_SUITE_PATTERN.test(suite))
|
|
21
|
+
throw new Error('Coverage transport suite is invalid');
|
|
22
|
+
const prefix = transportPrefix(identity, attempt);
|
|
23
|
+
return {
|
|
24
|
+
lcov: `${prefix}/coverage/${suite}/lcov.info`,
|
|
25
|
+
manifest: `${prefix}/coverage/${suite}/coverage-manifest.json`,
|
|
26
|
+
blob: `${prefix}/blobs/${suite}.tar.gz`,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export function parseTransportObjectKey(key, identity) {
|
|
30
|
+
const root = `coverage-transport/${identity.repository}/${identity.runId}/${identity.revision}/`;
|
|
31
|
+
if (!key.startsWith(root) || key.includes('..') || key.includes('\\'))
|
|
32
|
+
return null;
|
|
33
|
+
const match = /^attempt-([1-9][0-9]*)\/(?:coverage\/([a-z0-9]+(?:-[a-z0-9]+)*)\/(lcov\.info|coverage-manifest\.json)|blobs\/([a-z0-9]+(?:-[a-z0-9]+)*)\.tar\.gz)$/.exec(key.slice(root.length));
|
|
34
|
+
if (!match)
|
|
35
|
+
return null;
|
|
36
|
+
const attempt = Number(match[1]);
|
|
37
|
+
if (!Number.isSafeInteger(attempt) || attempt > identity.controlAttempt)
|
|
38
|
+
return null;
|
|
39
|
+
const coverageSuite = match[2];
|
|
40
|
+
if (coverageSuite) {
|
|
41
|
+
return { attempt, suite: coverageSuite, kind: match[3] === 'lcov.info' ? 'lcov' : 'manifest' };
|
|
42
|
+
}
|
|
43
|
+
return { attempt, suite: match[4], kind: 'blob' };
|
|
44
|
+
}
|
|
@@ -4,6 +4,7 @@ import { join } from 'node:path';
|
|
|
4
4
|
import { DEFAULT_COVERAGE_MANIFEST_FILENAME, DEFAULT_MAX_BODY_BYTES, assertCoverageManifestFilename, } from './constants.mjs';
|
|
5
5
|
import { readTransportControl, } from './control.mjs';
|
|
6
6
|
import { fetchGet, fetchPut, logTransport } from './http.mjs';
|
|
7
|
+
import { downloadPrefixBlobs, downloadPrefixCoverage, uploadPrefixTransport, } from './prefix-transfer.mjs';
|
|
7
8
|
import { downloadVitestBlobBundles, packVitestBlobBundle } from './vitest-blob-transport.mjs';
|
|
8
9
|
export { DEFAULT_COVERAGE_MANIFEST_FILENAME } from './constants.mjs';
|
|
9
10
|
export async function cmdUpload(controlPath, suite, options) {
|
|
@@ -15,6 +16,11 @@ export async function cmdUpload(controlPath, suite, options) {
|
|
|
15
16
|
const cwd = options.cwd ?? process.cwd();
|
|
16
17
|
const manifestFilename = options.coverageManifestFilename ?? DEFAULT_COVERAGE_MANIFEST_FILENAME;
|
|
17
18
|
assertCoverageManifestFilename(manifestFilename);
|
|
19
|
+
if (control.mode === 'prefix-upload') {
|
|
20
|
+
return uploadPrefixTransport(control, suite, cwd, options.expectedIdentity, manifestFilename, options);
|
|
21
|
+
}
|
|
22
|
+
if (control.mode === 'discovered-download')
|
|
23
|
+
throw new Error('Coverage transport download control cannot upload');
|
|
18
24
|
const coverageUrls = control.coverage[suite];
|
|
19
25
|
const lcovPath = join(cwd, 'coverage', 'lcov.info');
|
|
20
26
|
const manifestPath = join(cwd, 'coverage', manifestFilename);
|
|
@@ -58,6 +64,11 @@ export async function cmdDownloadCoverage(controlPath, destinationRoot, options)
|
|
|
58
64
|
}
|
|
59
65
|
const manifestFilename = options.coverageManifestFilename ?? DEFAULT_COVERAGE_MANIFEST_FILENAME;
|
|
60
66
|
assertCoverageManifestFilename(manifestFilename);
|
|
67
|
+
if (control.mode === 'prefix-upload')
|
|
68
|
+
throw new Error('Coverage transport upload control cannot download');
|
|
69
|
+
if (control.mode === 'discovered-download') {
|
|
70
|
+
return downloadPrefixCoverage(control, destinationRoot, manifestFilename, options);
|
|
71
|
+
}
|
|
61
72
|
await Promise.all(Object.entries(control.coverage).map(async ([suite, urls]) => {
|
|
62
73
|
const [lcov, manifest] = await Promise.all([
|
|
63
74
|
fetchGet(urls.lcovGet, options),
|
|
@@ -80,5 +91,9 @@ export async function cmdDownloadVitestBlobs(controlPath, destinationRoot, optio
|
|
|
80
91
|
logTransport(options, '[coverage-transport] S3 unavailable; artifact fallback required');
|
|
81
92
|
return;
|
|
82
93
|
}
|
|
94
|
+
if (control.mode === 'prefix-upload')
|
|
95
|
+
throw new Error('Coverage transport upload control cannot download');
|
|
96
|
+
if (control.mode === 'discovered-download')
|
|
97
|
+
return downloadPrefixBlobs(control, destinationRoot, options);
|
|
83
98
|
await downloadVitestBlobBundles(control.blobs, destinationRoot, options);
|
|
84
99
|
}
|
|
@@ -23,7 +23,9 @@ export function assertCoverageTransportOutcome(suite, primary, artifactAttempt1,
|
|
|
23
23
|
return true;
|
|
24
24
|
}
|
|
25
25
|
if (artifactSucceeded) {
|
|
26
|
-
|
|
26
|
+
if (primary !== 'skipped') {
|
|
27
|
+
emit(`::warning::Coverage persisted only to GitHub artifacts for suite=${suite}; S3 primary is degraded.`);
|
|
28
|
+
}
|
|
27
29
|
return true;
|
|
28
30
|
}
|
|
29
31
|
emit(`::error::COVERAGE_TRANSPORT_EXHAUSTED suite=${suite} Neither S3 nor GitHub artifacts persisted the coverage pair.`);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { DiscoveredDownloadTransportControl, PrefixUploadTransportControl } from './control-v2.mts';
|
|
2
|
+
import type { ExpectedTransportIdentity, RequestOptions } from './control.mts';
|
|
3
|
+
export declare function uploadPrefixTransport(control: PrefixUploadTransportControl, suite: string, cwd: string, expectedIdentity: ExpectedTransportIdentity, manifestFilename: string, options: RequestOptions): Promise<{
|
|
4
|
+
coverage: boolean;
|
|
5
|
+
blob: boolean;
|
|
6
|
+
}>;
|
|
7
|
+
export declare function downloadPrefixCoverage(control: DiscoveredDownloadTransportControl, destinationRoot: string, manifestFilename: string, options: RequestOptions): Promise<void>;
|
|
8
|
+
export declare function downloadPrefixBlobs(control: DiscoveredDownloadTransportControl, destinationRoot: string, options: RequestOptions): Promise<void>;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { DEFAULT_COVERAGE_MANIFEST_FILENAME } from './constants.mjs';
|
|
5
|
+
import { fetchGet, fetchPost, logTransport } from './http.mjs';
|
|
6
|
+
import { transportObjectKeysV2 } from './keys.mjs';
|
|
7
|
+
import { downloadVitestBlobBundles, packVitestBlobBundle } from './vitest-blob-transport.mjs';
|
|
8
|
+
function identity(control) {
|
|
9
|
+
return {
|
|
10
|
+
repository: control.repository,
|
|
11
|
+
revision: control.revision,
|
|
12
|
+
runId: control.run.id,
|
|
13
|
+
controlAttempt: control.run.controlAttempt,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
export async function uploadPrefixTransport(control, suite, cwd, expectedIdentity, manifestFilename, options) {
|
|
17
|
+
if (control.run.controlAttempt !== expectedIdentity.currentAttempt)
|
|
18
|
+
throw new Error('Prefix transport control attempt does not match the producer attempt');
|
|
19
|
+
if (manifestFilename !== DEFAULT_COVERAGE_MANIFEST_FILENAME)
|
|
20
|
+
throw new Error('Prefix transport requires the default coverage manifest filename');
|
|
21
|
+
const keys = transportObjectKeysV2(identity(control), suite);
|
|
22
|
+
const maxObjectBytes = Math.min(control.upload.maxObjectBytes, options.maxBodyBytes ?? control.upload.maxObjectBytes);
|
|
23
|
+
const lcovPath = join(cwd, 'coverage', 'lcov.info');
|
|
24
|
+
const manifestPath = join(cwd, 'coverage', manifestFilename);
|
|
25
|
+
const lcov = existsSync(lcovPath) && existsSync(manifestPath) ? await readFile(lcovPath) : null;
|
|
26
|
+
const manifest = lcov === null ? null : await readFile(manifestPath);
|
|
27
|
+
const storedLcov = lcov !== null && lcov.byteLength <= maxObjectBytes
|
|
28
|
+
? await fetchPost(control.upload.url, control.upload.fields, keys.lcov, lcov, options)
|
|
29
|
+
: false;
|
|
30
|
+
const coverage = storedLcov && manifest !== null && manifest.byteLength <= maxObjectBytes
|
|
31
|
+
? await fetchPost(control.upload.url, control.upload.fields, keys.manifest, manifest, options)
|
|
32
|
+
: false;
|
|
33
|
+
if (lcov !== null)
|
|
34
|
+
logTransport(options, coverage
|
|
35
|
+
? `[coverage-transport] Uploaded coverage pair for ${suite}`
|
|
36
|
+
: `[coverage-transport] Coverage pair upload failed for ${suite}`);
|
|
37
|
+
const blobData = packVitestBlobBundle(cwd, suite, expectedIdentity, options);
|
|
38
|
+
const blob = Boolean(blobData &&
|
|
39
|
+
blobData.byteLength <= maxObjectBytes &&
|
|
40
|
+
(await fetchPost(control.upload.url, control.upload.fields, keys.blob, blobData, options)));
|
|
41
|
+
if (blob)
|
|
42
|
+
logTransport(options, `[coverage-transport] Uploaded vitest blob for ${suite}`);
|
|
43
|
+
else if (blobData)
|
|
44
|
+
logTransport(options, `[coverage-transport] Vitest blob upload failed for ${suite}`);
|
|
45
|
+
return { coverage, blob };
|
|
46
|
+
}
|
|
47
|
+
export async function downloadPrefixCoverage(control, destinationRoot, manifestFilename, options) {
|
|
48
|
+
if (manifestFilename !== DEFAULT_COVERAGE_MANIFEST_FILENAME)
|
|
49
|
+
throw new Error('Prefix transport requires the default coverage manifest filename');
|
|
50
|
+
await Promise.all(Object.entries(control.coverage).map(async ([suite, pair]) => {
|
|
51
|
+
const [lcov, manifest] = await Promise.all([
|
|
52
|
+
fetchGet(pair.lcov.url, options),
|
|
53
|
+
fetchGet(pair.manifest.url, options),
|
|
54
|
+
]);
|
|
55
|
+
if (!lcov || !manifest)
|
|
56
|
+
return logTransport(options, `[coverage-transport] Skipped incomplete coverage pair for ${suite}`);
|
|
57
|
+
const destination = join(destinationRoot, `coverage-${suite}`);
|
|
58
|
+
mkdirSync(destination, { recursive: true });
|
|
59
|
+
writeFileSync(join(destination, 'lcov.info'), lcov);
|
|
60
|
+
writeFileSync(join(destination, manifestFilename), manifest, { mode: 0o600 });
|
|
61
|
+
logTransport(options, `[coverage-transport] Downloaded coverage pair for ${suite}`);
|
|
62
|
+
}));
|
|
63
|
+
}
|
|
64
|
+
export async function downloadPrefixBlobs(control, destinationRoot, options) {
|
|
65
|
+
await downloadVitestBlobBundles(Object.fromEntries(Object.entries(control.blobs).map(([suite, object]) => [
|
|
66
|
+
suite,
|
|
67
|
+
{ get: object.url, put: 'unused' },
|
|
68
|
+
])), destinationRoot, options);
|
|
69
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { PrefixPostTarget, PrefixUploadTransportControl } from './control-v2.mts';
|
|
2
|
+
import { type PrefixTransportIdentity } from './keys.mts';
|
|
3
|
+
export interface PrefixPostSigner {
|
|
4
|
+
signPost(keyPrefix: string, ttlSeconds: number, maxObjectBytes: number): Promise<PrefixPostTarget>;
|
|
5
|
+
}
|
|
6
|
+
export interface MintPrefixUploadOptions {
|
|
7
|
+
readonly ttlSeconds?: number;
|
|
8
|
+
readonly maxObjectBytes?: number;
|
|
9
|
+
readonly now?: () => Date;
|
|
10
|
+
}
|
|
11
|
+
export declare const DEFAULT_TRANSPORT_TTL_SECONDS = 14400;
|
|
12
|
+
export declare function transportExpiresAt(options: MintPrefixUploadOptions): string;
|
|
13
|
+
export declare function mintPrefixUploadControl(value: PrefixTransportIdentity, signer: PrefixPostSigner, options?: MintPrefixUploadOptions): Promise<PrefixUploadTransportControl>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { DEFAULT_MAX_BODY_BYTES } from './constants.mjs';
|
|
2
|
+
import { parseTransportControl } from './control.mjs';
|
|
3
|
+
import { transportPrefix } from './keys.mjs';
|
|
4
|
+
export const DEFAULT_TRANSPORT_TTL_SECONDS = 14_400;
|
|
5
|
+
export function transportExpiresAt(options) {
|
|
6
|
+
const ttl = options.ttlSeconds ?? DEFAULT_TRANSPORT_TTL_SECONDS;
|
|
7
|
+
if (!Number.isSafeInteger(ttl) || ttl < 1)
|
|
8
|
+
throw new Error('Coverage transport TTL is invalid');
|
|
9
|
+
return new Date((options.now?.() ?? new Date()).getTime() + ttl * 1000).toISOString();
|
|
10
|
+
}
|
|
11
|
+
export async function mintPrefixUploadControl(value, signer, options = {}) {
|
|
12
|
+
const ttlSeconds = options.ttlSeconds ?? DEFAULT_TRANSPORT_TTL_SECONDS;
|
|
13
|
+
const maxObjectBytes = options.maxObjectBytes ?? DEFAULT_MAX_BODY_BYTES;
|
|
14
|
+
if (!Number.isSafeInteger(maxObjectBytes) ||
|
|
15
|
+
maxObjectBytes < 1 ||
|
|
16
|
+
maxObjectBytes > DEFAULT_MAX_BODY_BYTES)
|
|
17
|
+
throw new Error('Coverage transport size limit is invalid');
|
|
18
|
+
const upload = await signer.signPost(`${transportPrefix(value)}/`, ttlSeconds, maxObjectBytes);
|
|
19
|
+
if (upload.maxObjectBytes > maxObjectBytes)
|
|
20
|
+
throw new Error('Coverage transport signer size limit is invalid');
|
|
21
|
+
return parseTransportControl({
|
|
22
|
+
version: 2,
|
|
23
|
+
mode: 'prefix-upload',
|
|
24
|
+
repository: value.repository,
|
|
25
|
+
revision: value.revision,
|
|
26
|
+
run: { id: value.runId, controlAttempt: value.controlAttempt },
|
|
27
|
+
expiresAt: transportExpiresAt(options),
|
|
28
|
+
upload,
|
|
29
|
+
});
|
|
30
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vouchington-tooling",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.22",
|
|
4
4
|
"description": "Vouchington CLI and extractable tooling libraries.",
|
|
5
5
|
"homepage": "https://github.com/vouchington/vouchington-tooling/tree/main/packages/vouchington-tooling#readme",
|
|
6
6
|
"bugs": {
|