urma-mcp 0.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +389 -0
- package/assets/urma-banner.png +0 -0
- package/assets/urma-logo.png +0 -0
- package/dist/src/acquisition/frame-panel.d.ts +22 -0
- package/dist/src/acquisition/frame-panel.js +110 -0
- package/dist/src/acquisition/frame-panel.js.map +1 -0
- package/dist/src/acquisition/frames.d.ts +37 -0
- package/dist/src/acquisition/frames.js +463 -0
- package/dist/src/acquisition/frames.js.map +1 -0
- package/dist/src/acquisition/media.d.ts +59 -0
- package/dist/src/acquisition/media.js +946 -0
- package/dist/src/acquisition/media.js.map +1 -0
- package/dist/src/acquisition/overview.d.ts +31 -0
- package/dist/src/acquisition/overview.js +335 -0
- package/dist/src/acquisition/overview.js.map +1 -0
- package/dist/src/acquisition/panel.d.ts +16 -0
- package/dist/src/acquisition/panel.js +125 -0
- package/dist/src/acquisition/panel.js.map +1 -0
- package/dist/src/acquisition/records.d.ts +24 -0
- package/dist/src/acquisition/records.js +127 -0
- package/dist/src/acquisition/records.js.map +1 -0
- package/dist/src/acquisition/remote-budget.d.ts +4 -0
- package/dist/src/acquisition/remote-budget.js +121 -0
- package/dist/src/acquisition/remote-budget.js.map +1 -0
- package/dist/src/acquisition/singleflight.d.ts +10 -0
- package/dist/src/acquisition/singleflight.js +75 -0
- package/dist/src/acquisition/singleflight.js.map +1 -0
- package/dist/src/acquisition/storyboard.d.ts +36 -0
- package/dist/src/acquisition/storyboard.js +324 -0
- package/dist/src/acquisition/storyboard.js.map +1 -0
- package/dist/src/acquisition/transcript.d.ts +25 -0
- package/dist/src/acquisition/transcript.js +367 -0
- package/dist/src/acquisition/transcript.js.map +1 -0
- package/dist/src/acquisition/transport-policy.d.ts +7 -0
- package/dist/src/acquisition/transport-policy.js +22 -0
- package/dist/src/acquisition/transport-policy.js.map +1 -0
- package/dist/src/acquisition/video-timing.d.ts +24 -0
- package/dist/src/acquisition/video-timing.js +249 -0
- package/dist/src/acquisition/video-timing.js.map +1 -0
- package/dist/src/app.d.ts +49 -0
- package/dist/src/app.js +31 -0
- package/dist/src/app.js.map +1 -0
- package/dist/src/cli/doctor.d.ts +24 -0
- package/dist/src/cli/doctor.js +368 -0
- package/dist/src/cli/doctor.js.map +1 -0
- package/dist/src/cli/main.d.ts +2 -0
- package/dist/src/cli/main.js +79 -0
- package/dist/src/cli/main.js.map +1 -0
- package/dist/src/config.d.ts +36 -0
- package/dist/src/config.js +110 -0
- package/dist/src/config.js.map +1 -0
- package/dist/src/core/coverage.d.ts +20 -0
- package/dist/src/core/coverage.js +89 -0
- package/dist/src/core/coverage.js.map +1 -0
- package/dist/src/core/diagnostics.d.ts +58 -0
- package/dist/src/core/diagnostics.js +451 -0
- package/dist/src/core/diagnostics.js.map +1 -0
- package/dist/src/core/errors.d.ts +13 -0
- package/dist/src/core/errors.js +57 -0
- package/dist/src/core/errors.js.map +1 -0
- package/dist/src/core/frame-schedule.d.ts +16 -0
- package/dist/src/core/frame-schedule.js +52 -0
- package/dist/src/core/frame-schedule.js.map +1 -0
- package/dist/src/core/ids.d.ts +48 -0
- package/dist/src/core/ids.js +122 -0
- package/dist/src/core/ids.js.map +1 -0
- package/dist/src/core/model.d.ts +216 -0
- package/dist/src/core/model.js +2 -0
- package/dist/src/core/model.js.map +1 -0
- package/dist/src/core/request-key.d.ts +1 -0
- package/dist/src/core/request-key.js +8 -0
- package/dist/src/core/request-key.js.map +1 -0
- package/dist/src/core/search-limits.d.ts +11 -0
- package/dist/src/core/search-limits.js +13 -0
- package/dist/src/core/search-limits.js.map +1 -0
- package/dist/src/distribution/active.d.ts +20 -0
- package/dist/src/distribution/active.js +169 -0
- package/dist/src/distribution/active.js.map +1 -0
- package/dist/src/distribution/archive.d.ts +8 -0
- package/dist/src/distribution/archive.js +368 -0
- package/dist/src/distribution/archive.js.map +1 -0
- package/dist/src/distribution/download.d.ts +8 -0
- package/dist/src/distribution/download.js +93 -0
- package/dist/src/distribution/download.js.map +1 -0
- package/dist/src/distribution/host-registration.d.ts +7 -0
- package/dist/src/distribution/host-registration.js +64 -0
- package/dist/src/distribution/host-registration.js.map +1 -0
- package/dist/src/distribution/installer.d.ts +38 -0
- package/dist/src/distribution/installer.js +445 -0
- package/dist/src/distribution/installer.js.map +1 -0
- package/dist/src/distribution/integrity.d.ts +8 -0
- package/dist/src/distribution/integrity.js +131 -0
- package/dist/src/distribution/integrity.js.map +1 -0
- package/dist/src/distribution/lock.d.ts +14 -0
- package/dist/src/distribution/lock.js +136 -0
- package/dist/src/distribution/lock.js.map +1 -0
- package/dist/src/distribution/manifest.d.ts +46 -0
- package/dist/src/distribution/manifest.js +202 -0
- package/dist/src/distribution/manifest.js.map +1 -0
- package/dist/src/distribution/paths.d.ts +30 -0
- package/dist/src/distribution/paths.js +154 -0
- package/dist/src/distribution/paths.js.map +1 -0
- package/dist/src/distribution/platform.d.ts +15 -0
- package/dist/src/distribution/platform.js +76 -0
- package/dist/src/distribution/platform.js.map +1 -0
- package/dist/src/distribution/qualification.d.ts +26 -0
- package/dist/src/distribution/qualification.js +265 -0
- package/dist/src/distribution/qualification.js.map +1 -0
- package/dist/src/distribution/receipt.d.ts +52 -0
- package/dist/src/distribution/receipt.js +142 -0
- package/dist/src/distribution/receipt.js.map +1 -0
- package/dist/src/distribution/runtime.d.ts +8 -0
- package/dist/src/distribution/runtime.js +173 -0
- package/dist/src/distribution/runtime.js.map +1 -0
- package/dist/src/evidence/service.d.ts +959 -0
- package/dist/src/evidence/service.js +1455 -0
- package/dist/src/evidence/service.js.map +1 -0
- package/dist/src/evidence/state.d.ts +52 -0
- package/dist/src/evidence/state.js +362 -0
- package/dist/src/evidence/state.js.map +1 -0
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.js +6 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/mcp/logo.d.ts +5 -0
- package/dist/src/mcp/logo.js +11 -0
- package/dist/src/mcp/logo.js.map +1 -0
- package/dist/src/mcp/projection.d.ts +4 -0
- package/dist/src/mcp/projection.js +144 -0
- package/dist/src/mcp/projection.js.map +1 -0
- package/dist/src/mcp/schemas.d.ts +1744 -0
- package/dist/src/mcp/schemas.js +662 -0
- package/dist/src/mcp/schemas.js.map +1 -0
- package/dist/src/mcp/server.d.ts +6 -0
- package/dist/src/mcp/server.js +470 -0
- package/dist/src/mcp/server.js.map +1 -0
- package/dist/src/mcp/stdio.d.ts +1 -0
- package/dist/src/mcp/stdio.js +36 -0
- package/dist/src/mcp/stdio.js.map +1 -0
- package/dist/src/remote/egress.d.ts +42 -0
- package/dist/src/remote/egress.js +611 -0
- package/dist/src/remote/egress.js.map +1 -0
- package/dist/src/remote/formats.d.ts +5 -0
- package/dist/src/remote/formats.js +30 -0
- package/dist/src/remote/formats.js.map +1 -0
- package/dist/src/remote/lease.d.ts +15 -0
- package/dist/src/remote/lease.js +2 -0
- package/dist/src/remote/lease.js.map +1 -0
- package/dist/src/remote/normalize.d.ts +14 -0
- package/dist/src/remote/normalize.js +243 -0
- package/dist/src/remote/normalize.js.map +1 -0
- package/dist/src/remote/policy.d.ts +13 -0
- package/dist/src/remote/policy.js +107 -0
- package/dist/src/remote/policy.js.map +1 -0
- package/dist/src/remote/worker.d.ts +14 -0
- package/dist/src/remote/worker.js +40 -0
- package/dist/src/remote/worker.js.map +1 -0
- package/dist/src/sources/candidates.d.ts +18 -0
- package/dist/src/sources/candidates.js +23 -0
- package/dist/src/sources/candidates.js.map +1 -0
- package/dist/src/sources/caption-tracks.d.ts +12 -0
- package/dist/src/sources/caption-tracks.js +75 -0
- package/dist/src/sources/caption-tracks.js.map +1 -0
- package/dist/src/sources/local.d.ts +42 -0
- package/dist/src/sources/local.js +203 -0
- package/dist/src/sources/local.js.map +1 -0
- package/dist/src/sources/metadata.d.ts +15 -0
- package/dist/src/sources/metadata.js +513 -0
- package/dist/src/sources/metadata.js.map +1 -0
- package/dist/src/sources/resolver.d.ts +22 -0
- package/dist/src/sources/resolver.js +363 -0
- package/dist/src/sources/resolver.js.map +1 -0
- package/dist/src/sources/timeline.d.ts +23 -0
- package/dist/src/sources/timeline.js +145 -0
- package/dist/src/sources/timeline.js.map +1 -0
- package/dist/src/sources/types.d.ts +67 -0
- package/dist/src/sources/types.js +2 -0
- package/dist/src/sources/types.js.map +1 -0
- package/dist/src/sources/youtube.d.ts +10 -0
- package/dist/src/sources/youtube.js +59 -0
- package/dist/src/sources/youtube.js.map +1 -0
- package/dist/src/store/blob-store.d.ts +20 -0
- package/dist/src/store/blob-store.js +156 -0
- package/dist/src/store/blob-store.js.map +1 -0
- package/dist/src/store/schema.d.ts +4 -0
- package/dist/src/store/schema.js +169 -0
- package/dist/src/store/schema.js.map +1 -0
- package/dist/src/store/sqlite-store.d.ts +47 -0
- package/dist/src/store/sqlite-store.js +545 -0
- package/dist/src/store/sqlite-store.js.map +1 -0
- package/dist/src/store/store.d.ts +138 -0
- package/dist/src/store/store.js +2 -0
- package/dist/src/store/store.js.map +1 -0
- package/dist/src/subprocess/ffmpeg.d.ts +18 -0
- package/dist/src/subprocess/ffmpeg.js +117 -0
- package/dist/src/subprocess/ffmpeg.js.map +1 -0
- package/dist/src/subprocess/ffprobe.d.ts +11 -0
- package/dist/src/subprocess/ffprobe.js +62 -0
- package/dist/src/subprocess/ffprobe.js.map +1 -0
- package/dist/src/subprocess/redaction.d.ts +3 -0
- package/dist/src/subprocess/redaction.js +31 -0
- package/dist/src/subprocess/redaction.js.map +1 -0
- package/dist/src/subprocess/remote-media.d.ts +3 -0
- package/dist/src/subprocess/remote-media.js +27 -0
- package/dist/src/subprocess/remote-media.js.map +1 -0
- package/dist/src/subprocess/runner.d.ts +23 -0
- package/dist/src/subprocess/runner.js +197 -0
- package/dist/src/subprocess/runner.js.map +1 -0
- package/dist/src/subprocess/versions.d.ts +8 -0
- package/dist/src/subprocess/versions.js +46 -0
- package/dist/src/subprocess/versions.js.map +1 -0
- package/dist/src/subprocess/ytdlp.d.ts +39 -0
- package/dist/src/subprocess/ytdlp.js +326 -0
- package/dist/src/subprocess/ytdlp.js.map +1 -0
- package/dist/src/version.d.ts +8 -0
- package/dist/src/version.js +44 -0
- package/dist/src/version.js.map +1 -0
- package/docs/ARCHITECTURE.md +273 -0
- package/docs/CONTRIBUTING.md +101 -0
- package/docs/EVIDENCE_MODEL.md +292 -0
- package/docs/PRODUCT_SCOPE.md +115 -0
- package/docs/SECURITY.md +182 -0
- package/docs/assets/architecture.png +0 -0
- package/docs/assets/contribution.png +0 -0
- package/docs/assets/evidence-model.png +0 -0
- package/docs/assets/product-scope.png +0 -0
- package/docs/assets/security.png +0 -0
- package/launcher-v1.mjs +430 -0
- package/package.json +45 -0
|
@@ -0,0 +1,946 @@
|
|
|
1
|
+
import { readdir, realpath, stat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { currentExactFrameDiagnosticTrace, diagnosticLog, measureDiagnosticAsync, } from "../core/diagnostics.js";
|
|
4
|
+
import { normalizeError, UrmaError } from "../core/errors.js";
|
|
5
|
+
import { deterministicRequestKey } from "../core/request-key.js";
|
|
6
|
+
import { candidateKeyForSourceFormat, safeFormatDescription } from "../sources/candidates.js";
|
|
7
|
+
import { Ffprobe } from "../subprocess/ffprobe.js";
|
|
8
|
+
import { YtDlp } from "../subprocess/ytdlp.js";
|
|
9
|
+
import { collectBinaryVersions, } from "../subprocess/versions.js";
|
|
10
|
+
import { BlobStore } from "../store/blob-store.js";
|
|
11
|
+
import { URMA_VERSION } from "../version.js";
|
|
12
|
+
import { assertExpectedRemoteBytes, assertRemoteDirectoryWithinBudget, withRemoteAcquisitionDirectory, } from "./remote-budget.js";
|
|
13
|
+
import { startAcquisition } from "./records.js";
|
|
14
|
+
import { parseStoredBoundedVideoCoverage, parseStoredVideoCoverage, parseVideoStreamCoverage, serializeVideoPtsCoverage, } from "./video-timing.js";
|
|
15
|
+
const SECTION_PREFIX = "URMA_SECTION\t";
|
|
16
|
+
const SECTION_OUTPUT_TEMPLATE = "media-%(section_start)010.3f-%(section_end)010.3f.%(ext)s";
|
|
17
|
+
// yt-dlp sections use seconds; Urma identities use integer milliseconds
|
|
18
|
+
const SECTION_METADATA_TOLERANCE_MS = 1;
|
|
19
|
+
function targetedDerivativeUnavailable(message, detail = {}) {
|
|
20
|
+
return new UrmaError("TARGETED_MEDIA_UNAVAILABLE", message, {
|
|
21
|
+
detail,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
function videoFormats(source) {
|
|
25
|
+
return source.formats.filter((item) => item.videoCodec !== null &&
|
|
26
|
+
item.videoCodec !== "none" &&
|
|
27
|
+
item.ext !== "mhtml");
|
|
28
|
+
}
|
|
29
|
+
function navFormat(source) {
|
|
30
|
+
const formats = videoFormats(source);
|
|
31
|
+
return ([...formats.filter((item) => (item.height ?? Infinity) <= 144)].sort((a, b) => (b.height ?? 0) - (a.height ?? 0) ||
|
|
32
|
+
(a.estimatedBytes ?? Infinity) - (b.estimatedBytes ?? Infinity))[0] ??
|
|
33
|
+
[...formats].sort((a, b) => (a.height ?? Infinity) - (b.height ?? Infinity) ||
|
|
34
|
+
(a.estimatedBytes ?? Infinity) - (b.estimatedBytes ?? Infinity))[0] ??
|
|
35
|
+
null);
|
|
36
|
+
}
|
|
37
|
+
function evidenceFormats(source, hls) {
|
|
38
|
+
return [
|
|
39
|
+
...videoFormats(source)
|
|
40
|
+
.filter((item) => !hls || item.protocol?.startsWith("m3u8"))
|
|
41
|
+
.filter((item) => (item.height ?? 0) <= 1080),
|
|
42
|
+
].sort((a, b) => (b.height ?? 0) - (a.height ?? 0) ||
|
|
43
|
+
(a.estimatedBytes ?? Infinity) - (b.estimatedBytes ?? Infinity));
|
|
44
|
+
}
|
|
45
|
+
function evidenceFormat(source, hls) {
|
|
46
|
+
return evidenceFormats(source, hls)[0] ?? null;
|
|
47
|
+
}
|
|
48
|
+
function formatIdentity(source, format) {
|
|
49
|
+
return format === null ? { candidateKey: null, id: null } : {
|
|
50
|
+
candidateKey: candidateKeyForSourceFormat(source, format),
|
|
51
|
+
id: format.id,
|
|
52
|
+
...safeFormatDescription(format),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/** Describe the selected exact-frame transport representation without acquiring media */
|
|
56
|
+
export function frameEvidenceRepresentation(config, source) {
|
|
57
|
+
if (source.kind === "local")
|
|
58
|
+
return { mode: "local-direct" };
|
|
59
|
+
const bounded = evidenceFormat(source, true);
|
|
60
|
+
if (bounded !== null) {
|
|
61
|
+
return {
|
|
62
|
+
mode: "hls-bounded-section",
|
|
63
|
+
format: formatIdentity(source, bounded),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
const formats = evidenceFormats(source, false);
|
|
67
|
+
const reusable = formats.find((item) => item.estimatedBytes === null ||
|
|
68
|
+
Math.ceil(item.estimatedBytes) <=
|
|
69
|
+
config.limits.maxReusableEvidenceMediaBytes) ??
|
|
70
|
+
formats[0] ??
|
|
71
|
+
null;
|
|
72
|
+
return {
|
|
73
|
+
mode: "reusable-evidence",
|
|
74
|
+
format: formatIdentity(source, reusable),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function mimeTypeFor(file) {
|
|
78
|
+
switch (path.extname(file).toLowerCase()) {
|
|
79
|
+
case ".mp4":
|
|
80
|
+
case ".m4v":
|
|
81
|
+
return "video/mp4";
|
|
82
|
+
case ".webm":
|
|
83
|
+
return "video/webm";
|
|
84
|
+
case ".mkv":
|
|
85
|
+
return "video/x-matroska";
|
|
86
|
+
case ".mov":
|
|
87
|
+
return "video/quicktime";
|
|
88
|
+
default:
|
|
89
|
+
return "application/octet-stream";
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function sectionArgument(startMs, endMs) {
|
|
93
|
+
return `*${(startMs / 1_000).toFixed(3)}-${(endMs / 1_000).toFixed(3)}`;
|
|
94
|
+
}
|
|
95
|
+
function withoutFormatSelector(args, formatId) {
|
|
96
|
+
const output = [];
|
|
97
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
98
|
+
if (args[index] === "-f" && args[index + 1] === formatId) {
|
|
99
|
+
index += 1;
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
output.push(args[index]);
|
|
103
|
+
}
|
|
104
|
+
return output;
|
|
105
|
+
}
|
|
106
|
+
async function leaseFor(downloader, source, format, signal) {
|
|
107
|
+
if (source.kind !== "remote")
|
|
108
|
+
return null;
|
|
109
|
+
if (typeof downloader.lease !== "function")
|
|
110
|
+
return null;
|
|
111
|
+
return await downloader.lease(source, format, signal);
|
|
112
|
+
}
|
|
113
|
+
function sectionSpec(source, startMs, endMs) {
|
|
114
|
+
const format = evidenceFormat(source, true);
|
|
115
|
+
if (!format) {
|
|
116
|
+
throw new UrmaError("TARGETED_MEDIA_UNAVAILABLE", "Source has no targetable HLS video format; use reusable evidence media");
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
operation: "media-section",
|
|
120
|
+
kind: "media_section",
|
|
121
|
+
startMs,
|
|
122
|
+
endMs,
|
|
123
|
+
format,
|
|
124
|
+
args: [
|
|
125
|
+
"--download-sections",
|
|
126
|
+
sectionArgument(startMs, endMs),
|
|
127
|
+
"-f",
|
|
128
|
+
format.id,
|
|
129
|
+
],
|
|
130
|
+
version: "bounded-section",
|
|
131
|
+
params: {
|
|
132
|
+
candidateKey: candidateKeyForSourceFormat(source, format),
|
|
133
|
+
formatId: format.id,
|
|
134
|
+
fidelity: "evidence",
|
|
135
|
+
requestedStartMs: startMs,
|
|
136
|
+
requestedEndMs: endMs,
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
function sectionRequestKey(requirement) {
|
|
141
|
+
const spec = sectionSpec(requirement.source, requirement.startMs, requirement.endMs);
|
|
142
|
+
return deterministicRequestKey(requirement.source.revision, spec.operation, { startMs: spec.startMs, endMs: spec.endMs, ...spec.params }, spec.version);
|
|
143
|
+
}
|
|
144
|
+
function sectionRequirementIdentity(requirement) {
|
|
145
|
+
return JSON.stringify([
|
|
146
|
+
requirement.source.sourceRef,
|
|
147
|
+
requirement.source.revision,
|
|
148
|
+
requirement.investigationRef,
|
|
149
|
+
sectionRequestKey(requirement),
|
|
150
|
+
]);
|
|
151
|
+
}
|
|
152
|
+
function compatibilityKey(config, requirement) {
|
|
153
|
+
const format = evidenceFormat(requirement.source, true);
|
|
154
|
+
return JSON.stringify([
|
|
155
|
+
requirement.investigationRef,
|
|
156
|
+
requirement.source.sourceRef,
|
|
157
|
+
requirement.source.revision,
|
|
158
|
+
requirement.source.canonicalLocator,
|
|
159
|
+
config.ytdlp,
|
|
160
|
+
"bounded-section",
|
|
161
|
+
format === null
|
|
162
|
+
? null
|
|
163
|
+
: candidateKeyForSourceFormat(requirement.source, format),
|
|
164
|
+
format?.id ?? null,
|
|
165
|
+
format?.protocol ?? null,
|
|
166
|
+
format?.height ?? null,
|
|
167
|
+
"evidence",
|
|
168
|
+
]);
|
|
169
|
+
}
|
|
170
|
+
/** Group section requests by the effective bounded-section acquisition key */
|
|
171
|
+
export function groupCompatibleSectionRequirements(config, requirements) {
|
|
172
|
+
const groups = new Map();
|
|
173
|
+
for (const requirement of requirements) {
|
|
174
|
+
const key = compatibilityKey(config, requirement);
|
|
175
|
+
const group = groups.get(key);
|
|
176
|
+
if (group)
|
|
177
|
+
group.push(requirement);
|
|
178
|
+
else
|
|
179
|
+
groups.set(key, [requirement]);
|
|
180
|
+
}
|
|
181
|
+
return [...groups.values()];
|
|
182
|
+
}
|
|
183
|
+
function milliseconds(seconds) {
|
|
184
|
+
const parsed = Number(seconds);
|
|
185
|
+
return Number.isFinite(parsed) ? Math.round(parsed * 1_000) : null;
|
|
186
|
+
}
|
|
187
|
+
function parseSectionEmissions(stdout) {
|
|
188
|
+
const emissions = [];
|
|
189
|
+
for (const line of stdout.toString("utf8").split(/\r?\n/u)) {
|
|
190
|
+
if (!line.startsWith(SECTION_PREFIX))
|
|
191
|
+
continue;
|
|
192
|
+
const fields = line.slice(SECTION_PREFIX.length).split("\t");
|
|
193
|
+
if (fields.length !== 3)
|
|
194
|
+
continue;
|
|
195
|
+
try {
|
|
196
|
+
const sectionStartMs = milliseconds(JSON.parse(fields[0]));
|
|
197
|
+
const sectionEndMs = milliseconds(JSON.parse(fields[1]));
|
|
198
|
+
const filepath = JSON.parse(fields[2]);
|
|
199
|
+
if (sectionStartMs !== null &&
|
|
200
|
+
sectionEndMs !== null &&
|
|
201
|
+
typeof filepath === "string") {
|
|
202
|
+
emissions.push({ sectionStartMs, sectionEndMs, filepath });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return emissions;
|
|
209
|
+
}
|
|
210
|
+
async function scanSectionEmissions(directory) {
|
|
211
|
+
const emissions = [];
|
|
212
|
+
for (const name of await readdir(directory)) {
|
|
213
|
+
const match = /^media-([0-9]+\.[0-9]{3})-([0-9]+\.[0-9]{3})\.[^.]+$/u.exec(name);
|
|
214
|
+
if (!match)
|
|
215
|
+
continue;
|
|
216
|
+
const sectionStartMs = milliseconds(match[1]);
|
|
217
|
+
const sectionEndMs = milliseconds(match[2]);
|
|
218
|
+
if (sectionStartMs !== null && sectionEndMs !== null) {
|
|
219
|
+
emissions.push({
|
|
220
|
+
sectionStartMs,
|
|
221
|
+
sectionEndMs,
|
|
222
|
+
filepath: path.join(directory, name),
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return emissions;
|
|
227
|
+
}
|
|
228
|
+
function matchingRequirements(requirements, emission) {
|
|
229
|
+
return requirements.filter((requirement) => Math.abs(requirement.startMs - emission.sectionStartMs) <=
|
|
230
|
+
SECTION_METADATA_TOLERANCE_MS &&
|
|
231
|
+
Math.abs(requirement.endMs - emission.sectionEndMs) <=
|
|
232
|
+
SECTION_METADATA_TOLERANCE_MS);
|
|
233
|
+
}
|
|
234
|
+
function uniqueCandidateMap(directory, requirements, emissions) {
|
|
235
|
+
const mapped = new Map();
|
|
236
|
+
const seen = new Set();
|
|
237
|
+
for (const emission of emissions) {
|
|
238
|
+
const matches = matchingRequirements(requirements, emission);
|
|
239
|
+
if (matches.length !== 1)
|
|
240
|
+
continue;
|
|
241
|
+
const identity = sectionRequirementIdentity(matches[0]);
|
|
242
|
+
const resolvedPath = path.resolve(directory, emission.filepath);
|
|
243
|
+
const candidateIdentity = JSON.stringify([
|
|
244
|
+
identity,
|
|
245
|
+
emission.sectionStartMs,
|
|
246
|
+
emission.sectionEndMs,
|
|
247
|
+
resolvedPath,
|
|
248
|
+
]);
|
|
249
|
+
if (seen.has(candidateIdentity))
|
|
250
|
+
continue;
|
|
251
|
+
seen.add(candidateIdentity);
|
|
252
|
+
const candidates = mapped.get(identity);
|
|
253
|
+
if (candidates)
|
|
254
|
+
candidates.push(emission);
|
|
255
|
+
else
|
|
256
|
+
mapped.set(identity, [emission]);
|
|
257
|
+
}
|
|
258
|
+
return mapped;
|
|
259
|
+
}
|
|
260
|
+
export function mediaBudgetBytes(config, kind) {
|
|
261
|
+
if (kind === "media_section")
|
|
262
|
+
return config.limits.maxTargetedMediaBytes;
|
|
263
|
+
if (kind === "navigation_media")
|
|
264
|
+
return config.limits.maxNavigationCopyBytes;
|
|
265
|
+
if (kind === "evidence_media") {
|
|
266
|
+
return config.limits.maxReusableEvidenceMediaBytes;
|
|
267
|
+
}
|
|
268
|
+
throw new RangeError(`Artifact kind ${kind} has no remote media budget`);
|
|
269
|
+
}
|
|
270
|
+
export function expectedMediaBytes(source, spec) {
|
|
271
|
+
const expected = spec.format.estimatedBytes;
|
|
272
|
+
if (expected === null)
|
|
273
|
+
return null;
|
|
274
|
+
if (spec.kind !== "media_section")
|
|
275
|
+
return Math.ceil(expected);
|
|
276
|
+
const requested = Math.max(0, spec.endMs - spec.startMs);
|
|
277
|
+
if (source.durationMs < 1 || requested < 1)
|
|
278
|
+
return null;
|
|
279
|
+
return Math.ceil(expected * Math.min(1, requested / source.durationMs));
|
|
280
|
+
}
|
|
281
|
+
export class MediaAcquirer {
|
|
282
|
+
config;
|
|
283
|
+
store;
|
|
284
|
+
blobs;
|
|
285
|
+
remoteContext;
|
|
286
|
+
downloader;
|
|
287
|
+
constructor(config, store, blobs, downloader, remoteContext = null) {
|
|
288
|
+
this.config = config;
|
|
289
|
+
this.store = store;
|
|
290
|
+
this.blobs = blobs;
|
|
291
|
+
this.remoteContext = remoteContext;
|
|
292
|
+
this.downloader = downloader ?? new YtDlp(config, undefined, remoteContext);
|
|
293
|
+
}
|
|
294
|
+
async navigation(source, ref, signal) {
|
|
295
|
+
const format = navFormat(source);
|
|
296
|
+
if (!format) {
|
|
297
|
+
throw new UrmaError("TARGETED_MEDIA_UNAVAILABLE", "Source has no usable video format for a navigation copy");
|
|
298
|
+
}
|
|
299
|
+
return await this.#download(source, ref, {
|
|
300
|
+
operation: "navigation-copy",
|
|
301
|
+
kind: "navigation_media",
|
|
302
|
+
startMs: 0,
|
|
303
|
+
endMs: source.durationMs,
|
|
304
|
+
format,
|
|
305
|
+
args: ["-f", format.id],
|
|
306
|
+
version: "navigation-copy",
|
|
307
|
+
params: {
|
|
308
|
+
candidateKey: candidateKeyForSourceFormat(source, format),
|
|
309
|
+
formatId: format.id,
|
|
310
|
+
fidelity: "navigation",
|
|
311
|
+
},
|
|
312
|
+
}, signal);
|
|
313
|
+
}
|
|
314
|
+
async reusableEvidence(source, ref, signal) {
|
|
315
|
+
const formats = evidenceFormats(source, false);
|
|
316
|
+
const format = formats.find((item) => item.estimatedBytes === null ||
|
|
317
|
+
Math.ceil(item.estimatedBytes) <=
|
|
318
|
+
this.config.limits.maxReusableEvidenceMediaBytes) ??
|
|
319
|
+
formats[0] ??
|
|
320
|
+
null;
|
|
321
|
+
if (!format) {
|
|
322
|
+
throw new UrmaError("TARGETED_MEDIA_UNAVAILABLE", "Source has no usable evidence-fidelity video format");
|
|
323
|
+
}
|
|
324
|
+
return await this.#download(source, ref, {
|
|
325
|
+
operation: "evidence-copy",
|
|
326
|
+
kind: "evidence_media",
|
|
327
|
+
startMs: 0,
|
|
328
|
+
endMs: source.durationMs,
|
|
329
|
+
format,
|
|
330
|
+
args: ["-f", format.id],
|
|
331
|
+
version: "evidence-copy",
|
|
332
|
+
params: {
|
|
333
|
+
candidateKey: candidateKeyForSourceFormat(source, format),
|
|
334
|
+
formatId: format.id,
|
|
335
|
+
fidelity: "evidence",
|
|
336
|
+
},
|
|
337
|
+
}, signal);
|
|
338
|
+
}
|
|
339
|
+
async section(source, ref, startMs, endMs, signal) {
|
|
340
|
+
return await this.#download(source, ref, sectionSpec(source, startMs, endMs), signal);
|
|
341
|
+
}
|
|
342
|
+
async sections(requirements, signal) {
|
|
343
|
+
if (requirements.length === 0)
|
|
344
|
+
return [];
|
|
345
|
+
const diagnostics = {
|
|
346
|
+
logicalSectionRequirements: requirements.length,
|
|
347
|
+
uniqueSectionRequirements: 0,
|
|
348
|
+
compatibleBatchCount: 0,
|
|
349
|
+
sectionsPerBatch: [],
|
|
350
|
+
outerYtDlpInvocations: 0,
|
|
351
|
+
multiSectionYtDlpInvocations: 0,
|
|
352
|
+
singleSectionInvocations: 0,
|
|
353
|
+
singleSectionFallbackInvocations: 0,
|
|
354
|
+
batchSectionsRequested: 0,
|
|
355
|
+
batchSectionsValid: 0,
|
|
356
|
+
batchSectionsInvalid: 0,
|
|
357
|
+
batchSectionsMissingUnmapped: 0,
|
|
358
|
+
fallbackSectionsRequested: 0,
|
|
359
|
+
fallbackSectionsSuccessful: 0,
|
|
360
|
+
fallbackSectionsFailed: 0,
|
|
361
|
+
batchElapsedMs: 0,
|
|
362
|
+
fallbackElapsedMs: 0,
|
|
363
|
+
cacheHits: 0,
|
|
364
|
+
};
|
|
365
|
+
const outcomes = new Map();
|
|
366
|
+
const unique = new Map();
|
|
367
|
+
for (const requirement of requirements) {
|
|
368
|
+
unique.set(sectionRequirementIdentity(requirement), requirement);
|
|
369
|
+
}
|
|
370
|
+
diagnostics.uniqueSectionRequirements = unique.size;
|
|
371
|
+
try {
|
|
372
|
+
for (const group of groupCompatibleSectionRequirements(this.config, [
|
|
373
|
+
...unique.values(),
|
|
374
|
+
])) {
|
|
375
|
+
const missing = [];
|
|
376
|
+
for (const requirement of group) {
|
|
377
|
+
const cached = await this.#cachedSection(requirement);
|
|
378
|
+
if (cached) {
|
|
379
|
+
diagnostics.cacheHits += 1;
|
|
380
|
+
outcomes.set(sectionRequirementIdentity(requirement), {
|
|
381
|
+
status: "fulfilled",
|
|
382
|
+
value: cached,
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
else
|
|
386
|
+
missing.push(requirement);
|
|
387
|
+
}
|
|
388
|
+
if (missing.length === 0)
|
|
389
|
+
continue;
|
|
390
|
+
if (missing.length === 1) {
|
|
391
|
+
await this.#acquireSingleIntoOutcomes(missing[0], outcomes, diagnostics, false, signal);
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
const attempted = await this.#acquireBatch(missing, signal);
|
|
395
|
+
if (attempted.invoked) {
|
|
396
|
+
diagnostics.compatibleBatchCount += 1;
|
|
397
|
+
diagnostics.sectionsPerBatch.push(attempted.requested);
|
|
398
|
+
diagnostics.outerYtDlpInvocations += 1;
|
|
399
|
+
if (attempted.requested > 1) {
|
|
400
|
+
diagnostics.multiSectionYtDlpInvocations += 1;
|
|
401
|
+
}
|
|
402
|
+
else
|
|
403
|
+
diagnostics.singleSectionInvocations += 1;
|
|
404
|
+
}
|
|
405
|
+
diagnostics.batchSectionsRequested += attempted.requested;
|
|
406
|
+
diagnostics.batchSectionsValid += attempted.valid;
|
|
407
|
+
diagnostics.batchSectionsInvalid += attempted.invalid;
|
|
408
|
+
diagnostics.batchSectionsMissingUnmapped += attempted.missingUnmapped;
|
|
409
|
+
diagnostics.batchElapsedMs += attempted.elapsedMs;
|
|
410
|
+
for (const [identity, value] of attempted.values) {
|
|
411
|
+
outcomes.set(identity, { status: "fulfilled", value });
|
|
412
|
+
}
|
|
413
|
+
for (const requirement of missing) {
|
|
414
|
+
const identity = sectionRequirementIdentity(requirement);
|
|
415
|
+
if (attempted.values.has(identity))
|
|
416
|
+
continue;
|
|
417
|
+
const batchFailure = attempted.failures.get(identity);
|
|
418
|
+
const batchFailureCode = batchFailure === undefined
|
|
419
|
+
? null
|
|
420
|
+
: normalizeError(batchFailure).code;
|
|
421
|
+
if (batchFailureCode === "CANCELLED")
|
|
422
|
+
throw batchFailure;
|
|
423
|
+
if (batchFailureCode === "MEDIA_BUDGET_EXCEEDED") {
|
|
424
|
+
outcomes.set(identity, {
|
|
425
|
+
status: "rejected",
|
|
426
|
+
reason: batchFailure,
|
|
427
|
+
});
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
if (!attempted.unresolved.has(identity)) {
|
|
431
|
+
outcomes.set(identity, {
|
|
432
|
+
status: "rejected",
|
|
433
|
+
reason: batchFailure ?? new UrmaError("TARGETED_MEDIA_UNAVAILABLE", `Batched bounded section [${requirement.startMs},${requirement.endMs}) was validated but unavailable`),
|
|
434
|
+
});
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
await this.#acquireSingleIntoOutcomes(requirement, outcomes, diagnostics, true, signal);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
finally {
|
|
442
|
+
diagnosticLog(this.config.debug, "bounded-section-batching", {
|
|
443
|
+
logicalSectionRequirements: diagnostics.logicalSectionRequirements,
|
|
444
|
+
uniqueSectionRequirements: diagnostics.uniqueSectionRequirements,
|
|
445
|
+
compatibleBatchCount: diagnostics.compatibleBatchCount,
|
|
446
|
+
sectionsPerBatch: diagnostics.sectionsPerBatch.join(","),
|
|
447
|
+
outerYtDlpInvocations: diagnostics.outerYtDlpInvocations,
|
|
448
|
+
multiSectionYtDlpInvocations: diagnostics.multiSectionYtDlpInvocations,
|
|
449
|
+
singleSectionInvocations: diagnostics.singleSectionInvocations,
|
|
450
|
+
singleSectionFallbackInvocations: diagnostics.singleSectionFallbackInvocations,
|
|
451
|
+
batchSectionsRequested: diagnostics.batchSectionsRequested,
|
|
452
|
+
batchSectionsValid: diagnostics.batchSectionsValid,
|
|
453
|
+
batchSectionsInvalid: diagnostics.batchSectionsInvalid,
|
|
454
|
+
batchSectionsMissingUnmapped: diagnostics.batchSectionsMissingUnmapped,
|
|
455
|
+
fallbackSectionsRequested: diagnostics.fallbackSectionsRequested,
|
|
456
|
+
fallbackSectionsSuccessful: diagnostics.fallbackSectionsSuccessful,
|
|
457
|
+
fallbackSectionsFailed: diagnostics.fallbackSectionsFailed,
|
|
458
|
+
batchElapsedMs: Math.round(diagnostics.batchElapsedMs),
|
|
459
|
+
fallbackElapsedMs: Math.round(diagnostics.fallbackElapsedMs),
|
|
460
|
+
cacheHits: diagnostics.cacheHits,
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
return requirements.map((requirement) => {
|
|
464
|
+
const outcome = outcomes.get(sectionRequirementIdentity(requirement));
|
|
465
|
+
return outcome
|
|
466
|
+
? { requirement, ...outcome }
|
|
467
|
+
: {
|
|
468
|
+
requirement,
|
|
469
|
+
status: "rejected",
|
|
470
|
+
reason: new UrmaError("TARGETED_MEDIA_UNAVAILABLE", `Bounded section [${requirement.startMs},${requirement.endMs}) was not resolved by batching or fallback`),
|
|
471
|
+
};
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
async #cachedSection(requirement) {
|
|
475
|
+
const trace = currentExactFrameDiagnosticTrace();
|
|
476
|
+
const started = performance.now();
|
|
477
|
+
try {
|
|
478
|
+
const existing = this.store.getArtifactByRequest(sectionRequestKey(requirement));
|
|
479
|
+
if (!existing ||
|
|
480
|
+
existing.kind !== "media_section" ||
|
|
481
|
+
parseStoredBoundedVideoCoverage(existing.producer) === null) {
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
484
|
+
try {
|
|
485
|
+
return {
|
|
486
|
+
artifact: existing,
|
|
487
|
+
path: await this.blobs.verify(existing.artifactId, existing.blobPath),
|
|
488
|
+
cacheHit: true,
|
|
489
|
+
};
|
|
490
|
+
}
|
|
491
|
+
catch {
|
|
492
|
+
return null;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
finally {
|
|
496
|
+
const elapsedMs = performance.now() - started;
|
|
497
|
+
trace?.addStage("artifactCacheLookupMs", elapsedMs);
|
|
498
|
+
trace?.addStage("boundedArtifactCacheLookupMs", elapsedMs);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
async #acquireSingleIntoOutcomes(requirement, outcomes, diagnostics, fallback, signal) {
|
|
502
|
+
const identity = sectionRequirementIdentity(requirement);
|
|
503
|
+
const started = performance.now();
|
|
504
|
+
if (fallback)
|
|
505
|
+
diagnostics.fallbackSectionsRequested += 1;
|
|
506
|
+
try {
|
|
507
|
+
const value = await this.#download(requirement.source, requirement.investigationRef, sectionSpec(requirement.source, requirement.startMs, requirement.endMs), signal, () => {
|
|
508
|
+
diagnostics.outerYtDlpInvocations += 1;
|
|
509
|
+
if (fallback)
|
|
510
|
+
diagnostics.singleSectionFallbackInvocations += 1;
|
|
511
|
+
else
|
|
512
|
+
diagnostics.singleSectionInvocations += 1;
|
|
513
|
+
});
|
|
514
|
+
outcomes.set(identity, { status: "fulfilled", value });
|
|
515
|
+
if (fallback)
|
|
516
|
+
diagnostics.fallbackSectionsSuccessful += 1;
|
|
517
|
+
}
|
|
518
|
+
catch (error) {
|
|
519
|
+
outcomes.set(identity, { status: "rejected", reason: error });
|
|
520
|
+
if (fallback)
|
|
521
|
+
diagnostics.fallbackSectionsFailed += 1;
|
|
522
|
+
if (normalizeError(error).code === "CANCELLED")
|
|
523
|
+
throw error;
|
|
524
|
+
}
|
|
525
|
+
finally {
|
|
526
|
+
if (fallback) {
|
|
527
|
+
diagnostics.fallbackElapsedMs += performance.now() - started;
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
async #acquireBatch(requirements, signal) {
|
|
532
|
+
const started = performance.now();
|
|
533
|
+
const trace = currentExactFrameDiagnosticTrace();
|
|
534
|
+
const values = new Map();
|
|
535
|
+
const failures = new Map();
|
|
536
|
+
const unresolved = new Set();
|
|
537
|
+
const handles = new Map();
|
|
538
|
+
const eligible = [];
|
|
539
|
+
const perSectionBudget = this.config.limits.maxTargetedMediaBytes;
|
|
540
|
+
for (const requirement of requirements) {
|
|
541
|
+
const identity = sectionRequirementIdentity(requirement);
|
|
542
|
+
const spec = sectionSpec(requirement.source, requirement.startMs, requirement.endMs);
|
|
543
|
+
const handle = startAcquisition(this.store, {
|
|
544
|
+
sourceRef: requirement.source.sourceRef,
|
|
545
|
+
sourceRevision: requirement.source.revision,
|
|
546
|
+
investigationRef: requirement.investigationRef,
|
|
547
|
+
operation: "acquire-media-section",
|
|
548
|
+
requestKey: sectionRequestKey(requirement),
|
|
549
|
+
method: "yt-dlp-bounded-section",
|
|
550
|
+
debug: this.config.debug,
|
|
551
|
+
});
|
|
552
|
+
handles.set(identity, handle);
|
|
553
|
+
try {
|
|
554
|
+
assertExpectedRemoteBytes(expectedMediaBytes(requirement.source, spec), perSectionBudget, spec.operation);
|
|
555
|
+
eligible.push(requirement);
|
|
556
|
+
}
|
|
557
|
+
catch (error) {
|
|
558
|
+
handle.fail(error);
|
|
559
|
+
failures.set(identity, error);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
let invalid = 0;
|
|
563
|
+
let missingUnmapped = 0;
|
|
564
|
+
let invoked = false;
|
|
565
|
+
if (eligible.length > 0) {
|
|
566
|
+
const source = eligible[0].source;
|
|
567
|
+
const spec = sectionSpec(source, eligible[0].startMs, eligible[0].endMs);
|
|
568
|
+
const totalBudget = perSectionBudget * eligible.length;
|
|
569
|
+
try {
|
|
570
|
+
const remoteStarted = performance.now();
|
|
571
|
+
try {
|
|
572
|
+
await withRemoteAcquisitionDirectory(this.config, "media-batch", totalBudget, signal, async (temporary, remoteSignal) => {
|
|
573
|
+
let result = null;
|
|
574
|
+
let processError = null;
|
|
575
|
+
try {
|
|
576
|
+
invoked = true;
|
|
577
|
+
trace?.markRemoteAcquisition("media_section");
|
|
578
|
+
const lease = await leaseFor(this.downloader, source, spec.format, remoteSignal);
|
|
579
|
+
result = await this.downloader.run([
|
|
580
|
+
...eligible.flatMap((requirement) => [
|
|
581
|
+
"--download-sections",
|
|
582
|
+
sectionArgument(requirement.startMs, requirement.endMs),
|
|
583
|
+
]),
|
|
584
|
+
...(lease ? [] : ["-f", spec.format.id]),
|
|
585
|
+
"--paths",
|
|
586
|
+
temporary,
|
|
587
|
+
"-o",
|
|
588
|
+
SECTION_OUTPUT_TEMPLATE,
|
|
589
|
+
"--print",
|
|
590
|
+
`after_move:${SECTION_PREFIX}%(section_start)j\t%(section_end)j\t%(filepath)j`,
|
|
591
|
+
lease?.deliveryUrl ?? source.canonicalLocator,
|
|
592
|
+
], {
|
|
593
|
+
signal: remoteSignal,
|
|
594
|
+
timeoutMs: this.config.limits.maxRemoteAcquisitionWallMs,
|
|
595
|
+
});
|
|
596
|
+
if (result.code !== 0) {
|
|
597
|
+
processError = new UrmaError("SOURCE_UNAVAILABLE", `yt-dlp batch exited with code ${result.code}; independently valid bounded sections were preserved and unresolved sections will use single-section fallback`, { retryable: true, detail: { exitCode: result.code } });
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
catch (error) {
|
|
601
|
+
processError = error;
|
|
602
|
+
}
|
|
603
|
+
if (remoteSignal.aborted) {
|
|
604
|
+
const aborted = processError ??
|
|
605
|
+
new UrmaError("CANCELLED", "Bounded-section batch was cancelled before outputs could be validated");
|
|
606
|
+
for (const requirement of eligible) {
|
|
607
|
+
const identity = sectionRequirementIdentity(requirement);
|
|
608
|
+
failures.set(identity, aborted);
|
|
609
|
+
handles.get(identity).fail(aborted);
|
|
610
|
+
}
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
await assertRemoteDirectoryWithinBudget(temporary, totalBudget, "media-section batch", perSectionBudget);
|
|
614
|
+
const emissions = [
|
|
615
|
+
...(result ? parseSectionEmissions(result.stdout) : []),
|
|
616
|
+
...(await scanSectionEmissions(temporary)),
|
|
617
|
+
];
|
|
618
|
+
const candidates = uniqueCandidateMap(temporary, eligible, emissions);
|
|
619
|
+
const versions = await collectBinaryVersions(this.config, ["ytdlp", "ffprobe"], remoteSignal);
|
|
620
|
+
const canonicalTemporary = await realpath(temporary);
|
|
621
|
+
for (const requirement of eligible) {
|
|
622
|
+
const identity = sectionRequirementIdentity(requirement);
|
|
623
|
+
const mapped = candidates.get(identity) ?? [];
|
|
624
|
+
if (mapped.length !== 1) {
|
|
625
|
+
missingUnmapped += 1;
|
|
626
|
+
unresolved.add(identity);
|
|
627
|
+
const error = processError ??
|
|
628
|
+
new UrmaError("TARGETED_MEDIA_UNAVAILABLE", `Batched bounded section [${requirement.startMs},${requirement.endMs}) had ${mapped.length} unambiguous emitted artifacts; single-section fallback is required`, { retryable: true });
|
|
629
|
+
failures.set(identity, error);
|
|
630
|
+
handles.get(identity).fail(error);
|
|
631
|
+
continue;
|
|
632
|
+
}
|
|
633
|
+
try {
|
|
634
|
+
const candidate = await this.#validateBatchCandidate(canonicalTemporary, mapped[0], requirement, perSectionBudget, remoteSignal);
|
|
635
|
+
const normalizedProcessError = processError === null
|
|
636
|
+
? null
|
|
637
|
+
: normalizeError(processError);
|
|
638
|
+
const errorExitCode = normalizedProcessError &&
|
|
639
|
+
typeof normalizedProcessError.detail.exitCode === "number"
|
|
640
|
+
? normalizedProcessError.detail.exitCode
|
|
641
|
+
: null;
|
|
642
|
+
const value = await this.#promoteDownloaded(requirement.source, sectionSpec(requirement.source, requirement.startMs, requirement.endMs), candidate.file, candidate.durationSeconds, versions, handles.get(identity), {
|
|
643
|
+
batch: true,
|
|
644
|
+
batchSize: eligible.length,
|
|
645
|
+
processExitCode: result?.code ?? errorExitCode,
|
|
646
|
+
}, candidate.coverage);
|
|
647
|
+
values.set(identity, value);
|
|
648
|
+
}
|
|
649
|
+
catch (error) {
|
|
650
|
+
invalid += 1;
|
|
651
|
+
failures.set(identity, error);
|
|
652
|
+
handles.get(identity).fail(error);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
}, perSectionBudget);
|
|
656
|
+
}
|
|
657
|
+
finally {
|
|
658
|
+
trace?.addStage("remoteBoundedAcquisitionMs", performance.now() - remoteStarted);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
catch (error) {
|
|
662
|
+
for (const requirement of eligible) {
|
|
663
|
+
const identity = sectionRequirementIdentity(requirement);
|
|
664
|
+
if (values.has(identity) || failures.has(identity))
|
|
665
|
+
continue;
|
|
666
|
+
unresolved.add(identity);
|
|
667
|
+
failures.set(identity, error);
|
|
668
|
+
handles.get(identity).fail(error);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
return {
|
|
673
|
+
values,
|
|
674
|
+
failures,
|
|
675
|
+
unresolved,
|
|
676
|
+
invoked,
|
|
677
|
+
requested: eligible.length,
|
|
678
|
+
valid: values.size,
|
|
679
|
+
invalid,
|
|
680
|
+
missingUnmapped: missingUnmapped +
|
|
681
|
+
Math.max(0, eligible.length - values.size - invalid - missingUnmapped),
|
|
682
|
+
elapsedMs: performance.now() - started,
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
async #validateBatchCandidate(canonicalTemporary, emission, requirement, budgetBytes, signal) {
|
|
686
|
+
const trace = currentExactFrameDiagnosticTrace();
|
|
687
|
+
const file = await measureDiagnosticAsync(trace, "artifactValidationMs", async () => await realpath(path.resolve(canonicalTemporary, emission.filepath)));
|
|
688
|
+
const relative = path.relative(canonicalTemporary, file);
|
|
689
|
+
if (relative.length === 0 ||
|
|
690
|
+
relative.startsWith("..") ||
|
|
691
|
+
path.isAbsolute(relative)) {
|
|
692
|
+
throw new UrmaError("MEDIA_INVALID", `Batched bounded section [${requirement.startMs},${requirement.endMs}) emitted a file outside its isolated staging directory`);
|
|
693
|
+
}
|
|
694
|
+
const info = await measureDiagnosticAsync(trace, "artifactValidationMs", async () => await stat(file));
|
|
695
|
+
if (!info.isFile() || info.size < 1) {
|
|
696
|
+
throw targetedDerivativeUnavailable(`Batched bounded section [${requirement.startMs},${requirement.endMs}) did not emit a non-empty regular media file`, { reason: "empty-or-nonregular-output" });
|
|
697
|
+
}
|
|
698
|
+
if (info.size > budgetBytes) {
|
|
699
|
+
throw new UrmaError("MEDIA_BUDGET_EXCEEDED", `Batched bounded section [${requirement.startMs},${requirement.endMs}) emitted ${info.size} bytes, exceeding its ${budgetBytes}-byte hard acquisition budget`);
|
|
700
|
+
}
|
|
701
|
+
const validated = await measureDiagnosticAsync(trace, "mediaProbeTimingValidationMs", async () => {
|
|
702
|
+
const probe = await new Ffprobe(this.config, this.remoteContext).inspect(file, signal);
|
|
703
|
+
const streams = Array.isArray(probe.streams)
|
|
704
|
+
? probe.streams
|
|
705
|
+
: [];
|
|
706
|
+
const videoStream = streams.find((item) => item.codec_type === "video");
|
|
707
|
+
if (!videoStream) {
|
|
708
|
+
throw targetedDerivativeUnavailable(`Batched bounded section [${requirement.startMs},${requirement.endMs}) has no video stream; the bounded target is unavailable`, { reason: "zero-video-stream" });
|
|
709
|
+
}
|
|
710
|
+
const coverage = parseVideoStreamCoverage(videoStream);
|
|
711
|
+
if (!coverage) {
|
|
712
|
+
throw targetedDerivativeUnavailable(`Batched bounded section [${requirement.startMs},${requirement.endMs}) has no valid finite video PTS coverage; the bounded target is unavailable`, { reason: "invalid-video-pts-coverage" });
|
|
713
|
+
}
|
|
714
|
+
const format = typeof probe.format === "object" && probe.format !== null
|
|
715
|
+
? probe.format
|
|
716
|
+
: {};
|
|
717
|
+
const durationSeconds = Number(format.duration);
|
|
718
|
+
if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) {
|
|
719
|
+
throw targetedDerivativeUnavailable(`Batched bounded section [${requirement.startMs},${requirement.endMs}) has no positive finite container duration; the bounded target is unavailable`, { reason: "invalid-container-duration" });
|
|
720
|
+
}
|
|
721
|
+
this.#assertBoundedDuration(requirement.startMs, requirement.endMs, durationSeconds);
|
|
722
|
+
return { durationSeconds, coverage };
|
|
723
|
+
});
|
|
724
|
+
return {
|
|
725
|
+
file,
|
|
726
|
+
durationSeconds: validated.durationSeconds,
|
|
727
|
+
coverage: validated.coverage,
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
#assertBoundedDuration(startMs, endMs, durationSeconds) {
|
|
731
|
+
const requestedSeconds = (endMs - startMs) / 1_000;
|
|
732
|
+
const maximumSeconds = Math.max(requestedSeconds * 3, requestedSeconds + 30);
|
|
733
|
+
if (durationSeconds > maximumSeconds) {
|
|
734
|
+
throw new UrmaError("MEDIA_INVALID", `Bounded section returned ${durationSeconds.toFixed(3)} seconds for a ${requestedSeconds.toFixed(3)}-second request; the accidental full download was rejected`);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
async #promoteDownloaded(source, spec, file, durationSeconds, versions, acquisition, acquisitionMetadata = {}, coverage = null) {
|
|
738
|
+
const trace = currentExactFrameDiagnosticTrace();
|
|
739
|
+
const started = performance.now();
|
|
740
|
+
try {
|
|
741
|
+
if ((spec.kind === "media_section" || spec.kind === "evidence_media") &&
|
|
742
|
+
coverage === null) {
|
|
743
|
+
throw new RangeError("Exact media promotion requires validated video PTS coverage");
|
|
744
|
+
}
|
|
745
|
+
const blob = await this.blobs.putFile(file);
|
|
746
|
+
const requestKey = deterministicRequestKey(source.revision, spec.operation, { startMs: spec.startMs, endMs: spec.endMs, ...spec.params }, spec.version);
|
|
747
|
+
const artifact = {
|
|
748
|
+
artifactId: blob.artifactId,
|
|
749
|
+
sourceRef: source.sourceRef,
|
|
750
|
+
sourceRevision: source.revision,
|
|
751
|
+
kind: spec.kind,
|
|
752
|
+
role: "transport",
|
|
753
|
+
mimeType: mimeTypeFor(file),
|
|
754
|
+
sha256: blob.sha256,
|
|
755
|
+
byteSize: blob.byteSize,
|
|
756
|
+
blobPath: blob.relativePath,
|
|
757
|
+
startMs: spec.startMs,
|
|
758
|
+
endMs: spec.endMs,
|
|
759
|
+
params: spec.params,
|
|
760
|
+
producer: {
|
|
761
|
+
version: spec.version,
|
|
762
|
+
urmaVersion: URMA_VERSION,
|
|
763
|
+
...versions,
|
|
764
|
+
formatId: spec.format.id,
|
|
765
|
+
candidateKey: candidateKeyForSourceFormat(source, spec.format),
|
|
766
|
+
height: spec.format.height,
|
|
767
|
+
protocol: spec.format.protocol,
|
|
768
|
+
validatedDurationMs: Math.round(durationSeconds * 1_000),
|
|
769
|
+
...(spec.kind === "media_section"
|
|
770
|
+
? { requestedStartMs: spec.startMs, requestedEndMs: spec.endMs }
|
|
771
|
+
: {}),
|
|
772
|
+
...(coverage === null ? {} : serializeVideoPtsCoverage(coverage)),
|
|
773
|
+
},
|
|
774
|
+
createdAt: new Date().toISOString(),
|
|
775
|
+
};
|
|
776
|
+
this.store.putArtifact(artifact, {
|
|
777
|
+
requestKey,
|
|
778
|
+
operation: spec.operation,
|
|
779
|
+
});
|
|
780
|
+
acquisition.succeed({
|
|
781
|
+
networkBytes: null,
|
|
782
|
+
networkAccountingComplete: false,
|
|
783
|
+
metadata: {
|
|
784
|
+
artifactId: artifact.artifactId,
|
|
785
|
+
...versions,
|
|
786
|
+
...acquisitionMetadata,
|
|
787
|
+
},
|
|
788
|
+
});
|
|
789
|
+
trace?.addNewTransportArtifactBytes(artifact.byteSize);
|
|
790
|
+
return { artifact, path: blob.absolutePath, cacheHit: false };
|
|
791
|
+
}
|
|
792
|
+
finally {
|
|
793
|
+
trace?.addStage("mediaArtifactCommitMs", performance.now() - started);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
async #download(source, ref, spec, signal, onInvoke) {
|
|
797
|
+
const trace = currentExactFrameDiagnosticTrace();
|
|
798
|
+
const requestKey = deterministicRequestKey(source.revision, spec.operation, { startMs: spec.startMs, endMs: spec.endMs, ...spec.params }, spec.version);
|
|
799
|
+
const cacheStarted = performance.now();
|
|
800
|
+
let existing = null;
|
|
801
|
+
try {
|
|
802
|
+
existing = this.store.getArtifactByRequest(requestKey);
|
|
803
|
+
const cachedCoverage = existing === null
|
|
804
|
+
? null
|
|
805
|
+
: spec.kind === "media_section"
|
|
806
|
+
? parseStoredBoundedVideoCoverage(existing.producer)
|
|
807
|
+
: spec.kind === "evidence_media"
|
|
808
|
+
? parseStoredVideoCoverage(existing.producer)
|
|
809
|
+
: true;
|
|
810
|
+
if (existing &&
|
|
811
|
+
cachedCoverage !== null) {
|
|
812
|
+
try {
|
|
813
|
+
return {
|
|
814
|
+
artifact: existing,
|
|
815
|
+
path: await this.blobs.verify(existing.artifactId, existing.blobPath),
|
|
816
|
+
cacheHit: true,
|
|
817
|
+
};
|
|
818
|
+
}
|
|
819
|
+
catch {
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
finally {
|
|
824
|
+
const elapsedMs = performance.now() - cacheStarted;
|
|
825
|
+
trace?.addStage("artifactCacheLookupMs", elapsedMs);
|
|
826
|
+
trace?.addStage(spec.kind === "media_section"
|
|
827
|
+
? "boundedArtifactCacheLookupMs"
|
|
828
|
+
: "reusableArtifactCacheLookupMs", elapsedMs);
|
|
829
|
+
}
|
|
830
|
+
const acquisition = startAcquisition(this.store, {
|
|
831
|
+
sourceRef: source.sourceRef,
|
|
832
|
+
sourceRevision: source.revision,
|
|
833
|
+
investigationRef: ref,
|
|
834
|
+
operation: `acquire-${spec.operation}`,
|
|
835
|
+
requestKey,
|
|
836
|
+
method: spec.kind === "media_section"
|
|
837
|
+
? "yt-dlp-bounded-section"
|
|
838
|
+
: "yt-dlp-reusable-media",
|
|
839
|
+
debug: this.config.debug,
|
|
840
|
+
});
|
|
841
|
+
const budget = mediaBudgetBytes(this.config, spec.kind);
|
|
842
|
+
try {
|
|
843
|
+
assertExpectedRemoteBytes(expectedMediaBytes(source, spec), budget, spec.operation);
|
|
844
|
+
const remoteStarted = performance.now();
|
|
845
|
+
try {
|
|
846
|
+
return await withRemoteAcquisitionDirectory(this.config, "media", budget, signal, async (temporary, remoteSignal) => {
|
|
847
|
+
onInvoke?.();
|
|
848
|
+
if (spec.kind === "media_section" || spec.kind === "evidence_media") {
|
|
849
|
+
trace?.markRemoteAcquisition(spec.kind);
|
|
850
|
+
}
|
|
851
|
+
const lease = await leaseFor(this.downloader, source, spec.format, remoteSignal);
|
|
852
|
+
await this.downloader.run([
|
|
853
|
+
...(lease
|
|
854
|
+
? [
|
|
855
|
+
...withoutFormatSelector(spec.args, spec.format.id),
|
|
856
|
+
"-o",
|
|
857
|
+
"media.%(ext)s",
|
|
858
|
+
]
|
|
859
|
+
: spec.args),
|
|
860
|
+
"--paths",
|
|
861
|
+
temporary,
|
|
862
|
+
lease?.deliveryUrl ?? source.canonicalLocator,
|
|
863
|
+
], {
|
|
864
|
+
signal: remoteSignal,
|
|
865
|
+
timeoutMs: this.config.limits.maxRemoteAcquisitionWallMs,
|
|
866
|
+
cwd: temporary,
|
|
867
|
+
});
|
|
868
|
+
const validated = await measureDiagnosticAsync(trace, "artifactValidationMs", async () => {
|
|
869
|
+
await assertRemoteDirectoryWithinBudget(temporary, budget, spec.operation);
|
|
870
|
+
const names = (await readdir(temporary)).filter((item) => !item.endsWith(".part") &&
|
|
871
|
+
!item.endsWith(".ytdl") &&
|
|
872
|
+
/^[A-Za-z0-9_.-]{1,200}$/u.test(item) &&
|
|
873
|
+
[
|
|
874
|
+
".mp4",
|
|
875
|
+
".m4v",
|
|
876
|
+
".webm",
|
|
877
|
+
".mkv",
|
|
878
|
+
".mov",
|
|
879
|
+
".ts",
|
|
880
|
+
".m2ts",
|
|
881
|
+
".avi",
|
|
882
|
+
].includes(path.extname(item).toLowerCase()));
|
|
883
|
+
if (names.length !== 1) {
|
|
884
|
+
throw targetedDerivativeUnavailable(names.length === 0
|
|
885
|
+
? `yt-dlp completed ${spec.operation} without a typed media artifact`
|
|
886
|
+
: `yt-dlp completed ${spec.operation} with ambiguous typed media outputs`, { reason: names.length === 0 ? "missing-targeted-output" : "ambiguous-targeted-output" });
|
|
887
|
+
}
|
|
888
|
+
const file = path.join(temporary, names[0]);
|
|
889
|
+
const media = await measureDiagnosticAsync(trace, "mediaProbeTimingValidationMs", async () => {
|
|
890
|
+
const probe = await new Ffprobe(this.config, this.remoteContext).inspect(file, remoteSignal);
|
|
891
|
+
const streams = Array.isArray(probe.streams)
|
|
892
|
+
? probe.streams
|
|
893
|
+
: [];
|
|
894
|
+
const videoStream = streams.find((item) => item.codec_type === "video");
|
|
895
|
+
if (!videoStream) {
|
|
896
|
+
if (spec.kind === "media_section") {
|
|
897
|
+
throw targetedDerivativeUnavailable(`${spec.operation} artifact has no video stream; the bounded target is unavailable`, { reason: "zero-video-stream" });
|
|
898
|
+
}
|
|
899
|
+
throw new UrmaError("MEDIA_INVALID", `${spec.operation} artifact has no video stream; retry with an updated yt-dlp`);
|
|
900
|
+
}
|
|
901
|
+
const requiresExactTiming = spec.kind === "media_section" || spec.kind === "evidence_media";
|
|
902
|
+
const coverage = requiresExactTiming
|
|
903
|
+
? parseVideoStreamCoverage(videoStream)
|
|
904
|
+
: null;
|
|
905
|
+
if (coverage === null && requiresExactTiming) {
|
|
906
|
+
if (spec.kind === "media_section") {
|
|
907
|
+
throw targetedDerivativeUnavailable(`${spec.operation} artifact has no valid finite video PTS coverage; the bounded target is unavailable`, { reason: "invalid-video-pts-coverage" });
|
|
908
|
+
}
|
|
909
|
+
throw new UrmaError("MEDIA_INVALID", `${spec.operation} artifact has no valid finite video PTS coverage; exact frames are unavailable`, { detail: { reason: "invalid-video-pts-coverage" } });
|
|
910
|
+
}
|
|
911
|
+
const format = typeof probe.format === "object" && probe.format !== null
|
|
912
|
+
? probe.format
|
|
913
|
+
: {};
|
|
914
|
+
const durationSeconds = Number(format.duration);
|
|
915
|
+
if (!Number.isFinite(durationSeconds) ||
|
|
916
|
+
durationSeconds <= 0) {
|
|
917
|
+
if (spec.kind === "media_section") {
|
|
918
|
+
throw targetedDerivativeUnavailable(`${spec.operation} artifact has no positive finite duration; the bounded target is unavailable`, { reason: "invalid-container-duration" });
|
|
919
|
+
}
|
|
920
|
+
throw new UrmaError("MEDIA_INVALID", `${spec.operation} artifact has no positive finite duration; retry with an updated yt-dlp`);
|
|
921
|
+
}
|
|
922
|
+
if (spec.kind === "media_section") {
|
|
923
|
+
this.#assertBoundedDuration(spec.startMs, spec.endMs, durationSeconds);
|
|
924
|
+
}
|
|
925
|
+
return { file, durationSeconds, coverage };
|
|
926
|
+
});
|
|
927
|
+
await assertRemoteDirectoryWithinBudget(temporary, budget, spec.operation);
|
|
928
|
+
return media;
|
|
929
|
+
});
|
|
930
|
+
const versions = await collectBinaryVersions(this.config, ["ytdlp", "ffprobe"], remoteSignal);
|
|
931
|
+
return await this.#promoteDownloaded(source, spec, validated.file, validated.durationSeconds, versions, acquisition, {}, validated.coverage);
|
|
932
|
+
});
|
|
933
|
+
}
|
|
934
|
+
finally {
|
|
935
|
+
trace?.addStage(spec.kind === "media_section"
|
|
936
|
+
? "remoteBoundedAcquisitionMs"
|
|
937
|
+
: "remoteReusableAcquisitionMs", performance.now() - remoteStarted);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
catch (error) {
|
|
941
|
+
acquisition.fail(error);
|
|
942
|
+
throw error;
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
//# sourceMappingURL=media.js.map
|