veodl 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -0
- package/CHANGELOG.md +264 -0
- package/LICENSE +21 -0
- package/README.md +725 -0
- package/bin/veo.js +4 -0
- package/docs/AGENT_GUIDE.md +66 -0
- package/package.json +59 -0
- package/src/backend-update.js +191 -0
- package/src/backend.js +520 -0
- package/src/cli.js +454 -0
- package/src/compatibility.js +39 -0
- package/src/config-clipboard.js +67 -0
- package/src/config-diagnostics.js +144 -0
- package/src/config-editor.js +281 -0
- package/src/config-errors.js +64 -0
- package/src/config-reset.js +39 -0
- package/src/config-template.js +171 -0
- package/src/config.js +213 -0
- package/src/disk-space.js +68 -0
- package/src/doctor.js +302 -0
- package/src/download-cache.js +31 -0
- package/src/downloader.js +600 -0
- package/src/execution.js +68 -0
- package/src/flush.js +66 -0
- package/src/history.js +173 -0
- package/src/inspect-media.js +168 -0
- package/src/interactive.js +82 -0
- package/src/jobs.js +172 -0
- package/src/legacy-config-comments.js +106 -0
- package/src/naming.js +50 -0
- package/src/open-file.js +15 -0
- package/src/output.js +61 -0
- package/src/paths.js +26 -0
- package/src/playlist.js +31 -0
- package/src/progress.js +152 -0
- package/src/run-archive.js +148 -0
- package/src/runs.js +333 -0
- package/src/state.js +27 -0
- package/src/stats.js +63 -0
- package/src/terminal-title.js +19 -0
- package/src/tool-setup.js +151 -0
- package/src/updater.js +227 -0
- package/src/utils.js +208 -0
- package/src/version.js +38 -0
|
@@ -0,0 +1,600 @@
|
|
|
1
|
+
import { ensureCompatibility } from './compatibility.js';
|
|
2
|
+
import { adaptiveRun, reducedLimit, phaseTimer, formatTimings } from './execution.js';
|
|
3
|
+
import { mediaDestination, prepareDestination } from './naming.js';
|
|
4
|
+
import { estimateMediaBytes, reserveSpace } from './disk-space.js';
|
|
5
|
+
import { spawn } from 'node:child_process';
|
|
6
|
+
import { createInterface } from 'node:readline';
|
|
7
|
+
import { lstat, mkdir, open, readdir, rm, rmdir, stat } from 'node:fs/promises';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { resolveBackend } from './backend.js';
|
|
10
|
+
import { availableHeights, cappedHeight, closestHeight, saveUnique, sanitizeTitle } from './utils.js';
|
|
11
|
+
import { digest, readJson, writeJson } from './state.js';
|
|
12
|
+
import { cleanupDownloadCache, downloadCacheRoot, TRANSFER_RETENTION_MS } from './download-cache.js';
|
|
13
|
+
import { selectedEntries, sizeEstimate, describeEstimate } from './playlist.js';
|
|
14
|
+
|
|
15
|
+
const PARTIAL_PREFIX = '.veo-part-';
|
|
16
|
+
const STAGED_MEDIA = /^media(?:-(\d+))?\.([A-Za-z0-9]{1,8})$/;
|
|
17
|
+
// Only real media extensions may count as the downloaded file, so a thumbnail
|
|
18
|
+
// or a subtitle that the backend leaves behind is never mistaken for it.
|
|
19
|
+
const MEDIA_EXTENSIONS = new Set(['mp4', 'mkv', 'webm', 'mov', 'avi', 'm4v', '3gp', 'ts', 'flv', 'mpg', 'mpeg', 'ogv',
|
|
20
|
+
'mp3', 'm4a', 'aac', 'opus', 'flac', 'wav', 'ogg', 'oga', 'wma', 'mka', 'aiff']);
|
|
21
|
+
|
|
22
|
+
export function isStagedMedia(name) {
|
|
23
|
+
const match = STAGED_MEDIA.exec(name);
|
|
24
|
+
return Boolean(match) && MEDIA_EXTENSIONS.has(match[2].toLowerCase());
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function runBackend(executable, args, { signal, onLine } = {}) {
|
|
28
|
+
return new Promise((resolve, reject) => {
|
|
29
|
+
signal?.throwIfAborted();
|
|
30
|
+
const child = spawn(executable, args, { shell: false, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'], signal: process.platform === 'win32' ? undefined : signal });
|
|
31
|
+
const cancelTree = () => {
|
|
32
|
+
if (!child.pid || child.exitCode !== null) return;
|
|
33
|
+
const killer = spawn('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { shell: false, windowsHide: true, stdio: 'ignore' });
|
|
34
|
+
killer.on('error', () => child.kill());
|
|
35
|
+
};
|
|
36
|
+
if (process.platform === 'win32') signal?.addEventListener('abort', cancelTree, { once: true });
|
|
37
|
+
let output = '';
|
|
38
|
+
let errors = '';
|
|
39
|
+
let failure;
|
|
40
|
+
const lines = createInterface({ input: child.stdout });
|
|
41
|
+
const errorLines = createInterface({ input: child.stderr });
|
|
42
|
+
errorLines.on('line', line => {
|
|
43
|
+
if (onLine && /^veo-(?:progress|postprocess):/.test(line)) {
|
|
44
|
+
try { onLine(line); } catch (error) { failure = error; child.kill(); }
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
lines.on('line', line => {
|
|
48
|
+
if (onLine) {
|
|
49
|
+
try { onLine(line); } catch (error) { failure = error; child.kill(); }
|
|
50
|
+
} else if (output.length < 32 * 1024 * 1024) output += `${line}\n`;
|
|
51
|
+
else { failure = new Error('Video metadata exceeded the supported size.'); child.kill(); }
|
|
52
|
+
});
|
|
53
|
+
child.stderr.on('data', data => { errors = (errors + data).slice(-16000); });
|
|
54
|
+
child.on('error', error => { failure = error; });
|
|
55
|
+
child.on('close', code => {
|
|
56
|
+
signal?.removeEventListener('abort', cancelTree);
|
|
57
|
+
lines.close();
|
|
58
|
+
errorLines.close();
|
|
59
|
+
if (failure) reject(failure);
|
|
60
|
+
else if (signal?.aborted) reject(new DOMException('Cancelled', 'AbortError'));
|
|
61
|
+
else if (code !== 0) {
|
|
62
|
+
const diagnostics = errors.split(/\r?\n/).filter(line => !/^veo-(?:progress|postprocess):/.test(line));
|
|
63
|
+
const failureLine = diagnostics.filter(line => /^ERROR:/i.test(line)).at(-1);
|
|
64
|
+
reject(new Error(failureLine || diagnostics.join('\n') || `Downloading backend exited with code ${code}.`));
|
|
65
|
+
}
|
|
66
|
+
else resolve(output);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function formatSelector(height) {
|
|
72
|
+
const filter = height ? `[height=${height}]` : '';
|
|
73
|
+
return `bv${filter}+ba/b${filter}/bv${filter}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Turn a requested quality into backend arguments and a user-facing label.
|
|
78
|
+
*
|
|
79
|
+
* Default (upper bound): the best resolution at or below the request, so
|
|
80
|
+
* `-q 720p` on a phone plan can never silently fetch 2160p. A source that
|
|
81
|
+
* offers nothing at or below the request fails before anything is downloaded,
|
|
82
|
+
* naming the resolutions that do exist.
|
|
83
|
+
* `--closest-quality` keeps the historical "nearest available height" rule.
|
|
84
|
+
* Sources without resolution metadata fall back to the best available stream,
|
|
85
|
+
* and collections use the backend's own resolution preference per entry.
|
|
86
|
+
*/
|
|
87
|
+
export function selectQuality({ quality = 'best', audio = false, closest = false, formats, playlist = false } = {}) {
|
|
88
|
+
if (audio || quality === 'best') return { format: null, sort: null, label: null };
|
|
89
|
+
const target = Number.parseInt(quality, 10);
|
|
90
|
+
if (playlist) {
|
|
91
|
+
// Entries differ in resolution, so cap them inside the backend instead.
|
|
92
|
+
return {
|
|
93
|
+
format: null,
|
|
94
|
+
sort: `res:${target}`,
|
|
95
|
+
label: closest ? `Quality: closest to ${target}p per entry` : `Quality: up to ${target}p per entry`,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
const heights = availableHeights(formats);
|
|
99
|
+
if (!heights.length) return { format: null, sort: null, label: 'Resolution unknown; using the best available stream.' };
|
|
100
|
+
if (closest) {
|
|
101
|
+
const height = closestHeight(formats, quality);
|
|
102
|
+
return {
|
|
103
|
+
format: formatSelector(height),
|
|
104
|
+
sort: null,
|
|
105
|
+
label: `Quality: ${height}p${height === target ? '' : ` (closest to ${quality})`}`,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
const height = cappedHeight(formats, quality);
|
|
109
|
+
if (!height) {
|
|
110
|
+
const offered = [...heights].sort((a, b) => a - b).map(value => `${value}p`).join(', ');
|
|
111
|
+
return {
|
|
112
|
+
format: null,
|
|
113
|
+
sort: null,
|
|
114
|
+
label: null,
|
|
115
|
+
error: `No stream at or below ${target}p is available (offered: ${offered}). Use a higher --quality or "best".`,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
format: formatSelector(height),
|
|
120
|
+
sort: null,
|
|
121
|
+
label: `Quality: ${height}p${height === target ? '' : ` (highest at or below ${quality})`}`,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* A stable, filesystem-safe key for one video, so a re-run with --resume finds
|
|
127
|
+
* the same staging directory. Site ids are untrusted input and never used raw.
|
|
128
|
+
*/
|
|
129
|
+
export function partialKey(metadata, url, options = {}) {
|
|
130
|
+
return digest({ source: sourceKey(metadata, url), settings: downloadSettings(options) });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function sourceKey(metadata, url) {
|
|
134
|
+
return metadata.id && (metadata.extractor_key || metadata.extractor)
|
|
135
|
+
? `${metadata.extractor_key || metadata.extractor}:${metadata.id}` : `${url}#${metadata.id || ''}`;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// A history record can only skip a download while every file it names still
|
|
139
|
+
// exists as a plain file inside the same output directory.
|
|
140
|
+
export async function historyRecordUsable(record, directory) {
|
|
141
|
+
return Boolean(record?.files?.length && record.files.every(file => typeof file === 'string' && path.dirname(file) === directory)
|
|
142
|
+
&& (await Promise.all(record.files.map(file => stat(file).then(info => info.isFile(), () => false)))).every(Boolean));
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Drops a duplicate-detection record whose files are gone and removes the
|
|
146
|
+
// history folder with its last record, so deleting media leaves no stale
|
|
147
|
+
// state behind. Returns whether the record is still usable.
|
|
148
|
+
export async function pruneHistoryRecord(historyFile, directory, history) {
|
|
149
|
+
const usable = await historyRecordUsable(history, directory);
|
|
150
|
+
if (history && !usable) await rm(historyFile, { force: true }).catch(() => {});
|
|
151
|
+
if (!usable) await rmdir(path.dirname(historyFile)).catch(() => {});
|
|
152
|
+
return usable;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function downloadSettings(options) {
|
|
156
|
+
options = { quality: 'best', audio: false, closestQuality: false, subs: false, embedSubs: false,
|
|
157
|
+
embedMetadata: false, embedThumbnail: false, ...options };
|
|
158
|
+
const settings = Object.fromEntries(['quality', 'audio', 'format', 'closestQuality', 'subs', 'subLangs', 'embedSubs',
|
|
159
|
+
'embedMetadata', 'embedThumbnail', 'sponsorblockRemove', 'section'].map(key => [key, options[key] ?? null]));
|
|
160
|
+
// Existing conversion jobs retain their keys; remux requests must not reuse them.
|
|
161
|
+
if (options.format && !options.audio && !options.recode) settings.videoRemux = true;
|
|
162
|
+
if (options.compatible) settings.compatible = true;
|
|
163
|
+
if (options.filenameTemplate) settings.filenameTemplate = options.filenameTemplate;
|
|
164
|
+
if (options.folderTemplate) settings.folderTemplate = options.folderTemplate;
|
|
165
|
+
return settings;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export function localRequestKey(options) {
|
|
169
|
+
return digest({ url: options.url, entry: options._entryIndex || null, settings: downloadSettings(options), output: path.resolve(options.output) });
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Resuming reuses a predictable directory name, so it must not follow a
|
|
174
|
+
* symlink or a file that another process placed there.
|
|
175
|
+
*/
|
|
176
|
+
async function prepareStaging(partials) {
|
|
177
|
+
try {
|
|
178
|
+
await mkdir(partials);
|
|
179
|
+
} catch (error) {
|
|
180
|
+
if (error.code !== 'EEXIST') throw error;
|
|
181
|
+
}
|
|
182
|
+
const info = await lstat(partials).catch(() => undefined);
|
|
183
|
+
if (!info?.isDirectory() || info.isSymbolicLink()) throw new Error(`The partial download path is not a plain directory: ${partials}`);
|
|
184
|
+
return partials;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* A finished download whose save step failed earlier is already complete; using
|
|
189
|
+
* it avoids relying on the backend's "--no-overwrites" skip behaviour.
|
|
190
|
+
* Returns the largest staged media file, ignoring fragments and .part files.
|
|
191
|
+
*/
|
|
192
|
+
export async function findFinishedMedia(directory) {
|
|
193
|
+
const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
|
|
194
|
+
const candidates = [];
|
|
195
|
+
for (const entry of entries) {
|
|
196
|
+
if (!entry.isFile() || !isStagedMedia(entry.name)) continue;
|
|
197
|
+
const file = path.join(directory, entry.name);
|
|
198
|
+
candidates.push({ file, size: (await stat(file).catch(() => ({ size: 0 }))).size });
|
|
199
|
+
}
|
|
200
|
+
return candidates.sort((a, b) => b.size - a.size)[0]?.file;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Formats the backend may leave beside the media file. An allowlist keeps
|
|
204
|
+
// merge fragments such as media.f137.mp4 out, since they are never deliverables.
|
|
205
|
+
const SIDECAR_EXTENSIONS = new Set(['vtt', 'srt', 'ass', 'lrc', 'ttml', 'srv1', 'srv2', 'srv3', 'json', 'jpg', 'jpeg', 'png', 'webp']);
|
|
206
|
+
|
|
207
|
+
/** Files a download may leave beside the media file (subtitles, thumbnails). */
|
|
208
|
+
export function isSidecar(name) {
|
|
209
|
+
if (name.startsWith('.') || isStagedMedia(name)) return false;
|
|
210
|
+
return SIDECAR_EXTENSIONS.has(path.extname(name).slice(1).toLowerCase());
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Title of one staged file. An explicit --rename wins, and collections use the
|
|
215
|
+
* matching entry title so every entry keeps its own name.
|
|
216
|
+
*/
|
|
217
|
+
export function stagedTitle(file, metadata, rename) {
|
|
218
|
+
const index = isStagedMedia(path.basename(file)) ? STAGED_MEDIA.exec(path.basename(file))[1] : undefined;
|
|
219
|
+
const fallback = metadata?.title || metadata?.id || 'video';
|
|
220
|
+
const title = index ? metadata?.entries?.[Number(index) - 1]?.title || `${fallback} - ${index}` : fallback;
|
|
221
|
+
if (rename?.includes('*')) return rename.replaceAll('*', () => title);
|
|
222
|
+
if (rename) return index ? `${rename} - ${index}` : rename;
|
|
223
|
+
return title;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function backendArgs(options, backend) {
|
|
227
|
+
const common = ['--ignore-config', '--no-plugin-dirs', '--no-colors', '--no-warnings',
|
|
228
|
+
'--socket-timeout', '30', '--retries', '3', '--fragment-retries', '3',
|
|
229
|
+
'--no-js-runtimes', '--js-runtimes', `node:${process.execPath}`,
|
|
230
|
+
'--ffmpeg-location', backend.ffmpegLocation];
|
|
231
|
+
if (!options.playlist) common.push('--no-playlist');
|
|
232
|
+
if (options._entryIndex) common.push('--playlist-items', String(options._entryIndex));
|
|
233
|
+
common.push('--concurrent-fragments', String(options.concurrentFragments ?? 8));
|
|
234
|
+
// Credentials are opt-in per invocation and only ever forwarded to the backend,
|
|
235
|
+
// which never bypasses access controls on its own.
|
|
236
|
+
if (options.cookies) common.push('--cookies', options.cookies);
|
|
237
|
+
if (options.cookiesFromBrowser) common.push('--cookies-from-browser', options.cookiesFromBrowser);
|
|
238
|
+
if (options.url) {
|
|
239
|
+
try {
|
|
240
|
+
const origin = new URL(options.url).origin;
|
|
241
|
+
common.push('--referer', `${origin}/`);
|
|
242
|
+
} catch {}
|
|
243
|
+
}
|
|
244
|
+
return common;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function mediaArgs(options, quality) {
|
|
248
|
+
const args = ['--no-overwrites', options.resume ? '--continue' : '--no-continue', '--newline',
|
|
249
|
+
'--progress', '--progress-delta', '0.2',
|
|
250
|
+
'--progress-template', 'download:veo-progress:{"progress":%(progress)j,"video":%(info.vcodec|null)j,"audio":%(info.acodec|null)j}',
|
|
251
|
+
'--progress-template', 'postprocess:veo-postprocess:%(progress)j',
|
|
252
|
+
'--print', 'after_move:veo-file:%(filepath)j', '--no-simulate'];
|
|
253
|
+
if (options.audio) args.push('-f', 'ba/b', '--extract-audio', '--audio-format', options.format || 'mp3', '--audio-quality', '0');
|
|
254
|
+
else {
|
|
255
|
+
if (quality.format) args.push('-f', quality.format);
|
|
256
|
+
// Merge into a permissive container first; e.g. H.264 cannot be merged
|
|
257
|
+
// straight into WebM before the requested codec conversion runs.
|
|
258
|
+
if (options.compatible) args.push('--merge-output-format', 'mkv');
|
|
259
|
+
else if (options.format) args.push('--merge-output-format', 'mkv', options.recode ? '--recode-video' : '--remux-video', options.format);
|
|
260
|
+
else args.push('--merge-output-format', 'mp4/mkv');
|
|
261
|
+
}
|
|
262
|
+
const sort = ['vcodec:h264,acodec:aac', quality.sort].filter(Boolean).join(',');
|
|
263
|
+
// Prefer broadly playable sources for MP4/MKV too. Explicit recoding and
|
|
264
|
+
// WebM keep their own source selection. Resolution remains the primary choice.
|
|
265
|
+
const selectedSort = options.recode || options.format === 'webm' ? quality.sort : sort;
|
|
266
|
+
if (!options.audio && selectedSort) args.push('-S', selectedSort);
|
|
267
|
+
if (options.subs || options.subLangs || options.embedSubs) {
|
|
268
|
+
args.push('--write-subs', '--sub-langs', options.subLangs || 'en.*,en');
|
|
269
|
+
}
|
|
270
|
+
if (options.embedSubs) args.push('--embed-subs');
|
|
271
|
+
if (options.embedMetadata) args.push('--embed-metadata');
|
|
272
|
+
if (options.embedThumbnail) args.push('--embed-thumbnail');
|
|
273
|
+
if (options.sponsorblockRemove) args.push('--sponsorblock-remove', options.sponsorblockRemove);
|
|
274
|
+
if (options.section) args.push('--download-sections', options.section, '--force-keyframes-at-cuts');
|
|
275
|
+
return args;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
export function stagingTemplate(directory, playlist) {
|
|
279
|
+
const name = playlist ? 'media-%(playlist_index)03d.%(ext)s' : 'media.%(ext)s';
|
|
280
|
+
return path.join(directory, name).replaceAll('%', '%%').replace('%%(playlist_index)03d', '%(playlist_index)03d').replace('%%(ext)s', '%(ext)s');
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Read video metadata once, with the same vetted arguments as a download. */
|
|
284
|
+
export async function fetchMetadata(options, { signal, backend, runner, reporter } = {}) {
|
|
285
|
+
const common = backendArgs(options, backend);
|
|
286
|
+
reporter?.status('Reading video…');
|
|
287
|
+
const args = [...common, '--dump-single-json', '--skip-download'];
|
|
288
|
+
if (options.playlist && !options._entryIndex) args.push('--flat-playlist');
|
|
289
|
+
args.push('--', options.url);
|
|
290
|
+
let metadata = JSON.parse(await runner(backend.ytDlp, args, { signal }));
|
|
291
|
+
if (options._entryIndex && metadata.entries) metadata = metadata.entries.find(Boolean);
|
|
292
|
+
if (!metadata) throw new Error('The selected playlist entry is unavailable.');
|
|
293
|
+
if (!options.playlist && (metadata._type === 'playlist' || metadata.entries)) {
|
|
294
|
+
throw new Error('This URL is a collection. Add --playlist to download every entry.');
|
|
295
|
+
}
|
|
296
|
+
if (metadata.is_live) throw new Error('Live streams are not supported. Please use a finished video.');
|
|
297
|
+
if (metadata.has_drm) throw new Error('This content is DRM-protected.');
|
|
298
|
+
return metadata;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export async function prepareBackend(options, { signal, backendResolver = resolveBackend, reporter } = {}) {
|
|
302
|
+
return backendResolver({ signal, onStatus: message => reporter?.status(message) });
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** The extension the finished file will most likely have, for previews. */
|
|
306
|
+
export function predictedExtension(options) {
|
|
307
|
+
if (options.audio) return options.format || 'mp3';
|
|
308
|
+
return options.format || 'mp4';
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** First free destination name, without creating anything (preview only). */
|
|
312
|
+
export async function previewPath(directory, title, extension, { exists = async () => false } = {}) {
|
|
313
|
+
const name = sanitizeTitle(title);
|
|
314
|
+
for (let number = 0; ; number++) {
|
|
315
|
+
const candidate = path.join(directory, `${name}${number ? ` (${number})` : ''}.${extension}`);
|
|
316
|
+
if (!await exists(candidate)) return candidate;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** `veo --list-formats`: print the backend's own format table and stop. */
|
|
321
|
+
export async function listFormats(options, { signal, backendResolver = resolveBackend, runner = runBackend, reporter } = {}) {
|
|
322
|
+
const backend = await prepareBackend(options, { signal, backendResolver, reporter });
|
|
323
|
+
reporter?.status('Reading available formats…');
|
|
324
|
+
return runner(backend.ytDlp, [...backendArgs(options, backend), '-F', '--', options.url], { signal });
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* `veo --dry-run`: inspect the video and report what would happen without
|
|
329
|
+
* downloading or writing anything.
|
|
330
|
+
*/
|
|
331
|
+
export async function planDownload(options, { signal, backendResolver = resolveBackend, runner = runBackend, reporter } = {}) {
|
|
332
|
+
const directory = path.resolve(options.output);
|
|
333
|
+
const backend = await prepareBackend(options, { signal, backendResolver, reporter });
|
|
334
|
+
const metadata = await fetchMetadata(options, { signal, backend, runner, reporter });
|
|
335
|
+
const quality = selectQuality({ quality: options.quality, audio: options.audio, closest: options.closestQuality, formats: metadata.formats, playlist: options.playlist });
|
|
336
|
+
if (quality.error) throw new Error(quality.error);
|
|
337
|
+
const selected = metadata.entries ? selectedEntries(metadata, options.playlistItems) : [{ entry: metadata, index: 1 }];
|
|
338
|
+
const titles = options.playlist && metadata.entries?.length
|
|
339
|
+
? selected.map(({ entry, index }) => options.rename?.includes('*') ? stagedTitle('media.mp4', entry, options.rename) : options.rename ? `${options.rename} - ${String(index).padStart(3, '0')}` : entry?.title || `${metadata.title || 'video'} - ${index}`)
|
|
340
|
+
: [stagedTitle('media.mp4', metadata, options.rename)];
|
|
341
|
+
const extension = predictedExtension(options);
|
|
342
|
+
const reserved = new Set();
|
|
343
|
+
const exists = async candidate => reserved.has(candidate) || Boolean(await stat(candidate).catch(() => undefined));
|
|
344
|
+
const planned = [];
|
|
345
|
+
for (const [offset, title] of titles.entries()) {
|
|
346
|
+
const { entry, index } = selected[offset];
|
|
347
|
+
const destination = mediaDestination({ ...options, _entryIndex: metadata.entries ? index : options._entryIndex, _playlistTitle: metadata.entries ? metadata.title : options._playlistTitle }, entry, title);
|
|
348
|
+
await prepareDestination(directory, destination.directory);
|
|
349
|
+
const target = await previewPath(destination.directory, destination.title, extension, { exists });
|
|
350
|
+
reserved.add(target);
|
|
351
|
+
planned.push({ title, path: target });
|
|
352
|
+
}
|
|
353
|
+
return { url: options.url, playlist: Boolean(options.playlist), quality: quality.label, entries: planned, ...sizeEstimate(selected) };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
async function downloadCollection(options, metadata, dependencies) {
|
|
357
|
+
const { reporter, signal } = dependencies;
|
|
358
|
+
const entries = selectedEntries(metadata, options.playlistItems);
|
|
359
|
+
reporter?.status(describeEstimate(sizeEstimate(entries)));
|
|
360
|
+
const results = new Array(entries.length), failures = [], entryTimings = [];
|
|
361
|
+
let saved = 0, skipped = 0;
|
|
362
|
+
let next = 0;
|
|
363
|
+
let notifications = Promise.resolve();
|
|
364
|
+
const concurrency = options.playlistConcurrency ?? 2;
|
|
365
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 4) throw new Error('Playlist concurrency must be between 1 and 4.');
|
|
366
|
+
const notify = entry => {
|
|
367
|
+
const pending = notifications.then(() => dependencies.onEntry?.(entry));
|
|
368
|
+
notifications = pending.catch(() => {});
|
|
369
|
+
return pending;
|
|
370
|
+
};
|
|
371
|
+
const worker = async workerIndex => {
|
|
372
|
+
while (next < entries.length && workerIndex < reducedLimit(concurrency, dependencies.adaptiveState)) {
|
|
373
|
+
signal?.throwIfAborted();
|
|
374
|
+
const offset = next++;
|
|
375
|
+
const { entry, index } = entries[offset];
|
|
376
|
+
const title = entry?.title || `Entry ${index}`;
|
|
377
|
+
const childReporter = concurrency > 1 ? reporter?.scoped?.(index, metadata.entries.length, title) || reporter : reporter;
|
|
378
|
+
if (concurrency === 1) reporter?.item?.(offset + 1, entries.length, title);
|
|
379
|
+
const itemOptions = { ...options, _entryIndex: index, _playlistTitle: metadata.title || metadata.id,
|
|
380
|
+
rename: options.rename?.includes('*') ? options.rename : options.rename ? `${options.rename} - ${String(index).padStart(3, '0')}` : undefined };
|
|
381
|
+
let outcome;
|
|
382
|
+
try {
|
|
383
|
+
const result = await download(itemOptions, { ...dependencies, reporter: childReporter, skipCacheCleanup: true });
|
|
384
|
+
results[offset] = result.files;
|
|
385
|
+
if (result.timings) entryTimings.push({ index, timings: result.timings });
|
|
386
|
+
if (result.status === 'skipped') skipped++; else saved++;
|
|
387
|
+
outcome = { index, status: result.status || 'saved', files: result.files, ...(result.timings ? { timings: result.timings } : {}) };
|
|
388
|
+
} catch (error) {
|
|
389
|
+
if (signal?.aborted) throw error;
|
|
390
|
+
failures.push({ index, error: error.message });
|
|
391
|
+
reporter?.status(`Entry ${index} failed: ${error.message}`);
|
|
392
|
+
outcome = { index, status: 'failed', error: error.message };
|
|
393
|
+
}
|
|
394
|
+
await notify(outcome);
|
|
395
|
+
}
|
|
396
|
+
};
|
|
397
|
+
// Drain every active worker before returning, including after cancellation.
|
|
398
|
+
const settled = await Promise.allSettled(Array.from({ length: Math.min(concurrency, entries.length) }, (_, index) => worker(index)));
|
|
399
|
+
const rejected = settled.find(result => result.status === 'rejected');
|
|
400
|
+
if (rejected) throw rejected.reason;
|
|
401
|
+
const files = results.flatMap(files => files || []);
|
|
402
|
+
failures.sort((a, b) => a.index - b.index);
|
|
403
|
+
return { url: options.url, title: metadata.title || metadata.id || 'Playlist', files, saved, skipped, failures, ...(entryTimings.length ? { entryTimings: entryTimings.sort((a, b) => a.index - b.index) } : {}),
|
|
404
|
+
status: failures.length ? 'failed' : saved ? 'saved' : 'skipped' };
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Download one URL. Returns every file that was saved, in the order the
|
|
409
|
+
* backend produced them, so collections and subtitle sidecars are reported.
|
|
410
|
+
*/
|
|
411
|
+
export async function download(options, { signal, reporter, backendResolver = resolveBackend, runner = runBackend, onEntry, localRoot = downloadCacheRoot(), skipCacheCleanup = false, adaptiveState = { divisor: 1 }, wait, diskChecker = reserveSpace, now, compatibilityRunner = runBackend } = {}) {
|
|
412
|
+
const timer = phaseTimer(now);
|
|
413
|
+
const timingResult = () => {
|
|
414
|
+
if (options.timings === false) return {};
|
|
415
|
+
const timings = timer.result(); reporter?.status(formatTimings(timings)); return { timings };
|
|
416
|
+
};
|
|
417
|
+
let directory = path.resolve(options.output);
|
|
418
|
+
localRoot = path.resolve(localRoot);
|
|
419
|
+
if (!skipCacheCleanup) await cleanupDownloadCache(localRoot);
|
|
420
|
+
const requestKey = localRequestKey(options);
|
|
421
|
+
const stagingPath = path.join(localRoot, `${PARTIAL_PREFIX}${requestKey}`);
|
|
422
|
+
const existingStage = await lstat(stagingPath).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
423
|
+
if (existingStage && (!existingStage.isDirectory() || existingStage.isSymbolicLink())) throw new Error(`The partial download path is not a plain directory: ${stagingPath}`);
|
|
424
|
+
const cached = await readJson(path.join(stagingPath, 'job.json'), null);
|
|
425
|
+
const readyToTransfer = cached?.requestKey === requestKey && cached.ready?.length && cached.metadata
|
|
426
|
+
&& (cached.backendSucceeded === true || cached.compatibilityChecked === true || cached.files?.length);
|
|
427
|
+
const backend = readyToTransfer ? null : await prepareBackend(options, { signal, backendResolver, reporter });
|
|
428
|
+
timer.switch('metadata');
|
|
429
|
+
const metadata = readyToTransfer ? cached.metadata : await adaptiveRun(() => fetchMetadata(options, { signal, backend, runner, reporter }), { enabled: options.adaptiveConcurrency !== false, state: adaptiveState, signal, reporter, wait, timer });
|
|
430
|
+
if (metadata.entries && !options._entryIndex) return downloadCollection(options, metadata, { signal, reporter, backendResolver: async () => backend, runner, onEntry, localRoot, adaptiveState, wait, diskChecker, now, compatibilityRunner });
|
|
431
|
+
const destination = mediaDestination(options, metadata, stagedTitle('media.mp4', metadata, options.rename));
|
|
432
|
+
directory = destination.directory;
|
|
433
|
+
await prepareDestination(options.output, directory);
|
|
434
|
+
const playlist = false;
|
|
435
|
+
reporter?.name?.(destination.title);
|
|
436
|
+
const quality = selectQuality({ quality: options.quality, audio: options.audio, closest: options.closestQuality, formats: metadata.formats, playlist });
|
|
437
|
+
if (quality.error) throw new Error(quality.error);
|
|
438
|
+
if (quality.label) reporter?.status(quality.label);
|
|
439
|
+
|
|
440
|
+
const key = partialKey(metadata, options.url, options);
|
|
441
|
+
const historyFile = path.join(directory, '.veo-history', `${key}.json`);
|
|
442
|
+
const history = options.skipExisting || (options.resume && options._entryIndex) ? await readJson(historyFile, null) : null;
|
|
443
|
+
if (await pruneHistoryRecord(historyFile, directory, history)) {
|
|
444
|
+
reporter?.status('Already downloaded; skipped.');
|
|
445
|
+
return { url: options.url, title: metadata.title, files: [], status: 'skipped', saved: 0, skipped: 1, ...timingResult() };
|
|
446
|
+
}
|
|
447
|
+
const staging = await prepareStaging(stagingPath);
|
|
448
|
+
const lockPath = path.join(staging, '.lock');
|
|
449
|
+
let lock;
|
|
450
|
+
try { lock = await open(lockPath, 'wx'); }
|
|
451
|
+
catch (error) {
|
|
452
|
+
if (error.code === 'EEXIST') throw new Error(`This download is already active, or a force-killed process left ${lockPath}. Remove that lock only after checking that no other veo process uses it.`);
|
|
453
|
+
throw error;
|
|
454
|
+
}
|
|
455
|
+
const manifestFile = path.join(staging, 'job.json');
|
|
456
|
+
let keepPartial = false;
|
|
457
|
+
let unconfirmedOutput = Boolean(!readyToTransfer && (cached?.ready?.length || cached?.unconfirmedOutput));
|
|
458
|
+
let manifest;
|
|
459
|
+
let releaseSpace = () => {};
|
|
460
|
+
try {
|
|
461
|
+
manifest = await readJson(manifestFile, { version: 1, requestKey, metadata: { id: metadata.id, title: metadata.title, channel: metadata.channel, uploader: metadata.uploader, upload_date: metadata.upload_date, playlist_title: options._playlistTitle || metadata.playlist_title, extractor: metadata.extractor, extractor_key: metadata.extractor_key, formats: metadata.formats?.map(({ height, vcodec, has_drm }) => ({ height, vcodec, has_drm })) }, key, source: sourceKey(metadata, options.url), settings: downloadSettings(options), ready: [], files: [] });
|
|
462
|
+
if (manifest.key !== key || manifest.version !== 1) throw new Error('Partial download belongs to different settings.');
|
|
463
|
+
if (!Array.isArray(manifest.ready) || !Array.isArray(manifest.files)
|
|
464
|
+
|| manifest.ready.some(file => typeof file !== 'string' || path.dirname(file) !== staging || !isStagedMedia(path.basename(file)))
|
|
465
|
+
|| manifest.files.some(item => typeof item?.source !== 'string' || path.dirname(item.source) !== staging || typeof item.destination !== 'string' || path.dirname(item.destination) !== directory)) {
|
|
466
|
+
throw new Error('Invalid partial download manifest.');
|
|
467
|
+
}
|
|
468
|
+
if (options.checkSpace !== false) {
|
|
469
|
+
const bytes = readyToTransfer ? (await Promise.all(manifest.ready.map(file => stat(file)))).reduce((sum, info) => sum + info.size, 0) : estimateMediaBytes(metadata, options);
|
|
470
|
+
releaseSpace = await diskChecker({ cache: localRoot, destination: directory, bytes, cached: Boolean(readyToTransfer), reporter });
|
|
471
|
+
}
|
|
472
|
+
delete manifest.expiresAt;
|
|
473
|
+
await writeJson(manifestFile, manifest);
|
|
474
|
+
// after_move can still be emitted by yt-dlp after a media stream failed.
|
|
475
|
+
// A successful process exit must confirm candidates before transfer/reuse.
|
|
476
|
+
let staged = readyToTransfer ? [...new Set(manifest.ready)] : [];
|
|
477
|
+
if (!staged.length) {
|
|
478
|
+
const unconfirmed = manifest.backendSucceeded === false || manifest.ready.length > 0;
|
|
479
|
+
manifest.ready = [];
|
|
480
|
+
manifest.backendSucceeded = false;
|
|
481
|
+
await writeJson(manifestFile, manifest);
|
|
482
|
+
reporter?.status(options.audio ? 'Downloading audio…' : 'Downloading…');
|
|
483
|
+
timer.switch('download');
|
|
484
|
+
await adaptiveRun(async attempt => {
|
|
485
|
+
const candidates = [];
|
|
486
|
+
const attemptOptions = { ...options, resume: options.resume || attempt > 0, concurrentFragments: reducedLimit(options.concurrentFragments ?? 8, adaptiveState) };
|
|
487
|
+
const args = [...backendArgs(attemptOptions, backend), ...mediaArgs(attemptOptions, quality), '-o', stagingTemplate(staging, false), '--', options.url];
|
|
488
|
+
// A failed process may have left a final filename: --no-overwrites
|
|
489
|
+
// would otherwise silently accept that unconfirmed file on retry.
|
|
490
|
+
if (unconfirmed || attempt > 0) args.splice(args.indexOf('--'), 0, '--force-overwrites');
|
|
491
|
+
await runner(backend.ytDlp, args, { signal, onLine(line) {
|
|
492
|
+
if (line.startsWith('veo-progress:')) {
|
|
493
|
+
if (timer.phase !== 'download') timer.switch('download');
|
|
494
|
+
const data = JSON.parse(line.slice('veo-progress:'.length));
|
|
495
|
+
reporter?.progress(data.progress ? { ...data.progress, stream: data.video === 'none' ? 'Audio' : data.audio === 'none' ? 'Video' : 'Media' } : data);
|
|
496
|
+
}
|
|
497
|
+
if (line.startsWith('veo-postprocess:')) {
|
|
498
|
+
if (timer.phase !== 'processing') timer.switch('processing');
|
|
499
|
+
reporter?.processing?.(JSON.parse(line.slice('veo-postprocess:'.length)));
|
|
500
|
+
}
|
|
501
|
+
if (line.startsWith('veo-file:')) {
|
|
502
|
+
const file = JSON.parse(line.slice('veo-file:'.length));
|
|
503
|
+
if (path.dirname(path.resolve(file)) !== staging || !isStagedMedia(path.basename(file))) throw new Error('The backend returned an invalid saved file path.');
|
|
504
|
+
candidates.push(file);
|
|
505
|
+
unconfirmedOutput = true;
|
|
506
|
+
}
|
|
507
|
+
} });
|
|
508
|
+
staged = [...new Set(candidates)];
|
|
509
|
+
}, { enabled: options.adaptiveConcurrency !== false, state: adaptiveState, signal, reporter, wait, timer });
|
|
510
|
+
manifest.ready = staged;
|
|
511
|
+
manifest.backendSucceeded = true;
|
|
512
|
+
delete manifest.unconfirmedOutput;
|
|
513
|
+
unconfirmedOutput = false;
|
|
514
|
+
await writeJson(manifestFile, manifest);
|
|
515
|
+
}
|
|
516
|
+
// Cancellation must win over any follow-up error so the caller can report
|
|
517
|
+
// "Cancelled." instead of a confusing backend message.
|
|
518
|
+
signal?.throwIfAborted();
|
|
519
|
+
if (!staged.length) throw new Error('The backend finished without producing a file.');
|
|
520
|
+
|
|
521
|
+
if (!options.audio && !manifest.compatibilityChecked && (!readyToTransfer || options.compatible)) {
|
|
522
|
+
timer.switch('processing');
|
|
523
|
+
const tools = backend || await prepareBackend(options, { signal, backendResolver, reporter });
|
|
524
|
+
for (let index = 0; index < staged.length; index++) staged[index] = await ensureCompatibility(staged[index], { backend: tools, runner: compatibilityRunner, signal, convert: Boolean(options.compatible), reporter });
|
|
525
|
+
manifest.ready = staged;
|
|
526
|
+
manifest.compatibilityChecked = true;
|
|
527
|
+
await writeJson(manifestFile, manifest);
|
|
528
|
+
}
|
|
529
|
+
const files = [];
|
|
530
|
+
timer.switch('saving');
|
|
531
|
+
reporter?.status(readyToTransfer ? 'Retrying transfer from local cache…' : 'Saving local download to destination…');
|
|
532
|
+
await prepareDestination(options.output, directory, { create: true });
|
|
533
|
+
for (const stagedFile of staged) {
|
|
534
|
+
const resolved = path.resolve(stagedFile);
|
|
535
|
+
// The backend must only ever hand back a file inside our staging directory.
|
|
536
|
+
if (path.dirname(resolved) !== staging || !(await lstat(resolved)).isFile()) throw new Error('The backend returned an invalid saved file path.');
|
|
537
|
+
let record = manifest.files.find(item => item.source === resolved);
|
|
538
|
+
if (record && !await stat(record.destination).then(info => info.isFile(), () => false)) record = null;
|
|
539
|
+
const saved = record?.destination || await saveUnique(resolved, directory, destination.title, { signal, keepSource: true });
|
|
540
|
+
if (!record) {
|
|
541
|
+
manifest.files = manifest.files.filter(item => item.source !== resolved);
|
|
542
|
+
manifest.files.push({ source: resolved, destination: saved });
|
|
543
|
+
await writeJson(manifestFile, manifest);
|
|
544
|
+
}
|
|
545
|
+
files.push(saved);
|
|
546
|
+
files.push(...await saveSidecars(staging, resolved, saved, { signal, manifest, manifestFile }));
|
|
547
|
+
}
|
|
548
|
+
await writeJson(historyFile, { version: 1, source: sourceKey(metadata, options.url), settings: downloadSettings(options), files });
|
|
549
|
+
return { url: options.url, title: metadata.title || metadata.id || 'video', files, status: 'saved', saved: 1, skipped: 0, ...timingResult() };
|
|
550
|
+
} catch (error) {
|
|
551
|
+
// A kept staging directory is the whole point of --resume, and cancellation
|
|
552
|
+
// is the most common reason to want one.
|
|
553
|
+
keepPartial = Boolean(options.resume || manifest?.ready?.length || unconfirmedOutput);
|
|
554
|
+
if (manifest && unconfirmedOutput) manifest.unconfirmedOutput = true;
|
|
555
|
+
if (manifest?.ready?.length || (unconfirmedOutput && !options.resume)) {
|
|
556
|
+
manifest.expiresAt = Date.now() + TRANSFER_RETENTION_MS;
|
|
557
|
+
await writeJson(manifestFile, manifest);
|
|
558
|
+
}
|
|
559
|
+
throw error;
|
|
560
|
+
} finally {
|
|
561
|
+
releaseSpace();
|
|
562
|
+
await lock.close();
|
|
563
|
+
await rm(lockPath, { force: true });
|
|
564
|
+
reporter?.finish();
|
|
565
|
+
if (keepPartial) reporter?.status(manifest?.expiresAt && !manifest?.ready?.length
|
|
566
|
+
? `Unconfirmed download kept locally in ${staging} for 15 minutes. Retry will rerun the backend; incomplete media may need to be downloaded again.`
|
|
567
|
+
: manifest?.expiresAt
|
|
568
|
+
? `Completed download kept locally in ${staging}. Retry within 15 minutes to transfer without downloading again. Expired files are cleaned on the next veo run.`
|
|
569
|
+
: `Partial download kept locally in ${staging}. Re-run with --resume to continue it.`);
|
|
570
|
+
else await rm(staging, { recursive: true, force: true });
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Move subtitle and thumbnail files beside the saved media file, keeping the
|
|
576
|
+
* media name as the prefix (Title.mp4 -> Title.en.vtt). Collections pair each
|
|
577
|
+
* file with the sidecars that share its index.
|
|
578
|
+
*/
|
|
579
|
+
async function saveSidecars(staging, stagedMedia, savedMedia, { signal, manifest, manifestFile } = {}) {
|
|
580
|
+
const written = [];
|
|
581
|
+
const stem = path.basename(stagedMedia, path.extname(stagedMedia));
|
|
582
|
+
const base = path.basename(savedMedia, path.extname(savedMedia));
|
|
583
|
+
const entries = await readdir(staging, { withFileTypes: true }).catch(() => []);
|
|
584
|
+
for (const entry of entries) {
|
|
585
|
+
if (!entry.isFile() || !isSidecar(entry.name) || !entry.name.startsWith(`${stem}.`)) continue;
|
|
586
|
+
const suffix = path.basename(entry.name, path.extname(entry.name)).slice(stem.length);
|
|
587
|
+
const source = path.join(staging, entry.name);
|
|
588
|
+
const record = manifest.files.find(item => item.source === source);
|
|
589
|
+
if (record && await stat(record.destination).then(info => info.isFile(), () => false)) {
|
|
590
|
+
written.push(record.destination);
|
|
591
|
+
continue;
|
|
592
|
+
}
|
|
593
|
+
const destination = await saveUnique(source, path.dirname(savedMedia), `${base}${suffix}`, { signal, keepSource: true });
|
|
594
|
+
manifest.files = manifest.files.filter(item => item.source !== source);
|
|
595
|
+
manifest.files.push({ source, destination });
|
|
596
|
+
await writeJson(manifestFile, manifest);
|
|
597
|
+
written.push(destination);
|
|
598
|
+
}
|
|
599
|
+
return written;
|
|
600
|
+
}
|