veodl 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -0
- package/CHANGELOG.md +264 -0
- package/LICENSE +21 -0
- package/README.md +725 -0
- package/bin/veo.js +4 -0
- package/docs/AGENT_GUIDE.md +66 -0
- package/package.json +59 -0
- package/src/backend-update.js +191 -0
- package/src/backend.js +520 -0
- package/src/cli.js +454 -0
- package/src/compatibility.js +39 -0
- package/src/config-clipboard.js +67 -0
- package/src/config-diagnostics.js +144 -0
- package/src/config-editor.js +281 -0
- package/src/config-errors.js +64 -0
- package/src/config-reset.js +39 -0
- package/src/config-template.js +171 -0
- package/src/config.js +213 -0
- package/src/disk-space.js +68 -0
- package/src/doctor.js +302 -0
- package/src/download-cache.js +31 -0
- package/src/downloader.js +600 -0
- package/src/execution.js +68 -0
- package/src/flush.js +66 -0
- package/src/history.js +173 -0
- package/src/inspect-media.js +168 -0
- package/src/interactive.js +82 -0
- package/src/jobs.js +172 -0
- package/src/legacy-config-comments.js +106 -0
- package/src/naming.js +50 -0
- package/src/open-file.js +15 -0
- package/src/output.js +61 -0
- package/src/paths.js +26 -0
- package/src/playlist.js +31 -0
- package/src/progress.js +152 -0
- package/src/run-archive.js +148 -0
- package/src/runs.js +333 -0
- package/src/state.js +27 -0
- package/src/stats.js +63 -0
- package/src/terminal-title.js +19 -0
- package/src/tool-setup.js +151 -0
- package/src/updater.js +227 -0
- package/src/utils.js +208 -0
- package/src/version.js +38 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { lstat, open, readdir } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { cacheBase } from './paths.js';
|
|
5
|
+
import { readJson, writeJson } from './state.js';
|
|
6
|
+
|
|
7
|
+
export const RUN_ARCHIVE_ID = /^[0-9a-z]{6}$/;
|
|
8
|
+
const SAMPLE_BYTES = 64 * 1024;
|
|
9
|
+
const MAX_SEARCH_FILES = 200_000;
|
|
10
|
+
const MEDIA_EXTENSIONS = new Set(['.mp4', '.mkv', '.webm', '.mov', '.m4v', '.mp3', '.m4a', '.aac', '.opus', '.ogg', '.flac', '.wav', '.mka', '.ts']);
|
|
11
|
+
|
|
12
|
+
export function runArchiveFile(id, root = cacheBase()) {
|
|
13
|
+
if (!RUN_ARCHIVE_ID.test(id)) throw new Error('Run ids have 6 letters or digits.');
|
|
14
|
+
return path.join(path.resolve(root), 'run-history', `${id}.json`);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function isMediaFile(file) { return MEDIA_EXTENSIONS.has(path.extname(file).toLowerCase()); }
|
|
18
|
+
|
|
19
|
+
async function regularFile(file) {
|
|
20
|
+
const info = await lstat(file).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
21
|
+
return info?.isFile() && !info.isSymbolicLink() ? info : null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Sample three parts of a file so renames can be found without rereading a huge video. */
|
|
25
|
+
export async function fileFingerprint(file, knownInfo) {
|
|
26
|
+
const info = knownInfo || await regularFile(file);
|
|
27
|
+
if (!info) return null;
|
|
28
|
+
const digest = createHash('sha256').update(`veo-file-v1:${info.size}:`);
|
|
29
|
+
const positions = [...new Set([0, Math.floor(Math.max(0, info.size - SAMPLE_BYTES) / 2), Math.max(0, info.size - SAMPLE_BYTES)])];
|
|
30
|
+
const handle = await open(file, 'r');
|
|
31
|
+
try {
|
|
32
|
+
for (const position of positions) {
|
|
33
|
+
const bytes = Math.min(SAMPLE_BYTES, info.size - position);
|
|
34
|
+
const buffer = Buffer.alloc(bytes);
|
|
35
|
+
const { bytesRead } = await handle.read(buffer, 0, bytes, position);
|
|
36
|
+
if (bytesRead !== bytes) throw new Error(`File changed while recording its identity: ${file}`);
|
|
37
|
+
digest.update(String(position)).update(':').update(buffer);
|
|
38
|
+
}
|
|
39
|
+
} finally { await handle.close(); }
|
|
40
|
+
return { size: info.size, sampleSha256: digest.digest('hex'),
|
|
41
|
+
device: info.dev, inode: info.ino };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function archiveRun(job, { root = cacheBase(), now = new Date() } = {}) {
|
|
45
|
+
const id = job.runId;
|
|
46
|
+
if (!RUN_ARCHIVE_ID.test(id || '')) return null;
|
|
47
|
+
const items = [];
|
|
48
|
+
for (const item of job.items || []) {
|
|
49
|
+
const files = [];
|
|
50
|
+
for (const file of item.files || []) {
|
|
51
|
+
const media = isMediaFile(file);
|
|
52
|
+
files.push({ originalPath: path.resolve(file), media,
|
|
53
|
+
fingerprint: media ? item.fingerprints?.[file] || await fileFingerprint(file).catch(() => null) : null });
|
|
54
|
+
}
|
|
55
|
+
items.push({ url: item.url, status: item.status, error: item.error || null,
|
|
56
|
+
output: path.resolve(item.options?.output || job.options?.output || process.cwd()), files });
|
|
57
|
+
}
|
|
58
|
+
const archived = { version: 1, id, startedAt: job.startedAt || null, finishedAt: now.toISOString(),
|
|
59
|
+
status: items.every(item => ['saved', 'skipped'].includes(item.status)) ? 'completed' : 'incomplete', items };
|
|
60
|
+
if (await lstat(runArchiveFile(id, root)).then(() => true, error => { if (error.code === 'ENOENT') return false; throw error; })) {
|
|
61
|
+
throw new Error(`Run archive ${id} already exists; refusing to overwrite it.`);
|
|
62
|
+
}
|
|
63
|
+
await writeJson(runArchiveFile(id, root), archived);
|
|
64
|
+
return archived;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export async function readRunArchive(id, root = cacheBase()) {
|
|
68
|
+
const archive = await readJson(runArchiveFile(id, root), null);
|
|
69
|
+
if (!archive) throw new Error(`No finished run with id ${id} was found. Active runs appear in veo runs --json.`);
|
|
70
|
+
if (archive.version !== 1 || archive.id !== id || !Array.isArray(archive.items)) throw new Error(`Invalid archived run: ${id}`);
|
|
71
|
+
return archive;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function scan(root, wantedSizes, signal) {
|
|
75
|
+
const info = await regularFile(root);
|
|
76
|
+
if (info) throw new Error(`Search root is a file: ${root}`);
|
|
77
|
+
const directory = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
78
|
+
if (!directory) return [];
|
|
79
|
+
if (!directory.isDirectory() || directory.isSymbolicLink()) throw new Error(`Search root is not a plain directory: ${root}`);
|
|
80
|
+
const candidates = [];
|
|
81
|
+
const pending = [root];
|
|
82
|
+
let seen = 0;
|
|
83
|
+
while (pending.length) {
|
|
84
|
+
signal?.throwIfAborted();
|
|
85
|
+
const current = pending.pop();
|
|
86
|
+
for (const entry of await readdir(current, { withFileTypes: true })) {
|
|
87
|
+
const file = path.join(current, entry.name);
|
|
88
|
+
if (entry.isDirectory()) pending.push(file);
|
|
89
|
+
else if (entry.isFile()) {
|
|
90
|
+
if (++seen > MAX_SEARCH_FILES) throw new Error(`Too many files under ${root}; use --search with a narrower directory.`);
|
|
91
|
+
const details = await regularFile(file);
|
|
92
|
+
if (details && wantedSizes.has(details.size)) candidates.push({ file, details });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return candidates;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Resolve original paths first, then use fingerprints to find renamed media. */
|
|
100
|
+
export async function locateRunFiles(archive, { searchRoot, signal } = {}) {
|
|
101
|
+
const items = [];
|
|
102
|
+
const missing = [];
|
|
103
|
+
for (const item of archive.items) {
|
|
104
|
+
signal?.throwIfAborted();
|
|
105
|
+
const files = [];
|
|
106
|
+
for (const entry of item.files || []) {
|
|
107
|
+
const details = await regularFile(entry.originalPath);
|
|
108
|
+
const fingerprint = details && entry.fingerprint ? await fileFingerprint(entry.originalPath, details).catch(error => { if (error.code === 'ENOENT') return null; throw error; }) : null;
|
|
109
|
+
const valid = details && (!entry.fingerprint || (fingerprint?.size === entry.fingerprint.size && fingerprint.sampleSha256 === entry.fingerprint.sampleSha256));
|
|
110
|
+
const resolved = { originalPath: entry.originalPath, path: valid ? entry.originalPath : null,
|
|
111
|
+
found: Boolean(valid), renamed: false, media: Boolean(entry.media) };
|
|
112
|
+
files.push(resolved);
|
|
113
|
+
if (!valid && entry.media && entry.fingerprint) missing.push({ resolved, entry, output: item.output });
|
|
114
|
+
}
|
|
115
|
+
items.push({ url: item.url, status: item.status, error: item.error || null, files });
|
|
116
|
+
}
|
|
117
|
+
if (missing.length) {
|
|
118
|
+
const roots = [...new Set(missing.map(item => item.output).concat(searchRoot ? [path.resolve(searchRoot)] : []))];
|
|
119
|
+
const wanted = new Set(missing.map(item => item.entry.fingerprint.size));
|
|
120
|
+
const matches = new Map();
|
|
121
|
+
for (const root of roots) for (const candidate of await scan(root, wanted, signal)) {
|
|
122
|
+
signal?.throwIfAborted();
|
|
123
|
+
const fingerprint = await fileFingerprint(candidate.file, candidate.details).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
124
|
+
if (!fingerprint) continue;
|
|
125
|
+
const key = `${fingerprint.size}:${fingerprint.sampleSha256}`;
|
|
126
|
+
const paths = matches.get(key) || new Set();
|
|
127
|
+
paths.add(candidate.file);
|
|
128
|
+
matches.set(key, paths);
|
|
129
|
+
}
|
|
130
|
+
for (const { resolved, entry } of missing) {
|
|
131
|
+
const key = `${entry.fingerprint.size}:${entry.fingerprint.sampleSha256}`;
|
|
132
|
+
const paths = [...(matches.get(key) || [])];
|
|
133
|
+
let selected = paths.length === 1 ? paths[0] : null;
|
|
134
|
+
if (!selected && paths.length > 1 && entry.fingerprint.inode > 0) {
|
|
135
|
+
const sameFile = [];
|
|
136
|
+
for (const file of paths) {
|
|
137
|
+
const details = await regularFile(file);
|
|
138
|
+
if (details?.dev === entry.fingerprint.device && details.ino === entry.fingerprint.inode) sameFile.push(file);
|
|
139
|
+
}
|
|
140
|
+
if (sameFile.length === 1) selected = sameFile[0];
|
|
141
|
+
}
|
|
142
|
+
if (selected) {
|
|
143
|
+
resolved.path = selected; resolved.found = true; resolved.renamed = selected !== entry.originalPath;
|
|
144
|
+
} else if (paths.length > 1) resolved.candidates = paths;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return items;
|
|
148
|
+
}
|
package/src/runs.js
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
import { commandOutput } from './output.js';
|
|
2
|
+
import { randomInt, randomUUID } from 'node:crypto';
|
|
3
|
+
import { lstat, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
6
|
+
import { cacheBase } from './paths.js';
|
|
7
|
+
import { writeJson } from './state.js';
|
|
8
|
+
import { runArchiveFile } from './run-archive.js';
|
|
9
|
+
import { cleanText, localStamp } from './utils.js';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Every veo run registers itself under the per-user cache's `runs` directory
|
|
13
|
+
* while it works, so another terminal can list it (`veo runs`), inspect it
|
|
14
|
+
* (`veo runs <id>`) or cancel it (`veo stop <id>`) without guessing at PIDs.
|
|
15
|
+
* Records hold an id, times, URLs, paths and settings — never credentials or
|
|
16
|
+
* cookie settings, and the record itself is always removed when the run ends.
|
|
17
|
+
*/
|
|
18
|
+
export const RUN_ID = /^[0-9a-z]{6}$/;
|
|
19
|
+
export const STOP_TIMEOUT_MS = 15000;
|
|
20
|
+
const RECORD = /^[a-f0-9-]{36}\.json$/;
|
|
21
|
+
const ID_ALPHABET = '0123456789abcdefghijklmnopqrstuvwxyz';
|
|
22
|
+
const ITEM_STATUSES = new Set(['pending', 'running', 'saved', 'skipped', 'failed', 'cancelled']);
|
|
23
|
+
const SHOWN_ITEMS = 10;
|
|
24
|
+
const OUTPUT_WIDTH = 42;
|
|
25
|
+
|
|
26
|
+
export const RUNS_HELP = `veo runs [id] [--json]
|
|
27
|
+
|
|
28
|
+
List active veo runs with their 6-character id, process id, start time and progress.
|
|
29
|
+
With an id, show one run in detail: URLs, media settings, output directory, job file
|
|
30
|
+
and the state of every item. A run appears while it works and disappears when it
|
|
31
|
+
finishes. Records of crashed runs are kept but marked stale.
|
|
32
|
+
Use --json for machine-readable run metadata and per-item progress.
|
|
33
|
+
Use veo stop <id> to stop one run.
|
|
34
|
+
`;
|
|
35
|
+
|
|
36
|
+
export const STOP_HELP = `veo stop [id]
|
|
37
|
+
|
|
38
|
+
Ask one run, or every run when no id is given, to stop, and wait until it exits.
|
|
39
|
+
A stopped run keeps its partial data and its retry job, so the printed
|
|
40
|
+
veo --retry-failed command still works; veo flush removes those later.
|
|
41
|
+
Stale records of crashed runs are removed. Exit status is 1 when a run does not
|
|
42
|
+
stop in time.
|
|
43
|
+
`;
|
|
44
|
+
|
|
45
|
+
export function runsDirectory(root = cacheBase()) {
|
|
46
|
+
return path.join(path.resolve(root), 'runs');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* A run record is only trusted when it names a plausible process. Older veo
|
|
51
|
+
* versions wrote nothing but a pid, so a missing version is still accepted.
|
|
52
|
+
*/
|
|
53
|
+
function validRecord(run) {
|
|
54
|
+
if (!run || typeof run !== 'object' || Array.isArray(run)) return false;
|
|
55
|
+
return (run.version === undefined || run.version === 1) && Number.isInteger(run.pid) && run.pid > 0;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const exists = file => lstat(file).then(() => true, error => { if (error.code === 'ENOENT') return false; throw error; });
|
|
59
|
+
const text = value => typeof value === 'string' ? cleanText(value) : '';
|
|
60
|
+
const newRunId = () => Array.from({ length: 6 }, () => ID_ALPHABET[randomInt(ID_ALPHABET.length)]).join('');
|
|
61
|
+
|
|
62
|
+
export function alive(pid) {
|
|
63
|
+
try { process.kill(pid, 0); return true; }
|
|
64
|
+
catch (error) { return error.code !== 'ESRCH'; }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function runFiles(root) {
|
|
68
|
+
const directory = runsDirectory(root);
|
|
69
|
+
const info = await lstat(directory).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
70
|
+
if (!info) return [];
|
|
71
|
+
if (!info.isDirectory() || info.isSymbolicLink()) throw new Error(`Invalid active run directory: ${directory}`);
|
|
72
|
+
return (await readdir(directory, { withFileTypes: true }))
|
|
73
|
+
.filter(entry => entry.isFile() && RECORD.test(entry.name)).map(entry => path.join(directory, entry.name));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Cache files are machine-written: a missing, truncated or foreign file must
|
|
77
|
+
// never take a veo command down, so unreadable JSON simply counts as unknown.
|
|
78
|
+
async function readRecordFile(file) {
|
|
79
|
+
try { return JSON.parse(await readFile(file, 'utf8')); } catch { return null; }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function readRecords(root) {
|
|
83
|
+
const records = [];
|
|
84
|
+
for (const file of await runFiles(root)) {
|
|
85
|
+
// Damaged or foreign files are ignored, never fatal: a single stray file
|
|
86
|
+
// must not block every later veo run. Downloads of a run that could not be
|
|
87
|
+
// recognized stay protected by the staging lock that run holds.
|
|
88
|
+
const run = await readRecordFile(file);
|
|
89
|
+
if (!validRecord(run)) continue;
|
|
90
|
+
records.push({ ...run, id: RUN_ID.test(run.id || '') ? run.id : null, file });
|
|
91
|
+
}
|
|
92
|
+
return records;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Every registered run, live ones first and then the most recently started. */
|
|
96
|
+
export async function listRuns(root = cacheBase()) {
|
|
97
|
+
const runs = await readRecords(root);
|
|
98
|
+
for (const run of runs) run.alive = alive(run.pid);
|
|
99
|
+
return runs.sort((a, b) => Number(b.alive) - Number(a.alive) || String(b.startedAt || '').localeCompare(String(a.startedAt || '')));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Register this process as a run and poll for a stop request. The returned
|
|
104
|
+
* handle exposes the run's id, its resolved settings for `veo runs <id>`, and
|
|
105
|
+
* the cleanup that removes the record when the run ends.
|
|
106
|
+
*/
|
|
107
|
+
export async function registerRun(cancel, root = cacheBase()) {
|
|
108
|
+
const directory = runsDirectory(root);
|
|
109
|
+
await mkdir(directory, { recursive: true });
|
|
110
|
+
const gate = path.join(path.resolve(root), '.flush');
|
|
111
|
+
if (await exists(gate)) throw new Error('veo flush is in progress. Try again after cleanup finishes.');
|
|
112
|
+
const taken = new Set((await readRecords(root)).map(run => run.id).filter(Boolean));
|
|
113
|
+
let id = null;
|
|
114
|
+
for (let attempt = 0; attempt < 20 && !id; attempt++) {
|
|
115
|
+
const candidate = newRunId();
|
|
116
|
+
if (!taken.has(candidate) && !await exists(runArchiveFile(candidate, root))) id = candidate;
|
|
117
|
+
}
|
|
118
|
+
if (!id) throw new Error('Could not allocate a free run id. Remove stale run records with veo stop or veo flush.');
|
|
119
|
+
const file = path.join(directory, `${randomUUID()}.json`);
|
|
120
|
+
const request = `${file}.cancel`;
|
|
121
|
+
const record = { version: 1, id, pid: process.pid, startedAt: new Date().toISOString(),
|
|
122
|
+
urls: [], output: null, media: null, quality: null, format: null, playlist: false, job: null };
|
|
123
|
+
await writeJson(file, record);
|
|
124
|
+
// A flush that started while this run registered must still win.
|
|
125
|
+
if (await exists(gate)) {
|
|
126
|
+
await rm(file, { force: true });
|
|
127
|
+
throw new Error('veo flush is in progress. Try again after cleanup finishes.');
|
|
128
|
+
}
|
|
129
|
+
const timer = setInterval(() => { exists(request).then(found => { if (found) cancel(); }).catch(() => {}); }, 200);
|
|
130
|
+
timer.unref();
|
|
131
|
+
return {
|
|
132
|
+
id,
|
|
133
|
+
file,
|
|
134
|
+
/** Merge what this run resolved, so other terminals can inspect it. */
|
|
135
|
+
async describe(details = {}) {
|
|
136
|
+
Object.assign(record, {
|
|
137
|
+
urls: (Array.isArray(details.urls) ? details.urls : []).filter(url => typeof url === 'string').slice(0, 200).map(cleanText),
|
|
138
|
+
output: text(details.output) || null,
|
|
139
|
+
media: details.audio ? 'audio' : 'video',
|
|
140
|
+
quality: details.audio ? null : text(details.quality) || null,
|
|
141
|
+
format: text(details.format) || null,
|
|
142
|
+
playlist: Boolean(details.playlist),
|
|
143
|
+
job: text(details.job) || null,
|
|
144
|
+
});
|
|
145
|
+
await writeJson(file, record);
|
|
146
|
+
},
|
|
147
|
+
async unregister() {
|
|
148
|
+
clearInterval(timer);
|
|
149
|
+
await rm(file, { force: true });
|
|
150
|
+
await rm(request, { force: true });
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Per-item progress of a run, read from the job file it writes as it works. */
|
|
156
|
+
async function readProgress(jobFile) {
|
|
157
|
+
if (!jobFile) return null;
|
|
158
|
+
const job = await readRecordFile(jobFile);
|
|
159
|
+
if (job?.version !== 1 || !Array.isArray(job.items)) return null;
|
|
160
|
+
const items = job.items.map(item => ({
|
|
161
|
+
url: text(item?.url),
|
|
162
|
+
status: ITEM_STATUSES.has(item?.status) ? item.status : 'pending',
|
|
163
|
+
files: (Array.isArray(item?.files) ? item.files : []).map(text).filter(Boolean),
|
|
164
|
+
error: text(item?.error) || null,
|
|
165
|
+
}));
|
|
166
|
+
const counts = { total: items.length, pending: 0, running: 0, saved: 0, skipped: 0, failed: 0, cancelled: 0 };
|
|
167
|
+
for (const item of items) counts[item.status]++;
|
|
168
|
+
return { items, counts };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
async function describeRuns(root) {
|
|
172
|
+
const runs = await listRuns(root);
|
|
173
|
+
return Promise.all(runs.map(async run => ({ ...run, progress: await readProgress(run.job) })));
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function summarizeProgress(progress) {
|
|
177
|
+
const counts = progress?.counts;
|
|
178
|
+
if (!counts?.total) return 'starting';
|
|
179
|
+
const parts = [];
|
|
180
|
+
const done = counts.saved + counts.skipped;
|
|
181
|
+
if (done) parts.push(`${done}/${counts.total} done`);
|
|
182
|
+
if (counts.running) parts.push(`${counts.running} running`);
|
|
183
|
+
if (counts.failed) parts.push(`${counts.failed} failed`);
|
|
184
|
+
if (counts.cancelled) parts.push(`${counts.cancelled} cancelled`);
|
|
185
|
+
if (counts.pending) parts.push(parts.length ? `${counts.pending} pending` : `${counts.total} pending`);
|
|
186
|
+
return parts.join(', ');
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function shortDuration(ms) {
|
|
190
|
+
if (!Number.isFinite(ms) || ms < 0) return '0s';
|
|
191
|
+
const seconds = Math.floor(ms / 1000);
|
|
192
|
+
if (seconds < 60) return `${seconds}s`;
|
|
193
|
+
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
|
|
194
|
+
return `${Math.floor(seconds / 3600)}h ${Math.floor(seconds % 3600 / 60)}m`;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const shorten = (value, width = OUTPUT_WIDTH) => value.length <= width ? value : `…${value.slice(-(width - 1))}`;
|
|
198
|
+
const label = run => run.id || `PID ${run.pid}`;
|
|
199
|
+
// Records of older versions have no start time, so no uptime can be shown.
|
|
200
|
+
const uptime = run => Number.isFinite(Date.parse(run.startedAt)) ? shortDuration(Date.now() - Date.parse(run.startedAt)) : null;
|
|
201
|
+
|
|
202
|
+
function describeMedia(run) {
|
|
203
|
+
const detail = run.media === 'audio' ? run.format : run.quality;
|
|
204
|
+
return `${run.media || 'not resolved yet'}${detail ? `, ${detail}` : ''}${run.playlist ? ', playlist' : ''}`;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export function formatRuns(runs) {
|
|
208
|
+
if (!runs.length) return 'veo runs\n\nNo veo runs are active. Start a download to see it here.\n';
|
|
209
|
+
const rows = runs.map(run => {
|
|
210
|
+
const running = uptime(run);
|
|
211
|
+
return [run.id || '-', String(run.pid), run.alive ? `running${running ? ` ${running}` : ''}` : 'stale',
|
|
212
|
+
localStamp(run.startedAt), summarizeProgress(run.progress), run.output ? shorten(text(run.output)) : '-'];
|
|
213
|
+
});
|
|
214
|
+
const header = ['ID', 'PID', 'STATE', 'STARTED', 'ITEMS', 'OUTPUT'];
|
|
215
|
+
const widths = header.map((title, index) => Math.max(title.length, ...rows.map(row => row[index].length)));
|
|
216
|
+
const line = cells => cells.map((cell, index) => cell.padEnd(widths[index])).join(' ').trimEnd();
|
|
217
|
+
return `${['veo runs', '', line(header), ...rows.map(line), '', 'Details: veo runs <id> Stop: veo stop [id]'].join('\n')}\n`;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
export function formatRunDetails(run) {
|
|
221
|
+
const lines = [`veo runs ${run.id || '-'}`, ''];
|
|
222
|
+
const running = uptime(run);
|
|
223
|
+
lines.push(`State: ${run.alive ? `running (PID ${run.pid})` : `stale (PID ${run.pid} is gone)`}`);
|
|
224
|
+
lines.push(`Started: ${localStamp(run.startedAt)}${run.alive && running ? ` (${running} ago)` : ''}`);
|
|
225
|
+
lines.push(`Media: ${describeMedia(run)}`);
|
|
226
|
+
lines.push(`Output: ${text(run.output) || 'not resolved yet'}`);
|
|
227
|
+
lines.push(`Job: ${text(run.job) || 'not created yet'}`);
|
|
228
|
+
const urls = (Array.isArray(run.urls) ? run.urls : []).map(text).filter(Boolean);
|
|
229
|
+
lines.push(`URLs: ${urls.length ? `${urls.length}` : 'not resolved yet'}`);
|
|
230
|
+
urls.slice(0, SHOWN_ITEMS).forEach((url, index) => lines.push(` ${index + 1}. ${url}`));
|
|
231
|
+
if (urls.length > SHOWN_ITEMS) lines.push(` … and ${urls.length - SHOWN_ITEMS} more`);
|
|
232
|
+
const items = run.progress?.items || [];
|
|
233
|
+
if (items.length) {
|
|
234
|
+
lines.push(`Items: ${summarizeProgress(run.progress)}`);
|
|
235
|
+
items.slice(0, SHOWN_ITEMS).forEach((item, index) => {
|
|
236
|
+
const files = item.files.length ? ` ${item.files[0]}${item.files.length > 1 ? ` (+${item.files.length - 1} file(s))` : ''}` : '';
|
|
237
|
+
lines.push(` ${String(index + 1).padStart(2)}. ${item.status}${item.status === 'saved' ? files : ` ${item.url}`}`);
|
|
238
|
+
if (item.error) lines.push(` ${item.error}`);
|
|
239
|
+
});
|
|
240
|
+
if (items.length > SHOWN_ITEMS) lines.push(` … and ${items.length - SHOWN_ITEMS} more item(s)`);
|
|
241
|
+
}
|
|
242
|
+
lines.push('');
|
|
243
|
+
lines.push(run.alive && run.id ? `Stop: veo stop ${run.id}` : 'Cleanup: veo stop (removes stale records)');
|
|
244
|
+
return `${lines.join('\n')}\n`;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function normalizeId(value) {
|
|
248
|
+
const id = cleanText(value).toLowerCase();
|
|
249
|
+
if (!RUN_ID.test(id)) throw new Error(`Invalid run id "${cleanText(value)}". Run ids have 6 characters and are listed by veo runs.`);
|
|
250
|
+
return id;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export async function runsMain(args = [], { stdout = process.stdout, root = cacheBase() } = {}) {
|
|
254
|
+
[args, stdout] = commandOutput(args, stdout);
|
|
255
|
+
if (args.length === 1 && ['--help', '-h'].includes(args[0])) {
|
|
256
|
+
stdout.write(RUNS_HELP);
|
|
257
|
+
return 0;
|
|
258
|
+
}
|
|
259
|
+
const json = args.includes('--json');
|
|
260
|
+
args = args.filter(arg => arg !== '--json');
|
|
261
|
+
if (args.length > 1) throw new Error('Usage: veo runs [id] [--json]');
|
|
262
|
+
const runs = await describeRuns(root);
|
|
263
|
+
if (!args.length) {
|
|
264
|
+
if (json) stdout.write(`${JSON.stringify({ count: runs.length, runs: runs.map(publicRun) })}\n`);
|
|
265
|
+
else stdout.write(formatRuns(runs));
|
|
266
|
+
return 0;
|
|
267
|
+
}
|
|
268
|
+
const id = normalizeId(args[0]);
|
|
269
|
+
const run = runs.find(item => item.id === id);
|
|
270
|
+
if (!run) throw new Error(`No veo run with id ${id} is active. List runs with veo runs.`);
|
|
271
|
+
if (json) stdout.write(`${JSON.stringify(publicRun(run))}\n`);
|
|
272
|
+
else stdout.write(formatRunDetails(run));
|
|
273
|
+
return 0;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function publicRun(run) {
|
|
277
|
+
return { id: run.id, pid: run.pid, state: run.alive ? 'running' : 'stale', startedAt: run.startedAt || null,
|
|
278
|
+
urls: Array.isArray(run.urls) ? run.urls : [], output: run.output || null, media: run.media || null,
|
|
279
|
+
quality: run.quality || null, format: run.format || null, playlist: Boolean(run.playlist), job: run.job || null,
|
|
280
|
+
progress: run.progress || null };
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* A run answers a stop request by cleaning up its own record, so a run counts as
|
|
285
|
+
* gone as soon as its record disappears — a live-looking process is not enough
|
|
286
|
+
* (the pid may already belong to something else).
|
|
287
|
+
*/
|
|
288
|
+
export async function waitForExit(runs, timeoutMs) {
|
|
289
|
+
const pending = async () => {
|
|
290
|
+
for (const run of runs) if (alive(run.pid) && await exists(run.file)) return true;
|
|
291
|
+
return false;
|
|
292
|
+
};
|
|
293
|
+
const deadline = Date.now() + timeoutMs;
|
|
294
|
+
while (await pending()) {
|
|
295
|
+
if (Date.now() >= deadline) return false;
|
|
296
|
+
await delay(100);
|
|
297
|
+
}
|
|
298
|
+
return true;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export async function stopMain(args = [], { stdout = process.stdout, root = cacheBase(), timeoutMs = STOP_TIMEOUT_MS } = {}) {
|
|
302
|
+
[args, stdout] = commandOutput(args, stdout);
|
|
303
|
+
if (args.length === 1 && ['--help', '-h'].includes(args[0])) {
|
|
304
|
+
stdout.write(STOP_HELP);
|
|
305
|
+
return 0;
|
|
306
|
+
}
|
|
307
|
+
if (args.length > 1) throw new Error('Usage: veo stop [id]');
|
|
308
|
+
let targets = await listRuns(root);
|
|
309
|
+
if (args.length) {
|
|
310
|
+
const id = normalizeId(args[0]);
|
|
311
|
+
targets = targets.filter(run => run.id === id);
|
|
312
|
+
if (!targets.length) throw new Error(`No veo run with id ${id} is active. List runs with veo runs.`);
|
|
313
|
+
}
|
|
314
|
+
const stale = targets.filter(run => !run.alive);
|
|
315
|
+
const live = targets.filter(run => run.alive);
|
|
316
|
+
for (const run of live) await writeFile(`${run.file}.cancel`, 'stop');
|
|
317
|
+
if (live.length) await waitForExit(live, timeoutMs);
|
|
318
|
+
const remaining = [];
|
|
319
|
+
for (const run of live) if (await exists(run.file) && alive(run.pid)) remaining.push(run);
|
|
320
|
+
const stopped = live.filter(run => !remaining.includes(run));
|
|
321
|
+
for (const run of [...stale, ...stopped]) {
|
|
322
|
+
await rm(run.file, { force: true });
|
|
323
|
+
await rm(`${run.file}.cancel`, { force: true });
|
|
324
|
+
}
|
|
325
|
+
const lines = ['veo stop', ''];
|
|
326
|
+
if (stopped.length) lines.push(`Stopped: ${stopped.map(label).join(', ')}`);
|
|
327
|
+
if (stale.length) lines.push(`Removed ${stale.length} stale record(s): ${stale.map(label).join(', ')}`);
|
|
328
|
+
if (remaining.length) lines.push(`Still running: ${remaining.map(label).join(', ')}`);
|
|
329
|
+
if (!stopped.length && !stale.length && !remaining.length) lines.push('No veo runs were active.');
|
|
330
|
+
if (stopped.length) lines.push('', 'Partial data and retry jobs are kept; veo flush removes them.');
|
|
331
|
+
stdout.write(`${lines.join('\n')}\n`);
|
|
332
|
+
return remaining.length ? 1 : 0;
|
|
333
|
+
}
|
package/src/state.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
export function digest(value) {
|
|
6
|
+
return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 24);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export async function readJson(file, fallback) {
|
|
10
|
+
try { return JSON.parse(await readFile(file, 'utf8')); }
|
|
11
|
+
catch (error) { if (error.code === 'ENOENT') return fallback; throw error; }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export async function writeJson(file, value) {
|
|
15
|
+
await mkdir(path.dirname(file), { recursive: true });
|
|
16
|
+
const temporary = `${file}.${randomUUID()}.tmp`;
|
|
17
|
+
try {
|
|
18
|
+
await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx' });
|
|
19
|
+
await rename(temporary, file);
|
|
20
|
+
} finally { await rm(temporary, { force: true }); }
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// Do not persist browser sessions or cookie paths in retry jobs.
|
|
24
|
+
export function publicOptions(options) {
|
|
25
|
+
const { cookies, cookiesFromBrowser, profile, urls, url, retryFailed, batchFile, ...safe } = options;
|
|
26
|
+
return { ...safe, output: path.resolve(options.output) };
|
|
27
|
+
}
|
package/src/stats.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { commandOutput } from './output.js';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { readdir, lstat, rm } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { cacheBase } from './paths.js';
|
|
6
|
+
import { readJson, writeJson } from './state.js';
|
|
7
|
+
|
|
8
|
+
const fields = ['videos', 'audio', 'failed', 'skipped', 'cancelled', 'elapsedMs'];
|
|
9
|
+
const pattern = /^[a-f0-9-]{36}\.json$/;
|
|
10
|
+
const empty = () => Object.fromEntries(fields.map(key => [key, 0]));
|
|
11
|
+
|
|
12
|
+
export function createStatsRecorder(root = cacheBase()) {
|
|
13
|
+
const file = path.join(root, 'stats', `${randomUUID()}.json`);
|
|
14
|
+
const totals = { version: 1, since: new Date().toISOString(), ...empty() };
|
|
15
|
+
return async delta => {
|
|
16
|
+
for (const key of fields) totals[key] += delta[key] || 0;
|
|
17
|
+
await writeJson(file, totals);
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function statFiles(root) {
|
|
22
|
+
const directory = path.resolve(root, 'stats');
|
|
23
|
+
const info = await lstat(directory).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
|
|
24
|
+
if (!info) return [];
|
|
25
|
+
if (!info.isDirectory() || info.isSymbolicLink()) throw new Error(`Invalid statistics directory: ${directory}`);
|
|
26
|
+
return (await readdir(directory, { withFileTypes: true }))
|
|
27
|
+
.filter(entry => entry.isFile() && pattern.test(entry.name)).map(entry => path.join(directory, entry.name));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function readStats(root = cacheBase()) {
|
|
31
|
+
const result = { since: null, ...empty() };
|
|
32
|
+
for (const file of await statFiles(root)) {
|
|
33
|
+
const value = await readJson(file, null);
|
|
34
|
+
if (!value) continue;
|
|
35
|
+
if (value.version !== 1 || !Number.isFinite(Date.parse(value.since)) || fields.some(key => !Number.isFinite(value[key]) || value[key] < 0)) {
|
|
36
|
+
throw new Error(`Invalid statistics file: ${file}`);
|
|
37
|
+
}
|
|
38
|
+
if (!result.since || value.since < result.since) result.since = value.since;
|
|
39
|
+
for (const key of fields) result[key] += value[key];
|
|
40
|
+
}
|
|
41
|
+
return result;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Called by flush only after registered runs have stopped and while its gate is held.
|
|
45
|
+
export async function resetStats(root = cacheBase()) {
|
|
46
|
+
for (const file of await statFiles(root)) await rm(file, { force: true });
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export async function statsMain(args = [], { stdout = process.stdout, root = cacheBase() } = {}) {
|
|
50
|
+
[args, stdout] = commandOutput(args, stdout);
|
|
51
|
+
if (args.length === 1 && ['--help', '-h'].includes(args[0])) {
|
|
52
|
+
stdout.write('veo stats [--json]\n\nShow recorded download totals. Time includes preparation, processing and saving,\nincluding failed attempts. Parallel run times are added together.\nStats start with this version; old downloads are not imported.\nUse veo flush --stats to clear temporary data and reset statistics.\n');
|
|
53
|
+
return 0;
|
|
54
|
+
}
|
|
55
|
+
if (args.length && (args.length !== 1 || args[0] !== '--json')) throw new Error('Usage: veo stats [--json]');
|
|
56
|
+
const stats = await readStats(root);
|
|
57
|
+
if (args[0] === '--json') stdout.write(`${JSON.stringify(stats)}\n`);
|
|
58
|
+
else {
|
|
59
|
+
const seconds = Math.floor(stats.elapsedMs / 1000);
|
|
60
|
+
stdout.write(`veo stats\n\nTracking since: ${stats.since || 'No downloads recorded yet'}\nVideos saved: ${stats.videos}\nAudio saved: ${stats.audio}\nTotal failures: ${stats.failed}\nSkipped: ${stats.skipped}\nCancelled: ${stats.cancelled}\nDownload time: ${Math.floor(seconds / 3600)}h ${Math.floor(seconds / 60) % 60}m ${seconds % 60}s\n\nTime includes preparation, processing and saving. Active requests appear when finished.\n`);
|
|
61
|
+
}
|
|
62
|
+
return 0;
|
|
63
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { cleanText } from './utils.js';
|
|
2
|
+
|
|
3
|
+
// PowerShell's console title API on Windows; OSC 0 on xterm-compatible Unix
|
|
4
|
+
// terminals. Never put escape sequences into redirected output or dumb terminals.
|
|
5
|
+
export function createTerminalTitle(stream, { platform = process.platform, env = process.env, processInfo = process } = {}) {
|
|
6
|
+
let previous;
|
|
7
|
+
return title => {
|
|
8
|
+
if (!stream.isTTY || env.TERM === 'dumb') return;
|
|
9
|
+
const safe = cleanText(title).slice(0, 240);
|
|
10
|
+
if (safe === previous) return;
|
|
11
|
+
try {
|
|
12
|
+
if (platform === 'win32') processInfo.title = safe;
|
|
13
|
+
else stream.write(`\x1b]0;${safe}\x07`);
|
|
14
|
+
previous = safe;
|
|
15
|
+
} catch {
|
|
16
|
+
// Cosmetic only: a terminal that rejects titles must not break downloads.
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
}
|