rushes 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/AGENTS.md +17 -3
- package/README.md +32 -1
- package/dist/cli/doctor.js +1 -1
- package/dist/cli/main.js +100 -8
- package/dist/core/found.js +333 -0
- package/dist/core/media.js +2 -0
- package/dist/core/schema.js +19 -0
- package/dist/core/store.js +13 -1
- package/dist/mcp/client.js +13 -1
- package/dist/mcp/tools.js +74 -5
- package/dist/server/app.js +132 -11
- package/dist/server/files.js +29 -2
- package/dist/server/found.js +1199 -0
- package/dist/server/start.js +15 -2
- package/package.json +1 -1
- package/skills/rushes/SKILL.md +4 -2
- package/web-dist/assets/index-DpiDziCn.js +3 -0
- package/web-dist/assets/index-DqCXH_lZ.css +1 -0
- package/web-dist/index.html +2 -2
- package/web-dist/assets/index-ChXjhVC5.js +0 -3
- package/web-dist/assets/index-Cj_tr1I-.css +0 -1
|
@@ -0,0 +1,1199 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { lstat, realpath, stat } from "node:fs/promises";
|
|
3
|
+
import { basename, extname, isAbsolute, join, resolve, sep } from "node:path";
|
|
4
|
+
import { AUDIO_EXT, DEFAULT_LIMITS, SKIPPED_FOLDERS, VIDEO_EXT, commonWordsOf, kindOf, pickCurrentSet, scanFolder, scoreCandidate, wordsOf, } from "../core/found.js";
|
|
5
|
+
import { InvalidError, RevConflictError, RushesError } from "../core/errors.js";
|
|
6
|
+
import { slugify } from "../core/ids.js";
|
|
7
|
+
import { probe as ffprobe } from "../core/media.js";
|
|
8
|
+
import { fromManifestPath, toManifestPath } from "../core/paths.js";
|
|
9
|
+
import { addFile, addVariant, addVersion, resolveVideo } from "../core/project.js";
|
|
10
|
+
/** The audio kinds, which an include can settle. */
|
|
11
|
+
const AUDIO_KINDS = new Set(["voice", "music", "sfx"]);
|
|
12
|
+
/** The most cuts' settled kinds found.json keeps (the newest). */
|
|
13
|
+
const MAX_SETTLED = 200;
|
|
14
|
+
/** The most items one bring-in request takes (§20.6's route, and bringIn itself). */
|
|
15
|
+
export const BRING_IN_MAX = 60;
|
|
16
|
+
/** §20.5: at most this many of one kind per request. */
|
|
17
|
+
export const BRING_IN_PER_KIND = 12;
|
|
18
|
+
/** The default for `adoptWaitMs`: how long adoption waits for the lengths it needs. */
|
|
19
|
+
export const ADOPT_WAIT_MS = 15_000;
|
|
20
|
+
/** The longest settled() waits by default, so it can never hang. */
|
|
21
|
+
export const SETTLED_MAX_MS = 30_000;
|
|
22
|
+
/** Why an item wasn't brought in. The first four are §20.5's own lines. */
|
|
23
|
+
export const BRING_IN_REASONS = {
|
|
24
|
+
outside: "That file isn't in the project folder",
|
|
25
|
+
gone: "That file has gone",
|
|
26
|
+
already: "Already in the project.",
|
|
27
|
+
cap: "Up to 12 at a time, so the tabs stay quick.",
|
|
28
|
+
link: "That file is a link. Bring in the file it points to",
|
|
29
|
+
folder: "That's a folder, not a file",
|
|
30
|
+
hiddenFile: "That's a hidden file",
|
|
31
|
+
skippedFolder: "Rushes leaves that folder alone",
|
|
32
|
+
chooseKind: "Choose a kind for this file",
|
|
33
|
+
notAudio: "That isn't an audio file",
|
|
34
|
+
notVideo: "That isn't a video file",
|
|
35
|
+
notDoc: "That isn't a script or document (md, txt or pdf)",
|
|
36
|
+
unknownType: "That isn't an audio, video or document file",
|
|
37
|
+
};
|
|
38
|
+
const R = BRING_IN_REASONS;
|
|
39
|
+
/** A refusal with its code: the key whose words these are, else "other". */
|
|
40
|
+
function refusal(path, reason) {
|
|
41
|
+
const code = Object.keys(BRING_IN_REASONS).find((k) => BRING_IN_REASONS[k] === reason);
|
|
42
|
+
return { path, reason, code: code ?? "other" };
|
|
43
|
+
}
|
|
44
|
+
/** §20.2: scripts and documents, brought in as library docs. */
|
|
45
|
+
const DOC_EXT = new Set([".md", ".txt", ".pdf"]);
|
|
46
|
+
/** A variant's or film's id comes from its name; this keeps it, plus uniqueId's suffix, inside the 64-character id limit. */
|
|
47
|
+
const NAME_SLUG_MAX = 58;
|
|
48
|
+
const ROUND_MAX = 64;
|
|
49
|
+
/** The default lanes' ids, which a voice round never takes. */
|
|
50
|
+
const STAGE_LANES = { voice: "voice", music: "music", sfx: "sfx" };
|
|
51
|
+
const STAGE_WORDS = { voice: "voiceover", music: "music", sfx: "sound effects" };
|
|
52
|
+
/** At most `max` characters (code points), trimmed. */
|
|
53
|
+
function cut(text, max) {
|
|
54
|
+
return [...text].slice(0, max).join("").trim();
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* A folder's name as a voice round: its last segment split on spaces, `_`, `-`, `.` and camel
|
|
58
|
+
* case, each word capitalised. "vo_jules" and "voJules" both become "Vo Jules". Undefined for the
|
|
59
|
+
* project folder itself, or a name with no words.
|
|
60
|
+
*/
|
|
61
|
+
export function roundName(folder) {
|
|
62
|
+
const last = folder.split("/").filter((p) => p !== "").pop();
|
|
63
|
+
if (!last)
|
|
64
|
+
return undefined;
|
|
65
|
+
const words = last
|
|
66
|
+
.normalize("NFC")
|
|
67
|
+
.replace(/(\p{Ll})(\p{Lu})/gu, "$1 $2")
|
|
68
|
+
.replace(/(\p{Lu}+)(\p{Lu}\p{Ll})/gu, "$1 $2")
|
|
69
|
+
.split(/[\s_.-]+/u)
|
|
70
|
+
.filter((w) => w !== "")
|
|
71
|
+
.map((w) => {
|
|
72
|
+
const [first, ...rest] = [...w];
|
|
73
|
+
return first.toUpperCase() + rest.join("");
|
|
74
|
+
});
|
|
75
|
+
const name = cut(words.join(" "), ROUND_MAX);
|
|
76
|
+
return name === "" ? undefined : name;
|
|
77
|
+
}
|
|
78
|
+
/** The lane id addVariant gives a round or lane name. */
|
|
79
|
+
function laneIdOf(name) {
|
|
80
|
+
return slugify(name).slice(0, 64).replace(/-+$/, "");
|
|
81
|
+
}
|
|
82
|
+
/** A manifest path's folder ("" at the top of the project). */
|
|
83
|
+
function folderOf(file) {
|
|
84
|
+
const slash = file.lastIndexOf("/");
|
|
85
|
+
return slash < 0 ? "" : file.slice(0, slash);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The round a voice file from `folder` joins (§20.5: each voiceover folder is its own round).
|
|
89
|
+
* Its folder's name, humanised, unless a round of that name holds another folder's files; then
|
|
90
|
+
* the parent folder's name is added ("Vo · B"), then a number ("Vo 2"). A folder named like
|
|
91
|
+
* the music or sfx lane, or with no name, joins the default voiceover lane (undefined).
|
|
92
|
+
*/
|
|
93
|
+
function voiceRound(p, folder) {
|
|
94
|
+
const base = roundName(folder);
|
|
95
|
+
if (base === undefined)
|
|
96
|
+
return undefined;
|
|
97
|
+
const reserved = new Set([STAGE_LANES.music, STAGE_LANES.sfx]);
|
|
98
|
+
if (reserved.has(laneIdOf(base)))
|
|
99
|
+
return undefined;
|
|
100
|
+
const parentFolder = folderOf(folder);
|
|
101
|
+
const parent = parentFolder === "" ? undefined : roundName(parentFolder);
|
|
102
|
+
const options = [base];
|
|
103
|
+
if (parent !== undefined)
|
|
104
|
+
options.push(cut(`${base} · ${parent}`, ROUND_MAX));
|
|
105
|
+
for (let n = 2; n <= 99; n++)
|
|
106
|
+
options.push(`${cut(base, ROUND_MAX - 3)} ${n}`);
|
|
107
|
+
for (const name of options) {
|
|
108
|
+
const id = laneIdOf(name);
|
|
109
|
+
if (reserved.has(id))
|
|
110
|
+
continue;
|
|
111
|
+
const lane = p.lanes.find((l) => l.id === id);
|
|
112
|
+
if (!lane)
|
|
113
|
+
return name;
|
|
114
|
+
if (lane.stage !== "voice")
|
|
115
|
+
continue;
|
|
116
|
+
// An empty round of that name, or one already holding this folder's files, is this folder's.
|
|
117
|
+
if (lane.variants.length === 0 || lane.variants.some((v) => folderOf(v.file) === folder))
|
|
118
|
+
return name;
|
|
119
|
+
}
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
/** Why a variant of `stage` can't go into the lane `round` (or the stage's own lane) names, or null when it can. */
|
|
123
|
+
function laneClash(p, stage, round) {
|
|
124
|
+
if (round !== undefined) {
|
|
125
|
+
const id = laneIdOf(round);
|
|
126
|
+
const lane = p.lanes.find((l) => l.id === id);
|
|
127
|
+
if (lane && lane.stage !== stage)
|
|
128
|
+
return `“${round}” is already the ${STAGE_WORDS[lane.stage]} lane. Choose another round name`;
|
|
129
|
+
if (!lane && (id === STAGE_LANES.music || id === STAGE_LANES.sfx)) {
|
|
130
|
+
return `“${round}” is kept for the ${STAGE_WORDS[id === STAGE_LANES.music ? "music" : "sfx"]} lane. Choose another round name`;
|
|
131
|
+
}
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
const lane = p.lanes.find((l) => l.id === STAGE_LANES[stage]);
|
|
135
|
+
if (lane && lane.stage !== stage)
|
|
136
|
+
return `The ${STAGE_WORDS[stage]} lane's name is taken by the ${STAGE_WORDS[lane.stage]} round “${lane.name}”`;
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
/** `name`, shortened only as far as its id needs. */
|
|
140
|
+
function fitName(name) {
|
|
141
|
+
let chars = [...name];
|
|
142
|
+
while (chars.length > 1 && slugify(chars.join("")).length > NAME_SLUG_MAX)
|
|
143
|
+
chars = chars.slice(0, -1);
|
|
144
|
+
return chars.join("").trim() || name;
|
|
145
|
+
}
|
|
146
|
+
function isInside(realRoot, real) {
|
|
147
|
+
return real === realRoot || real.startsWith(realRoot.endsWith(sep) ? realRoot : realRoot + sep);
|
|
148
|
+
}
|
|
149
|
+
/** A version's added time as a number, for ordering (a hand-edited, unreadable one counts as oldest). */
|
|
150
|
+
function addedTime(v) {
|
|
151
|
+
const t = Date.parse(v.addedAt);
|
|
152
|
+
return Number.isFinite(t) ? t : -Infinity;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* §20.4's cut: the named film's newest version when it has one, otherwise the newest registered
|
|
156
|
+
* cut of any film (latest `addedAt`; on a tie, the later one in the project).
|
|
157
|
+
*/
|
|
158
|
+
export function anchorCut(p, film) {
|
|
159
|
+
if (film !== undefined) {
|
|
160
|
+
try {
|
|
161
|
+
const video = resolveVideo(p, film);
|
|
162
|
+
const version = video.versions[video.versions.length - 1];
|
|
163
|
+
if (version)
|
|
164
|
+
return { video, version };
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Not a film in this project: the newest cut stands in.
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
let best = null;
|
|
171
|
+
for (const video of p.videos) {
|
|
172
|
+
for (const version of video.versions) {
|
|
173
|
+
if (!best || addedTime(version) >= addedTime(best.version))
|
|
174
|
+
best = { video, version };
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return best;
|
|
178
|
+
}
|
|
179
|
+
/** The film a found cut joins: the anchor cut's film, else the first film. Null with no films. */
|
|
180
|
+
function filmFor(p, film) {
|
|
181
|
+
return anchorCut(p, film)?.video.id ?? p.videos[0]?.id ?? null;
|
|
182
|
+
}
|
|
183
|
+
/** Thrown inside a project update when nothing in it succeeded, so nothing is written. */
|
|
184
|
+
class NothingAdded extends Error {
|
|
185
|
+
refused;
|
|
186
|
+
constructor(refused) {
|
|
187
|
+
super("nothing added");
|
|
188
|
+
this.refused = refused;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/** How many times a bring-in re-reads and retries after another writer changed the project. */
|
|
192
|
+
const BRING_IN_ATTEMPTS = 5;
|
|
193
|
+
/** The most entries `list()` returns (§20.6). */
|
|
194
|
+
export const FOUND_LIST_CAP = 2000;
|
|
195
|
+
/** Background probes run this many at a time (global constraints). */
|
|
196
|
+
export const FOUND_PROBE_CONCURRENCY = 2;
|
|
197
|
+
/** While probes keep landing, a `change` goes out after this many new durations (and always when they're all done). */
|
|
198
|
+
export const FOUND_ANNOUNCE_EVERY = 20;
|
|
199
|
+
/** The time budget for carrying a scan on in the background after the first pass ran out of time (§20.2). */
|
|
200
|
+
export const CONTINUE_BUDGET_MS = 30_000;
|
|
201
|
+
/** The most dismissed paths kept (the schema's limit): the oldest go first. */
|
|
202
|
+
const MAX_DISMISSED = 10000;
|
|
203
|
+
/** How many probe results are remembered. */
|
|
204
|
+
const DURATION_CACHE_LIMIT = 10000;
|
|
205
|
+
/** The default probe: ffprobe's duration, or null without ffprobe. */
|
|
206
|
+
export const ffprobeDuration = async (abs, signal) => (await ffprobe(abs, { signal })).duration;
|
|
207
|
+
/**
|
|
208
|
+
* The scanner's `announce` for a server: a `change` event on the store, which the SSE route sends
|
|
209
|
+
* to every open tab so it reads the state again. Its `file` is "found"; its `rev` counts the
|
|
210
|
+
* scanner's announcements this session (found.json's own writes announce its real rev as usual).
|
|
211
|
+
*/
|
|
212
|
+
export function foundAnnouncer(store) {
|
|
213
|
+
let seq = 0;
|
|
214
|
+
return () => {
|
|
215
|
+
store.emit("change", { file: "found", rev: ++seq });
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
/** Every manifest path the project has registered: cuts, variants, library files and script takes. */
|
|
219
|
+
export function registeredPaths(project, script) {
|
|
220
|
+
const out = [];
|
|
221
|
+
for (const v of project.videos)
|
|
222
|
+
for (const ver of v.versions)
|
|
223
|
+
out.push(ver.file);
|
|
224
|
+
for (const l of project.lanes)
|
|
225
|
+
for (const variant of l.variants)
|
|
226
|
+
out.push(variant.file);
|
|
227
|
+
for (const f of project.files)
|
|
228
|
+
out.push(f.file);
|
|
229
|
+
for (const s of script.sections)
|
|
230
|
+
for (const t of s.takes)
|
|
231
|
+
out.push(t.file);
|
|
232
|
+
return out;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* A path's identity on disk: its real path when it exists (case, symlinks, `..` and Unicode
|
|
236
|
+
* spelling all resolved by the file system), otherwise the plain resolved spelling.
|
|
237
|
+
*/
|
|
238
|
+
export async function realIdentity(abs) {
|
|
239
|
+
try {
|
|
240
|
+
return await realpath(abs);
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
return resolve(abs);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* `path` (relative to `root`, or absolute inside it) as a manifest path in its spelling on disk,
|
|
248
|
+
* when it names a plain file inside the project reached without any symlink on the way. Null for
|
|
249
|
+
* anything else: missing, a folder, a symlink or a path through one, outside the root, or a path
|
|
250
|
+
* with an empty, `.` or `..` segment.
|
|
251
|
+
*/
|
|
252
|
+
export async function plainFileInside(root, path) {
|
|
253
|
+
try {
|
|
254
|
+
const realRoot = await realpath(root);
|
|
255
|
+
let rel = path;
|
|
256
|
+
if (isAbsolute(path)) {
|
|
257
|
+
rel = toManifestPath(realRoot, path);
|
|
258
|
+
if (isAbsolute(rel))
|
|
259
|
+
rel = toManifestPath(root, path);
|
|
260
|
+
if (isAbsolute(rel))
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
if (rel === "" || rel.includes("\\") || rel.includes("\0"))
|
|
264
|
+
return null;
|
|
265
|
+
const parts = rel.split("/");
|
|
266
|
+
if (parts.some((p) => p === "" || p === "." || p === ".."))
|
|
267
|
+
return null;
|
|
268
|
+
// Every folder on the way, then the file itself, must be the real thing, never a symlink.
|
|
269
|
+
let at = realRoot;
|
|
270
|
+
for (let i = 0; i < parts.length; i++) {
|
|
271
|
+
at = join(at, parts[i]);
|
|
272
|
+
const info = await lstat(at);
|
|
273
|
+
if (info.isSymbolicLink())
|
|
274
|
+
return null;
|
|
275
|
+
if (i < parts.length - 1 ? !info.isDirectory() : !info.isFile())
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
const out = toManifestPath(realRoot, await realpath(at));
|
|
279
|
+
return isAbsolute(out) ? null : out;
|
|
280
|
+
}
|
|
281
|
+
catch {
|
|
282
|
+
return null;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
const revKey = (f) => `${f.abs}\0${f.size}\0${f.modified}`;
|
|
286
|
+
/**
|
|
287
|
+
* The probe queue's order. Adoption needs the anchor's length and every voice and music file's
|
|
288
|
+
* (registered ones first: they hold the current set's place), so those go before the lengths
|
|
289
|
+
* the list only shows. Sound effects get no length score (§20.4).
|
|
290
|
+
*/
|
|
291
|
+
const PROBE_TIER = { anchor: 0, competitor: 1, candidate: 2, listed: 3, cut: 4 };
|
|
292
|
+
export class FoundScanner {
|
|
293
|
+
store;
|
|
294
|
+
probeFile;
|
|
295
|
+
announceChange;
|
|
296
|
+
limits;
|
|
297
|
+
continueBudgetMs;
|
|
298
|
+
/** Everything the last walk found, registered or not. */
|
|
299
|
+
files = [];
|
|
300
|
+
scannedAt = null;
|
|
301
|
+
complete = false;
|
|
302
|
+
walking = false;
|
|
303
|
+
/** The first pass in flight, which a second scan() joins. */
|
|
304
|
+
firstPass = null;
|
|
305
|
+
/** The whole walk in flight, continuation included. */
|
|
306
|
+
walk = null;
|
|
307
|
+
film;
|
|
308
|
+
closing = false;
|
|
309
|
+
/** Aborted by close(): a walk in flight stops before its next folder. */
|
|
310
|
+
walkAbort = new AbortController();
|
|
311
|
+
/**
|
|
312
|
+
* Bumped by every dismiss and restore. A read of found.json that started before one of them is
|
|
313
|
+
* stale by the time it lands, and is dropped rather than undoing the change in has().
|
|
314
|
+
*/
|
|
315
|
+
dismissedGeneration = 0;
|
|
316
|
+
/** Dismissed manifest paths, as last read from or written to found.json. */
|
|
317
|
+
dismissed = new Set();
|
|
318
|
+
/** The paths `has()` answers yes for: found, not registered, not dismissed. */
|
|
319
|
+
current = new Set();
|
|
320
|
+
/** Registered real paths, by project and script rev. */
|
|
321
|
+
registeredCache = null;
|
|
322
|
+
/** Durations by path, size and mtime. */
|
|
323
|
+
durations = new Map();
|
|
324
|
+
queue = [];
|
|
325
|
+
queued = new Set();
|
|
326
|
+
probing = new Set();
|
|
327
|
+
inflight = new Set();
|
|
328
|
+
landedSinceAnnounce = 0;
|
|
329
|
+
idle = null;
|
|
330
|
+
/** Paths brought in this session, by hand or adopted. Not persisted. */
|
|
331
|
+
broughtIn = [];
|
|
332
|
+
/** The same, with where each went and why (§20.7), in the order they came in. */
|
|
333
|
+
broughtInLog = [];
|
|
334
|
+
adoptWaitMs;
|
|
335
|
+
/** Bring-ins (and adoptions) run one at a time, so two never add the same file. */
|
|
336
|
+
bringing = Promise.resolve();
|
|
337
|
+
/** Adoptions still running, which settled() waits for. */
|
|
338
|
+
adoptions = new Set();
|
|
339
|
+
/** Woken each time a probe lands (and on close), for adoption's wait on lengths. */
|
|
340
|
+
lengthWaiters = new Set();
|
|
341
|
+
constructor(opts) {
|
|
342
|
+
this.store = opts.store;
|
|
343
|
+
this.probeFile = opts.probe;
|
|
344
|
+
this.announceChange = opts.announce;
|
|
345
|
+
this.limits = opts.limits ?? {};
|
|
346
|
+
this.continueBudgetMs = opts.continueBudgetMs ?? CONTINUE_BUDGET_MS;
|
|
347
|
+
this.adoptWaitMs = opts.adoptWaitMs ?? ADOPT_WAIT_MS;
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Scans now, or joins a scan whose first pass is still running. Resolves when the first pass
|
|
351
|
+
* (the walk and the kinds) is done; probing, and a continuation when the first pass ran out of
|
|
352
|
+
* time, carry on in the background. `film` names the video (id, slug or name) whose newest cut
|
|
353
|
+
* anchors the scoring, and is remembered for later lists. Never rejects.
|
|
354
|
+
*/
|
|
355
|
+
scan(opts = {}) {
|
|
356
|
+
if (opts.film !== undefined)
|
|
357
|
+
this.film = opts.film;
|
|
358
|
+
if (this.firstPass)
|
|
359
|
+
return this.firstPass;
|
|
360
|
+
// The first pass is done and its background continuation is still walking: that walk is
|
|
361
|
+
// already the fresh look a second scan would take.
|
|
362
|
+
if (this.walk || this.closing)
|
|
363
|
+
return Promise.resolve();
|
|
364
|
+
let firstDone;
|
|
365
|
+
const first = new Promise((r) => (firstDone = r));
|
|
366
|
+
this.firstPass = first;
|
|
367
|
+
this.walking = true;
|
|
368
|
+
this.announce();
|
|
369
|
+
this.walk = this.runWalk(() => {
|
|
370
|
+
this.firstPass = null;
|
|
371
|
+
firstDone();
|
|
372
|
+
})
|
|
373
|
+
.catch(() => undefined)
|
|
374
|
+
.finally(() => {
|
|
375
|
+
this.walking = false;
|
|
376
|
+
this.walk = null;
|
|
377
|
+
this.firstPass = null;
|
|
378
|
+
firstDone();
|
|
379
|
+
this.announce();
|
|
380
|
+
this.resolveIdle();
|
|
381
|
+
});
|
|
382
|
+
return first;
|
|
383
|
+
}
|
|
384
|
+
async runWalk(firstPassDone) {
|
|
385
|
+
let result = await this.walkOnce(this.limits);
|
|
386
|
+
await this.land(result);
|
|
387
|
+
firstPassDone();
|
|
388
|
+
if (this.closing)
|
|
389
|
+
return;
|
|
390
|
+
// §20.2: a first pass that ran out of time (not out of entries or room) carries on in the
|
|
391
|
+
// background, and the tabs hear when it lands (the walk's end announces).
|
|
392
|
+
const limits = { ...DEFAULT_LIMITS, ...this.limits };
|
|
393
|
+
const outOfTime = !result.complete && result.examined < limits.maxExamined && result.files.length < limits.maxKept;
|
|
394
|
+
if (outOfTime) {
|
|
395
|
+
this.announce();
|
|
396
|
+
result = await this.walkOnce({ ...this.limits, budgetMs: this.continueBudgetMs });
|
|
397
|
+
await this.land(result);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
async walkOnce(limits) {
|
|
401
|
+
try {
|
|
402
|
+
return await scanFolder(this.store.root, { limits, signal: this.walkAbort.signal });
|
|
403
|
+
}
|
|
404
|
+
catch {
|
|
405
|
+
// The root has gone or can't be read: nothing found, and not complete.
|
|
406
|
+
return { files: [], complete: false, examined: 0 };
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
/** Takes a walk's result as the current one, and queues probes for what's new. */
|
|
410
|
+
async land(result) {
|
|
411
|
+
if (this.closing)
|
|
412
|
+
return;
|
|
413
|
+
this.files = result.files;
|
|
414
|
+
this.complete = result.complete;
|
|
415
|
+
this.scannedAt = new Date().toISOString();
|
|
416
|
+
try {
|
|
417
|
+
const ctx = await this.context();
|
|
418
|
+
const candidates = this.candidates(ctx.registered);
|
|
419
|
+
const listed = new Set(candidates.map((c) => c.path));
|
|
420
|
+
// In the order adoption needs them (see PROBE_TIER): the anchor's own length, when the
|
|
421
|
+
// project doesn't know it; registered and dismissed voice and music, which still compete
|
|
422
|
+
// for the current set; voice and music candidates; then the rest of the list.
|
|
423
|
+
await this.anchor(ctx.project);
|
|
424
|
+
const lengthScored = (f) => f.kind === "voice" || f.kind === "music";
|
|
425
|
+
for (const f of this.files.filter((c) => lengthScored(c) && !listed.has(c.path)))
|
|
426
|
+
this.enqueue(f, PROBE_TIER.competitor);
|
|
427
|
+
for (const f of candidates.filter(lengthScored))
|
|
428
|
+
this.enqueue(f, PROBE_TIER.candidate);
|
|
429
|
+
for (const f of candidates.filter((c) => c.kind === "sfx" || c.kind === "other"))
|
|
430
|
+
this.enqueue(f, PROBE_TIER.listed);
|
|
431
|
+
for (const f of candidates.filter((c) => c.kind === "cut"))
|
|
432
|
+
this.enqueue(f, PROBE_TIER.cut);
|
|
433
|
+
}
|
|
434
|
+
catch {
|
|
435
|
+
// project.json or script.json unreadable right now: the watcher reports that. has() still
|
|
436
|
+
// only knows this walk's files (less the dismissed), and the next list() tries again.
|
|
437
|
+
this.current = new Set(this.files.filter((f) => !this.dismissed.has(f.path)).map((f) => f.path));
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
async summary() {
|
|
441
|
+
const counts = { voice: 0, music: 0, sfx: 0, cut: 0, other: 0 };
|
|
442
|
+
const identities = [];
|
|
443
|
+
try {
|
|
444
|
+
const ctx = await this.context();
|
|
445
|
+
for (const f of this.candidates(ctx.registered)) {
|
|
446
|
+
counts[f.kind]++;
|
|
447
|
+
identities.push(`${f.kind}:${f.path}`);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
catch {
|
|
451
|
+
// Unreadable project files: counts stay at zero; the watcher reports the file.
|
|
452
|
+
}
|
|
453
|
+
const digest = createHash("sha1")
|
|
454
|
+
.update([...identities.sort(), "|", ...[...this.broughtIn].sort()].join("\n"))
|
|
455
|
+
.digest("hex")
|
|
456
|
+
.slice(0, 16);
|
|
457
|
+
return { scanning: this.walking, scannedAt: this.scannedAt, complete: this.complete, counts, broughtIn: [...this.broughtIn], digest };
|
|
458
|
+
}
|
|
459
|
+
/** Everything brought in this server session, with where it went and why (§20.7). */
|
|
460
|
+
broughtInDetail() {
|
|
461
|
+
return this.broughtInLog.map((e) => ({ ...e, reasons: [...e.reasons] }));
|
|
462
|
+
}
|
|
463
|
+
/** Candidates not registered and not dismissed, best score first, at most 2,000. */
|
|
464
|
+
async list() {
|
|
465
|
+
const { candidates, byPath, suggested } = await this.scoring(await this.context());
|
|
466
|
+
const entries = candidates.map((f) => {
|
|
467
|
+
const s = byPath.get(f.path);
|
|
468
|
+
return { ...f, score: s ? s.score : null, reasons: s ? s.reasons : [], suggested: suggested.has(f.path) };
|
|
469
|
+
});
|
|
470
|
+
entries.sort((a, b) => {
|
|
471
|
+
if (a.score !== b.score) {
|
|
472
|
+
if (a.score === null)
|
|
473
|
+
return 1;
|
|
474
|
+
if (b.score === null)
|
|
475
|
+
return -1;
|
|
476
|
+
return b.score - a.score;
|
|
477
|
+
}
|
|
478
|
+
return a.path < b.path ? -1 : a.path > b.path ? 1 : 0;
|
|
479
|
+
});
|
|
480
|
+
return entries.slice(0, FOUND_LIST_CAP);
|
|
481
|
+
}
|
|
482
|
+
/** The files the user hid with "Not these" that the last walk still found, so the dashboard can offer to restore them. By path. */
|
|
483
|
+
async hidden() {
|
|
484
|
+
const { registered } = await this.context();
|
|
485
|
+
return this.files
|
|
486
|
+
.filter((f) => this.dismissed.has(f.path) && !registered.has(f.abs))
|
|
487
|
+
.map((f) => ({ ...f, duration: this.durations.get(revKey(f)) ?? null }))
|
|
488
|
+
.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* §20.5's Bring in: registers each item as an unpicked variant (voice, music, sfx), a new
|
|
492
|
+
* version of the film (cut) or a library doc, in one write for the whole request, and no write
|
|
493
|
+
* when nothing can come in. An item that can't come in fails with a reason and the rest still
|
|
494
|
+
* go in. Nothing is ever picked. At most 60 items. `reasons` (by manifest path) describe the
|
|
495
|
+
* variants; without it, the current scoring's are used.
|
|
496
|
+
*/
|
|
497
|
+
bringIn(items, opts = {}) {
|
|
498
|
+
if (items.length > BRING_IN_MAX)
|
|
499
|
+
return Promise.reject(new InvalidError(`Up to ${BRING_IN_MAX} files at a time`));
|
|
500
|
+
return this.oneAtATime(() => this.bringInNow(items, opts));
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* §20.4's auto-add: brings in the current set's file for each kind (at most one per kind, and
|
|
504
|
+
* none on a tie or a low score), unpicked, described by its reasons. Adds nothing without an
|
|
505
|
+
* anchor cut, and nothing twice: the winner keeps its place once it's in, so the runner-up never
|
|
506
|
+
* moves up. It waits for the walk to end, then up to `adoptWaitMs` for the lengths scoring needs
|
|
507
|
+
* (the anchor's, and every voice and music file's). A kind whose lengths are still unknown then
|
|
508
|
+
* is skipped, never guessed from part of the picture; the next scan decides it. `skipKinds`
|
|
509
|
+
* leaves out kinds the caller has already settled (§20.7: what an agent includes wins).
|
|
510
|
+
*/
|
|
511
|
+
adoptCurrentSet(opts = {}) {
|
|
512
|
+
if (opts.film !== undefined)
|
|
513
|
+
this.film = opts.film;
|
|
514
|
+
const run = this.adopt(opts.film, new Set(opts.skipKinds ?? []));
|
|
515
|
+
const tracked = run.then(() => this.adoptions.delete(tracked), () => this.adoptions.delete(tracked));
|
|
516
|
+
this.adoptions.add(tracked);
|
|
517
|
+
return run;
|
|
518
|
+
}
|
|
519
|
+
async adopt(film, skip) {
|
|
520
|
+
const none = () => ({ added: [], failed: [] });
|
|
521
|
+
if (this.closing)
|
|
522
|
+
return none();
|
|
523
|
+
// The walk is bounded by its own budgets (3 s, then 30 s), and close() stops it.
|
|
524
|
+
if (this.walk)
|
|
525
|
+
await this.walk;
|
|
526
|
+
await this.lengthsKnown(Date.now() + this.adoptWaitMs);
|
|
527
|
+
return this.oneAtATime(async () => {
|
|
528
|
+
if (this.closing)
|
|
529
|
+
return none();
|
|
530
|
+
let picked;
|
|
531
|
+
let byPath;
|
|
532
|
+
try {
|
|
533
|
+
const ctx = await this.context();
|
|
534
|
+
const unknown = await this.pendingLengths(ctx.project);
|
|
535
|
+
const s = await this.scoring(ctx);
|
|
536
|
+
if (!s.anchor)
|
|
537
|
+
return none();
|
|
538
|
+
// What an agent's include settled for this cut (§20.7), read here so a start-up adoption,
|
|
539
|
+
// a later scan and a restart all honour it.
|
|
540
|
+
const settled = await this.settledFor(ctx.project);
|
|
541
|
+
picked = s.candidates.filter((f) => s.suggested.has(f.path) && !unknown.has(f.kind) && !skip.has(f.kind) && !settled.has(f.kind));
|
|
542
|
+
byPath = s.byPath;
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
return none(); // project files unreadable right now: the watcher reports them
|
|
546
|
+
}
|
|
547
|
+
if (picked.length === 0)
|
|
548
|
+
return none();
|
|
549
|
+
const reasons = new Map(picked.map((f) => [f.path, byPath.get(f.path)?.reasons ?? []]));
|
|
550
|
+
return this.bringInNow(picked.map((f) => ({ path: f.path, kind: f.kind })), { reasons, film, origin: "auto" });
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
/** The kinds an agent's include settled for the current anchor cut. */
|
|
554
|
+
async settledFor(project) {
|
|
555
|
+
const cut = anchorCut(project, this.film);
|
|
556
|
+
if (!cut)
|
|
557
|
+
return new Set();
|
|
558
|
+
const entry = (await this.store.read("found")).settled.find((e) => e.video === cut.video.id && e.version === cut.version.id);
|
|
559
|
+
return new Set(entry?.kinds ?? []);
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* §20.7: an agent's `include` wins over the scoring. The audio kinds of the items (the kind
|
|
563
|
+
* given, else what the name says, else the lane a file already registered sits in) are kept in
|
|
564
|
+
* found.json against the anchor cut, so no adoption adds a file of that kind for that cut again.
|
|
565
|
+
* It runs ahead of every adoption not yet at its write, and registrations are never removed.
|
|
566
|
+
* Items outside the project settle nothing. Returns the audio kinds the items settle, once they're written (or none to write: with no cut, nothing is recorded).
|
|
567
|
+
*/
|
|
568
|
+
settleKinds(items, film) {
|
|
569
|
+
if (film !== undefined)
|
|
570
|
+
this.film = film;
|
|
571
|
+
return this.oneAtATime(async () => {
|
|
572
|
+
const project = await this.store.read("project");
|
|
573
|
+
const cut = anchorCut(project, this.film);
|
|
574
|
+
const root = this.store.root;
|
|
575
|
+
const realRoot = await realIdentity(root);
|
|
576
|
+
const kinds = new Set();
|
|
577
|
+
let lanes = null;
|
|
578
|
+
for (const item of items) {
|
|
579
|
+
const given = isAbsolute(item.path) ? resolve(item.path) : resolve(root, ...item.path.split("/"));
|
|
580
|
+
let rel = toManifestPath(realRoot, given);
|
|
581
|
+
if (isAbsolute(rel))
|
|
582
|
+
rel = toManifestPath(root, given);
|
|
583
|
+
if (isAbsolute(rel) || rel === "" || rel.startsWith("../") || rel === "..")
|
|
584
|
+
continue;
|
|
585
|
+
const ext = extname(rel).toLowerCase();
|
|
586
|
+
let kind = item.kind !== undefined && AUDIO_KINDS.has(item.kind) ? item.kind : undefined;
|
|
587
|
+
if (kind === undefined && item.kind === undefined && AUDIO_EXT.has(ext)) {
|
|
588
|
+
const guess = kindOf(rel);
|
|
589
|
+
if (AUDIO_KINDS.has(guess))
|
|
590
|
+
kind = guess;
|
|
591
|
+
}
|
|
592
|
+
if (kind === undefined && item.kind === undefined && AUDIO_EXT.has(ext)) {
|
|
593
|
+
// A name that says nothing, but the file may be registered already: its lane says.
|
|
594
|
+
if (!lanes) {
|
|
595
|
+
lanes = new Map();
|
|
596
|
+
for (const l of project.lanes)
|
|
597
|
+
for (const v of l.variants)
|
|
598
|
+
lanes.set(await realIdentity(fromManifestPath(root, v.file)), l.stage);
|
|
599
|
+
}
|
|
600
|
+
kind = lanes.get(await realIdentity(fromManifestPath(root, rel)));
|
|
601
|
+
}
|
|
602
|
+
if (kind !== undefined)
|
|
603
|
+
kinds.add(kind);
|
|
604
|
+
}
|
|
605
|
+
const settled = [...kinds];
|
|
606
|
+
// With no cut there is nothing to record against yet; the caller records again once one exists.
|
|
607
|
+
if (kinds.size === 0 || !cut)
|
|
608
|
+
return settled;
|
|
609
|
+
await this.store.update("found", (f) => {
|
|
610
|
+
const entry = f.settled.find((e) => e.video === cut.video.id && e.version === cut.version.id);
|
|
611
|
+
if (entry) {
|
|
612
|
+
const add = [...kinds].filter((k) => !entry.kinds.includes(k));
|
|
613
|
+
if (add.length === 0)
|
|
614
|
+
return;
|
|
615
|
+
entry.kinds.push(...add);
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
618
|
+
f.settled.push({ video: cut.video.id, version: cut.version.id, kinds: [...kinds] });
|
|
619
|
+
if (f.settled.length > MAX_SETTLED)
|
|
620
|
+
f.settled = f.settled.slice(-MAX_SETTLED);
|
|
621
|
+
});
|
|
622
|
+
return settled;
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
/** The kinds whose scoring still waits on a length: voice and music files, or the anchor (both). */
|
|
626
|
+
async pendingLengths(project) {
|
|
627
|
+
const out = new Set();
|
|
628
|
+
const { anchor, pending } = await this.anchorState(project);
|
|
629
|
+
if (!anchor)
|
|
630
|
+
return out;
|
|
631
|
+
if (pending)
|
|
632
|
+
return new Set(["voice", "music"]);
|
|
633
|
+
for (const f of this.files) {
|
|
634
|
+
if ((f.kind === "voice" || f.kind === "music") && !this.durations.has(revKey(f)))
|
|
635
|
+
out.add(f.kind);
|
|
636
|
+
}
|
|
637
|
+
return out;
|
|
638
|
+
}
|
|
639
|
+
/** Waits until no length adoption needs is pending, or until `deadline`, or close(). */
|
|
640
|
+
async lengthsKnown(deadline) {
|
|
641
|
+
while (!this.closing) {
|
|
642
|
+
let pending;
|
|
643
|
+
try {
|
|
644
|
+
pending = await this.pendingLengths(await this.store.read("project"));
|
|
645
|
+
}
|
|
646
|
+
catch {
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
const left = deadline - Date.now();
|
|
650
|
+
if (pending.size === 0 || left <= 0)
|
|
651
|
+
return;
|
|
652
|
+
let timer;
|
|
653
|
+
await Promise.race([
|
|
654
|
+
new Promise((r) => this.lengthWaiters.add(r)),
|
|
655
|
+
new Promise((r) => {
|
|
656
|
+
timer = setTimeout(r, left);
|
|
657
|
+
timer.unref?.();
|
|
658
|
+
}),
|
|
659
|
+
]);
|
|
660
|
+
clearTimeout(timer);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
wakeLengthWaiters() {
|
|
664
|
+
const waiters = [...this.lengthWaiters];
|
|
665
|
+
this.lengthWaiters.clear();
|
|
666
|
+
for (const w of waiters)
|
|
667
|
+
w();
|
|
668
|
+
}
|
|
669
|
+
oneAtATime(job) {
|
|
670
|
+
const run = this.bringing.then(job, job);
|
|
671
|
+
this.bringing = run.catch(() => undefined);
|
|
672
|
+
return run;
|
|
673
|
+
}
|
|
674
|
+
async bringInNow(items, opts) {
|
|
675
|
+
const reasons = opts.reasons ?? (await this.currentReasons());
|
|
676
|
+
const film = opts.film ?? this.film;
|
|
677
|
+
for (let attempt = 1;; attempt++) {
|
|
678
|
+
const [project, script] = await Promise.all([this.store.read("project"), this.store.read("script")]);
|
|
679
|
+
// "Already in the project" by real path, freshly read: never has(), which can lag a
|
|
680
|
+
// registration. The write below only lands if the project is still this rev.
|
|
681
|
+
const registered = new Set(await Promise.all(registeredPaths(project, script).map((p) => realIdentity(fromManifestPath(this.store.root, p)))));
|
|
682
|
+
const failed = [];
|
|
683
|
+
const plan = [];
|
|
684
|
+
for (const item of items) {
|
|
685
|
+
const where = await this.locate(item.path);
|
|
686
|
+
if ("reason" in where) {
|
|
687
|
+
failed.push(refusal(item.path, where.reason));
|
|
688
|
+
continue;
|
|
689
|
+
}
|
|
690
|
+
// Already in wins over a kind it can't be given: a file the agent registered itself and
|
|
691
|
+
// names again has nothing to choose.
|
|
692
|
+
if (registered.has(where.abs)) {
|
|
693
|
+
failed.push(refusal(item.path, R.already));
|
|
694
|
+
continue;
|
|
695
|
+
}
|
|
696
|
+
const kind = this.kindFor(item, where.rel);
|
|
697
|
+
if (typeof kind !== "string") {
|
|
698
|
+
failed.push(refusal(item.path, kind.reason));
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
registered.add(where.abs); // another spelling later in this request is the same file
|
|
702
|
+
plan.push({ ...where, item, kind, reasons: reasons.get(where.rel) ?? reasons.get(item.path) ?? [] });
|
|
703
|
+
}
|
|
704
|
+
if (plan.length === 0)
|
|
705
|
+
return { added: [], failed };
|
|
706
|
+
try {
|
|
707
|
+
const { result } = await this.store.update("project", (p) => {
|
|
708
|
+
const out = this.applyPlan(p, plan, film);
|
|
709
|
+
if (out.ok.length === 0)
|
|
710
|
+
throw new NothingAdded(out.refused);
|
|
711
|
+
return out;
|
|
712
|
+
}, project.rev);
|
|
713
|
+
for (const a of result.ok) {
|
|
714
|
+
if (!this.broughtIn.includes(a.path))
|
|
715
|
+
this.broughtIn.push(a.path);
|
|
716
|
+
this.broughtInLog.push({ ...a, reasons: plan.find((it) => it.rel === a.path)?.reasons ?? [], origin: opts.origin ?? "hand" });
|
|
717
|
+
this.current.delete(a.path);
|
|
718
|
+
}
|
|
719
|
+
this.announce();
|
|
720
|
+
return { added: result.ok, failed: [...failed, ...result.refused] };
|
|
721
|
+
}
|
|
722
|
+
catch (e) {
|
|
723
|
+
if (e instanceof NothingAdded)
|
|
724
|
+
return { added: [], failed: [...failed, ...e.refused] };
|
|
725
|
+
// Another writer changed the project since it was read: check everything again.
|
|
726
|
+
if (e instanceof RevConflictError && attempt < BRING_IN_ATTEMPTS)
|
|
727
|
+
continue;
|
|
728
|
+
throw e;
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
/** Registers the plan in the project draft, in order. The cap counts only what goes in. */
|
|
733
|
+
applyPlan(p, plan, film) {
|
|
734
|
+
const ok = [];
|
|
735
|
+
const refused = [];
|
|
736
|
+
const perKind = new Map();
|
|
737
|
+
for (const it of plan) {
|
|
738
|
+
if ((perKind.get(it.kind) ?? 0) >= BRING_IN_PER_KIND) {
|
|
739
|
+
refused.push(refusal(it.item.path, R.cap));
|
|
740
|
+
continue;
|
|
741
|
+
}
|
|
742
|
+
const out = this.register(p, it, film);
|
|
743
|
+
if ("reason" in out) {
|
|
744
|
+
refused.push(refusal(it.item.path, out.reason));
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
747
|
+
ok.push(out);
|
|
748
|
+
perKind.set(it.kind, (perKind.get(it.kind) ?? 0) + 1);
|
|
749
|
+
}
|
|
750
|
+
return { ok, refused };
|
|
751
|
+
}
|
|
752
|
+
/** Registers one planned item in the project draft, or says why it can't, changing nothing. */
|
|
753
|
+
register(p, it, film) {
|
|
754
|
+
const stem = basename(it.rel, extname(it.rel));
|
|
755
|
+
const name = fitName(stem);
|
|
756
|
+
switch (it.kind) {
|
|
757
|
+
case "voice":
|
|
758
|
+
case "music":
|
|
759
|
+
case "sfx": {
|
|
760
|
+
const round = it.kind !== "voice" ? undefined : it.item.round !== undefined ? it.item.round : voiceRound(p, folderOf(it.rel));
|
|
761
|
+
const clash = laneClash(p, it.kind, round);
|
|
762
|
+
if (clash)
|
|
763
|
+
return { reason: clash };
|
|
764
|
+
const description = it.reasons.length > 0 ? it.reasons.join(" · ") : undefined;
|
|
765
|
+
try {
|
|
766
|
+
const { lane, variant } = addVariant(p, { stage: it.kind, round, name, file: it.rel, description });
|
|
767
|
+
return { path: it.rel, kind: it.kind, lane: lane.id, variant: variant.id };
|
|
768
|
+
}
|
|
769
|
+
catch (e) {
|
|
770
|
+
return { reason: e instanceof RushesError ? e.message : "That file couldn't be brought in" };
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
case "cut": {
|
|
774
|
+
const duration = this.durations.get(revKey(it)) ?? null;
|
|
775
|
+
const { video, version } = addVersion(p, { video: filmFor(p, film) ?? (film ? fitName(film) : name), file: it.rel, duration });
|
|
776
|
+
return { path: it.rel, kind: it.kind, video: video.id, version: version.id };
|
|
777
|
+
}
|
|
778
|
+
case "doc":
|
|
779
|
+
addFile(p, { kind: "doc", file: it.rel });
|
|
780
|
+
return { path: it.rel, kind: it.kind };
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
/**
|
|
784
|
+
* Where a bring-in path is: a plain file inside the project, reached without a symlink, not
|
|
785
|
+
* hidden and not in a folder the scan leaves alone, in its spelling on disk. Symlinks are never
|
|
786
|
+
* followed (§20.2); where one leads only decides the reason.
|
|
787
|
+
*/
|
|
788
|
+
async locate(path) {
|
|
789
|
+
if (path.includes("\0"))
|
|
790
|
+
return { reason: R.outside };
|
|
791
|
+
const root = this.store.root;
|
|
792
|
+
const realRoot = await realIdentity(root);
|
|
793
|
+
// `..` and `.` resolve by name only. The path that results is the one checked and registered.
|
|
794
|
+
const given = isAbsolute(path) ? resolve(path) : resolve(root, ...path.split("/"));
|
|
795
|
+
let rel = toManifestPath(realRoot, given);
|
|
796
|
+
if (isAbsolute(rel))
|
|
797
|
+
rel = toManifestPath(root, given);
|
|
798
|
+
if (isAbsolute(rel))
|
|
799
|
+
return { reason: R.outside };
|
|
800
|
+
if (rel === "")
|
|
801
|
+
return { reason: R.folder };
|
|
802
|
+
const parts = rel.split("/");
|
|
803
|
+
// §20.2's skipped folders and hidden names: the scan never offers them, so neither does this.
|
|
804
|
+
if (parts.slice(0, -1).some((d) => d.startsWith(".") || SKIPPED_FOLDERS.has(d.toLowerCase())))
|
|
805
|
+
return { reason: R.skippedFolder };
|
|
806
|
+
if (parts[parts.length - 1].startsWith("."))
|
|
807
|
+
return { reason: R.hiddenFile };
|
|
808
|
+
let at = realRoot;
|
|
809
|
+
for (let i = 0; i < parts.length; i++) {
|
|
810
|
+
at = join(at, parts[i]);
|
|
811
|
+
let info;
|
|
812
|
+
try {
|
|
813
|
+
info = await lstat(at);
|
|
814
|
+
}
|
|
815
|
+
catch {
|
|
816
|
+
return { reason: R.gone };
|
|
817
|
+
}
|
|
818
|
+
if (info.isSymbolicLink()) {
|
|
819
|
+
const target = await realpath(at).catch(() => null);
|
|
820
|
+
if (target === null)
|
|
821
|
+
return { reason: R.gone };
|
|
822
|
+
return { reason: isInside(realRoot, target) ? R.link : R.outside };
|
|
823
|
+
}
|
|
824
|
+
if (i < parts.length - 1) {
|
|
825
|
+
if (!info.isDirectory())
|
|
826
|
+
return { reason: R.gone };
|
|
827
|
+
continue;
|
|
828
|
+
}
|
|
829
|
+
if (info.isDirectory())
|
|
830
|
+
return { reason: R.folder };
|
|
831
|
+
if (!info.isFile())
|
|
832
|
+
return { reason: R.unknownType };
|
|
833
|
+
// The spelling on disk (case and Unicode form), as the scan records it.
|
|
834
|
+
const real = await realpath(at).catch(() => null);
|
|
835
|
+
if (real === null)
|
|
836
|
+
return { reason: R.gone };
|
|
837
|
+
const onDisk = toManifestPath(realRoot, real);
|
|
838
|
+
if (isAbsolute(onDisk))
|
|
839
|
+
return { reason: R.outside };
|
|
840
|
+
return { rel: onDisk, abs: real, size: info.size, modified: info.mtimeMs };
|
|
841
|
+
}
|
|
842
|
+
return { reason: R.gone };
|
|
843
|
+
}
|
|
844
|
+
/** The kind an item comes in as: the one asked for, if the file suits it, or the one its name suggests. */
|
|
845
|
+
kindFor(item, rel) {
|
|
846
|
+
const ext = extname(rel).toLowerCase();
|
|
847
|
+
const audio = AUDIO_EXT.has(ext);
|
|
848
|
+
const video = VIDEO_EXT.has(ext);
|
|
849
|
+
const doc = DOC_EXT.has(ext);
|
|
850
|
+
let kind = item.kind;
|
|
851
|
+
if (kind === undefined) {
|
|
852
|
+
if (video)
|
|
853
|
+
return "cut";
|
|
854
|
+
if (doc)
|
|
855
|
+
return "doc";
|
|
856
|
+
if (!audio)
|
|
857
|
+
return { reason: R.unknownType };
|
|
858
|
+
const guess = kindOf(rel);
|
|
859
|
+
if (guess !== "voice" && guess !== "music" && guess !== "sfx")
|
|
860
|
+
return { reason: R.chooseKind };
|
|
861
|
+
kind = guess;
|
|
862
|
+
}
|
|
863
|
+
if ((kind === "voice" || kind === "music" || kind === "sfx") && !audio)
|
|
864
|
+
return { reason: R.notAudio };
|
|
865
|
+
if (kind === "cut" && !video)
|
|
866
|
+
return { reason: R.notVideo };
|
|
867
|
+
if (kind === "doc" && !doc)
|
|
868
|
+
return { reason: R.notDoc };
|
|
869
|
+
return kind;
|
|
870
|
+
}
|
|
871
|
+
/** The current scoring's reasons by path, for describing what's brought in by hand. */
|
|
872
|
+
async currentReasons() {
|
|
873
|
+
try {
|
|
874
|
+
return new Map((await this.list()).filter((e) => e.reasons.length > 0).map((e) => [e.path, e.reasons]));
|
|
875
|
+
}
|
|
876
|
+
catch {
|
|
877
|
+
return new Map();
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
/**
|
|
881
|
+
* §20.4 against the anchor cut. Every audio file the walk found competes for the current set,
|
|
882
|
+
* registered and dismissed ones included, so bringing the winner in (or hiding it) never hands
|
|
883
|
+
* its place to the runner-up: adoption stays idempotent, across restarts too. Only candidates
|
|
884
|
+
* are suggested. Common words are counted over the whole walk, so a score doesn't shift as
|
|
885
|
+
* files are brought in.
|
|
886
|
+
*/
|
|
887
|
+
async scoring(ctx) {
|
|
888
|
+
const withDuration = (f) => ({ ...f, duration: this.durations.get(revKey(f)) ?? null });
|
|
889
|
+
const candidates = this.candidates(ctx.registered).map(withDuration);
|
|
890
|
+
const anchor = await this.anchor(ctx.project);
|
|
891
|
+
const byPath = new Map();
|
|
892
|
+
const suggested = new Set();
|
|
893
|
+
if (anchor) {
|
|
894
|
+
const all = this.files.map(withDuration);
|
|
895
|
+
const common = commonWordsOf(all);
|
|
896
|
+
const scored = all.filter((f) => f.kind !== "cut").map((f) => scoreCandidate(f, anchor, common));
|
|
897
|
+
for (const s of scored)
|
|
898
|
+
byPath.set(s.file.path, s);
|
|
899
|
+
const listed = new Set(candidates.map((c) => c.path));
|
|
900
|
+
for (const s of Object.values(pickCurrentSet(scored)))
|
|
901
|
+
if (s && listed.has(s.file.path))
|
|
902
|
+
suggested.add(s.file.path);
|
|
903
|
+
}
|
|
904
|
+
return { candidates, anchor, byPath, suggested };
|
|
905
|
+
}
|
|
906
|
+
/** Whether `rel` is a current found candidate, exactly as the scan recorded it (for /media). */
|
|
907
|
+
has(rel) {
|
|
908
|
+
return this.current.has(rel);
|
|
909
|
+
}
|
|
910
|
+
/** Hides paths for good (found.json). Each path may be absolute or relative; it's kept as a manifest path. */
|
|
911
|
+
async dismiss(paths) {
|
|
912
|
+
const rels = await this.dismissablePaths(paths);
|
|
913
|
+
if (rels.length === 0)
|
|
914
|
+
return;
|
|
915
|
+
const already = new Set((await this.store.read("found")).dismissed);
|
|
916
|
+
if (rels.every((p) => already.has(p))) {
|
|
917
|
+
this.setDismissed(already);
|
|
918
|
+
for (const p of rels)
|
|
919
|
+
this.current.delete(p);
|
|
920
|
+
return;
|
|
921
|
+
}
|
|
922
|
+
const { data } = await this.store.update("found", (f) => {
|
|
923
|
+
const set = new Set(f.dismissed);
|
|
924
|
+
for (const p of rels) {
|
|
925
|
+
set.delete(p); // re-added at the end, so the newest are the ones kept
|
|
926
|
+
set.add(p);
|
|
927
|
+
}
|
|
928
|
+
f.dismissed = [...set].slice(-MAX_DISMISSED);
|
|
929
|
+
});
|
|
930
|
+
this.setDismissed(new Set(data.dismissed));
|
|
931
|
+
for (const p of rels)
|
|
932
|
+
this.current.delete(p);
|
|
933
|
+
}
|
|
934
|
+
/**
|
|
935
|
+
* The paths worth keeping as dismissed: a current candidate as listed, or a plain file inside
|
|
936
|
+
* the project (in its spelling on disk). Junk, a folder, a path with an empty or dotted segment,
|
|
937
|
+
* and anything reached through a symlink are dropped. No duplicates.
|
|
938
|
+
*/
|
|
939
|
+
async dismissablePaths(paths) {
|
|
940
|
+
const out = [];
|
|
941
|
+
for (const p of paths) {
|
|
942
|
+
const rel = this.current.has(p) ? p : await plainFileInside(this.store.root, p);
|
|
943
|
+
if (rel !== null && !out.includes(rel))
|
|
944
|
+
out.push(rel);
|
|
945
|
+
}
|
|
946
|
+
return out;
|
|
947
|
+
}
|
|
948
|
+
/** Records the dismissed set this server just wrote or read, and makes reads already in flight stale. */
|
|
949
|
+
setDismissed(set) {
|
|
950
|
+
this.dismissed = set;
|
|
951
|
+
this.dismissedGeneration++;
|
|
952
|
+
}
|
|
953
|
+
/** Brings dismissed paths back into the list. */
|
|
954
|
+
async restore(paths) {
|
|
955
|
+
const rels = new Set(await this.manifestPaths(paths));
|
|
956
|
+
if (rels.size === 0)
|
|
957
|
+
return;
|
|
958
|
+
const before = await this.store.read("found");
|
|
959
|
+
if (!before.dismissed.some((p) => rels.has(p))) {
|
|
960
|
+
this.setDismissed(new Set(before.dismissed));
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
const { data } = await this.store.update("found", (f) => {
|
|
964
|
+
f.dismissed = f.dismissed.filter((p) => !rels.has(p));
|
|
965
|
+
});
|
|
966
|
+
this.setDismissed(new Set(data.dismissed));
|
|
967
|
+
// Back in has() straight away, unless it's registered: the next context() settles that.
|
|
968
|
+
try {
|
|
969
|
+
const ctx = await this.context();
|
|
970
|
+
this.candidates(ctx.registered);
|
|
971
|
+
}
|
|
972
|
+
catch {
|
|
973
|
+
// Left for the next list().
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
/** Stops probing (nothing queued starts, and probes in flight are aborted) and stops announcing. */
|
|
977
|
+
close() {
|
|
978
|
+
this.closing = true;
|
|
979
|
+
this.walkAbort.abort();
|
|
980
|
+
this.queue.length = 0;
|
|
981
|
+
this.queued.clear();
|
|
982
|
+
for (const c of this.inflight)
|
|
983
|
+
c.abort();
|
|
984
|
+
this.resolveIdle();
|
|
985
|
+
this.wakeLengthWaiters();
|
|
986
|
+
}
|
|
987
|
+
/**
|
|
988
|
+
* Resolves once nothing runs in the background: no walk, no probe queued or running, and no
|
|
989
|
+
* adoption of the current set (which reads and may write the project) still to finish. Never
|
|
990
|
+
* waits longer than `maxMs` (30 s by default), so a hung probe can't hang a caller.
|
|
991
|
+
*/
|
|
992
|
+
async settled(maxMs = SETTLED_MAX_MS) {
|
|
993
|
+
let timer;
|
|
994
|
+
const bound = new Promise((r) => {
|
|
995
|
+
timer = setTimeout(r, maxMs);
|
|
996
|
+
timer.unref?.();
|
|
997
|
+
});
|
|
998
|
+
const all = (async () => {
|
|
999
|
+
await this.workSettled();
|
|
1000
|
+
while (this.adoptions.size > 0) {
|
|
1001
|
+
await Promise.all([...this.adoptions]);
|
|
1002
|
+
await this.workSettled();
|
|
1003
|
+
}
|
|
1004
|
+
})();
|
|
1005
|
+
try {
|
|
1006
|
+
await Promise.race([all, bound]);
|
|
1007
|
+
}
|
|
1008
|
+
finally {
|
|
1009
|
+
clearTimeout(timer);
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
/** Resolves once no walk is running and no probe is queued or running. */
|
|
1013
|
+
workSettled() {
|
|
1014
|
+
if (this.isIdle())
|
|
1015
|
+
return Promise.resolve();
|
|
1016
|
+
if (!this.idle) {
|
|
1017
|
+
let resolveIt;
|
|
1018
|
+
const promise = new Promise((r) => (resolveIt = r));
|
|
1019
|
+
this.idle = { promise, resolve: resolveIt };
|
|
1020
|
+
}
|
|
1021
|
+
return this.idle.promise;
|
|
1022
|
+
}
|
|
1023
|
+
// ---- internals ----
|
|
1024
|
+
isIdle() {
|
|
1025
|
+
if (this.walk)
|
|
1026
|
+
return false;
|
|
1027
|
+
if (this.closing)
|
|
1028
|
+
return true; // probes in flight are abandoned once closing
|
|
1029
|
+
return this.probing.size === 0 && this.queue.length === 0;
|
|
1030
|
+
}
|
|
1031
|
+
resolveIdle() {
|
|
1032
|
+
if (this.idle && this.isIdle()) {
|
|
1033
|
+
this.idle.resolve();
|
|
1034
|
+
this.idle = null;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
announce() {
|
|
1038
|
+
if (this.closing)
|
|
1039
|
+
return;
|
|
1040
|
+
try {
|
|
1041
|
+
this.announceChange();
|
|
1042
|
+
}
|
|
1043
|
+
catch {
|
|
1044
|
+
// An announcement is a courtesy; it never breaks a scan.
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
/** The project, and the real paths of everything it has registered. Refreshes the dismissed set too. */
|
|
1048
|
+
async context() {
|
|
1049
|
+
const [project, script] = await Promise.all([this.store.read("project"), this.store.read("script")]);
|
|
1050
|
+
await this.refreshDismissed();
|
|
1051
|
+
const key = `${project.rev}\0${script.rev}\0${this.scannedAt}`;
|
|
1052
|
+
if (this.registeredCache?.key !== key) {
|
|
1053
|
+
const real = new Set(await Promise.all(registeredPaths(project, script).map((p) => realIdentity(fromManifestPath(this.store.root, p)))));
|
|
1054
|
+
this.registeredCache = { key, real };
|
|
1055
|
+
}
|
|
1056
|
+
return { project, registered: this.registeredCache.real };
|
|
1057
|
+
}
|
|
1058
|
+
/** found.json's dismissed paths. A file that can't be read leaves the last known set in place (the watcher reports it). */
|
|
1059
|
+
async refreshDismissed() {
|
|
1060
|
+
const generation = this.dismissedGeneration;
|
|
1061
|
+
try {
|
|
1062
|
+
const { dismissed } = await this.store.read("found");
|
|
1063
|
+
// A dismiss or restore landed while this read was in flight: what it read may predate that
|
|
1064
|
+
// write, so it must not replace the set the write left (Review: the has() race).
|
|
1065
|
+
if (generation !== this.dismissedGeneration)
|
|
1066
|
+
return;
|
|
1067
|
+
this.dismissed = new Set(dismissed);
|
|
1068
|
+
}
|
|
1069
|
+
catch {
|
|
1070
|
+
// Corrupt by hand: keep what we had.
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
/** The walk's files minus registered and dismissed ones. Keeps `has()` in step. */
|
|
1074
|
+
candidates(registered) {
|
|
1075
|
+
const out = this.files.filter((f) => !registered.has(f.abs) && !this.dismissed.has(f.path));
|
|
1076
|
+
this.current = new Set(out.map((f) => f.path));
|
|
1077
|
+
return out;
|
|
1078
|
+
}
|
|
1079
|
+
/** Each path as the manifest path found.json keeps: inside the root, relative, real spelling when it exists. */
|
|
1080
|
+
async manifestPaths(paths) {
|
|
1081
|
+
const root = this.store.root;
|
|
1082
|
+
const realRoot = await realIdentity(root);
|
|
1083
|
+
const out = [];
|
|
1084
|
+
for (const p of paths) {
|
|
1085
|
+
const abs = isAbsolute(p) ? p : resolve(root, ...p.split("/"));
|
|
1086
|
+
// The spelling on disk when the file is there, so a dismissed path matches what the scan lists.
|
|
1087
|
+
let rel = toManifestPath(realRoot, await realIdentity(abs));
|
|
1088
|
+
if (isAbsolute(rel))
|
|
1089
|
+
rel = toManifestPath(root, abs);
|
|
1090
|
+
// Only paths inside the project folder mean anything here.
|
|
1091
|
+
if (isAbsolute(rel))
|
|
1092
|
+
continue;
|
|
1093
|
+
if (!out.includes(rel))
|
|
1094
|
+
out.push(rel);
|
|
1095
|
+
}
|
|
1096
|
+
return out;
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* §20.4's anchor: the named film's newest cut, otherwise the newest cut of any film (§20.4; see anchorCut). Its
|
|
1100
|
+
* duration comes from the version, else a probe (queued, not waited on). Null when there's no
|
|
1101
|
+
* registered cut, or its file can't be read.
|
|
1102
|
+
*/
|
|
1103
|
+
async anchor(project) {
|
|
1104
|
+
return (await this.anchorState(project)).anchor;
|
|
1105
|
+
}
|
|
1106
|
+
/** The anchor, and whether its length is still waiting on a probe. */
|
|
1107
|
+
async anchorState(project) {
|
|
1108
|
+
const version = anchorCut(project, this.film)?.version;
|
|
1109
|
+
if (!version)
|
|
1110
|
+
return { anchor: null, pending: false };
|
|
1111
|
+
const abs = fromManifestPath(this.store.root, version.file);
|
|
1112
|
+
let info;
|
|
1113
|
+
try {
|
|
1114
|
+
info = await stat(abs);
|
|
1115
|
+
}
|
|
1116
|
+
catch {
|
|
1117
|
+
return { anchor: null, pending: false };
|
|
1118
|
+
}
|
|
1119
|
+
let duration = version.duration;
|
|
1120
|
+
let pending = false;
|
|
1121
|
+
if (duration === null) {
|
|
1122
|
+
const item = { abs, size: info.size, modified: info.mtimeMs };
|
|
1123
|
+
const key = revKey(item);
|
|
1124
|
+
if (this.durations.has(key))
|
|
1125
|
+
duration = this.durations.get(key);
|
|
1126
|
+
else {
|
|
1127
|
+
this.enqueue(item, PROBE_TIER.anchor);
|
|
1128
|
+
pending = true;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
const name = basename(version.file, extname(version.file));
|
|
1132
|
+
return { anchor: { duration, modified: info.mtimeMs, words: wordsOf(name) }, pending };
|
|
1133
|
+
}
|
|
1134
|
+
/** Queues a probe in its tier's place (after others of the same tier), or moves a queued one up. */
|
|
1135
|
+
enqueue(f, tier) {
|
|
1136
|
+
if (this.closing)
|
|
1137
|
+
return;
|
|
1138
|
+
const key = revKey(f);
|
|
1139
|
+
if (this.durations.has(key) || this.probing.has(key))
|
|
1140
|
+
return;
|
|
1141
|
+
if (this.queued.has(key)) {
|
|
1142
|
+
const at = this.queue.findIndex((q) => q.key === key);
|
|
1143
|
+
if (at < 0 || this.queue[at].tier <= tier)
|
|
1144
|
+
return;
|
|
1145
|
+
this.queue.splice(at, 1);
|
|
1146
|
+
}
|
|
1147
|
+
this.queued.add(key);
|
|
1148
|
+
const before = this.queue.findIndex((q) => q.tier > tier);
|
|
1149
|
+
const item = { key, abs: f.abs, tier };
|
|
1150
|
+
if (before < 0)
|
|
1151
|
+
this.queue.push(item);
|
|
1152
|
+
else
|
|
1153
|
+
this.queue.splice(before, 0, item);
|
|
1154
|
+
this.pump();
|
|
1155
|
+
}
|
|
1156
|
+
pump() {
|
|
1157
|
+
while (!this.closing && this.probing.size < FOUND_PROBE_CONCURRENCY && this.queue.length > 0) {
|
|
1158
|
+
const item = this.queue.shift();
|
|
1159
|
+
this.queued.delete(item.key);
|
|
1160
|
+
this.probing.add(item.key);
|
|
1161
|
+
void this.probeOne(item);
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
async probeOne(item) {
|
|
1165
|
+
const controller = new AbortController();
|
|
1166
|
+
this.inflight.add(controller);
|
|
1167
|
+
let duration = null;
|
|
1168
|
+
try {
|
|
1169
|
+
const d = await this.probeFile(item.abs, controller.signal);
|
|
1170
|
+
duration = typeof d === "number" && Number.isFinite(d) && d >= 0 ? d : null;
|
|
1171
|
+
}
|
|
1172
|
+
catch {
|
|
1173
|
+
duration = null;
|
|
1174
|
+
}
|
|
1175
|
+
finally {
|
|
1176
|
+
this.inflight.delete(controller);
|
|
1177
|
+
}
|
|
1178
|
+
if (this.closing)
|
|
1179
|
+
return;
|
|
1180
|
+
this.remember(item.key, duration);
|
|
1181
|
+
if (duration !== null)
|
|
1182
|
+
this.landedSinceAnnounce++;
|
|
1183
|
+
this.probing.delete(item.key);
|
|
1184
|
+
this.wakeLengthWaiters();
|
|
1185
|
+
const drained = this.probing.size === 0 && this.queue.length === 0;
|
|
1186
|
+
if (this.landedSinceAnnounce >= FOUND_ANNOUNCE_EVERY || (drained && this.landedSinceAnnounce > 0)) {
|
|
1187
|
+
this.landedSinceAnnounce = 0;
|
|
1188
|
+
this.announce();
|
|
1189
|
+
}
|
|
1190
|
+
this.pump();
|
|
1191
|
+
this.resolveIdle();
|
|
1192
|
+
}
|
|
1193
|
+
remember(key, duration) {
|
|
1194
|
+
this.durations.delete(key);
|
|
1195
|
+
this.durations.set(key, duration);
|
|
1196
|
+
while (this.durations.size > DURATION_CACHE_LIMIT)
|
|
1197
|
+
this.durations.delete(this.durations.keys().next().value);
|
|
1198
|
+
}
|
|
1199
|
+
}
|