transloadit 6.4.0 → 6.5.0

package/dist/cli/commands/assemblies.js

@@ -9,13 +9,15 @@ import { pipeline } from 'node:stream/promises';
 import { setTimeout as delay } from 'node:timers/promises';
 import { promisify } from 'node:util';
 import { Command, Option } from 'clipanion';
-import got from 'got';
+import got, { RequestError } from 'got';
 import PQueue from 'p-queue';
 import * as t from 'typanion';
 import { z } from 'zod';
 import { formatLintIssue } from "../../alphalib/assembly-linter.lang.en.js";
 import { tryCatch } from "../../alphalib/tryCatch.js";
+import { assemblyStatusResultSchema } from "../../alphalib/types/assemblyStatus.js";
 import { ensureUniqueCounterValue } from "../../ensureUniqueCounter.js";
+import { prepareInputFiles } from "../../inputFiles.js";
 import { lintAssemblyInstructions } from "../../lintAssemblyInstructions.js";
 import { lintingExamples } from "../docs/assemblyLintingExamples.js";
 import { concurrencyOption, deleteAfterProcessingOption, inputPathsOption, printUrlsOption, recursiveOption, reprocessStaleOption, singleAssemblyOption, validateSharedFileProcessingOptions, watchOption, } from "../fileProcessingOptions.js";
@@ -23,7 +25,7 @@ import { formatAPIError, readCliInput } from "../helpers.js";
 import { normalizeAssemblyResults } from "../resultFiles.js";
 import { collectNormalizedResultUrlRows, printResultUrls } from "../resultUrls.js";
 import { parseStepsInputJson, readStepsInputFile } from "../stepsInput.js";
-import { ensureError, isErrnoException } from "../types.js";
+import { assemblySummarySchema, ensureError, isErrnoException } from "../types.js";
 import { AuthenticatedCommand, UnauthenticatedCommand } from "./BaseCommand.js";
 function formatAssemblyReference({ assemblyId, assemblyUrl, }) {
     if (assemblyUrl != null) {
@@ -95,11 +97,7 @@ export function list(output, client, { before, after, fields, keywords, pagesize
 export async function get(output, client, { assemblies }) {
     for (const assembly of assemblies) {
         await delay(1000);
-        const [err, result] = await tryCatch(client.getAssembly(assembly));
-        if (err) {
-            output.error(formatAPIError(err));
-            throw ensureError(err);
-        }
+        const result = await client.getAssembly(assembly);
         output.print(result, result);
     }
 }
@@ -317,16 +315,32 @@ async function createExistingPathOutputPlan(outputPath) {
     const [, stats] = await tryCatch(fsp.stat(outputPath));
     return createOutputPlan(outputPath, stats?.mtime ?? new Date(0));
 }
-function dirProvider(output) {
-    return async (inpath, indir = process.cwd()) => {
+function dirProvider(output, transientInputPaths) {
+    const namesByInput = new Map();
+    const usedNames = new Set();
+    return async (inpath, indir) => {
         if (inpath == null) {
             return await createExistingPathOutputPlan(output);
         }
         if (inpath === '-') {
             throw new Error('You must provide an input to output to a directory');
         }
-        let relpath = path.relative(indir, inpath);
-        relpath = relpath.replace(/^(\.\.\/)+/, '');
+        const inputKey = path.resolve(inpath);
+        let relpath = namesByInput.get(inputKey);
+        if (relpath == null) {
+            const requestedPath = transientInputPaths?.has(inputKey)
+                ? path.basename(inpath)
+                : path.relative(indir ?? process.cwd(), inpath).replace(/^(\.\.\/)+/, '');
+            const parsed = path.parse(requestedPath);
+            relpath = await ensureUniqueCounterValue({
+                initialValue: requestedPath,
+                isTaken: (candidate) => usedNames.has(candidate),
+                reserve: (candidate) => usedNames.add(candidate),
+                nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`),
+                scope: usedNames,
+            });
+            namesByInput.set(inputKey, relpath);
+        }
         const outpath = path.join(output, relpath);
         return await createExistingPathOutputPlan(outpath);
     };
@@ -342,7 +356,7 @@ function fileProvider(output) {
 function nullProvider() {
     return async (_inpath) => null;
 }
-async function downloadResultToFile(resultUrl, outPath, signal) {
+async function downloadResultToFile(resultUrl, outPath, signal, isCurrent) {
     await fsp.mkdir(path.dirname(outPath), { recursive: true });
     const tempPath = path.join(path.dirname(outPath), `.${path.basename(outPath)}.${randomUUID()}.tmp`);
     const outStream = fs.createWriteStream(tempPath);
@@ -352,6 +366,10 @@ async function downloadResultToFile(resultUrl, outPath, signal) {
         await fsp.rm(tempPath, { force: true });
         throw dlErr;
     }
+    if (!isCurrent()) {
+        await fsp.rm(tempPath, { force: true });
+        return;
+    }
     await fsp.rename(tempPath, outPath);
 }
 async function downloadResultToStdout(resultUrl, signal) {
@@ -386,45 +404,336 @@ async function downloadResultToStdout(resultUrl, signal) {
     });
     await pipeline(got.stream(resultUrl, { signal }), stdoutStream);
 }
-function sanitizeResultName(value) {
+function sanitizeResultName(value, fallback) {
     const base = path.basename(value);
-    return base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '');
+    const name = base.replaceAll('\\', '_').replaceAll('/', '_').replaceAll('\u0000', '');
+    if (name === '' || name === '.' || name === '..') {
+        if (fallback != null)
+            return sanitizeResultName(fallback);
+        throw new Error('Assembly result has an unsafe name');
+    }
+    return name;
 }
-async function ensureUniquePath(targetPath, reservedPaths) {
+async function outputNameSensitivity(root) {
+    await fsp.mkdir(root, { recursive: true });
+    const probe = await fsp.mkdtemp(path.join(root, '.transloadit-names-'));
+    async function distinguishesNames(first, second) {
+        await fsp.writeFile(path.join(probe, first), '');
+        const [error] = await tryCatch(fsp.lstat(path.join(probe, second)));
+        if (error && (!isErrnoException(error) || error.code !== 'ENOENT'))
+            throw error;
+        return error != null;
+    }
+    try {
+        // Assume this tree shares the root's naming rules; nested filesystems are tracked in #615.
+        return {
+            caseSensitive: await distinguishesNames('case', 'CASE'),
+            normalizationSensitive: await distinguishesNames('\u00e9', 'e\u0301'),
+        };
+    }
+    finally {
+        await fsp.rm(probe, { recursive: true, force: true });
+    }
+}
+function downloadPathKey(target, reservedPaths) {
+    const absolutePath = path.resolve(target);
+    const folded = reservedPaths.caseSensitive ? absolutePath : absolutePath.toLowerCase();
+    return reservedPaths.normalizationSensitive ? folded : folded.normalize('NFC');
+}
+function hasReservedFileAncestor(target, reservedPaths) {
+    let current = downloadPathKey(target, reservedPaths);
+    while (true) {
+        if (reservedPaths.ownersByPath.has(current) && !reservedPaths.directories.has(current)) {
+            return true;
+        }
+        const parent = path.dirname(current);
+        if (parent === current)
+            return false;
+        current = parent;
+    }
+}
+function reserveDownloadPath(target, owner, reservedPaths, directory = false) {
+    const key = downloadPathKey(target, reservedPaths);
+    reservedPaths.ownersByPath.set(key, owner);
+    let current = directory ? key : path.dirname(key);
+    while (true) {
+        reservedPaths.directories.add(current);
+        const parent = path.dirname(current);
+        if (parent === current)
+            return;
+        current = parent;
+    }
+}
+async function ensureUniquePath(targetPath, reservedPaths, owner, directory = false, overwriteExistingFile = false) {
+    const selectedPath = reservedPaths.pathsByOwner.get(owner);
+    if (selectedPath != null)
+        return selectedPath;
     const parsed = path.parse(targetPath);
     return await ensureUniqueCounterValue({
         initialValue: targetPath,
-        isTaken: async (candidate) => {
-            if (reservedPaths.has(candidate)) {
-                return true;
-            }
-            const [statErr] = await tryCatch(fsp.stat(candidate));
-            return statErr == null;
-        },
+        isTaken: async (candidate) => await isDownloadPathTaken(candidate, reservedPaths, owner, directory, overwriteExistingFile && candidate === targetPath),
         reserve: (candidate) => {
-            reservedPaths.add(candidate);
+            reserveDownloadPath(candidate, owner, reservedPaths, directory);
+            reservedPaths.pathsByOwner.set(owner, candidate);
         },
-        nextValue: (counter) => path.join(parsed.dir, `${parsed.name}__${counter}${parsed.ext}`),
+        nextValue: (counter) => path.join(parsed.dir, directory ? `${parsed.base}__${counter}` : `${parsed.name}__${counter}${parsed.ext}`),
         scope: reservedPaths,
     });
 }
-function getResultFileName(file) {
-    return sanitizeResultName(file.name);
+const resultPathSchema = z.object({
+    meta: z.object({ relative_path: z.string().nullish() }).nullish(),
+});
+function getResultRelativePath(file) {
+    const parsed = resultPathSchema.safeParse(file.file);
+    if (!parsed.success)
+        throw new Error('Assembly result has invalid relative path metadata');
+    const relativePath = parsed.data.meta?.relative_path;
+    if (relativePath == null)
+        return null;
+    const name = sanitizeResultName(file.name);
+    // Adaptive video results use an empty directory for the master playlist.
+    if (relativePath === '')
+        return name;
+    const normalizedPath = relativePath.replaceAll('\\', '/');
+    if (normalizedPath.includes('\u0000') ||
+        path.win32.isAbsolute(normalizedPath) ||
+        /^[a-z]:/i.test(normalizedPath) ||
+        normalizedPath.split('/').some((part) => part === '' || part === '.' || part === '..')) {
+        throw new Error('Assembly result has an unsafe relative path');
+    }
+    return path.join(normalizedPath, name);
+}
+async function downloadResultBundle(targets, signal, isCurrent, commits, onDownload) {
+    const stagingDirs = new Map();
+    const staged = [];
+    try {
+        for (const [index, target] of targets.entries()) {
+            if (!isCurrent())
+                return;
+            const parent = path.dirname(target.targetPath);
+            await fsp.mkdir(parent, { recursive: true });
+            let stagingDir = stagingDirs.get(parent);
+            if (stagingDir == null) {
+                // Stage beside each destination so publication also works across nested filesystems.
+                stagingDir = await fsp.mkdtemp(path.join(parent, '.transloadit-bundle-'));
+                stagingDirs.set(parent, stagingDir);
+            }
+            const stagingPath = path.join(stagingDir, String(index));
+            staged.push({ ...target, stagingPath });
+            onDownload();
+            await downloadResultToFile(target.resultUrl, stagingPath, signal, () => true);
+        }
+        await commits.add(async () => {
+            if (!isCurrent())
+                return;
+            // Finish publication once it starts, even if superseded. A failed newer job must not
+            // strand this playlist without its segments, and newer commits must wait for this one.
+            for (const { stagingPath, targetPath } of staged) {
+                await fsp.mkdir(path.dirname(targetPath), { recursive: true });
+                await fsp.rename(stagingPath, targetPath);
+            }
+        });
+    }
+    finally {
+        await Promise.all([...stagingDirs.values()].map(async (dir) => await fsp.rm(dir, { recursive: true, force: true })));
+    }
+}
+async function assertNoOutputSymlinks(root, target) {
+    // Reject preexisting descendant symlinks; another process can still race filesystem checks.
+    const relativePath = path.relative(root, target);
+    // The caller explicitly selected the root; only descendants come from result/input metadata.
+    if (relativePath === '')
+        return;
+    if (relativePath.startsWith(`..${path.sep}`) ||
+        relativePath === '..' ||
+        path.isAbsolute(relativePath)) {
+        throw new Error('Assembly result path escapes the output directory');
+    }
+    let current = root;
+    for (const part of relativePath.split(path.sep)) {
+        current = path.join(current, part);
+        const [error, stats] = await tryCatch(fsp.lstat(current));
+        if (error) {
+            if (isErrnoException(error) && (error.code === 'ENOENT' || error.code === 'ENOTDIR'))
+                return;
+            throw error;
+        }
+        if (stats.isSymbolicLink()) {
+            throw new Error('Assembly result path contains a symbolic link');
+        }
+    }
+}
+async function isDownloadPathTaken(target, reservedPaths, owner, directory = false, overwriteExistingFile = false) {
+    const key = downloadPathKey(target, reservedPaths);
+    if (directory && reservedPaths.directories.has(key))
+        return false;
+    const existingOwner = reservedPaths.ownersByPath.get(key);
+    if (existingOwner != null)
+        return existingOwner !== owner;
+    if (reservedPaths.directories.has(key) || hasReservedFileAncestor(target, reservedPaths)) {
+        return true;
+    }
+    const [error, stats] = await tryCatch(fsp.lstat(target));
+    if (isErrnoException(error) && error.code === 'ENOTDIR')
+        return true;
+    if (error && (!isErrnoException(error) || error.code !== 'ENOENT'))
+        throw error;
+    if (error != null)
+        return false;
+    if (directory)
+        return !stats?.isDirectory();
+    return !overwriteExistingFile || !stats?.isFile();
+}
+async function prepareDownloadDirectory(root, target, reservedPaths) {
+    await assertNoOutputSymlinks(root, target);
+    if (hasReservedFileAncestor(target, reservedPaths)) {
+        throw new Error('Output directory conflicts with a result file');
+    }
+    await fsp.mkdir(target, { recursive: true });
 }
 const STALE_OUTPUT_GRACE_MS = 1000;
 function isMeaningfullyNewer(newer, older) {
     return newer.getTime() - older.getTime() > STALE_OUTPUT_GRACE_MS;
 }
-async function buildDirectoryDownloadTargets({ allFiles, baseDir, groupByStep, reservedPaths, }) {
-    await fsp.mkdir(baseDir, { recursive: true });
+const resultSourceSchema = assemblyStatusResultSchema.pick({
+    original_basename: true,
+    original_id: true,
+    original_name: true,
+    original_path: true,
+});
+function groupDirectoryResults(allFiles) {
+    const steps = new Map();
+    for (const file of allFiles) {
+        const files = steps.get(file.stepName) ?? [];
+        files.push({ file, relativePath: getResultRelativePath(file) });
+        steps.set(file.stepName, files);
+    }
+    const groups = [];
+    for (const [stepName, stepFiles] of steps) {
+        if (!stepFiles.some((file) => file.relativePath != null)) {
+            groups.push({ directoryName: null, files: stepFiles, ownerKey: null, stepName });
+            continue;
+        }
+        const sources = new Map();
+        for (const file of stepFiles) {
+            const parsed = resultSourceSchema.safeParse(file.file.file);
+            const source = parsed.success ? parsed.data : undefined;
+            const sourceId = JSON.stringify(source?.original_id ?? null);
+            const name = source?.original_name ?? source?.original_basename ?? null;
+            const group = sources.get(sourceId) ?? {
+                // original_path can be a shared directory, so the source filename is also required.
+                identity: JSON.stringify([source?.original_path ?? null, name]),
+                name,
+                files: [],
+            };
+            group.files.push(file);
+            sources.set(sourceId, group);
+        }
+        const sourceOccurrences = new Map();
+        for (const { identity, name, files } of sources.values()) {
+            const occurrence = sourceOccurrences.get(identity) ?? 0;
+            sourceOccurrences.set(identity, occurrence + 1);
+            // Ordinary files can be renamed independently, even when they share an HLS source.
+            for (const linked of [true, false]) {
+                const groupFiles = files.filter(({ relativePath }) => (relativePath != null) === linked);
+                if (groupFiles.length === 0)
+                    continue;
+                groups.push({
+                    directoryName: sources.size > 1
+                        ? sanitizeResultName(path.parse(name ?? stepName).name, stepName)
+                        : null,
+                    files: groupFiles,
+                    // Upload IDs change between watch runs. Paths and names keep source ownership stable
+                    // even when results arrive in a different order.
+                    ownerKey: JSON.stringify([identity, occurrence, linked]),
+                    stepName,
+                });
+            }
+        }
+    }
+    return groups;
+}
+async function buildDirectoryDownloadTargets({ allFiles, baseDir, groupByStep, owner, outputRoot, overwritePath, reservedPaths, }) {
+    await prepareDownloadDirectory(outputRoot, baseDir, reservedPaths);
     const targets = [];
-    for (const resultFile of allFiles) {
-        const targetDir = groupByStep ? path.join(baseDir, resultFile.stepName) : baseDir;
-        await fsp.mkdir(targetDir, { recursive: true });
-        targets.push({
-            resultUrl: resultFile.url,
-            targetPath: await ensureUniquePath(path.join(targetDir, getResultFileName(resultFile)), reservedPaths),
-        });
+    for (const { directoryName, files, ownerKey, stepName } of groupDirectoryResults(allFiles)) {
+        const safeStepName = sanitizeResultName(stepName);
+        let targetDir = groupByStep ? path.join(baseDir, safeStepName) : baseDir;
+        if (directoryName != null)
+            targetDir = path.join(targetDir, directoryName);
+        const linked = files.some((file) => file.relativePath != null);
+        const groupOwner = JSON.stringify([owner, stepName, ownerKey]);
+        const selectedGroupDir = linked ? reservedPaths.pathsByOwner.get(groupOwner) : undefined;
+        if (selectedGroupDir != null)
+            targetDir = selectedGroupDir;
+        const names = files.map(({ file, relativePath }) => relativePath ?? sanitizeResultName(file.name));
+        if (linked &&
+            new Set(names.map((name) => downloadPathKey(path.join(targetDir, name), reservedPaths)))
+                .size !== names.length) {
+            // Merged Steps can hide distinct bundles sharing one upload (#619). Reject ambiguous
+            // linked paths rather than guessing bundle boundaries and overwriting their contents.
+            throw new Error('Assembly results have duplicate relative paths');
+        }
+        for (const name of names) {
+            await assertNoOutputSymlinks(outputRoot, path.join(targetDir, name));
+        }
+        if (linked) {
+            // Source folders share this parent, so changing their suffix cannot fix a blocked parent.
+            if (directoryName != null) {
+                await prepareDownloadDirectory(outputRoot, path.dirname(targetDir), reservedPaths);
+            }
+            const initialDir = targetDir;
+            // Selection and reservation share the same lock as ordinary result paths. A bundle must
+            // move together on collisions, while later watch updates reuse its owned destinations.
+            targetDir = await ensureUniqueCounterValue({
+                initialValue: initialDir,
+                isTaken: async (candidate) => {
+                    if (candidate !== initialDir &&
+                        (await isDownloadPathTaken(candidate, reservedPaths, groupOwner)))
+                        return true;
+                    const collisions = await Promise.all(names.map((name) => isDownloadPathTaken(path.join(candidate, name), reservedPaths, groupOwner)));
+                    const collides = collisions.some(Boolean);
+                    // Moving an already downloaded bundle would leave its previous playlist stale.
+                    if (collides && selectedGroupDir != null) {
+                        throw new Error('Assembly result path conflicts with an existing download');
+                    }
+                    return collides;
+                },
+                reserve: (candidate) => {
+                    if (candidate !== initialDir)
+                        reserveDownloadPath(candidate, groupOwner, reservedPaths, true);
+                    for (const name of names) {
+                        reserveDownloadPath(path.join(candidate, name), groupOwner, reservedPaths);
+                    }
+                    reservedPaths.pathsByOwner.set(groupOwner, candidate);
+                },
+                nextValue: (counter) => {
+                    if (directoryName != null) {
+                        return path.join(path.dirname(initialDir), `${directoryName}__${counter}`);
+                    }
+                    const suffix = groupByStep ? counter : counter - 1;
+                    return path.join(baseDir, suffix === 0 ? safeStepName : `${safeStepName}__${suffix}`);
+                },
+                scope: reservedPaths,
+            });
+        }
+        await prepareDownloadDirectory(outputRoot, targetDir, reservedPaths);
+        const nameOccurrences = new Map();
+        for (const [index, { file }] of files.entries()) {
+            const name = names[index];
+            const occurrence = nameOccurrences.get(name) ?? 0;
+            nameOccurrences.set(name, occurrence + 1);
+            const targetPath = path.join(targetDir, name);
+            const selectedPath = linked
+                ? targetPath
+                : await ensureUniquePath(targetPath, reservedPaths, JSON.stringify([groupOwner, name, occurrence]), false, overwritePath != null && path.resolve(targetPath) === path.resolve(overwritePath));
+            await assertNoOutputSymlinks(outputRoot, selectedPath);
+            targets.push({
+                bundleRoot: linked ? targetDir : undefined,
+                resultUrl: file.url,
+                targetPath: selectedPath,
+            });
+        }
     }
     return targets;
 }
@@ -436,20 +745,28 @@ function getSingleResultDownloadTarget(allFiles, targetPath) {
     }
     return [{ resultUrl, targetPath }];
 }
-async function resolveResultDownloadTargets({ hasDirectoryInput, inPath, inputs, normalizedResults, outputMode, outputPath, outputRoot, outputRootIsDirectory, reservedPaths, singleAssembly, }) {
+async function resolveResultDownloadTargets({ hasDirectoryInput, inPath, inputs, normalizedResults, outputMode, outputPath, outputRoot, outputRootIsDirectory, owner, reservedPaths, singleAssembly, }) {
     const { allFiles, entries } = normalizedResults;
-    const shouldGroupByInput = !singleAssembly && inPath != null && (hasDirectoryInput || inputs.length > 1);
+    const linked = allFiles.some((file) => getResultRelativePath(file) != null);
+    // Ordinary CLI outputs retain their input/Step layout. HLS uses linked result paths so
+    // playlist references remain reachable; moving those formerly broken outputs is intentional.
+    const keepInputStepLayout = outputMode !== 'directory' && inPath != null && allFiles.length > 1 && !linked;
+    const shouldGroupByInput = !singleAssembly &&
+        inPath != null &&
+        (hasDirectoryInput ||
+            (inputs.length > 1 && (allFiles.length > 1 || outputMode === 'directory')));
     const resolveDirectoryBaseDir = () => {
-        if (!shouldGroupByInput || inPath == null) {
-            return outputRoot;
+        if (outputPath != null && (keepInputStepLayout || hasDirectoryInput)) {
+            return allFiles.length === 1 && outputMode !== 'directory'
+                ? path.dirname(outputPath)
+                : path.join(path.dirname(outputPath), path.parse(outputPath).name);
         }
-        if (hasDirectoryInput && outputPath != null) {
-            const mappedRelative = path.relative(outputRoot, outputPath);
-            const mappedDir = path.dirname(mappedRelative);
-            const mappedStem = path.parse(mappedRelative).name;
-            return path.join(outputRoot, mappedDir === '.' ? '' : mappedDir, mappedStem);
+        if (!linked && outputMode !== 'directory' && inPath != null && outputPath != null) {
+            return path.dirname(outputPath);
         }
-        return path.join(outputRoot, path.parse(path.basename(inPath)).name);
+        if (!shouldGroupByInput || inPath == null)
+            return outputRoot;
+        return path.join(outputRoot, path.parse(path.basename(outputPath ?? inPath)).name);
     };
     if (!outputRootIsDirectory) {
         if (allFiles.length > 1) {
@@ -465,24 +782,28 @@ async function resolveResultDownloadTargets({ hasDirectoryInput, inPath, inputs,
             allFiles,
             baseDir: outputRoot,
             groupByStep: inputs.length === 0,
+            owner,
+            outputRoot,
             reservedPaths,
         });
     }
-    if (outputMode === 'directory' || outputPath == null || inPath == null) {
-        return await buildDirectoryDownloadTargets({
-            allFiles,
-            baseDir: resolveDirectoryBaseDir(),
-            groupByStep: inPath == null || entries.length > 1,
-            reservedPaths,
-        });
-    }
-    if (allFiles.length === 1) {
-        return getSingleResultDownloadTarget(allFiles, outputPath);
+    let baseDir = resolveDirectoryBaseDir();
+    if (path.resolve(baseDir) !== path.resolve(outputRoot)) {
+        // Input grouping directories share the result namespace, including files already reserved
+        // by another Assembly. Reuse existing directories and suffix a conflicting filename.
+        await assertNoOutputSymlinks(outputRoot, baseDir);
+        await prepareDownloadDirectory(outputRoot, path.dirname(baseDir), reservedPaths);
+        baseDir = await ensureUniquePath(baseDir, reservedPaths, JSON.stringify([owner, 'input-directory']), true);
     }
     return await buildDirectoryDownloadTargets({
         allFiles,
-        baseDir: path.join(path.dirname(outputPath), path.parse(outputPath).name),
-        groupByStep: true,
+        baseDir,
+        groupByStep: keepInputStepLayout || inPath == null || entries.length > 1,
+        owner,
+        outputRoot,
+        // The mapped single-file destination is also the historical freshness marker. Updating it
+        // lets an unchanged rerun skip processing; differently named results retain collision suffixes.
+        overwritePath: inPath != null && allFiles.length === 1 && !linked ? (outputPath ?? undefined) : undefined,
         reservedPaths,
     });
 }
@@ -521,11 +842,12 @@ async function shouldSkipStaleOutput({ inputPaths, outputPath, outputPlanMtime,
         return inputStat != null && isMeaningfullyNewer(outputStat.mtime, inputStat.mtime);
     });
 }
-async function materializeAssemblyResults({ abortSignal, hasDirectoryInput, inPath, inputs, normalizedResults, outputMode, outputPath, outputRoot, outputRootIsDirectory, outputctl, reservedPaths, singleAssembly, }) {
+async function materializeAssemblyResults({ abortSignal, hasDirectoryInput, inPath, isCurrent, inputs, normalizedResults, outputMode, outputPath, outputRoot, outputRootIsDirectory, outputctl, owner, reservedPaths, singleAssembly, }) {
     if (outputRoot == null) {
         return;
     }
-    const targets = await resolveResultDownloadTargets({
+    // Directory creation and filename reservation must share a lock across concurrent jobs.
+    const targets = await reservedPaths.allocations.add(async () => await resolveResultDownloadTargets({
         hasDirectoryInput,
         inPath,
         inputs,
@@ -534,14 +856,32 @@ async function materializeAssemblyResults({ abortSignal, hasDirectoryInput, inPa
         outputPath,
         outputRoot,
         outputRootIsDirectory,
+        owner,
         reservedPaths,
         singleAssembly,
-    });
-    for (const { resultUrl, targetPath } of targets) {
-        outputctl.debug('DOWNLOADING');
-        const [dlErr] = await tryCatch(targetPath == null
-            ? downloadResultToStdout(resultUrl, abortSignal)
-            : downloadResultToFile(resultUrl, targetPath, abortSignal));
+    }));
+    const bundles = new Map();
+    const downloads = [];
+    for (const { bundleRoot, resultUrl, targetPath } of targets) {
+        if (bundleRoot != null && targetPath != null) {
+            const files = bundles.get(bundleRoot) ?? [];
+            files.push({ resultUrl, targetPath });
+            bundles.set(bundleRoot, files);
+            continue;
+        }
+        downloads.push(async () => {
+            outputctl.debug('DOWNLOADING');
+            if (targetPath == null)
+                await downloadResultToStdout(resultUrl, abortSignal);
+            else
+                await downloadResultToFile(resultUrl, targetPath, abortSignal, isCurrent);
+        });
+    }
+    for (const files of bundles.values()) {
+        downloads.push(async () => await downloadResultBundle(files, abortSignal, isCurrent, reservedPaths.bundleCommits, () => outputctl.debug('DOWNLOADING')));
+    }
+    for (const download of downloads) {
+        const [dlErr] = await tryCatch(download());
         if (dlErr) {
             if (dlErr.name === 'AbortError') {
                 continue;
@@ -784,7 +1124,8 @@ function dismissStaleJobs(jobEmitter) {
     jobEmitter.on('end', () => Promise.all(pendingChecks).then(() => emitter.emit('end')));
     jobEmitter.on('error', (err) => emitter.emit('error', err));
     jobEmitter.on('job', (job) => {
-        if (job.inputPath == null || job.out == null) {
+        // An older download can finish after a watch change and have a misleading newer mtime.
+        if (job.watchEvent || job.inputPath == null || job.out == null) {
             emitter.emit('job', job);
             return;
         }
@@ -867,8 +1208,11 @@ function makeJobEmitter(inputs, { allowOutputCollisions, recursive, outputPlanPr
     return staleFilter(conflictFilter(emitter));
 }
 const DEFAULT_CONCURRENCY = 5;
-// --- Main assembly create function ---
-export async function create(outputctl, client, { steps, stepsData, template, fields, outputMode, watch: watchOption, recursive, inputs, output, del, reprocessStale, singleAssembly, concurrency = DEFAULT_CONCURRENCY, }) {
+const DEFAULT_URL_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000;
+function isUrlInput(input) {
+    return /^https?:\/\//i.test(input);
+}
+async function prepareCreateOptions(outputctl, { steps, stepsData, outputMode, output, inputs, singleAssembly, transientInputPaths, }) {
     // Quick fix for https://github.com/transloadit/transloadify/issues/13
     // Only default to stdout when output is undefined (not provided), not when explicitly null
     let resolvedOutput = output;
@@ -901,7 +1245,9 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
         if (!outstat.isDirectory() && inputs.length !== 0 && !singleAssembly) {
             const firstInput = inputs[0];
             if (firstInput) {
-                const firstInputStat = await myStat(process.stdin, firstInput);
+                const firstInputStat = isUrlInput(firstInput)
+                    ? { isDirectory: () => false }
+                    : await myStat(process.stdin, firstInput);
                 if (inputs.length > 1 || firstInputStat.isDirectory()) {
                     const msg = 'Output must be a directory when specifying multiple inputs';
                     outputctl.error(msg);
@@ -911,11 +1257,70 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
         }
     }
     const inputStats = await Promise.all(inputs.map(async (input) => {
-        if (input === '-')
+        if (input === '-' || isUrlInput(input))
             return null;
         return await myStat(process.stdin, input);
     }));
     const hasDirectoryInput = inputStats.some((stat) => stat?.isDirectory() === true);
+    return {
+        effectiveStepsData,
+        hasDirectoryInput,
+        outstat,
+        resolvedOutput,
+        transientInputPaths: transientInputPaths == null
+            ? undefined
+            : new Set(transientInputPaths.map((inputPath) => path.resolve(inputPath))),
+    };
+}
+// --- Main assembly create function ---
+/** Create Assemblies after materializing URL inputs as temporary local uploads. */
+export async function create(outputctl, client, options) {
+    const preparedOptions = await prepareCreateOptions(outputctl, options);
+    const urlDownloadTimeoutMs = toPositiveInteger(options.urlDownloadTimeoutMs, '--url-download-timeout') ??
+        DEFAULT_URL_DOWNLOAD_TIMEOUT_MS;
+    if (!options.inputs.some(isUrlInput))
+        return await createWithLocalInputs(outputctl, client, options, preparedOptions);
+    if (options.watch)
+        throw new Error('--watch requires local inputs');
+    const preparedInputs = [];
+    const localInputs = [];
+    const transientInputPaths = preparedOptions.transientInputPaths ?? new Set();
+    try {
+        for (const [index, input] of options.inputs.entries()) {
+            if (!isUrlInput(input)) {
+                localInputs.push(input);
+                continue;
+            }
+            const field = String(index);
+            // Separate staging roots protect case-only filenames and repeated URLs from aliasing.
+            const prepared = await prepareInputFiles({
+                inputFiles: [{ kind: 'url', field, url: input }],
+                allowPrivateUrls: false,
+                // Uploading preserves :original and Template semantics, just as Intents do.
+                urlStrategy: 'download',
+                urlDownloadTimeoutMs,
+            });
+            preparedInputs.push(prepared);
+            const localPath = prepared.files[field];
+            if (localPath == null)
+                throw new Error('URL input could not be prepared');
+            localInputs.push(localPath);
+            transientInputPaths.add(path.resolve(localPath));
+        }
+        return await createWithLocalInputs(outputctl, client, {
+            ...options,
+            inputs: localInputs,
+        }, { ...preparedOptions, transientInputPaths });
+    }
+    finally {
+        await Promise.all(preparedInputs.flatMap((prepared) => prepared.cleanup.map((cleanup) => cleanup())));
+    }
+}
+async function createWithLocalInputs(outputctl, client, { template, fields, outputMode, watch: watchOption, recursive, inputs, del, reprocessStale, singleAssembly, concurrency = DEFAULT_CONCURRENCY, printUrls = false, }, preparedOptions) {
+    const { effectiveStepsData, hasDirectoryInput, outstat, resolvedOutput } = preparedOptions;
+    const nameSensitivity = resolvedOutput != null && outstat?.isDirectory()
+        ? await outputNameSensitivity(resolvedOutput)
+        : { caseSensitive: true, normalizationSensitive: true };
     return new Promise((resolve, reject) => {
         const params = (effectiveStepsData
             ? { steps: effectiveStepsData }
@@ -926,7 +1331,7 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
         const outputPlanProvider = resolvedOutput == null
             ? nullProvider()
             : outstat?.isDirectory()
-                ? dirProvider(resolvedOutput)
+                ? dirProvider(resolvedOutput, preparedOptions.transientInputPaths)
                 : fileProvider(resolvedOutput);
         const emitter = makeJobEmitter(inputs, {
             allowOutputCollisions: singleAssembly,
@@ -939,8 +1344,16 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
         // Use p-queue for concurrency management
         const queue = new PQueue({ concurrency });
         const results = [];
+        const assemblies = [];
         const resultUrls = [];
-        const reservedResultPaths = new Set();
+        const reservedResultPaths = {
+            allocations: new PQueue({ concurrency: 1 }),
+            bundleCommits: new PQueue({ concurrency: 1 }),
+            ...nameSensitivity,
+            directories: new Set(),
+            ownersByPath: new Map(),
+            pathsByOwner: new Map(),
+        };
         const latestWatchJobTokenByOutputPath = new Map();
         let hasFailures = false;
         let nextWatchJobToken = 0;
@@ -975,28 +1388,88 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
             return createOptions;
         }
         async function awaitCompletedAssembly(createOptions) {
-            const result = await client.createAssembly(createOptions);
-            const assemblyId = result.assembly_id;
-            if (!assemblyId)
+            let assemblyReference;
+            function reportCreatedAssembly(rawAssembly) {
+                const assembly = assemblySummarySchema.parse(rawAssembly);
+                if (assemblyReference || !assembly.assembly_id)
+                    return;
+                assemblyReference = formatAssemblyReference({
+                    assemblyId: assembly.assembly_id,
+                    assemblyUrl: assembly.assembly_ssl_url ?? assembly.assembly_url ?? null,
+                });
+                outputctl.notice(assemblyReference);
+            }
+            const creation = client.createAssembly({
+                ...createOptions,
+                onAssemblyCreated: reportCreatedAssembly,
+            });
+            const [creationError, result] = await tryCatch(creation);
+            if (creationError) {
+                if (!assemblyReference && creationError instanceof RequestError && creation.assemblyId) {
+                    const requestUrl = creationError.options.url;
+                    assemblyReference = formatAssemblyReference({
+                        assemblyId: creation.assemblyId,
+                        assemblyUrl: requestUrl instanceof URL ? `${requestUrl.origin}${requestUrl.pathname}` : null,
+                    });
+                }
+                if (!assemblyReference)
+                    throw creationError;
+                throw new Error(`${formatAPIError(creationError)} (${assemblyReference})`, {
+                    cause: creationError,
+                });
+            }
+            reportCreatedAssembly(result);
+            const createdSummary = assemblySummarySchema.parse(result);
+            const assemblyId = createdSummary.assembly_id;
+            if (!assemblyId || !assemblyReference)
                 throw new Error('No assembly_id in result');
-            const assemblyUrl = result.assembly_ssl_url ?? result.assembly_url ?? null;
-            const assemblyReference = formatAssemblyReference({ assemblyId, assemblyUrl });
             let assembly;
             try {
                 assembly = await client.awaitAssemblyCompletion(assemblyId, {
                     signal: abortController.signal,
                     onPoll: () => true,
                     onAssemblyProgress: (status) => {
-                        outputctl.debug(`Assembly status: ${status.ok}`);
+                        const summary = assemblySummarySchema.parse(status);
+                        outputctl.debug(`Assembly status: ${summary.ok ?? 'unknown'}`);
                     },
                 });
             }
             catch (err) {
                 const error = ensureError(err);
-                throw new Error(`${error.message} (${assemblyReference})`, { cause: error });
+                const message = error instanceof RequestError ? formatAPIError(error) : error.message;
+                throw new Error(`${message} (${assemblyReference})`, { cause: error });
             }
+            const summary = assemblySummarySchema.parse(assembly);
+            summary.assembly_id = assemblyId;
+            summary.assembly_url ??= createdSummary.assembly_url;
+            summary.assembly_ssl_url ??= createdSummary.assembly_ssl_url;
+            const ignoredErrors = summary.ignored_errors;
+            const ignoredErrorCount = Math.max(summary.ignored_error_count, ignoredErrors.length);
+            summary.ignored_error_count = ignoredErrorCount;
+            // Preserve stdout file bytes. URL mode combines all summaries with its existing JSON object.
+            if (resolvedOutput !== '-') {
+                if (printUrls)
+                    assemblies.push(summary);
+                else
+                    outputctl.print(undefined, summary);
+            }
+            if (ignoredErrorCount > 0) {
+                const details = ignoredErrors
+                    .map((ignored) => {
+                    const code = typeof ignored.error === 'string' ? ignored.error : ignored.error?.code;
+                    const message = ignored.message ||
+                        (typeof ignored.error === 'object' ? ignored.error.message : undefined);
+                    return [ignored.step, code, message].filter(Boolean).join(': ');
+                })
+                    .filter(Boolean)
+                    .slice(0, 5);
+                outputctl.warn(`${ignoredErrorCount} ignored error${ignoredErrorCount === 1 ? '' : 's'} (${assemblyReference})${details.length > 0 ? `: ${details.join('; ')}` : ''}`);
+            }
+            // Malformed error fields still signal failure, but only sanitized values enter the message.
             if (assembly.error || (assembly.ok && assembly.ok !== 'ASSEMBLY_COMPLETED')) {
-                const msg = `Assembly failed: ${assembly.error || assembly.message} (Status: ${assembly.ok}, ${assemblyReference})`;
+                const details = [summary.error ?? summary.ok, summary.message].filter(Boolean).join(': ') ||
+                    'Unknown Assembly error';
+                const msg = `Assembly failed: ${details} (${assemblyReference})`;
                 throw new Error(msg);
             }
             return { assembly, assemblyId };
@@ -1028,16 +1501,26 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
                 abortSignal: abortController.signal,
                 hasDirectoryInput: singleAssemblyMode ? false : hasDirectoryInput,
                 inPath,
-                inputs: inputPaths,
+                isCurrent: () => !isSupersededWatchJob(outputPlan?.path ?? null, outputToken),
+                inputs: singleAssemblyMode ? inputPaths : inputs,
                 normalizedResults,
                 outputMode,
                 outputPath: outputPlan?.path ?? null,
                 outputRoot: resolvedOutput ?? null,
                 outputRootIsDirectory,
                 outputctl,
+                // Only watch updates reuse destinations; independent Assemblies retain every result.
+                owner: JSON.stringify([
+                    inPath,
+                    singleAssemblyMode ? inputPaths : [],
+                    watchOption ? null : assemblyId,
+                ]),
                 reservedPaths: reservedResultPaths,
                 singleAssembly: singleAssemblyMode,
             });
+            // Only the current watch job may delete its input; a failed newer job must retain it.
+            if (isSupersededWatchJob(outputPlan?.path ?? null, outputToken))
+                return assembly;
             outputctl.debug(`COMPLETED ${inPath ?? 'null'} ${outputPlan?.path ?? 'null'}`);
             if (del) {
                 for (const inputPath of inputPaths) {
@@ -1072,7 +1555,7 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
         function handleEmitterError(err) {
             abortController.abort();
             queue.clear();
-            outputctl.error(err);
+            outputctl.error(formatAPIError(err));
             reject(err);
         }
         function runSingleAssemblyEmitter() {
@@ -1097,7 +1580,7 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
                     singleInputReference: 'input',
                 })) {
                     outputctl.debug(`SKIPPED STALE SINGLE ASSEMBLY ${resolvedOutput ?? 'null'}`);
-                    resolve({ resultUrls, results: [], hasFailures: false });
+                    resolve({ assemblies, resultUrls, results: [], hasFailures: false });
                     return;
                 }
                 // Preserve original basenames/extensions for filesystem uploads so the backend
@@ -1140,9 +1623,9 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
                 }
                 catch (err) {
                     hasFailures = true;
-                    outputctl.error(err);
+                    outputctl.error(formatAPIError(err));
                 }
-                resolve({ resultUrls, results, hasFailures });
+                resolve({ assemblies, resultUrls, results, hasFailures });
             });
         }
         function runPerFileEmitter() {
@@ -1160,12 +1643,12 @@ export async function create(outputctl, client, { steps, stepsData, template, fi
                 })
                     .catch((err) => {
                     hasFailures = true;
-                    outputctl.error(err);
+                    outputctl.error(formatAPIError(err));
                 });
             });
             emitter.on('end', async () => {
                 await queue.onIdle();
-                resolve({ resultUrls, results, hasFailures });
+                resolve({ assemblies, resultUrls, results, hasFailures });
             });
         }
         emitter.on('error', handleEmitterError);
@@ -1276,6 +1759,9 @@ export class AssembliesCreateCommand extends AuthenticatedCommand {
     reprocessStale = reprocessStaleOption();
     singleAssembly = singleAssemblyOption();
     concurrency = concurrencyOption();
+    urlDownloadTimeoutMs = Option.String('--url-download-timeout', {
+        description: 'Maximum time per URL input download in milliseconds (default: 300000)',
+    });
     printUrls = printUrlsOption();
     async run() {
         if (!this.steps && !this.template) {
@@ -1305,7 +1791,7 @@ export class AssembliesCreateCommand extends AuthenticatedCommand {
             this.output.error(sharedValidationError);
             return 1;
         }
-        const { hasFailures, resultUrls } = await create(this.output, this.client, {
+        const { assemblies, hasFailures, resultUrls } = await create(this.output, this.client, {
             steps: this.steps,
             template: this.template,
             fields: fieldsMap ?? {},
@@ -1317,9 +1803,11 @@ export class AssembliesCreateCommand extends AuthenticatedCommand {
             reprocessStale: this.reprocessStale,
             singleAssembly: this.singleAssembly,
             concurrency: this.concurrency == null ? undefined : Number(this.concurrency),
+            printUrls: this.printUrls,
+            urlDownloadTimeoutMs: this.urlDownloadTimeoutMs == null ? undefined : Number(this.urlDownloadTimeoutMs),
         });
         if (this.printUrls) {
-            printResultUrls(this.output, resultUrls);
+            printResultUrls(this.output, resultUrls, assemblies);
         }
         return hasFailures ? 1 : undefined;
     }
@@ -1371,6 +1859,9 @@ export class RunCommand extends AuthenticatedCommand {
     reprocessStale = reprocessStaleOption();
     singleAssembly = singleAssemblyOption();
     concurrency = concurrencyOption();
+    urlDownloadTimeoutMs = Option.String('--url-download-timeout', {
+        description: 'Maximum time per URL input download in milliseconds (default: 300000)',
+    });
     printUrls = printUrlsOption();
     async run() {
         const inputList = this.inputs ?? [];
@@ -1405,7 +1896,7 @@ export class RunCommand extends AuthenticatedCommand {
             this.output.error(formatAPIError(error));
             return 1;
         }
-        const { hasFailures, resultUrls } = await create(this.output, this.client, {
+        const { assemblies, hasFailures, resultUrls } = await create(this.output, this.client, {
             stepsData: parseStepsInputJson(JSON.stringify(compiled.instructions.steps), this.output),
             fields: fieldsMap ?? {},
             watch: this.watch,
@@ -1416,9 +1907,11 @@ export class RunCommand extends AuthenticatedCommand {
             reprocessStale: this.reprocessStale,
             singleAssembly: this.singleAssembly,
             concurrency: this.concurrency == null ? undefined : Number(this.concurrency),
+            printUrls: this.printUrls,
+            urlDownloadTimeoutMs: this.urlDownloadTimeoutMs == null ? undefined : Number(this.urlDownloadTimeoutMs),
         });
         if (this.printUrls) {
-            printResultUrls(this.output, resultUrls);
+            printResultUrls(this.output, resultUrls, assemblies);
         }
         return hasFailures ? 1 : undefined;
     }