transloadit 6.4.0 → 6.5.0

package/src/cli/commands/templates.ts

@@ -11,11 +11,13 @@ import { Command, Option } from 'clipanion'
 import rreaddir from 'recursive-readdir'
 import { z } from 'zod'
 
+import { ApiError } from '../../ApiError.ts'
 import { tryCatch } from '../../alphalib/tryCatch.ts'
+import { CliInputError } from '../errors.ts'
 import { createReadStream, formatAPIError, streamToBuffer } from '../helpers.ts'
 import { createCliStepsSchema, parseStepsInputJson } from '../stepsInput.ts'
 import ModifiedLookup from '../template-last-modified.ts'
-import { ensureError, isTransloaditAPIError } from '../types.ts'
+import { isTransloaditAPIError } from '../types.ts'
 import { AuthenticatedCommand } from './BaseCommand.ts'
 
 const rreaddirAsync = promisify(rreaddir)
@@ -54,27 +56,29 @@ export interface TemplateSyncOptions {
   recursive?: boolean
 }
 
+async function readTemplateInput<T>(input: Promise<T>): Promise<T> {
+  const [error, content] = await tryCatch(input)
+  if (error) {
+    throw new CliInputError(`Cannot read Template input: ${error.message}`, { cause: error })
+  }
+  return content
+}
+
 export async function create(
   output: IOutputCtl,
   client: Transloadit,
   { name, file }: TemplateCreateOptions,
 ): Promise<unknown> {
-  try {
-    const buf = await streamToBuffer(createReadStream(file))
+  const buf = await readTemplateInput(streamToBuffer(createReadStream(file)))
 
-    const steps = parseStepsInputJson(buf.toString(), output)
+  const steps = parseStepsInputJson(buf.toString(), output)
 
-    const result = await client.createTemplate({
-      name,
-      template: { steps } as TemplateContent,
-    })
-    output.print(result.id, result)
-    return result
-  } catch (err) {
-    const error = ensureError(err)
-    output.error(error.message)
-    throw err
-  }
+  const result = await client.createTemplate({
+    name,
+    template: { steps } as TemplateContent,
+  })
+  output.print(result.id, result)
+  return result
 }
 
 export async function get(
@@ -84,11 +88,7 @@ export async function get(
 ): Promise<void> {
   const requests = templates.map((template) => client.getTemplate(template))
 
-  const [err, results] = await tryCatch(Promise.all(requests))
-  if (err) {
-    output.error(formatAPIError(err))
-    throw err
-  }
+  const results = await Promise.all(requests)
 
   for (const result of results) {
     output.print(result, result)
@@ -100,53 +100,49 @@ export async function modify(
   client: Transloadit,
   { template, name, file }: TemplateModifyOptions,
 ): Promise<void> {
-  try {
-    const buf = await streamToBuffer(createReadStream(file))
+  const buf = await readTemplateInput(streamToBuffer(createReadStream(file)))
 
-    let steps: TemplateContent['steps'] | null = null
-    let newName = name
+  let steps: TemplateContent['steps'] | null = null
+  let newName = name
 
-    if (buf.length > 0) {
-      steps = parseStepsInputJson(buf.toString(), output) as TemplateContent['steps']
-    }
-
-    if (!name || buf.length === 0) {
-      const tpl = await client.getTemplate(template)
-      if (!name) newName = tpl.name
-      if (buf.length === 0 && tpl.content.steps) {
-        steps = tpl.content.steps as TemplateContent['steps']
-      }
-    }
+  if (buf.length > 0) {
+    steps = parseStepsInputJson(buf.toString(), output) as TemplateContent['steps']
+  }
 
-    if (steps === null) {
-      throw new Error('No steps to update template with')
+  if (!name || buf.length === 0) {
+    const tpl = await client.getTemplate(template)
+    if (!name) newName = tpl.name
+    if (buf.length === 0 && tpl.content.steps) {
+      steps = tpl.content.steps as TemplateContent['steps']
     }
+  }
 
-    await client.editTemplate(template, {
-      name: newName,
-      // Steps (validated) is assignable to StepsInput at runtime; cast for TS
-      template: { steps } as TemplateContent,
-    })
-  } catch (err) {
-    output.error(formatAPIError(err))
-    throw err
+  if (steps === null) {
+    throw new CliInputError('No steps to update template with')
   }
+
+  await client.editTemplate(template, {
+    name: newName,
+    // Steps (validated) is assignable to StepsInput at runtime; cast for TS
+    template: { steps } as TemplateContent,
+  })
 }
 
 async function deleteTemplates(
   output: IOutputCtl,
   client: Transloadit,
   { templates }: TemplateDeleteOptions,
-): Promise<void> {
-  await Promise.all(
-    templates.map(async (template) => {
-      const [err] = await tryCatch(client.deleteTemplate(template))
-      if (err) {
-        output.error(formatAPIError(err))
-        throw err
-      }
-    }),
+): Promise<number | undefined> {
+  const results = await Promise.allSettled(
+    templates.map((template) => client.deleteTemplate(template)),
   )
+  let exitCode: number | undefined
+  for (const [index, result] of results.entries()) {
+    if (result.status === 'fulfilled') continue
+    output.error(`Template ${templates[index]}: ${formatAPIError(result.reason)}`)
+    exitCode = 1
+  }
+  return exitCode
 }
 
 // Export with `delete` alias for external consumers
@@ -281,54 +277,54 @@ export async function sync(
 
   const modified = new ModifiedLookup(client)
 
-  const [err] = await tryCatch(
-    Promise.all(
-      templates.map(async (template) => {
-        if (!('steps' in template.data)) {
-          if (!template.data.transloadit_template_id) {
-            throw new Error(`Template file has no id and no steps: ${template.file}`)
-          }
-          return download(template)
+  await Promise.all(
+    templates.map(async (template) => {
+      if (!('steps' in template.data)) {
+        if (!template.data.transloadit_template_id) {
+          throw new CliInputError(`Template file has no id and no steps: ${template.file}`)
         }
+        return download(template)
+      }
 
-        if (!template.data.transloadit_template_id) return upload(template)
-
-        const stats = await fsp.stat(template.file)
-        const fileModified = stats.mtime
-
-        let templateModified: Date
-        const templateId = template.data.transloadit_template_id
-        try {
-          await client.getTemplate(templateId)
-          templateModified = await new Promise<Date>((resolve, reject) =>
-            modified.byId(templateId, (err, res) => {
-              if (err) {
-                reject(err)
-              } else if (res) {
-                resolve(res)
-              } else {
-                reject(new Error('No date returned'))
-              }
-            }),
-          )
-        } catch (err) {
-          if (isTransloaditAPIError(err)) {
-            if (err.code === 'SERVER_404' || (err.response && err.response.statusCode === 404)) {
-              throw new Error(`Template file references nonexistent template: ${template.file}`)
+      if (!template.data.transloadit_template_id) return upload(template)
+
+      const stats = await readTemplateInput(fsp.stat(template.file))
+      const fileModified = stats.mtime
+
+      let templateModified: Date
+      const templateId = template.data.transloadit_template_id
+      try {
+        await client.getTemplate(templateId)
+        templateModified = await new Promise<Date>((resolve, reject) =>
+          modified.byId(templateId, (err, res) => {
+            if (err) {
+              reject(err)
+            } else if (res) {
+              resolve(res)
+            } else {
+              reject(new Error('No date returned'))
             }
-          }
-          throw err
+          }),
+        )
+      } catch (err) {
+        if (
+          (err instanceof ApiError &&
+            (err.code === 'TEMPLATE_NOT_FOUND' || err.cause?.response?.statusCode === 404)) ||
+          (isTransloaditAPIError(err) &&
+            (err.code === 'SERVER_404' || err.response?.statusCode === 404))
+        ) {
+          throw new CliInputError(
+            `Template file references nonexistent template: ${template.file}`,
+            { cause: err },
+          )
         }
+        throw err
+      }
 
-        if (fileModified > templateModified) return upload(template)
-        return download(template)
-      }),
-    ),
+      if (fileModified > templateModified) return upload(template)
+      return download(template)
+    }),
   )
-  if (err) {
-    output.error(err)
-    throw err
-  }
 
   async function upload(template: TemplateFile): Promise<void> {
     const params = {
@@ -479,10 +475,9 @@ export class TemplatesDeleteCommand extends AuthenticatedCommand {
   templateIds = Option.Rest({ required: 1 })
 
   protected async run(): Promise<number | undefined> {
-    await deleteTemplates(this.output, this.client, {
+    return await deleteTemplates(this.output, this.client, {
       templates: this.templateIds,
     })
-    return undefined
   }
 }
 

package/src/cli/errors.ts

@@ -0,0 +1,34 @@
+import type { ZodIssue } from 'zod'
+
+/** An input error the CLI can explain without an internal-error stack trace. */
+export class CliInputError extends Error {
+  override name = 'CliInputError'
+}
+
+function describeIssues(issues: ZodIssue[]): string[] {
+  return issues.flatMap((issue) => {
+    if (issue.code === 'invalid_union') {
+      const descriptions = issue.unionErrors.flatMap((error) => describeIssues(error.issues))
+      // Interpolation is an alternative to literal Robot parameters, not a required input format.
+      const literals = descriptions.filter(
+        (description) => !description.endsWith(': Must be a full interpolation string'),
+      )
+      return literals.length > 0 ? literals : descriptions
+    }
+    const message =
+      issue.code === 'invalid_enum_value'
+        ? `Expected one of: ${issue.options.join(', ')}`
+        : issue.message
+    return [`${issue.path.join('.') || '(root)'}: ${message}`]
+  })
+}
+
+/** Summarize validation paths and remedies without duplicating Zod's union diagnostics. */
+export function formatSchemaIssues(issues: ZodIssue[]): string {
+  const descriptions = [...new Set(describeIssues(issues))]
+  const remaining = descriptions.length - 10
+  return [
+    ...descriptions.slice(0, 10),
+    ...(remaining > 0 ? [`${remaining} more validation error${remaining === 1 ? '' : 's'}`] : []),
+  ].join('; ')
+}

package/src/cli/helpers.ts

@@ -8,8 +8,10 @@ import { homedir, userInfo } from 'node:os'
 import path from 'node:path'
 
 import { parse as parseDotenv } from 'dotenv'
+import { ParseError, RequestError } from 'got'
 import { z } from 'zod'
 
+import { ApiError } from '../ApiError.ts'
 import { isAPIError } from './types.ts'
 
 /** API signing algorithms supported by CLI credentials and device authorization. */
@@ -535,6 +537,25 @@ export async function readCliInput(options: ReadCliInputOptions): Promise<CliInp
 }
 
 export function formatAPIError(err: unknown): string {
+  if (err instanceof RequestError) {
+    // ParseError messages contain response fragments and signed request URLs.
+    if (err instanceof ParseError) return `${err.code}: The API returned invalid JSON.`
+    const requestUrl = err.options.url
+    const message =
+      requestUrl instanceof URL
+        ? err.message.replaceAll(requestUrl.href, `${requestUrl.origin}${requestUrl.pathname}`)
+        : err.message
+    return `${err.code}: ${message}`
+  }
+  if (err instanceof ApiError && !err.assemblySslUrl) {
+    const requestUrl = err.cause?.options.url
+    const url = typeof requestUrl === 'string' ? new URL(requestUrl) : requestUrl
+    if (url && /\/assemblies\/[^/]+$/.test(url.pathname)) {
+      // Request URLs can carry signed query parameters; only expose the Assembly location.
+      return `${err.message} (Assembly URL: ${url.origin}${url.pathname})`
+    }
+    if (err.assemblyId) return `${err.message} (Assembly ID: ${err.assemblyId})`
+  }
   if (isAPIError(err)) {
     return `${err.error}: ${err.message}`
   }

package/src/cli/intentRuntime.ts

@@ -33,6 +33,7 @@ export interface PreparedIntentInputs {
   cleanup: Array<() => Promise<void>>
   hasTransientInputs: boolean
   inputs: string[]
+  transientInputPaths?: string[]
 }
 
 export interface IntentSingleStepExecutionDefinition {
@@ -252,6 +253,7 @@ export async function prepareIntentInputs({
     cleanup: prepared.cleanup,
     hasTransientInputs: true,
     inputs,
+    transientInputPaths: Object.values(prepared.files),
   }
 }
 
@@ -383,14 +385,15 @@ async function executeIntentCommand({
           } as AssembliesCreateOptions['stepsData'],
         }
 
-  const { hasFailures, resultUrls } = await assembliesCommands.create(output, client, {
+  const { assemblies, hasFailures, resultUrls } = await assembliesCommands.create(output, client, {
     ...createOptions,
     output: outputPath ?? null,
     outputMode,
     ...executionOptions,
+    printUrls,
   })
   if (printUrls) {
-    printResultUrls(output, resultUrls)
+    printResultUrls(output, resultUrls, assemblies)
   }
   return hasFailures ? 1 : undefined
 }
@@ -766,9 +769,13 @@ abstract class GeneratedFileIntentCommandBase extends GeneratedIntentCommandBase
     }
 
     const effectiveOutputPath = this.getEffectiveOutputPath(rawValues)
+    const createOptions = this.getCreateOptions(effectivePreparedInputs.inputs)
+    if (effectivePreparedInputs.transientInputPaths != null) {
+      createOptions.transientInputPaths = effectivePreparedInputs.transientInputPaths
+    }
     return await executeIntentCommand({
       client: this.client,
-      createOptions: this.getCreateOptions(effectivePreparedInputs.inputs),
+      createOptions,
       definition: this.getIntentDefinition(),
       output: this.output,
       outputMode: this.getEffectiveOutputMode(rawValues, effectiveOutputPath),

package/src/cli/resultFiles.ts

@@ -1,6 +1,7 @@
 export interface AssemblyResultEntryLike {
   basename?: unknown
   ext?: unknown
+  meta?: unknown
   name?: unknown
   ssl_url?: unknown
   url?: unknown

package/src/cli/resultUrls.ts

@@ -1,5 +1,6 @@
 import type { IOutputCtl } from './OutputCtl.ts'
 import type { NormalizedAssemblyResults } from './resultFiles.ts'
+import type { AssemblySummary } from './types.ts'
 
 import { normalizeAssemblyResults } from './resultFiles.ts'
 
@@ -64,10 +65,17 @@ export function formatResultUrlRows(rows: readonly ResultUrlRow[]): string {
     .join('\n')
 }
 
-export function printResultUrls(output: IOutputCtl, rows: readonly ResultUrlRow[]): void {
-  if (rows.length === 0) {
+export function printResultUrls(
+  output: IOutputCtl,
+  rows: readonly ResultUrlRow[],
+  assemblies?: readonly AssemblySummary[],
+): void {
+  if (rows.length === 0 && !assemblies?.length) {
     return
   }
 
-  output.print(formatResultUrlRows(rows), { urls: rows })
+  output.print(rows.length > 0 ? formatResultUrlRows(rows) : undefined, {
+    urls: rows,
+    ...(assemblies?.length ? { assemblies } : {}),
+  })
 }

package/src/cli/stepsInput.ts

@@ -5,9 +5,11 @@ import fsp from 'node:fs/promises'
 
 import { z } from 'zod'
 
+import { tryCatch } from '../alphalib/tryCatch.ts'
 import { robotsSchema } from '../alphalib/types/robots/_index.ts'
 import { refineDocumentExtractOcr } from '../alphalib/types/robots/document-extract.ts'
 import { refineTransloaditImportSelector } from '../alphalib/types/robots/transloadit-import.ts'
+import { CliInputError, formatSchemaIssues } from './errors.ts'
 
 const stepsWrapperSchema = z.object({ steps: z.record(z.unknown()) })
 
@@ -35,7 +37,12 @@ export function parseStepsInputJson(
   content: string,
   output: Pick<IOutputCtl, 'warn'> = console,
 ): StepsInput {
-  const parsed: unknown = JSON.parse(content)
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(content)
+  } catch (error) {
+    throw new CliInputError('Invalid steps format: expected valid JSON', { cause: error })
+  }
   const wrapped = stepsWrapperSchema.safeParse(parsed)
   let stepsInput = parsed
   // A step named “steps” has a robot, unlike the top-level instructions wrapper.
@@ -43,13 +50,18 @@ export function parseStepsInputJson(
     // --steps only supplies steps; other instructions must not be silently discarded.
     const validatedWrapper = stepsWrapperSchema.strict().safeParse(parsed)
     if (!validatedWrapper.success) {
-      throw new Error(`Invalid steps format: ${validatedWrapper.error.message}`)
+      throw new CliInputError(
+        `Invalid steps format: ${formatSchemaIssues(validatedWrapper.error.issues)}`,
+        { cause: validatedWrapper.error },
+      )
     }
     stepsInput = validatedWrapper.data.steps
   }
   const validated = createCliStepsSchema(output).safeParse(stepsInput)
   if (!validated.success) {
-    throw new Error(`Invalid steps format: ${validated.error.message}`)
+    throw new CliInputError(`Invalid steps format: ${formatSchemaIssues(validated.error.issues)}`, {
+      cause: validated.error,
+    })
   }
 
   const parsedSteps = stepsInput as Record<string, Record<string, unknown>>
@@ -72,6 +84,9 @@ export async function readStepsInputFile(
   filePath: string,
   output: Pick<IOutputCtl, 'warn'> = console,
 ): Promise<StepsInput> {
-  const content = await fsp.readFile(filePath, 'utf8')
+  const [error, content] = await tryCatch(fsp.readFile(filePath, 'utf8'))
+  if (error) {
+    throw new CliInputError(`Cannot read steps input: ${error.message}`, { cause: error })
+  }
   return parseStepsInputJson(content, output)
 }

package/src/cli/types.ts

@@ -2,6 +2,41 @@ import type { Steps } from '../alphalib/types/template.ts'
 
 import { z } from 'zod'
 
+const ignoredErrorDetailsSchema = z
+  .union([
+    z.string(),
+    z.object({
+      code: z.string().optional().catch(undefined),
+      message: z.string().optional().catch(undefined),
+    }),
+  ])
+  .optional()
+  .catch(undefined)
+
+const ignoredErrorSummarySchema = z
+  .object({
+    step: z.string().nullable().optional().catch(undefined),
+    phase: z.string().optional().catch(undefined),
+    error: ignoredErrorDetailsSchema,
+    message: z.string().optional().catch(undefined),
+  })
+  .catch({})
+
+/** Project unchecked API responses to CLI diagnostics without nested request metadata. */
+export const assemblySummarySchema = z.object({
+  assembly_id: z.string().optional().catch(undefined),
+  assembly_url: z.string().nullable().optional().catch(undefined),
+  assembly_ssl_url: z.string().nullable().optional().catch(undefined),
+  ok: z.string().nullable().optional().catch(undefined),
+  error: z.string().optional().catch(undefined),
+  message: z.string().optional().catch(undefined),
+  ignored_error_count: z.number().finite().int().nonnegative().default(0).catch(0),
+  ignored_errors: z.array(ignoredErrorSummarySchema).default([]).catch([]),
+})
+
+/** Public diagnostics from one Assembly, excluding instructions and request internals. */
+export type AssemblySummary = z.infer<typeof assemblySummarySchema>
+
 // Zod schemas for runtime validation
 const APIErrorSchema = z.object({
   error: z.string(),

package/src/tus.ts

@@ -1,16 +1,24 @@
 import type { Readable } from 'node:stream'
 
-import type { OnSuccessPayload, UploadOptions } from 'tus-js-client'
+import type {
+  FileSource,
+  HttpRequest,
+  HttpResponse,
+  OnSuccessPayload,
+  SliceResult,
+  UploadOptions,
+} from 'tus-js-client'
 
 import type { AssemblyStatus } from './alphalib/types/assemblyStatus.ts'
 import type { UploadProgress } from './Transloadit.ts'
 
+import { ReadStream } from 'node:fs'
 import { stat } from 'node:fs/promises'
 import { basename } from 'node:path'
 
 import debug from 'debug'
 import pMap from 'p-map'
-import { Upload } from 'tus-js-client'
+import { defaultOptions, Upload } from 'tus-js-client'
 
 const log = debug('transloadit')
 const logWarn = debug('transloadit:warn')
@@ -31,6 +39,13 @@ interface SendTusRequestOptions {
   signal?: AbortSignal
   uploadUrls?: Record<string, string>
   uploadBehavior?: UploadBehavior
+  onFileStreamOpened?: (stream: ReadStream) => void
+  onFileSliceStarted?: (slice: Promise<unknown>) => void
+}
+
+interface SendTusRequestResult {
+  uploadUrls: Record<string, string>
+  uploadsComplete?: Promise<unknown>
 }
 
 export async function sendTusRequest({
@@ -42,7 +57,9 @@ export async function sendTusRequest({
   signal,
   uploadUrls,
   uploadBehavior = 'await',
-}: SendTusRequestOptions) {
+  onFileStreamOpened,
+  onFileSliceStarted,
+}: SendTusRequestOptions): Promise<SendTusRequestResult> {
   const streamLabels = Object.keys(streamsMap)
 
   let totalBytes = 0
@@ -203,6 +220,25 @@ export async function sendTusRequest({
           fieldname: label,
           filename,
         },
+        httpStack: {
+          createRequest(method: string, url: string): HttpRequest {
+            const request = defaultOptions.httpStack.createRequest(method, url)
+            const send = request.send.bind(request)
+            request.send = (body: unknown): Promise<HttpResponse> => {
+              // tus awaits pre-request hooks, so queued aborts must be checked at send itself.
+              if (signal?.aborted) {
+                // File setup can finish after abort() and reset tus's aborted state.
+                tusUpload.abort()
+                return Promise.reject(new Error('Upload aborted'))
+              }
+              return send(body)
+            }
+            return request
+          },
+          getName(): string {
+            return defaultOptions.httpStack.getName()
+          },
+        },
         onError: reject,
         onProgress: onTusProgress,
         onSuccess: resolve,
@@ -218,6 +254,36 @@ export async function sendTusRequest({
           }
         },
       }
+      if (onFileStreamOpened) {
+        // tus opens a separate ReadStream for each chunk; abort() leaves those streams open.
+        // Track those chunks even when the original file stream belongs to the caller.
+        tusOptions.fileReader = {
+          async openFile(input: unknown, chunkSize: number): Promise<FileSource> {
+            const source = await defaultOptions.fileReader.openFile(input, chunkSize)
+            return {
+              size: source.size,
+              async slice(start: number, end: number): Promise<SliceResult> {
+                if (signal?.aborted) throw new Error('Upload aborted')
+                const reading = source.slice(start, end).then((chunk) => {
+                  // tus-js-client declares platform-specific chunk values as any.
+                  const value: unknown = chunk.value
+                  if (value instanceof ReadStream) onFileStreamOpened(value)
+                  if (signal?.aborted) throw new Error('Upload aborted')
+                  return chunk
+                })
+                // A file slice can open its stream before the promise resolves.
+                if (input instanceof ReadStream && input.path != null) {
+                  onFileSliceStarted?.(reading)
+                }
+                return await reading
+              },
+              close() {
+                source.close()
+              },
+            }
+          },
+        }
+      }
       // tus-js-client doesn't like undefined/null
       if (size != null) tusOptions.uploadSize = size
       if (chunkSize) tusOptions.chunkSize = chunkSize
@@ -255,13 +321,14 @@ export async function sendTusRequest({
 
   await Promise.all(uploadUrlPromises)
 
+  const uploadsComplete = Promise.allSettled(completionPromises)
   if (uploadBehavior === 'await') {
     await Promise.all(completionPromises)
   } else {
-    Promise.allSettled(completionPromises).catch((err) => {
+    uploadsComplete.catch((err) => {
       logWarn('Background upload failed', err)
     })
   }
 
-  return { uploadUrls: uploadUrlsResult }
+  return { uploadUrls: uploadUrlsResult, uploadsComplete }
 }