transloadit 4.11.1 → 4.13.0

package/src/storageImage.ts

@@ -0,0 +1,266 @@
+import type { AssemblyStatus } from './alphalib/types/assemblyStatus.ts'
+import type { StoredAsset } from './alphalib/types/storageAsset.ts'
+import type { CreateAssemblyOptions, Transloadit } from './Transloadit.ts'
+
+import { createHash } from 'node:crypto'
+import { createReadStream } from 'node:fs'
+
+import debug from 'debug'
+import { z } from 'zod'
+
+import { ApiError } from './ApiError.ts'
+import { storedAssetSchema } from './alphalib/types/storageAsset.ts'
+import InconsistentResponseError from './InconsistentResponseError.ts'
+
+/**
+ * Verified Storage metadata that can be saved and passed directly to an image renderer.
+ * Width and height reflect EXIF auto-orientation, matching Storage preview delivery.
+ */
+export interface StoredImageReceipt extends Readonly<StoredAsset> {
+  /** Present as true only when the locally decoded original has an alpha channel. */
+  readonly hasAlpha?: boolean
+  readonly height: number
+  readonly md5hash: string
+  /** Optional base64 ThumbHash of the original pixels, for an inline blur placeholder. */
+  readonly thumbhash?: string
+  readonly width: number
+}
+
+/** One explicit destination, with the existing Assembly upload and polling controls. */
+export interface StoreImageOptions
+  extends Pick<
+    CreateAssemblyOptions,
+    'chunkSize' | 'onAssemblyProgress' | 'onUploadProgress' | 'signal' | 'timeout'
+  > {
+  /** Complete relative Storage filename. Directories and interpolation expressions are rejected. */
+  path: string
+  /** Explicit opt-in replacement of an existing path; defaults to false. */
+  overwrite?: boolean
+  /** Observe verified metadata. Not awaited; sync and async observer errors cannot undo a write. */
+  onReceipt?: (
+    receipt: StoredImageReceipt,
+    input: StoredImageExpectation,
+    assemblyId: string | undefined,
+  ) => void | Promise<void>
+}
+
+/** Trusted upload facts used to correlate a stored result with an application-owned upload. */
+export interface StoredImageExpectation {
+  path: string
+  size: number
+  md5hash: string
+}
+
+/** Recovers one original's receipt without re-uploading or trusting notification payloads. */
+export interface GetStoredImageReceiptOptions {
+  assemblyId: string
+  expected: StoredImageExpectation
+}
+
+const positiveIntegerSchema = z.number().int().positive().max(Number.MAX_SAFE_INTEGER)
+const receiptRequestSchema = z.object({
+  assemblyId: z.string().regex(/^[A-Za-z0-9_-]+$/),
+  expected: z.object({
+    path: z.string(),
+    size: positiveIntegerSchema,
+    md5hash: z.string().regex(/^[a-f0-9]{32}$/),
+  }),
+})
+const completedImageSchema = z.object({
+  ok: z.literal('ASSEMBLY_COMPLETED'),
+  results: z.object({
+    ':original': z.tuple([
+      storedAssetSchema.extend({
+        md5hash: storedAssetSchema.shape.md5hash.unwrap(),
+        height: positiveIntegerSchema,
+        width: positiveIntegerSchema,
+        size: positiveIntegerSchema,
+      }),
+    ]),
+  }),
+})
+
+function validateDestination(path: string): void {
+  if (
+    typeof path !== 'string' ||
+    path.trim() !== path ||
+    Buffer.byteLength(path) > 1024 ||
+    path.normalize('NFC') !== path ||
+    /[\p{Cc}\p{Cs}\\|]/u.test(path) ||
+    path.includes('${') ||
+    path.split('/').some((segment) => segment.trim() === '' || segment === '.' || segment === '..')
+  ) {
+    throw new TypeError('Storage path must be a complete relative filename without interpolation')
+  }
+}
+
+async function imagePlaceholderMetadata(
+  bytes: Buffer,
+): Promise<Pick<StoredImageReceipt, 'thumbhash' | 'hasAlpha'> | undefined> {
+  try {
+    const [{ default: sharp }, { rgbaToThumbHash }] = await Promise.all([
+      import('sharp'),
+      import('thumbhash'),
+    ])
+    const image = sharp(bytes, { limitInputPixels: 40_000_000 })
+    // Read original-channel metadata, not the alpha channel added for ThumbHash's RGBA input.
+    const { hasAlpha } = await image.metadata()
+    const { data, info } = await image
+      .autoOrient()
+      .resize(100, 100, { fit: 'inside', withoutEnlargement: true })
+      .ensureAlpha()
+      .raw()
+      .timeout({ seconds: 2 })
+      .toBuffer({ resolveWithObject: true })
+    return {
+      thumbhash: Buffer.from(rgbaToThumbHash(info.width, info.height, data)).toString('base64'),
+      ...(hasAlpha ? { hasAlpha: true } : {}),
+    }
+  } catch {
+    // A locally unsupported/oversized image can still be stored and decoded by the origin.
+    debug('transloadit:warn')('Omitted optional ThumbHash: local image decoding was unavailable')
+    return undefined
+  }
+}
+
+/** Stores one original without overwriting; receipt validation occurs after the Storage write. */
+export async function storeImage(
+  client: Transloadit,
+  filePath: string,
+  options: StoreImageOptions,
+): Promise<StoredImageReceipt> {
+  const {
+    path,
+    chunkSize,
+    onAssemblyProgress,
+    onUploadProgress,
+    onReceipt,
+    overwrite,
+    signal,
+    timeout,
+  } = options
+  validateDestination(path)
+  if (overwrite !== undefined && typeof overwrite !== 'boolean')
+    throw new TypeError('overwrite must be a boolean')
+  signal?.throwIfAborted()
+  const checksum = createHash('md5')
+  // Reuse the checksum read, bounded independently of file size; uploads still stream from disk.
+  let thumbnailChunks: Buffer[] | undefined = []
+  let size = 0
+  for await (const chunk of createReadStream(filePath, { signal })) {
+    checksum.update(chunk)
+    size += chunk.length
+    if (size > 32 * 1024 * 1024) thumbnailChunks = undefined
+    else thumbnailChunks?.push(chunk)
+  }
+  if (size === 0) throw new Error('Cannot store an empty image')
+  const md5hash = checksum.digest('hex')
+  signal?.throwIfAborted()
+  const placeholder =
+    thumbnailChunks === undefined
+      ? undefined
+      : await imagePlaceholderMetadata(Buffer.concat(thumbnailChunks))
+  thumbnailChunks = undefined
+  signal?.throwIfAborted()
+  const assembly = await client.createAssembly({
+    chunkSize,
+    files: { image: filePath },
+    onAssemblyProgress,
+    onUploadProgress,
+    params: {
+      steps: {
+        stored: {
+          robot: '/transloadit/store',
+          use: ':original',
+          path,
+          conflict_strategy: overwrite === true ? 'overwrite' : 'error',
+        },
+      },
+    },
+    signal,
+    timeout,
+    waitForCompletion: true,
+  })
+  const input = { path, size, md5hash }
+  // API2 can watermark uploads before Robots run; this exact write's result describes stored bytes.
+  const receipt = validateReceipt(assembly, input, true)
+  // A watermark or other origin-side rewrite makes the local pixels the wrong placeholder.
+  const withPlaceholder =
+    placeholder === undefined || receipt.md5hash !== md5hash || receipt.size !== size
+      ? receipt
+      : { ...receipt, ...placeholder }
+  const observerFailed = (): void => {
+    debug('transloadit:warn')('Ignored onReceipt observer failure after a completed Storage write')
+  }
+  try {
+    // Neither a stalled observer nor its rejection may hide a successfully stored receipt.
+    void Promise.resolve(onReceipt?.({ ...withPlaceholder }, input, assembly.assembly_id)).catch(
+      observerFailed,
+    )
+  } catch {
+    observerFailed()
+  }
+  return withPlaceholder
+}
+
+/** Fetches authoritative Assembly status and applies the same validation as a local image store. */
+export async function getStoredImageReceipt(
+  client: Transloadit,
+  options: GetStoredImageReceiptOptions,
+): Promise<StoredImageReceipt> {
+  const { assemblyId, expected } = receiptRequestSchema.parse(options)
+  validateDestination(expected.path)
+  const assembly = await client.getAssembly(assemblyId)
+  if (assembly.assembly_id !== assemblyId) {
+    throw new InconsistentResponseError('The response did not match the requested Assembly', {
+      cause: { assemblyId },
+    })
+  }
+  return validateReceipt(assembly, expected)
+}
+
+function validateReceipt(
+  assembly: AssemblyStatus,
+  expected: StoredImageExpectation,
+  acceptTransformed = false,
+): StoredImageReceipt {
+  const { path, size, md5hash } = expected
+  if (typeof assembly.error === 'string') throw new ApiError({ body: assembly })
+  if (assembly.ok === 'ASSEMBLY_CANCELED') {
+    throw new InconsistentResponseError('The Storage Assembly ended with ASSEMBLY_CANCELED', {
+      cause: { assemblyId: assembly.assembly_id },
+    })
+  }
+  if (
+    assembly.ok === 'ASSEMBLY_UPLOADING' ||
+    assembly.ok === 'ASSEMBLY_EXECUTING' ||
+    assembly.ok === 'ASSEMBLY_REPLAYING'
+  ) {
+    throw new InconsistentResponseError(`The Storage Assembly is not complete (${assembly.ok})`, {
+      cause: { assemblyId: assembly.assembly_id },
+    })
+  }
+  const parsed = completedImageSchema.safeParse(assembly)
+  const result = parsed.success ? parsed.data.results[':original'][0] : undefined
+  const originals = assembly.results?.[':original']
+  const receiptCheck = {
+    originalCount: Array.isArray(originals) ? originals.length : 0,
+    metadataValid: parsed.success,
+    pathMatches: result === undefined ? undefined : result.path === path,
+    sizeMatches: result === undefined ? undefined : result.size === size,
+    md5Matches: result === undefined ? undefined : result.md5hash === md5hash,
+  }
+  if (
+    result === undefined ||
+    result.path !== path ||
+    (!acceptTransformed && (!receiptCheck.sizeMatches || !receiptCheck.md5Matches))
+  ) {
+    throw new InconsistentResponseError(
+      'The Assembly did not return a matching Storage image receipt',
+      {
+        cause: { assemblyId: assembly.assembly_id, receiptCheck },
+      },
+    )
+  }
+  return result
+}

package/src/storagePublicPrefixes.ts

@@ -0,0 +1,39 @@
+import { validateStoragePathPrefix } from '@transloadit/utils'
+import { z } from 'zod'
+
+/** Normalize a public directory, never implicit workspace-wide access. */
+export function normalizeStoragePublicPrefix(prefix: string): string {
+  if (typeof prefix !== 'string' || prefix === '' || prefix === '/')
+    throw new TypeError('A public prefix must name a non-root directory')
+  const normalized = prefix.endsWith('/') ? prefix : `${prefix}/`
+  validateStoragePathPrefix(normalized, 0, 'public prefix')
+  if (new TextEncoder().encode(normalized).byteLength > 512)
+    throw new TypeError('A public prefix must be at most 512 UTF-8 bytes')
+  return normalized
+}
+
+const prefixSchema = z.string().refine((prefix) => {
+  try {
+    return normalizeStoragePublicPrefix(prefix) === prefix
+  } catch {
+    return false
+  }
+}, 'Expected a normalized public directory')
+const publicPrefixSchema = z.object({ prefix: prefixSchema, created_at: z.string() })
+export const storagePublicPrefixesSchema = z.object({
+  ok: z.literal('STORAGE_PUBLIC_PREFIXES_LISTED'),
+  public_prefixes: z.array(publicPrefixSchema),
+})
+export const storagePublicPrefixDeclaredSchema = publicPrefixSchema.extend({
+  ok: z.literal('STORAGE_PUBLIC_PREFIX_DECLARED'),
+  created: z.boolean(),
+})
+export const storagePublicPrefixRevokedSchema = z.object({
+  ok: z.literal('STORAGE_PUBLIC_PREFIX_REVOKED'),
+  prefix: prefixSchema,
+  deleted: z.boolean(),
+})
+
+export type StoragePublicPrefixes = z.infer<typeof storagePublicPrefixesSchema>
+export type StoragePublicPrefixDeclared = z.infer<typeof storagePublicPrefixDeclaredSchema>
+export type StoragePublicPrefixRevoked = z.infer<typeof storagePublicPrefixRevokedSchema>

package/src/storageResults.ts

@@ -0,0 +1,119 @@
+import type { StoredAsset } from './alphalib/types/storageAsset.ts'
+import type { Transloadit } from './Transloadit.ts'
+
+import { z } from 'zod'
+
+import { ApiError } from './ApiError.ts'
+import { robotFileServeInstructionsSchema } from './alphalib/types/robots/file-serve.ts'
+import { storedAssetSchema } from './alphalib/types/storageAsset.ts'
+import InconsistentResponseError from './InconsistentResponseError.ts'
+
+/** Original-byte delivery, not a preview or transcode; authorize the application's asset first. */
+export interface StoredAssetUrlOptions {
+  /** True uses the receipt filename; a string supplies a different attachment filename. */
+  download?: boolean | string
+  /** Maximum signed lifetime, from one second to 48 hours. Defaults to five minutes. */
+  lifetimeMs?: number
+}
+
+/** Signs a retained original, including non-image media, without fetching or proxying its bytes. */
+export function getStoredAssetUrl(
+  client: Transloadit,
+  input: StoredAsset,
+  options: StoredAssetUrlOptions = {},
+): string {
+  const asset = storedAssetSchema.parse(input)
+  const lifetimeMs = options.lifetimeMs ?? 5 * 60_000
+  if (!Number.isSafeInteger(lifetimeMs) || lifetimeMs < 1000 || lifetimeMs > 48 * 3600_000) {
+    throw new RangeError('lifetimeMs must be an integer between 1000 and 172800000')
+  }
+  const download = options.download === true ? asset.path.split('/').at(-1) : options.download
+  if (download !== undefined && download !== false) {
+    robotFileServeInstructionsSchema.shape.download_name.unwrap().min(1).parse(download)
+    if (typeof download !== 'string' || /[\uD800-\uDFFF]/u.test(download))
+      throw new TypeError('download must contain a valid Unicode filename')
+  }
+  // Bunny includes the full query in its cache key. Rotate at most once a minute, never at
+  // the lifetime boundary where rounding could issue an almost-expired grant.
+  const rotationMs = Math.min(60_000, Math.floor(lifetimeMs / 2))
+  const expiresAt = Math.floor(Date.now() / rotationMs) * rotationMs + lifetimeMs
+  return client.getSignedSmartCDNUrl({
+    workspace: asset.workspace,
+    template: 'builtin/storage-serve@0.0.3',
+    input: asset.asset_id,
+    urlParams: { v: asset.version_id, ...(typeof download === 'string' ? { download } : {}) },
+    expiresAt,
+  })
+}
+
+/** Server-owned Assembly identity and expected Workspace, not values from an untrusted notification. */
+export interface GetStoredAssemblyResultsOptions {
+  assemblyId: string
+  workspace: string
+}
+
+/** A canonical stored asset with the producing result's provenance; one input can have many outputs. */
+export interface StoredAssemblyResult {
+  assembly_id: string
+  step: string
+  result_id: string
+  original_id?: string | (string | null)[]
+  asset: StoredAsset
+}
+
+const requestSchema = z.object({
+  assemblyId: z.string().regex(/^[A-Za-z0-9_-]+$/),
+  workspace: z.string().min(1),
+})
+const resultSchema = storedAssetSchema.extend({
+  id: z.string().min(1),
+  original_id: z.union([z.string(), z.array(z.string().nullable())]).optional(),
+})
+
+/**
+ * Fetches a completed Assembly and verifies all retained results, including non-image media.
+ * The caller must first bind the Assembly to an authorized application upload. Register results
+ * idempotently by Assembly/step/result ID (or asset/version), not by filename or notification count.
+ */
+export async function getStoredAssemblyResults(
+  client: Transloadit,
+  options: GetStoredAssemblyResultsOptions,
+): Promise<StoredAssemblyResult[]> {
+  const { assemblyId, workspace } = requestSchema.parse(options)
+  const assembly = await client.getAssembly(assemblyId)
+  if (assembly.assembly_id !== assemblyId) {
+    throw new InconsistentResponseError('The response did not match the requested Assembly')
+  }
+  if (typeof assembly.error === 'string') throw new ApiError({ body: assembly })
+  if (assembly.ok !== 'ASSEMBLY_COMPLETED') {
+    throw new InconsistentResponseError(`The Storage Assembly is not complete (${assembly.ok})`)
+  }
+  if (assembly.results === undefined) {
+    throw new InconsistentResponseError('The completed Assembly did not contain results')
+  }
+  const results: StoredAssemblyResult[] = []
+  for (const [step, files] of Object.entries(assembly.results)) {
+    for (const file of files) {
+      // Ordinary temporary/exported outputs are not Storage receipts. Partial Storage records
+      // must fail validation rather than disappearing from an otherwise successful batch.
+      if (
+        file.asset_id === undefined &&
+        file.version_id === undefined &&
+        file.workspace === undefined
+      )
+        continue
+      const parsed = resultSchema.safeParse(file)
+      if (!parsed.success || parsed.data.workspace !== workspace) {
+        throw new InconsistentResponseError(
+          'The Assembly returned an invalid or cross-Workspace Storage result',
+          {
+            cause: { assemblyId, step },
+          },
+        )
+      }
+      const { id, original_id, ...asset } = parsed.data
+      results.push({ assembly_id: assemblyId, step, result_id: id, original_id, asset })
+    }
+  }
+  return results
+}