pipework 0.252.0 → 0.253.0

package/src/infra/storage/s3.ts

@@ -0,0 +1,257 @@
+import { Readable } from 'node:stream'
+import { createHash } from 'node:crypto'
+import { signV4, presignV4, type AwsCredentials } from '../aws/sigv4.js'
+import { awsCredentials } from '../aws/credentials.js'
+import { retryUnsent } from '../fetch/connect-retry.js'
+import { contentDisposition } from '../../request/http/file.js'
+import type { BackendObject, BackendSignedUrlOptions, StorageBackend } from './backend.js'
+
+export type S3Encryption =
+  | { readonly type: 'AES256' }
+  | { readonly type: 'aws:kms'; readonly keyId?: string }
+
+export interface S3Config {
+  readonly bucket: string
+  readonly region: string
+  /** Base URL of an S3-compatible server, e.g. `http://127.0.0.1:9000`. Default `https://s3.<region>.amazonaws.com`. */
+  readonly endpoint?: string
+  /** Address the bucket in the path (`<endpoint>/<bucket>/<key>`) rather than the host. Default: true when `endpoint` is given, false otherwise. */
+  readonly forcePathStyle?: boolean
+  /** Called for every request. Default: `awsCredentials()` (environment, ECS task role, EC2 instance role). */
+  readonly credentials?: () => Promise<AwsCredentials>
+  /** Prepended to every key, e.g. `verity/`. */
+  readonly prefix?: string
+  /** Server-side encryption, sent with every write (PutObject, CreateMultipartUpload, CopyObject). */
+  readonly encryption?: S3Encryption
+}
+
+/** Size of each multipart upload part; a body that fits in one is sent with a single PutObject. */
+export const PART_SIZE = 8 * 1024 * 1024
+const EMPTY_SHA256 = createHash('sha256').update('').digest('hex')
+const CONNECT_RETRY_DELAYS_MS = [250, 1_000, 3_000] as const
+
+/** The headers that ask S3 to encrypt a written object. */
+export function encryptionHeaders(encryption: S3Encryption | undefined): Record<string, string> {
+  if (encryption === undefined) return {}
+  if (encryption.type === 'AES256') return { 'x-amz-server-side-encryption': 'AES256' }
+  return {
+    'x-amz-server-side-encryption': 'aws:kms',
+    ...(encryption.keyId !== undefined ? { 'x-amz-server-side-encryption-aws-kms-key-id': encryption.keyId } : {}),
+  }
+}
+
+function encodePath(key: string): string {
+  return key.split('/').map(seg => encodeURIComponent(seg).replace(/[!'()*]/g, c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)).join('/')
+}
+
+function unescapeXml(s: string): string {
+  return s.replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, '&')
+}
+
+function xmlTag(xml: string, tag: string): string | null {
+  const m = new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`).exec(xml)
+  return m === null ? null : unescapeXml(m[1]!)
+}
+
+interface Call {
+  readonly method: string
+  readonly key?: string
+  readonly query?: Record<string, string>
+  readonly headers?: Record<string, string>
+  readonly body?: Buffer
+}
+
+/**
+ * S3-compatible storage (AWS S3, MinIO, R2, ...) over `fetch` with SigV4; no AWS SDK.
+ * Bodies are uploaded in 8 MiB parts (multipart) unless the whole body fits in one,
+ * and the signature does not cover the payload (`UNSIGNED-PAYLOAD`), so use `https`
+ * endpoints in production. A failed upload aborts its multipart upload.
+ */
+export function s3(config: S3Config): StorageBackend {
+  const getCredentials = config.credentials ?? awsCredentials()
+  const endpoint = (config.endpoint ?? `https://s3.${config.region}.amazonaws.com`).replace(/\/+$/, '')
+  const pathStyle = config.forcePathStyle ?? config.endpoint !== undefined
+  const prefix = config.prefix ?? ''
+  const write = encryptionHeaders(config.encryption)
+
+  function urlFor(key: string | undefined, query: Record<string, string> = {}): string {
+    const base = new URL(endpoint)
+    const objectPath = key === undefined ? '' : `/${encodePath(prefix + key)}`
+    let origin = base.origin
+    let path: string
+    if (pathStyle) path = `${base.pathname.replace(/\/+$/, '')}/${encodeURIComponent(config.bucket)}${objectPath}`
+    else {
+      origin = `${base.protocol}//${config.bucket}.${base.host}`
+      path = `${base.pathname.replace(/\/+$/, '')}${objectPath === '' ? '/' : objectPath}`
+    }
+    const qs = Object.entries(query).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&')
+    return `${origin}${path}${qs === '' ? '' : `?${qs}`}`
+  }
+
+  async function call(c: Call): Promise<Response> {
+    const credentials = await getCredentials()
+    const body = c.body ?? Buffer.alloc(0)
+    const payloadHash = c.body === undefined ? EMPTY_SHA256 : 'UNSIGNED-PAYLOAD'
+    const url = urlFor(c.key, c.query)
+    const headers = signV4({
+      method: c.method, url, region: config.region, service: 's3', credentials, body, payloadHash,
+      headers: { ...c.headers, 'x-amz-content-sha256': payloadHash },
+    })
+    const init: RequestInit = {
+      method: c.method, headers,
+      ...(c.body !== undefined ? { body: new Uint8Array(c.body.buffer, c.body.byteOffset, c.body.byteLength) } : {}),
+    }
+    // Only connect faults are retried, with short waits: storage calls run inside requests. The signature stays valid for the tries.
+    return retryUnsent(() => fetch(url, init), { delaysMs: CONNECT_RETRY_DELAYS_MS })
+  }
+
+  async function ok(op: string, res: Response, allowed: readonly number[] = []): Promise<string> {
+    const text = await res.text()
+    if ((res.status >= 200 && res.status < 300 && !(text.startsWith('<?xml') && /<Error>/.test(text))) || allowed.includes(res.status)) return text
+    const code = xmlTag(text, 'Code')
+    const message = xmlTag(text, 'Message')
+    throw new Error(
+      `[pipework] storage s3 ${op} failed: HTTP ${res.status}${code !== null ? ` ${code}` : ''}${message !== null ? `: ${message}` : ''}.\n`,
+    )
+  }
+
+  async function putWhole(key: string, body: Buffer): Promise<void> {
+    await ok('PutObject', await call({ method: 'PUT', key, body, headers: { 'content-type': 'application/octet-stream', ...write } }))
+  }
+
+  async function putMultipart(key: string, first: Buffer, rest: AsyncIterator<Buffer>, pending: Buffer[]): Promise<void> {
+    const created = await ok('CreateMultipartUpload', await call({
+      method: 'POST', key, query: { uploads: '' }, headers: { 'content-type': 'application/octet-stream', ...write },
+    }))
+    const uploadId = xmlTag(created, 'UploadId')
+    if (uploadId === null) throw new Error('[pipework] storage s3 CreateMultipartUpload returned no UploadId.\n')
+    const etags: string[] = []
+    try {
+      const sendPart = async (part: Buffer): Promise<void> => {
+        const number = etags.length + 1
+        const res = await call({ method: 'PUT', key, query: { partNumber: String(number), uploadId }, body: part })
+        await ok('UploadPart', res)
+        const etag = res.headers.get('etag')
+        if (etag === null) throw new Error('[pipework] storage s3 UploadPart returned no ETag.\n')
+        etags.push(etag)
+      }
+      await sendPart(first)
+      // `pending` holds what the first read left over beyond one part.
+      let buffered = pending
+      let size = buffered.reduce((n, b) => n + b.length, 0)
+      for (;;) {
+        while (size >= PART_SIZE) {
+          const all = Buffer.concat(buffered)
+          await sendPart(all.subarray(0, PART_SIZE))
+          const remainder = all.subarray(PART_SIZE)
+          buffered = remainder.length > 0 ? [remainder] : []
+          size = remainder.length
+        }
+        const next = await rest.next()
+        if (next.done === true) break
+        buffered.push(next.value)
+        size += next.value.length
+      }
+      if (size > 0) await sendPart(Buffer.concat(buffered))
+      const xml = '<CompleteMultipartUpload>' +
+        etags.map((e, i) => `<Part><PartNumber>${i + 1}</PartNumber><ETag>${e}</ETag></Part>`).join('') +
+        '</CompleteMultipartUpload>'
+      const payload = Buffer.from(xml, 'utf8')
+      await ok('CompleteMultipartUpload', await call({ method: 'POST', key, query: { uploadId }, body: payload, headers: { 'content-type': 'application/xml' } }))
+    } catch (err) {
+      try {
+        await ok('AbortMultipartUpload', await call({ method: 'DELETE', key, query: { uploadId } }), [404])
+      } catch {
+        // the original failure is the one to report
+      }
+      throw err
+    }
+  }
+
+  return {
+    name: 's3',
+
+    async put(key, body) {
+      const iterator = (async function* (): AsyncGenerator<Buffer> {
+        for await (const chunk of body) yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+      })()
+      const chunks: Buffer[] = []
+      let size = 0
+      try {
+        while (size < PART_SIZE) {
+          const next = await iterator.next()
+          if (next.done === true) {
+            await putWhole(key, Buffer.concat(chunks))
+            return
+          }
+          chunks.push(next.value)
+          size += next.value.length
+        }
+      } catch (err) {
+        await iterator.return(undefined).catch(() => undefined)
+        throw err
+      }
+      const all = Buffer.concat(chunks)
+      const rest = all.length > PART_SIZE ? [all.subarray(PART_SIZE)] : []
+      try {
+        await putMultipart(key, all.subarray(0, PART_SIZE), iterator, rest)
+      } finally {
+        await iterator.return(undefined).catch(() => undefined)
+      }
+    },
+
+    async exists(key) {
+      const res = await call({ method: 'HEAD', key })
+      await res.arrayBuffer()
+      if (res.status === 404) return false
+      if (res.status >= 200 && res.status < 300) return true
+      throw new Error(`[pipework] storage s3 HeadObject failed: HTTP ${res.status}.\n`)
+    },
+
+    async copy(from, to) {
+      const source = `/${encodePath(config.bucket)}/${encodePath(prefix + from)}`
+      await ok('CopyObject', await call({ method: 'PUT', key: to, headers: { 'x-amz-copy-source': source, ...write } }))
+    },
+
+    async remove(key) {
+      await ok('DeleteObject', await call({ method: 'DELETE', key }), [404])
+    },
+
+    async read(key) {
+      const res = await call({ method: 'GET', key })
+      if (res.status < 200 || res.status >= 300 || res.body === null) {
+        await ok('GetObject', res)
+        throw new Error('[pipework] storage s3 GetObject returned no body.\n')
+      }
+      return Readable.fromWeb(res.body as never)
+    },
+
+    async *list(listPrefix): AsyncGenerator<BackendObject> {
+      let token: string | undefined
+      do {
+        const query: Record<string, string> = { 'list-type': '2', prefix: prefix + listPrefix }
+        if (token !== undefined) query['continuation-token'] = token
+        const xml = await ok('ListObjectsV2', await call({ method: 'GET', query }))
+        for (const m of xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)) {
+          const key = xmlTag(m[1]!, 'Key')
+          const modified = xmlTag(m[1]!, 'LastModified')
+          if (key === null || modified === null || !key.startsWith(prefix)) continue
+          yield { key: key.slice(prefix.length), lastModified: new Date(modified) }
+        }
+        token = xmlTag(xml, 'IsTruncated') === 'true' ? (xmlTag(xml, 'NextContinuationToken') ?? undefined) : undefined
+      } while (token !== undefined)
+    },
+
+    async signedUrl(key: string, options: BackendSignedUrlOptions) {
+      const credentials = await getCredentials()
+      return presignV4({
+        method: 'GET',
+        url: urlFor(key, {
+          'response-content-disposition': contentDisposition(options.disposition, options.filename),
+          'response-content-type': options.mediaType,
+        }),
+        region: config.region, service: 's3', credentials, expiresInSeconds: options.expiresInSeconds,
+      })
+    },
+  }
+}

package/src/infra/storage/schema.ts

@@ -0,0 +1,24 @@
+// The table behind file storage: one metadata row per stored file, tenant-scoped
+// (forced row-level security in generated migrations). The bytes live in the backend.
+
+import { define } from '../../data/domain/index.js'
+import { field } from '../../data/domain/field.js'
+import { schema } from '../../data/schema/namespace.js'
+import { index } from '../../data/db/idx/index.js'
+
+export const storageFile = define('storage_file', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  tenantId: field.uuid().tenant(),
+  sha256: field.text(),
+  size: field.bigint(),
+  mediaType: field.text(),
+  filename: field.text().nullable(),
+  createdAt: field.timestamp().default('now'),
+  createdBy: field.text().nullable(),
+  expiresAt: field.timestamp().nullable(),
+  metadata: field.jsonb(schema.check.json()).nullable(),
+}, {
+  indexes: (t) => ({
+    bySha: index('idx_storage_file_sha').on(t['tenantId'], t['sha256']),
+  }),
+})