was-teaching-server 0.9.1 → 0.10.0

package/src/backends/postgresSchema.ts

@@ -21,7 +21,7 @@ import type pg from 'pg'
  * never edit an applied entry.
  */
 export const MIGRATIONS: string[] = [
-  // v1: the full WAS + WebKMS surface.
+  // v1: the full WAS + WebKMS + chunked-storage surface.
   `
   -- The Spaces tree. 'description' is NULL for a placeholder row created by a
   -- write below the Space level (a resource/policy write to a Space whose
@@ -29,17 +29,29 @@ export const MIGRATIONS: string[] = [
   -- without a '.space.' file; getSpaceDescription treats it as absent.
   -- 'usage_bytes' is the transactional quota counter (spec "Quotas"),
   -- maintained in the same transaction as every content write and delete.
+  -- 'controller' denormalizes the Space controller (also present as
+  -- description->>'controller') onto its own indexed column, keeping the
+  -- per-controller COUNT(*) for the Spaces count quota cheap and lockable;
+  -- writeSpace maintains it on every insert and update.
   CREATE TABLE spaces (
     space_id    text COLLATE "C" PRIMARY KEY,
     description jsonb,
-    usage_bytes bigint NOT NULL DEFAULT 0
+    usage_bytes bigint NOT NULL DEFAULT 0,
+    controller  text
   );
+  CREATE INDEX spaces_controller_idx ON spaces (controller);
 
+  -- 'description_version' is the monotonic Collection Description version --
+  -- the ETag validator behind conditional (If-Match) Collection Description
+  -- writes, so concurrent recipient edits compare-and-swap instead of
+  -- clobbering. Kept out of the stored 'description' jsonb (it travels only
+  -- as the ETag header); writeCollection bumps it on every write.
   CREATE TABLE collections (
-    space_id      text COLLATE "C" NOT NULL
-                  REFERENCES spaces ON DELETE CASCADE,
-    collection_id text COLLATE "C" NOT NULL,
-    description   jsonb,
+    space_id            text COLLATE "C" NOT NULL
+                        REFERENCES spaces ON DELETE CASCADE,
+    collection_id       text COLLATE "C" NOT NULL,
+    description         jsonb,
+    description_version integer NOT NULL DEFAULT 1,
     PRIMARY KEY (space_id, collection_id)
   );
 
@@ -49,6 +61,15 @@ export const MIGRATIONS: string[] = [
   -- 'content_type' records the last-known type on a tombstone. 'version' /
   -- 'meta_version' are the two ETag validators; 'custom' is the user-writable
   -- metadata (or the opaque encryption envelope on an encrypted Collection).
+  -- 'created_by' is the Resource's creator -- the DID of the invoker of its
+  -- FIRST content write, set once and preserved verbatim thereafter (the
+  -- spec's OPTIONAL 'createdBy'); NULL for a row written by a caller with no
+  -- resolved invoker. 'epoch' is the client-declared key epoch the content
+  -- was encrypted under (multi-recipient encrypted Collections), stored
+  -- opaquely (the server never computes or verifies it); NULL when no epoch
+  -- was declared (a plaintext Collection, or an encrypted write that omitted
+  -- the stamp). Neither is COLLATE "C": unlike 'created_at' / 'updated_at',
+  -- they never participate in an ORDER BY or keyset comparison.
   CREATE TABLE resources (
     space_id      text COLLATE "C" NOT NULL,
     collection_id text COLLATE "C" NOT NULL,
@@ -63,6 +84,8 @@ export const MIGRATIONS: string[] = [
     deleted       boolean NOT NULL DEFAULT false,
     created_at    text COLLATE "C" NOT NULL,
     updated_at    text COLLATE "C" NOT NULL,
+    created_by    text,
+    epoch         text,
     PRIMARY KEY (space_id, collection_id, resource_id),
     FOREIGN KEY (space_id, collection_id)
       REFERENCES collections ON DELETE CASCADE
@@ -73,6 +96,30 @@ export const MIGRATIONS: string[] = [
   CREATE INDEX resources_changes_idx
     ON resources (space_id, collection_id, updated_at, resource_id);
 
+  -- Chunk storage for chunked Resources. One row per addressed chunk
+  -- (space, collection, resource, index); 'bytes' is the opaque chunk
+  -- representation (stored exactly like a binary Resource's content, never
+  -- parsed), 'size' its byte length (the quota-counter input), and 'version'
+  -- the chunk's own monotonic ETag validator (independent of the parent
+  -- Resource's). The foreign key to 'resources' gives chunk rows the same
+  -- ON DELETE CASCADE the Space/Collection tree already uses, so a HARD
+  -- delete of the parent Resource (or its Collection or Space) removes its
+  -- chunks with it; a SOFT delete (the tombstone UPDATE in deleteResource)
+  -- removes them explicitly in the same transaction instead.
+  CREATE TABLE chunks (
+    space_id      text COLLATE "C" NOT NULL,
+    collection_id text COLLATE "C" NOT NULL,
+    resource_id   text COLLATE "C" NOT NULL,
+    chunk_index   integer NOT NULL,
+    content_type  text NOT NULL,
+    bytes         bytea NOT NULL,
+    size          bigint NOT NULL DEFAULT 0,
+    version       integer NOT NULL,
+    PRIMARY KEY (space_id, collection_id, resource_id, chunk_index),
+    FOREIGN KEY (space_id, collection_id, resource_id)
+      REFERENCES resources ON DELETE CASCADE
+  );
+
   -- Policies for all three levels in one table; '' sentinel columns keep the
   -- primary key total (Postgres PKs reject NULL). Space policy: ('', '');
   -- collection policy: (cid, ''); resource policy: (cid, rid).
@@ -94,6 +141,22 @@ export const MIGRATIONS: string[] = [
     PRIMARY KEY (space_id, backend_id)
   );
 
+  -- Space-scoped zcap revocations -- the WAS route families' sibling of the
+  -- keystore-scoped 'revocations' table below. A separate table (rather than
+  -- a nullable-FK union over one table) lets each keep its own
+  -- ON DELETE CASCADE to its parent and its own composite primary key;
+  -- deleting a Space here deletes its revocations. Columns mirror
+  -- 'revocations', with 'space_id' referencing the spaces tree in place of
+  -- 'keystore_id'.
+  CREATE TABLE space_revocations (
+    space_id      text COLLATE "C" NOT NULL REFERENCES spaces ON DELETE CASCADE,
+    delegator     text COLLATE "C" NOT NULL,
+    capability_id text COLLATE "C" NOT NULL,
+    record        jsonb NOT NULL,
+    expires       text COLLATE "C",
+    PRIMARY KEY (space_id, delegator, capability_id)
+  );
+
   -- WebKMS facet: a sibling tree to spaces, exactly as on the filesystem.
   -- 'controller' / 'sequence' / 'kms_module' are denormalized from the
   -- verbatim config for the list filter and the update gates.
@@ -126,57 +189,6 @@ export const MIGRATIONS: string[] = [
     expires       text COLLATE "C",
     PRIMARY KEY (keystore_id, delegator, capability_id)
   );
-  `,
-  // v2: denormalize a Space's controller onto its own column for the Spaces
-  // count quota (spec "Quotas"). The controller also lives inside 'description'
-  // (description->>'controller'), but a dedicated, indexed column keeps the
-  // per-controller COUNT(*) cheap and lockable. Backfill from existing rows;
-  // writeSpace maintains it on every insert and update.
-  `
-  ALTER TABLE spaces ADD COLUMN controller text;
-  UPDATE spaces SET controller = description->>'controller';
-  CREATE INDEX spaces_controller_idx ON spaces (controller);
-  `,
-  // v3: record a Resource's creator -- the DID of the invoker of its FIRST
-  // content write, set once and preserved verbatim thereafter (the spec's
-  // OPTIONAL `createdBy`). NULL for a pre-existing row and for one written by
-  // a caller with no resolved invoker. Not `COLLATE "C"`: unlike `created_at` /
-  // `updated_at`, this column never participates in an `ORDER BY` or keyset
-  // comparison.
-  `
-  ALTER TABLE resources ADD COLUMN created_by text;
-  `,
-  // v4: Space-scoped zcap revocations -- the WAS route families' sibling of the
-  // keystore-scoped `revocations` table above. A separate table (rather than a
-  // nullable-FK union over one table) lets each keep its own `ON DELETE CASCADE`
-  // to its parent and its own composite primary key; deleting a Space here
-  // deletes its revocations. Columns mirror `revocations`, with 'space_id'
-  // referencing the spaces tree in place of 'keystore_id'.
-  `
-  CREATE TABLE space_revocations (
-    space_id      text COLLATE "C" NOT NULL REFERENCES spaces ON DELETE CASCADE,
-    delegator     text COLLATE "C" NOT NULL,
-    capability_id text COLLATE "C" NOT NULL,
-    record        jsonb NOT NULL,
-    expires       text COLLATE "C",
-    PRIMARY KEY (space_id, delegator, capability_id)
-  );
-  `,
-  // v5: the client-declared key epoch a Resource's content was encrypted under
-  // (multi-recipient encrypted Collections). Stored opaquely: the server never
-  // computes or verifies it. NULL when no epoch was declared (a plaintext
-  // Collection, or an encrypted write that omitted the stamp). Not `COLLATE "C"`
-  // -- it never participates in an `ORDER BY` or keyset comparison.
-  `
-  ALTER TABLE resources ADD COLUMN epoch text;
-  `,
-  // v6: the monotonic Collection Description version -- the ETag validator behind
-  // conditional (`If-Match`) Collection Description writes, so concurrent
-  // recipient edits compare-and-swap instead of clobbering. Kept out of the
-  // stored 'description' jsonb (it travels only as the `ETag` header). Backfill
-  // existing rows to 1; writeCollection bumps it on every write.
-  `
-  ALTER TABLE collections ADD COLUMN description_version integer NOT NULL DEFAULT 1;
   `
 ]
 

package/src/errors.ts

@@ -124,6 +124,25 @@ export class InvalidResourceIdError extends ProblemError {
   }
 }
 
+/**
+ * 400 — the `:chunkIndex` path param of a chunk operation (the
+ * `chunked-streams` feature) is not a canonical non-negative integer. The
+ * canonical-form requirement (no leading zeros, no sign) keeps every chunk
+ * addressable at exactly one URL.
+ * @param options {object}
+ * @param [options.requestName] {string}   request name used in the error title
+ */
+export class InvalidChunkIndexError extends ProblemError {
+  constructor({ requestName }: { requestName?: string } = {}) {
+    super({
+      type: ProblemTypes.INVALID_ID,
+      title: `Invalid ${requestName || 'Chunk'} request`,
+      detail: 'Invalid chunk index (must be a canonical non-negative integer).',
+      statusCode: 400
+    })
+  }
+}
+
 /**
  * 409 — a `POST` create operation supplied an `id` that already exists.
  * Create-or-replace at a client-chosen id is the idempotent `PUT` path, which

package/src/lib/importTar.ts

@@ -2,7 +2,11 @@ import * as tar from 'tar-stream'
 import YAML from 'yaml'
 import type { Readable } from 'node:stream'
 import { assertValidId } from './validateId.js'
-import { parseResourceFileName } from './resourceFileName.js'
+import {
+  parseChunkDirName,
+  parseChunkIndexSegment,
+  parseResourceFileName
+} from './resourceFileName.js'
 import { InvalidImportError } from '../errors.js'
 import type {
   CollectionDescription,
@@ -53,6 +57,60 @@ export function metaSidecarFileId(fileName: string): string | undefined {
   return dotFileId(fileName, META_PREFIX)
 }
 
+/**
+ * If `fileName` is a chunk entry relative to its Collection dir
+ * (`.chunks.<encodedResourceId>/<chunkFile>`, the `chunked-streams` feature),
+ * returns the parent `resourceId` (decoded from the directory name) and the
+ * `chunkFileName` (its single-segment basename inside the chunk directory);
+ * otherwise undefined. Any deeper nesting or a traversal segment in the chunk
+ * file name is rejected (undefined), so the caller never builds a path from it.
+ * @param fileName {string}   the entry name relative to the Collection dir
+ * @returns {{ resourceId: string, chunkFileName: string } | undefined}
+ */
+function chunkEntryName(
+  fileName: string
+): { resourceId: string; chunkFileName: string } | undefined {
+  const slash = fileName.indexOf('/')
+  if (slash === -1) {
+    return undefined
+  }
+  const resourceId = parseChunkDirName(fileName.slice(0, slash))
+  if (resourceId === undefined) {
+    return undefined
+  }
+  const chunkFileName = fileName.slice(slash + 1)
+  if (
+    chunkFileName.length === 0 ||
+    chunkFileName.includes('/') ||
+    chunkFileName === '..'
+  ) {
+    return undefined
+  }
+  return { resourceId, chunkFileName }
+}
+
+/**
+ * The canonical-chunk-index gate for an archive chunk file: returns true only
+ * when `chunkFileName` is a chunk representation (`r.<index>.<encType>.<ext>`)
+ * or its version sidecar (`.meta.<index>.json`) whose RAW `<index>` segment
+ * passes {@link parseChunkIndexSegment} -- the same predicate the live route
+ * enforces. Validating the raw (undecoded) segment rejects both non-canonical
+ * spellings (`r.01.*`, which would alias chunk 1) and percent-encoded ones
+ * (`r.%31.*`), so an imported chunk file is always reachable at exactly the
+ * URL the listing advertises. Anything else in a chunk directory is dropped.
+ * @param chunkFileName {string}   the file's basename inside the chunk dir
+ * @returns {boolean}
+ */
+function isCanonicalChunkFileName(chunkFileName: string): boolean {
+  const indexSegment =
+    metaSidecarFileId(chunkFileName) ??
+    (chunkFileName.startsWith('r.') ? chunkFileName.split('.')[1] : undefined)
+  return (
+    indexSegment !== undefined &&
+    parseChunkIndexSegment(indexSegment) !== undefined
+  )
+}
+
 /** One extracted archive entry, keyed by its archive path. */
 export interface TarEntry {
   type: 'file' | 'directory'
@@ -66,6 +124,19 @@ export interface ImportPlanResource {
   body: Buffer
 }
 
+/**
+ * One chunk file of a chunked Resource staged for import (the `chunked-streams`
+ * feature): the raw bytes carried verbatim, keyed by the parent `resourceId`
+ * (decoded from the `.chunks.<encId>/` directory name) and the chunk file's
+ * basename inside that directory (`r.<index>...` bytes or its `.meta.<index>.json`
+ * version sidecar).
+ */
+export interface ImportPlanChunkFile {
+  resourceId: string
+  fileName: string
+  body: Buffer
+}
+
 /** One collection (plus its resources and policies) staged for import. */
 export interface ImportPlanCollection {
   collectionId: string
@@ -77,6 +148,8 @@ export interface ImportPlanCollection {
   resourcePolicies: Map<string, PolicyDocument>
   /** Resource metadata sidecars (raw `.meta.<id>.json` bytes), keyed by resourceId. */
   resourceMetadata: Map<string, Buffer>
+  /** Chunk files of chunked Resources in this Collection, carried verbatim. */
+  chunkFiles: ImportPlanChunkFile[]
 }
 
 /** The merge plan produced by {@link buildImportPlan}. */
@@ -185,10 +258,12 @@ export function validateManifest(entries: Map<string, TarEntry>): void {
  * - space/<sourceSpaceId>/<collectionId>/.policy.<collectionId>.json (collection policy)
  * - space/<sourceSpaceId>/<collectionId>/.policy.<resourceId>.json (resource policy)
  * - space/<sourceSpaceId>/<collectionId>/r.<resourceId>.<encodedContentType>.<ext>
+ * - space/<sourceSpaceId>/<collectionId>/.chunks.<encodedResourceId>/<chunkFile>
+ *   (a chunked Resource's chunk files; the `chunked-streams` feature)
  *
  * The source space id in the path may differ from the import target; collection
- * metadata, r.* resource files, and `.policy.*` policy files are merged into the
- * destination.
+ * metadata, r.* resource files, `.chunks.*` chunk files, and `.policy.*` policy
+ * files are merged into the destination.
  *
  * @param entries {Map<string, TarEntry>}
  * @returns {ImportPlan}
@@ -246,6 +321,7 @@ export function buildImportPlan(entries: Map<string, TarEntry>): ImportPlan {
     let collectionPolicy: PolicyDocument | undefined
     const resourcePolicies = new Map<string, PolicyDocument>()
     const resourceMetadata = new Map<string, Buffer>()
+    const chunkFiles: ImportPlanChunkFile[] = []
     for (const [entryName, entry] of entries) {
       if (
         !entryName.startsWith(collectionPrefix) ||
@@ -256,6 +332,32 @@ export function buildImportPlan(entries: Map<string, TarEntry>): ImportPlan {
       }
 
       const fileName = entryName.slice(collectionPrefix.length)
+
+      // Chunk files of a chunked Resource live in a per-Resource subdirectory
+      // (`.chunks.<encId>/<chunkFile>`), so they are the one Collection entry
+      // carrying a slash. Carry them verbatim, keyed by the parent resourceId
+      // decoded from the directory name (validated against path traversal).
+      const chunk = chunkEntryName(fileName)
+      if (chunk) {
+        assertValidId(chunk.resourceId, {
+          kind: 'resource',
+          requestName: 'Import Space'
+        })
+        // Drop a chunk file with a non-canonical index (or a stray file that
+        // is neither a representation nor a sidecar): written verbatim it
+        // would be unreachable at the canonical member URL yet advertised by
+        // the listing.
+        if (!isCanonicalChunkFileName(chunk.chunkFileName)) {
+          continue
+        }
+        chunkFiles.push({
+          resourceId: chunk.resourceId,
+          fileName: chunk.chunkFileName,
+          body: entry.body
+        })
+        continue
+      }
+
       if (fileName.includes('/')) {
         continue
       }
@@ -325,7 +427,8 @@ export function buildImportPlan(entries: Map<string, TarEntry>): ImportPlan {
       collectionPolicy,
       resources,
       resourcePolicies,
-      resourceMetadata
+      resourceMetadata,
+      chunkFiles
     }
   })
 

package/src/lib/paths.ts

@@ -209,6 +209,54 @@ export function metaPath({
   return `${resourcePath({ spaceId, collectionId, resourceId })}/meta`
 }
 
+/**
+ * `/space/:spaceId/:collectionId/:resourceId/chunks/:chunkIndex` -- a single
+ * chunk of a chunked Resource (the `chunked-streams` feature). Chunks are
+ * addressed under their parent Resource; like `meta`, the `chunks` segment
+ * needs no reserved-id entry (it sits a level below any Resource route).
+ * @param options {object}
+ * @param options.spaceId {string}
+ * @param options.collectionId {string}
+ * @param options.resourceId {string}
+ * @param options.chunkIndex {number}   non-negative integer chunk position
+ * @returns {string}
+ */
+export function chunkPath({
+  spaceId,
+  collectionId,
+  resourceId,
+  chunkIndex
+}: {
+  spaceId: string
+  collectionId: string
+  resourceId: string
+  chunkIndex: number
+}): string {
+  return `${resourcePath({ spaceId, collectionId, resourceId })}/chunks/${chunkIndex}`
+}
+
+/**
+ * `/space/:spaceId/:collectionId/:resourceId/chunks/` -- the chunk listing
+ * (container) path of a chunked Resource. Always the trailing-slash container
+ * form: a reader discovers the chunk count here before fetching `0..count-1`.
+ * @param options {object}
+ * @param options.spaceId {string}
+ * @param options.collectionId {string}
+ * @param options.resourceId {string}
+ * @returns {string}
+ */
+export function chunksContainerPath({
+  spaceId,
+  collectionId,
+  resourceId
+}: {
+  spaceId: string
+  collectionId: string
+  resourceId: string
+}): string {
+  return `${resourcePath({ spaceId, collectionId, resourceId })}/chunks/`
+}
+
 /**
  * `/space/:spaceId/:collectionId/backend` -- the "Collection Backend Selected"
  * resource path (reserved `backend` segment at the Resource level).

package/src/lib/resourceFileName.ts

@@ -67,3 +67,72 @@ export function parseResourceFileName(fileName: string): {
       : 'application/octet-stream'
   }
 }
+
+/**
+ * Prefix of a Resource's per-Resource chunk directory
+ * (`.chunks.<encodedResourceId>/`), which holds the chunk representations of a
+ * chunked Resource (the `chunked-streams` feature). The leading `.` keeps the
+ * directory out of the `r.`-prefixed Collection listing, and dot-escaping the
+ * id segment keeps it in one filesystem-name namespace with the Resource files.
+ */
+export const CHUNK_DIR_PREFIX = '.chunks.'
+
+/**
+ * Builds the on-disk directory name for a Resource's chunk directory:
+ * `.chunks.<encodedResourceId>`. The id segment is dot-escaped (see
+ * {@link encodeFilenameSegment}) so a dotted id round-trips.
+ * @param resourceId {string}
+ * @returns {string}
+ */
+export function chunkDirName(resourceId: string): string {
+  return `${CHUNK_DIR_PREFIX}${encodeFilenameSegment(resourceId)}`
+}
+
+/**
+ * Largest addressable chunk index. 2^31-1 (`int4` max): the Postgres backend
+ * stores the index in an `integer` column, so the shared validation caps it
+ * there and both backends agree on the addressable range.
+ */
+export const MAX_CHUNK_INDEX = 2 ** 31 - 1
+
+/**
+ * Canonical non-negative decimal integer: `0`, or a digit run with no leading
+ * zero. Rejecting non-canonical spellings (`01`, `+1`, `1e3`) keeps every chunk
+ * addressable at exactly one URL (and at exactly one archive file name).
+ */
+const CHUNK_INDEX_PATTERN = /^(0|[1-9][0-9]*)$/
+
+/**
+ * Parses a chunk-index segment -- the `:chunkIndex` path param, or the
+ * `<index>` segment of a chunk file name (`r.<index>.<encType>.<ext>` /
+ * `.meta.<index>.json`) -- into its number. Returns `undefined` unless the
+ * segment is the canonical decimal spelling of an integer in
+ * `[0, MAX_CHUNK_INDEX]`. The single shared predicate for the live route and
+ * both backends' import paths, so a chunk index means the same thing
+ * everywhere.
+ * @param segment {string}
+ * @returns {number | undefined}
+ */
+export function parseChunkIndexSegment(segment: string): number | undefined {
+  if (!CHUNK_INDEX_PATTERN.test(segment)) {
+    return undefined
+  }
+  const chunkIndex = Number(segment)
+  return chunkIndex <= MAX_CHUNK_INDEX ? chunkIndex : undefined
+}
+
+/**
+ * Parses a chunk directory name (`.chunks.<encodedResourceId>`) back into its
+ * parent `resourceId`, reversing {@link chunkDirName}. Returns `undefined` when
+ * the name is not a chunk directory (no matching prefix, or an empty id
+ * segment).
+ * @param dirName {string}   the basename of the directory
+ * @returns {string | undefined}
+ */
+export function parseChunkDirName(dirName: string): string | undefined {
+  if (!dirName.startsWith(CHUNK_DIR_PREFIX)) {
+    return undefined
+  }
+  const encodedId = dirName.slice(CHUNK_DIR_PREFIX.length)
+  return encodedId.length > 0 ? decodeURIComponent(encodedId) : undefined
+}