was-teaching-server 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +50 -1
  2. package/dist/backends/filesystem.d.ts +161 -2
  3. package/dist/backends/filesystem.d.ts.map +1 -1
  4. package/dist/backends/filesystem.js +491 -60
  5. package/dist/backends/filesystem.js.map +1 -1
  6. package/dist/backends/postgres.d.ts +166 -0
  7. package/dist/backends/postgres.d.ts.map +1 -1
  8. package/dist/backends/postgres.js +625 -29
  9. package/dist/backends/postgres.js.map +1 -1
  10. package/dist/backends/postgresSchema.d.ts.map +1 -1
  11. package/dist/backends/postgresSchema.js +25 -0
  12. package/dist/backends/postgresSchema.js.map +1 -1
  13. package/dist/errors.d.ts +13 -0
  14. package/dist/errors.d.ts.map +1 -1
  15. package/dist/errors.js +18 -0
  16. package/dist/errors.js.map +1 -1
  17. package/dist/lib/exportManifest.d.ts +9 -0
  18. package/dist/lib/exportManifest.d.ts.map +1 -1
  19. package/dist/lib/exportManifest.js +9 -0
  20. package/dist/lib/exportManifest.js.map +1 -1
  21. package/dist/lib/importTar.d.ts +18 -2
  22. package/dist/lib/importTar.d.ts.map +1 -1
  23. package/dist/lib/importTar.js +77 -4
  24. package/dist/lib/importTar.js.map +1 -1
  25. package/dist/lib/paths.d.ts +33 -0
  26. package/dist/lib/paths.d.ts.map +1 -1
  27. package/dist/lib/paths.js +28 -0
  28. package/dist/lib/paths.js.map +1 -1
  29. package/dist/lib/resourceFileName.d.ts +43 -0
  30. package/dist/lib/resourceFileName.d.ts.map +1 -1
  31. package/dist/lib/resourceFileName.js +63 -0
  32. package/dist/lib/resourceFileName.js.map +1 -1
  33. package/dist/requests/ChunkRequest.d.ts +104 -0
  34. package/dist/requests/ChunkRequest.d.ts.map +1 -0
  35. package/dist/requests/ChunkRequest.js +396 -0
  36. package/dist/requests/ChunkRequest.js.map +1 -0
  37. package/dist/routes.d.ts.map +1 -1
  38. package/dist/routes.js +21 -0
  39. package/dist/routes.js.map +1 -1
  40. package/dist/types.d.ts +81 -1
  41. package/dist/types.d.ts.map +1 -1
  42. package/package.json +1 -1
  43. package/src/backends/filesystem.ts +638 -57
  44. package/src/backends/postgres.ts +867 -21
  45. package/src/backends/postgresSchema.ts +69 -57
  46. package/src/errors.ts +19 -0
  47. package/src/lib/importTar.ts +107 -4
  48. package/src/lib/paths.ts +48 -0
  49. package/src/lib/resourceFileName.ts +69 -0
  50. package/src/requests/ChunkRequest.ts +478 -0
  51. package/src/routes.ts +46 -0
  52. package/src/types.ts +78 -1
@@ -19,12 +19,12 @@ import YAML from 'yaml';
19
19
  import { DEFAULT_MAX_UPLOAD_BYTES, DEFAULT_MAX_SPACES_PER_CONTROLLER, DEFAULT_MAX_COLLECTIONS_PER_SPACE, DEFAULT_MAX_RESOURCES_PER_SPACE, QUOTA_USAGE_CACHE_TTL, normalizeCountLimit } from '../config.default.js';
20
20
  import { extractTarEntries, buildImportPlan, metaSidecarFileId } from '../lib/importTar.js';
21
21
  import { collectionPath, resourcePath } from '../lib/paths.js';
22
- import { encodeFilenameSegment, fileNameFor, parseResourceFileName } from '../lib/resourceFileName.js';
22
+ import { encodeFilenameSegment, fileNameFor, parseResourceFileName, chunkDirName, CHUNK_DIR_PREFIX } from '../lib/resourceFileName.js';
23
23
  import { sanitizeBackendRecord } from '../lib/backends.js';
24
24
  import { backendUsageFields } from '../lib/backendUsage.js';
25
25
  import { assertEncryptedWriteConforms } from '../lib/encryption.js';
26
26
  import { encodeCursor, decodeCursor } from '../lib/cursor.js';
27
- import { buildExportManifest } from '../lib/exportManifest.js';
27
+ import { buildExportManifest, EXPORT_ENTRY_MTIME } from '../lib/exportManifest.js';
28
28
  import { revocationFileName } from '../lib/revocations.js';
29
29
  import { KeyedMutex } from '../lib/keyedMutex.js';
30
30
  import { isJson } from '../lib/isJson.js';
@@ -203,8 +203,9 @@ export class FileSystemBackend {
203
203
  * It advertises the `conditional-writes` affordance: it exposes a per-Resource
204
204
  * `version` as an HTTP `ETag` validator and honors `If-Match` / `If-None-Match`
205
205
  * write preconditions atomically (returning `412 precondition-failed` on a
206
- * mismatch). The remaining `features` vocabulary token (`chunked-streams`)
207
- * is not implemented yet and is added when it lands.
206
+ * mismatch). It also advertises `chunked-streams`: chunk addressing for large
207
+ * Resources (`/{resourceId}/chunks/{n}`), each chunk stored opaquely like a
208
+ * binary Resource representation.
208
209
  * (Client-side encryption is deliberately not a backend feature: encrypted
209
210
  * documents are opaque client-encrypted JSON this backend already stores
210
211
  * faithfully, with no server cooperation.)
@@ -229,11 +230,14 @@ export class FileSystemBackend {
229
230
  // `key-epochs`: multi-recipient encrypted Collections -- per-epoch wrapped
230
231
  // keys on the `encryption` marker, a client-declared `epoch` stamp on
231
232
  // Resources, and conditional (`If-Match`) Collection Description writes.
233
+ // `chunked-streams`: chunk addressing (`/{resourceId}/chunks/{n}`) for a
234
+ // large Resource, each chunk stored opaquely (bytes + content-type).
232
235
  features: [
233
236
  'conditional-writes',
234
237
  'changes-query',
235
238
  'blinded-index-query',
236
- 'key-epochs'
239
+ 'key-epochs',
240
+ 'chunked-streams'
237
241
  ]
238
242
  };
239
243
  }
@@ -812,6 +816,11 @@ export class FileSystemBackend {
812
816
  entry => !(entry.isFile() && entry.name.startsWith('.backend.')));
813
817
  spaceEntries.sort((a, b) => a.name.localeCompare(b.name));
814
818
  const collectionEntriesByDir = {};
819
+ // Per-Resource chunk directories (`.chunks.<encId>/`; the `chunked-streams`
820
+ // feature) are subdirectories of a Collection dir, so the file filter below
821
+ // skips them. Gather each one's files here so a chunked Resource's chunks
822
+ // travel in the export, keyed by Collection then by chunk-directory name.
823
+ const chunkDirsByCollection = {};
815
824
  for (const entry of spaceEntries) {
816
825
  if (!entry.isDirectory()) {
817
826
  continue;
@@ -820,6 +829,17 @@ export class FileSystemBackend {
820
829
  collectionEntriesByDir[entry.name] = entries
821
830
  .filter(e => e.isFile())
822
831
  .sort((a, b) => a.name.localeCompare(b.name));
832
+ const chunkDirs = [];
833
+ for (const sub of entries
834
+ .filter(e => e.isDirectory() && e.name.startsWith(CHUNK_DIR_PREFIX))
835
+ .sort((a, b) => a.name.localeCompare(b.name))) {
836
+ const files = (await fs.promises.readdir(path.join(sourceSpaceDir, entry.name, sub.name), { withFileTypes: true }))
837
+ .filter(e => e.isFile())
838
+ .map(e => e.name)
839
+ .sort((a, b) => a.localeCompare(b));
840
+ chunkDirs.push({ dirName: sub.name, files });
841
+ }
842
+ chunkDirsByCollection[entry.name] = chunkDirs;
823
843
  }
824
844
  // Space-scoped zcap revocations travel with the export. They live in the
825
845
  // sibling `spaceRevocationsDir` root (see that property's doc), so the
@@ -844,35 +864,58 @@ export class FileSystemBackend {
844
864
  entries: spaceEntries.map(entry => entry.isDirectory()
845
865
  ? {
846
866
  name: entry.name,
847
- files: (collectionEntriesByDir[entry.name] ?? []).map(file => file.name)
867
+ // The Collection's own files, then each chunked Resource's chunk
868
+ // files listed by their `.chunks.<encId>/<file>` relative path
869
+ // (the manifest mirrors the pack order below).
870
+ files: [
871
+ ...(collectionEntriesByDir[entry.name] ?? []).map(file => file.name),
872
+ ...(chunkDirsByCollection[entry.name] ?? []).flatMap(({ dirName, files }) => files.map(file => `${dirName}/${file}`))
873
+ ]
848
874
  }
849
875
  : // top-level files in space (e.g. .space.<spaceId>.json)
850
876
  { name: entry.name }),
851
877
  revocationFiles
852
878
  });
879
+ // Fixed mtime on every entry so the archive is byte-reproducible (see
880
+ // EXPORT_ENTRY_MTIME).
881
+ const mtime = EXPORT_ENTRY_MTIME;
853
882
  const pack = tar.pack();
854
- pack.entry({ name: 'manifest.yml' }, YAML.stringify(manifest));
855
- pack.entry({ name: 'space/', type: 'directory' });
856
- pack.entry({ name: `space/${spaceId}/`, type: 'directory' });
883
+ pack.entry({ name: 'manifest.yml', mtime }, YAML.stringify(manifest));
884
+ pack.entry({ name: 'space/', type: 'directory', mtime });
885
+ pack.entry({ name: `space/${spaceId}/`, type: 'directory', mtime });
857
886
  for (const entry of spaceEntries) {
858
887
  const entryTarget = `space/${spaceId}/${entry.name}`;
859
888
  if (entry.isDirectory()) {
860
- pack.entry({ name: `${entryTarget}/`, type: 'directory' });
889
+ pack.entry({ name: `${entryTarget}/`, type: 'directory', mtime });
861
890
  for (const file of collectionEntriesByDir[entry.name] ?? []) {
862
891
  const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name, file.name));
863
- pack.entry({ name: `${entryTarget}/${file.name}` }, bytes);
892
+ pack.entry({ name: `${entryTarget}/${file.name}`, mtime }, bytes);
893
+ }
894
+ // A chunked Resource's chunk directory and its files, packed under the
895
+ // Collection so `.chunks.<encId>/<file>` round-trips on import.
896
+ for (const { dirName, files } of chunkDirsByCollection[entry.name] ??
897
+ []) {
898
+ pack.entry({
899
+ name: `${entryTarget}/${dirName}/`,
900
+ type: 'directory',
901
+ mtime
902
+ });
903
+ for (const file of files) {
904
+ const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name, dirName, file));
905
+ pack.entry({ name: `${entryTarget}/${dirName}/${file}`, mtime }, bytes);
906
+ }
864
907
  }
865
908
  }
866
909
  else if (entry.isFile()) {
867
910
  const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name));
868
- pack.entry({ name: entryTarget }, bytes);
911
+ pack.entry({ name: entryTarget, mtime }, bytes);
869
912
  }
870
913
  }
871
914
  if (revocationFiles.length > 0) {
872
- pack.entry({ name: 'revocations/', type: 'directory' });
915
+ pack.entry({ name: 'revocations/', type: 'directory', mtime });
873
916
  for (const name of revocationFiles) {
874
917
  const bytes = await fs.promises.readFile(path.join(revocationsDir, name));
875
- pack.entry({ name: `revocations/${name}` }, bytes);
918
+ pack.entry({ name: `revocations/${name}`, mtime }, bytes);
876
919
  }
877
920
  }
878
921
  pack.finalize();
@@ -903,7 +946,7 @@ export class FileSystemBackend {
903
946
  // (existing ids) are counted conservatively, as for the quota estimate.
904
947
  const { capacityBytes, maxUploadBytes, maxCollectionsPerSpace, maxResourcesPerSpace } = this;
905
948
  let incomingBytes = 0;
906
- for (const { collectionId, collectionDescription, resources } of collections) {
949
+ for (const { collectionId, collectionDescription, resources, chunkFiles } of collections) {
907
950
  const existing = await this.getCollectionDescription({
908
951
  spaceId,
909
952
  collectionId
@@ -936,6 +979,19 @@ export class FileSystemBackend {
936
979
  }
937
980
  incomingBytes += body.length;
938
981
  }
982
+ // Chunk files (the `chunked-streams` feature) inherit the per-upload cap
983
+ // and quota estimate, but NOT the encryption-conformance check: a chunk is
984
+ // opaque bytes, not a JSON envelope of the Collection's scheme.
985
+ for (const { body } of chunkFiles) {
986
+ if (maxUploadBytes !== undefined && body.length > maxUploadBytes) {
987
+ throw new PayloadTooLargeError({
988
+ maxUploadBytes,
989
+ backendId: this.describe().id,
990
+ uploadBytes: body.length
991
+ });
992
+ }
993
+ incomingBytes += body.length;
994
+ }
939
995
  }
940
996
  if (capacityBytes !== undefined) {
941
997
  await this._assertSpaceHeadroom({
@@ -974,7 +1030,7 @@ export class FileSystemBackend {
974
1030
  let liveResourceCount = maxResourcesPerSpace !== undefined
975
1031
  ? await this._countLiveResources({ spaceId })
976
1032
  : 0;
977
- for (const { collectionId, collectionDescription, collectionPolicy, resources, resourcePolicies, resourceMetadata } of collections) {
1033
+ for (const { collectionId, collectionDescription, collectionPolicy, resources, resourcePolicies, resourceMetadata, chunkFiles } of collections) {
978
1034
  // check if collection already exists
979
1035
  const collectionExisted = Boolean(await this.getCollectionDescription({ spaceId, collectionId }));
980
1036
  if (collectionExisted) {
@@ -1104,6 +1160,43 @@ export class FileSystemBackend {
1104
1160
  });
1105
1161
  stats.resourcesCreated++;
1106
1162
  }
1163
+ // Restore chunk files of chunked Resources (the `chunked-streams` feature)
1164
+ // into their per-Resource chunk directories. Skip-not-overwrite, per chunk
1165
+ // file: an existing chunk file is left untouched, so a re-import never
1166
+ // clobbers stored chunk bytes (or their version sidecar). An ORPHAN chunk
1167
+ // file -- one whose parent Resource is absent or a tombstone (no live
1168
+ // representation on the destination after the Resource apply loop above)
1169
+ // -- is skipped rather than resurrected, matching the live write path's
1170
+ // parent-exists rule (and the Postgres import).
1171
+ const parentIsLive = new Map();
1172
+ for (const { resourceId, fileName, body } of chunkFiles) {
1173
+ let live = parentIsLive.get(resourceId);
1174
+ if (live === undefined) {
1175
+ live = Boolean(await this._findFile({ collectionDir, resourceId }));
1176
+ parentIsLive.set(resourceId, live);
1177
+ }
1178
+ if (!live) {
1179
+ continue;
1180
+ }
1181
+ const chunkDir = this._chunkDir({ collectionDir, resourceId });
1182
+ const target = path.join(chunkDir, fileName);
1183
+ this._assertContained(target);
1184
+ let present = false;
1185
+ try {
1186
+ await fsStat(target);
1187
+ present = true;
1188
+ }
1189
+ catch (err) {
1190
+ if (err.code !== 'ENOENT') {
1191
+ throw err;
1192
+ }
1193
+ }
1194
+ if (present) {
1195
+ continue;
1196
+ }
1197
+ await mkdir(chunkDir, { recursive: true });
1198
+ await atomicWriteFile({ filePath: target, data: body });
1199
+ }
1107
1200
  }
1108
1201
  // Restore the archive's Space-scoped zcap revocations under this Space's
1109
1202
  // scope: a capability revoked before the export must stay revoked after
@@ -1515,6 +1608,70 @@ export class FileSystemBackend {
1515
1608
  }
1516
1609
  }
1517
1610
  }
1611
+ await this._writeRepresentationBytes({ spaceId, filePath, input });
1612
+ // A Resource has a single current representation: remove any prior
1613
+ // representation stored under a different content-type (its filename
1614
+ // differs). Write-new-then-prune (not delete-then-write) so the resource is
1615
+ // never momentarily absent.
1616
+ const existing = await this._resourceFilesFor({ collectionDir, resourceId });
1617
+ await Promise.all(existing
1618
+ .filter(name => path.resolve(name) !== path.resolve(filePath))
1619
+ .map(name => rm(name)));
1620
+ // Maintain the server-managed timestamps and the monotonic `version`: a
1621
+ // content write sets `createdAt` on first write, bumps `updatedAt`, and
1622
+ // increments `version` (the ETag validator) from its prior value, preserving
1623
+ // any user-writable `custom` and the independent `metaVersion` already stored
1624
+ // in the sidecar (a content write does not touch the metadata sub-resource).
1625
+ //
1626
+ // `createdBy` pairs with `createdAt`: taken from this write's invoker only
1627
+ // when this write creates the sidecar, so it names the creator rather than
1628
+ // the last writer, and is preserved verbatim afterward -- including
1629
+ // preserved-as-absent, so a Resource created with no invoker never has a
1630
+ // later writer backfilled into it. A tombstone keeps both, so re-creating a
1631
+ // deleted id under a different invoker preserves the original creator, as
1632
+ // it does the original `createdAt`.
1633
+ const now = new Date().toISOString();
1634
+ const prior = await this._readMetaSidecar({ collectionDir, resourceId });
1635
+ const version = (prior?.version ?? 0) + 1;
1636
+ const creator = prior ? prior.createdBy : createdBy;
1637
+ await this._writeMetaSidecar({
1638
+ collectionDir,
1639
+ resourceId,
1640
+ sidecar: {
1641
+ createdAt: prior?.createdAt ?? now,
1642
+ updatedAt: now,
1643
+ ...(creator !== undefined && { createdBy: creator }),
1644
+ version,
1645
+ ...(prior?.metaVersion !== undefined && {
1646
+ metaVersion: prior.metaVersion
1647
+ }),
1648
+ ...(prior?.custom && { custom: prior.custom }),
1649
+ // The key-epoch stamp is set from this write's declaration and CLEARED
1650
+ // when absent (the new ciphertext's epoch is unknown -- a stale stamp is
1651
+ // worse than none), so it is NOT preserved from `prior` like `custom`.
1652
+ ...(epoch !== undefined && { epoch })
1653
+ }
1654
+ });
1655
+ return { version };
1656
+ }
1657
+ /**
1658
+ * Writes a representation body (JSON value or byte stream) to `filePath`,
1659
+ * applying the same size guards every write path shares: the per-upload cap
1660
+ * (413 `PayloadTooLargeError`) and, when a byte quota is configured, the
1661
+ * Space headroom (507). A JSON body is fully in memory, so its size is checked
1662
+ * up front and written atomically; a binary body is streamed through the cap /
1663
+ * quota guards into a temp file and durably committed (fsync + rename + dir
1664
+ * fsync), removing the partial file on any failure. Shared by the Resource
1665
+ * write path (`_writeResourceLocked`) and the chunk write path
1666
+ * (`_writeChunkLocked`); the caller has already resolved `filePath` and ensured
1667
+ * its parent directory exists for the streamed case.
1668
+ * @param options {object}
1669
+ * @param options.spaceId {string}
1670
+ * @param options.filePath {string} absolute path of the representation file
1671
+ * @param options.input {ResourceInput}
1672
+ * @returns {Promise<void>}
1673
+ */
1674
+ async _writeRepresentationBytes({ spaceId, filePath, input }) {
1518
1675
  const { capacityBytes, maxUploadBytes } = this;
1519
1676
  if (input.kind === 'json') {
1520
1677
  // JSON bodies are fully in memory, so their serialized size is known up
@@ -1603,50 +1760,6 @@ export class FileSystemBackend {
1603
1760
  throw err;
1604
1761
  }
1605
1762
  }
1606
- // A Resource has a single current representation: remove any prior
1607
- // representation stored under a different content-type (its filename
1608
- // differs). Write-new-then-prune (not delete-then-write) so the resource is
1609
- // never momentarily absent.
1610
- const existing = await this._resourceFilesFor({ collectionDir, resourceId });
1611
- await Promise.all(existing
1612
- .filter(name => path.resolve(name) !== path.resolve(filePath))
1613
- .map(name => rm(name)));
1614
- // Maintain the server-managed timestamps and the monotonic `version`: a
1615
- // content write sets `createdAt` on first write, bumps `updatedAt`, and
1616
- // increments `version` (the ETag validator) from its prior value, preserving
1617
- // any user-writable `custom` and the independent `metaVersion` already stored
1618
- // in the sidecar (a content write does not touch the metadata sub-resource).
1619
- //
1620
- // `createdBy` pairs with `createdAt`: taken from this write's invoker only
1621
- // when this write creates the sidecar, so it names the creator rather than
1622
- // the last writer, and is preserved verbatim afterward -- including
1623
- // preserved-as-absent, so a Resource created with no invoker never has a
1624
- // later writer backfilled into it. A tombstone keeps both, so re-creating a
1625
- // deleted id under a different invoker preserves the original creator, as
1626
- // it does the original `createdAt`.
1627
- const now = new Date().toISOString();
1628
- const prior = await this._readMetaSidecar({ collectionDir, resourceId });
1629
- const version = (prior?.version ?? 0) + 1;
1630
- const creator = prior ? prior.createdBy : createdBy;
1631
- await this._writeMetaSidecar({
1632
- collectionDir,
1633
- resourceId,
1634
- sidecar: {
1635
- createdAt: prior?.createdAt ?? now,
1636
- updatedAt: now,
1637
- ...(creator !== undefined && { createdBy: creator }),
1638
- version,
1639
- ...(prior?.metaVersion !== undefined && {
1640
- metaVersion: prior.metaVersion
1641
- }),
1642
- ...(prior?.custom && { custom: prior.custom }),
1643
- // The key-epoch stamp is set from this write's declaration and CLEARED
1644
- // when absent (the new ciphertext's epoch is unknown -- a stale stamp is
1645
- // worse than none), so it is NOT preserved from `prior` like `custom`.
1646
- ...(epoch !== undefined && { epoch })
1647
- }
1648
- });
1649
- return { version };
1650
1763
  }
1651
1764
  /**
1652
1765
  * Builds the per-Resource serialization key for `_writeMutex`
@@ -1979,6 +2092,15 @@ export class FileSystemBackend {
1979
2092
  const { contentType } = parseResourceFileName(path.basename(filesForResource[0]));
1980
2093
  // Drop the content representation(s) but KEEP the sidecar as the tombstone.
1981
2094
  await Promise.all(filesForResource.map(filename => rm(filename)));
2095
+ // Cascade-delete the Resource's chunks (the `chunked-streams` feature): a
2096
+ // chunk must never outlive its parent Resource, so its whole chunk
2097
+ // directory goes with the content. Runs under the same per-Resource lock a
2098
+ // `writeChunk` takes, so a chunk write racing this delete cannot re-create
2099
+ // an orphan directory. `force` makes an absent chunk directory a no-op.
2100
+ await rm(this._chunkDir({ collectionDir, resourceId }), {
2101
+ recursive: true,
2102
+ force: true
2103
+ });
1982
2104
  // Bump `version` / `updatedAt` so the tombstone sorts after the Resource's
1983
2105
  // prior state in the change feed, and continues the monotonic version (a
1984
2106
  // later re-create reads this sidecar and keeps counting up). `custom` is
@@ -2007,6 +2129,315 @@ export class FileSystemBackend {
2007
2129
  // for a conditional delete, as the old unconditional removal did).
2008
2130
  return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), softDelete);
2009
2131
  }
2132
+ // Chunks (the `chunked-streams` feature)
2133
+ /**
2134
+ * Builds the on-disk path for a Resource's chunk directory
2135
+ * (`.chunks.<encodedResourceId>/`) inside its Collection dir. A hidden
2136
+ * subdirectory (leading `.`, so it is invisible to the `r.`-prefixed Collection
2137
+ * listing and the live-Resource count) that holds the Resource's chunk
2138
+ * representations; inside it a chunk is stored exactly like a Resource keyed by
2139
+ * its index (`r.<index>.<encodedContentType>.<ext>` plus a `.meta.<index>.json`
2140
+ * version sidecar), so the Resource file / sidecar helpers are reused verbatim
2141
+ * with the chunk directory as the `collectionDir`.
2142
+ * @param options {object}
2143
+ * @param options.collectionDir {string}
2144
+ * @param options.resourceId {string}
2145
+ * @returns {string}
2146
+ */
2147
+ _chunkDir({ collectionDir, resourceId }) {
2148
+ const chunkDir = path.join(collectionDir, chunkDirName(resourceId));
2149
+ this._assertContained(chunkDir);
2150
+ return chunkDir;
2151
+ }
2152
+ /**
2153
+ * Writes one chunk of a chunked Resource, keyed by
2154
+ * `(spaceId, collectionId, resourceId, chunkIndex)`. The parent Resource MUST
2155
+ * already exist (else `ResourceNotFoundError`, 404), checked under the same
2156
+ * per-Resource lock a `deleteResource` cascade takes so a chunk can never be
2157
+ * orphaned by a racing delete. The body is stored opaquely (bytes +
2158
+ * content-type) through the shared upload-cap / quota guards, and the chunk's
2159
+ * own monotonic `version` (its ETag validator, independent of the parent's) is
2160
+ * bumped; any `If-Match` / `If-None-Match` precondition is evaluated on that
2161
+ * version atomically with the write (`PreconditionFailedError`, 412).
2162
+ * @param options {object}
2163
+ * @param options.spaceId {string}
2164
+ * @param options.collectionId {string}
2165
+ * @param options.resourceId {string}
2166
+ * @param options.chunkIndex {number} non-negative integer chunk position
2167
+ * @param options.input {ResourceInput}
2168
+ * @param [options.ifMatch] {string} `If-Match` precondition (a quoted ETag)
2169
+ * @param [options.ifNoneMatch] {boolean} `If-None-Match: *` (create-if-absent)
2170
+ * @returns {Promise<{ version: number }>} the chunk's new version
2171
+ */
2172
+ async writeChunk({ spaceId, collectionId, resourceId, chunkIndex, input, ifMatch, ifNoneMatch }) {
2173
+ const collectionDir = this._collectionDir({ spaceId, collectionId });
2174
+ // Serialize on the parent Resource's lock key -- the same key
2175
+ // `deleteResource` takes -- so the parent-exists check, the write, and the
2176
+ // cascade delete cannot interleave (no orphan chunk).
2177
+ return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), () => this._writeChunkLocked({
2178
+ spaceId,
2179
+ collectionDir,
2180
+ resourceId,
2181
+ chunkIndex,
2182
+ input,
2183
+ ifMatch,
2184
+ ifNoneMatch
2185
+ }));
2186
+ }
2187
+ /**
2188
+ * The critical section of `writeChunk`, run under the per-Resource lock. See
2189
+ * `writeChunk` for the parameters.
2190
+ * @returns {Promise<{ version: number }>}
2191
+ */
2192
+ async _writeChunkLocked({ spaceId, collectionDir, resourceId, chunkIndex, input, ifMatch, ifNoneMatch }) {
2193
+ // The parent Resource must exist: writing a chunk of an absent Resource
2194
+ // rejects, so orphan chunks cannot accumulate.
2195
+ const parentExists = (await this._findFile({ collectionDir, resourceId })) !== undefined;
2196
+ if (!parentExists) {
2197
+ throw new ResourceNotFoundError({ requestName: 'Write Chunk' });
2198
+ }
2199
+ // Inside the chunk directory a chunk is a Resource keyed by its index, so the
2200
+ // Resource file / sidecar helpers apply with `chunkDir` as the collectionDir
2201
+ // and the stringified index as the resourceId.
2202
+ const chunkDir = this._chunkDir({ collectionDir, resourceId });
2203
+ const chunkId = String(chunkIndex);
2204
+ const filename = fileNameFor({
2205
+ resourceId: chunkId,
2206
+ contentType: input.contentType
2207
+ });
2208
+ const filePath = path.join(chunkDir, filename);
2209
+ this._assertContained(filePath);
2210
+ // Evaluate any precondition against the chunk's current version before
2211
+ // writing (still inside the lock, so check and write are atomic).
2212
+ if (ifMatch !== undefined || ifNoneMatch) {
2213
+ await this._assertWritePrecondition({
2214
+ collectionDir: chunkDir,
2215
+ resourceId: chunkId,
2216
+ ifMatch,
2217
+ ifNoneMatch
2218
+ });
2219
+ }
2220
+ // The chunk directory is created lazily on first write (a binary body is
2221
+ // streamed into a temp file in it, so it must exist first).
2222
+ await mkdir(chunkDir, { recursive: true });
2223
+ await this._writeRepresentationBytes({ spaceId, filePath, input });
2224
+ // A chunk has a single current representation: remove any prior one stored
2225
+ // under a different content-type (write-new-then-prune).
2226
+ const existing = await this._resourceFilesFor({
2227
+ collectionDir: chunkDir,
2228
+ resourceId: chunkId
2229
+ });
2230
+ await Promise.all(existing
2231
+ .filter(name => path.resolve(name) !== path.resolve(filePath))
2232
+ .map(name => rm(name)));
2233
+ // Bump the chunk's monotonic `version` (its ETag validator), preserving its
2234
+ // `createdAt`. A chunk carries no user Metadata / `createdBy` / epoch stamp.
2235
+ const now = new Date().toISOString();
2236
+ const prior = await this._readMetaSidecar({
2237
+ collectionDir: chunkDir,
2238
+ resourceId: chunkId
2239
+ });
2240
+ const version = (prior?.version ?? 0) + 1;
2241
+ await this._writeMetaSidecar({
2242
+ collectionDir: chunkDir,
2243
+ resourceId: chunkId,
2244
+ sidecar: {
2245
+ createdAt: prior?.createdAt ?? now,
2246
+ updatedAt: now,
2247
+ version
2248
+ }
2249
+ });
2250
+ return { version };
2251
+ }
2252
+ /**
2253
+ * Reads a chunk's bytes, resolving a `ResourceResult` (stream + resolved
2254
+ * content-type + the chunk's `version`). Throws `ResourceNotFoundError` (404)
2255
+ * when the chunk is absent.
2256
+ * @param options {object}
2257
+ * @param options.spaceId {string}
2258
+ * @param options.collectionId {string}
2259
+ * @param options.resourceId {string}
2260
+ * @param options.chunkIndex {number}
2261
+ * @returns {Promise<ResourceResult>}
2262
+ */
2263
+ async getChunk({ spaceId, collectionId, resourceId, chunkIndex }) {
2264
+ const collectionDir = this._collectionDir({ spaceId, collectionId });
2265
+ const chunkDir = this._chunkDir({ collectionDir, resourceId });
2266
+ const chunkId = String(chunkIndex);
2267
+ const filePath = await this._findFile({
2268
+ collectionDir: chunkDir,
2269
+ resourceId: chunkId
2270
+ });
2271
+ if (!filePath) {
2272
+ throw new ResourceNotFoundError({ requestName: 'Get Chunk' });
2273
+ }
2274
+ try {
2275
+ await fsStat(filePath);
2276
+ }
2277
+ catch (err) {
2278
+ if (err.code === 'ENOENT') {
2279
+ throw new ResourceNotFoundError({ requestName: 'Get Chunk' });
2280
+ }
2281
+ throw err;
2282
+ }
2283
+ const { contentType: storedResourceType } = parseResourceFileName(path.basename(filePath));
2284
+ const sidecar = await this._readMetaSidecar({
2285
+ collectionDir: chunkDir,
2286
+ resourceId: chunkId
2287
+ });
2288
+ return {
2289
+ resourceStream: await openFileStream(filePath, this.logger),
2290
+ storedResourceType,
2291
+ ...(sidecar?.version !== undefined && { version: sidecar.version })
2292
+ };
2293
+ }
2294
+ /**
2295
+ * Reads a chunk's stored content-type / size / version (the HEAD payload
2296
+ * headers). Resolves `undefined` when the chunk is absent.
2297
+ * @param options {object}
2298
+ * @param options.spaceId {string}
2299
+ * @param options.collectionId {string}
2300
+ * @param options.resourceId {string}
2301
+ * @param options.chunkIndex {number}
2302
+ * @returns {Promise<{ contentType: string, size: number, version?: number } |
2303
+ * undefined>}
2304
+ */
2305
+ async getChunkMetadata({ spaceId, collectionId, resourceId, chunkIndex }) {
2306
+ const collectionDir = this._collectionDir({ spaceId, collectionId });
2307
+ const chunkDir = this._chunkDir({ collectionDir, resourceId });
2308
+ const chunkId = String(chunkIndex);
2309
+ const filePath = await this._findFile({
2310
+ collectionDir: chunkDir,
2311
+ resourceId: chunkId
2312
+ });
2313
+ if (!filePath) {
2314
+ return undefined;
2315
+ }
2316
+ let stats;
2317
+ try {
2318
+ stats = await fsStat(filePath);
2319
+ }
2320
+ catch (err) {
2321
+ if (err.code === 'ENOENT') {
2322
+ return undefined;
2323
+ }
2324
+ throw err;
2325
+ }
2326
+ const { contentType } = parseResourceFileName(path.basename(filePath));
2327
+ const sidecar = await this._readMetaSidecar({
2328
+ collectionDir: chunkDir,
2329
+ resourceId: chunkId
2330
+ });
2331
+ return {
2332
+ contentType,
2333
+ size: stats.size,
2334
+ ...(sidecar?.version !== undefined && { version: sidecar.version })
2335
+ };
2336
+ }
2337
+ /**
2338
+ * Deletes one chunk (a hard delete: its bytes and version sidecar both go, and
2339
+ * -- unlike a Resource -- it leaves no tombstone, since chunks are not part of
2340
+ * the change feed). Resolves `true` when a chunk was removed and `false` when
2341
+ * none was stored at that index. When `ifMatch` is supplied it is evaluated on
2342
+ * the chunk's current version atomically with the removal (under the same
2343
+ * per-Resource lock), throwing `PreconditionFailedError` (412) on a mismatch.
2344
+ * @param options {object}
2345
+ * @param options.spaceId {string}
2346
+ * @param options.collectionId {string}
2347
+ * @param options.resourceId {string}
2348
+ * @param options.chunkIndex {number}
2349
+ * @param [options.ifMatch] {string} `If-Match` precondition (a quoted ETag)
2350
+ * @returns {Promise<boolean>}
2351
+ */
2352
+ async deleteChunk({ spaceId, collectionId, resourceId, chunkIndex, ifMatch }) {
2353
+ const collectionDir = this._collectionDir({ spaceId, collectionId });
2354
+ const chunkDir = this._chunkDir({ collectionDir, resourceId });
2355
+ const chunkId = String(chunkIndex);
2356
+ return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), async () => {
2357
+ const files = await this._resourceFilesFor({
2358
+ collectionDir: chunkDir,
2359
+ resourceId: chunkId
2360
+ });
2361
+ if (files.length === 0) {
2362
+ // Absent: the handler 404s on `false` (chunk deletes are not silently
2363
+ // idempotent, mirroring the EDV chunk contract).
2364
+ return false;
2365
+ }
2366
+ if (ifMatch !== undefined) {
2367
+ await this._assertWritePrecondition({
2368
+ collectionDir: chunkDir,
2369
+ resourceId: chunkId,
2370
+ ifMatch
2371
+ });
2372
+ }
2373
+ await Promise.all(files.map(name => rm(name)));
2374
+ // Remove the version sidecar too: a chunk keeps no tombstone.
2375
+ await rm(this._metaSidecarPath({
2376
+ collectionDir: chunkDir,
2377
+ resourceId: chunkId
2378
+ }), { force: true });
2379
+ // When that was the last chunk, remove the now-empty chunk directory
2380
+ // itself: a lingering empty `.chunks.<encId>/` would otherwise appear
2381
+ // in the export walk (diverging from a Postgres export of the same
2382
+ // logical state) and count its allocated block toward the du-based
2383
+ // quota measurement. Safe under the per-Resource lock (`writeChunk`
2384
+ // serializes on the same key, so nothing lands in the directory
2385
+ // between the check and the rmdir).
2386
+ const remaining = await fs.promises.readdir(chunkDir);
2387
+ if (remaining.length === 0) {
2388
+ await fs.promises.rmdir(chunkDir);
2389
+ }
2390
+ // Freed bytes: drop the cached quota usage so the next write re-measures.
2391
+ this._usageCache.delete(spaceId);
2392
+ return true;
2393
+ });
2394
+ }
2395
+ /**
2396
+ * Lists a Resource's stored chunks in ascending `index` order -- the
2397
+ * discovery/reassembly listing (the server never reassembles). Resolves an
2398
+ * empty listing when the Resource has no chunk directory (including when the
2399
+ * Resource itself is absent -- existence is the parent routes' concern).
2400
+ * @param options {object}
2401
+ * @param options.spaceId {string}
2402
+ * @param options.collectionId {string}
2403
+ * @param options.resourceId {string}
2404
+ * @returns {Promise<{ count: number, chunks: Array<{ index: number, size:
2405
+ * number, contentType: string, version?: number }> }>}
2406
+ */
2407
+ async listChunks({ spaceId, collectionId, resourceId }) {
2408
+ const collectionDir = this._collectionDir({ spaceId, collectionId });
2409
+ const chunkDir = this._chunkDir({ collectionDir, resourceId });
2410
+ let entries;
2411
+ try {
2412
+ entries = await fs.promises.readdir(chunkDir, { withFileTypes: true });
2413
+ }
2414
+ catch (err) {
2415
+ if (err.code === 'ENOENT') {
2416
+ return { count: 0, chunks: [] };
2417
+ }
2418
+ throw err;
2419
+ }
2420
+ // Keep only chunk representations (`r.<index>.<type>.<ext>`), dropping the
2421
+ // `.meta.<index>.json` version sidecars.
2422
+ const chunkEntries = entries.filter(entry => entry.isFile() && entry.name.startsWith('r.'));
2423
+ const chunks = await Promise.all(chunkEntries.map(async (entry) => {
2424
+ const { resourceId: indexStr, contentType } = parseResourceFileName(entry.name);
2425
+ const filePath = path.join(chunkDir, entry.name);
2426
+ const stats = await fsStat(filePath);
2427
+ const sidecar = await this._readMetaSidecar({
2428
+ collectionDir: chunkDir,
2429
+ resourceId: indexStr
2430
+ });
2431
+ return {
2432
+ index: Number(indexStr),
2433
+ size: stats.size,
2434
+ contentType,
2435
+ ...(sidecar?.version !== undefined && { version: sidecar.version })
2436
+ };
2437
+ }));
2438
+ chunks.sort((left, right) => left.index - right.index);
2439
+ return { count: chunks.length, chunks };
2440
+ }
2010
2441
  /**
2011
2442
  * Replication change feed (the `changes` query profile; see the
2012
2443
  * `StorageBackend.changesSince` contract.