was-teaching-server 0.9.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +118 -1
  2. package/README.md +23 -11
  3. package/dist/backends/filesystem.d.ts +247 -4
  4. package/dist/backends/filesystem.d.ts.map +1 -1
  5. package/dist/backends/filesystem.js +711 -81
  6. package/dist/backends/filesystem.js.map +1 -1
  7. package/dist/backends/postgres.d.ts +240 -2
  8. package/dist/backends/postgres.d.ts.map +1 -1
  9. package/dist/backends/postgres.js +800 -42
  10. package/dist/backends/postgres.js.map +1 -1
  11. package/dist/backends/postgresSchema.d.ts.map +1 -1
  12. package/dist/backends/postgresSchema.js +69 -57
  13. package/dist/backends/postgresSchema.js.map +1 -1
  14. package/dist/errors.d.ts +26 -9
  15. package/dist/errors.d.ts.map +1 -1
  16. package/dist/errors.js +40 -12
  17. package/dist/errors.js.map +1 -1
  18. package/dist/lib/equalityIndex.d.ts +334 -0
  19. package/dist/lib/equalityIndex.d.ts.map +1 -0
  20. package/dist/lib/equalityIndex.js +552 -0
  21. package/dist/lib/equalityIndex.js.map +1 -0
  22. package/dist/lib/exportManifest.d.ts +9 -0
  23. package/dist/lib/exportManifest.d.ts.map +1 -1
  24. package/dist/lib/exportManifest.js +9 -0
  25. package/dist/lib/exportManifest.js.map +1 -1
  26. package/dist/lib/importTar.d.ts +18 -2
  27. package/dist/lib/importTar.d.ts.map +1 -1
  28. package/dist/lib/importTar.js +77 -4
  29. package/dist/lib/importTar.js.map +1 -1
  30. package/dist/lib/paths.d.ts +33 -0
  31. package/dist/lib/paths.d.ts.map +1 -1
  32. package/dist/lib/paths.js +28 -0
  33. package/dist/lib/paths.js.map +1 -1
  34. package/dist/lib/resourceFileName.d.ts +43 -0
  35. package/dist/lib/resourceFileName.d.ts.map +1 -1
  36. package/dist/lib/resourceFileName.js +63 -0
  37. package/dist/lib/resourceFileName.js.map +1 -1
  38. package/dist/plugin.d.ts.map +1 -1
  39. package/dist/plugin.js +5 -2
  40. package/dist/plugin.js.map +1 -1
  41. package/dist/requests/ChunkRequest.d.ts +104 -0
  42. package/dist/requests/ChunkRequest.d.ts.map +1 -0
  43. package/dist/requests/ChunkRequest.js +396 -0
  44. package/dist/requests/ChunkRequest.js.map +1 -0
  45. package/dist/requests/CollectionRequest.d.ts +22 -5
  46. package/dist/requests/CollectionRequest.d.ts.map +1 -1
  47. package/dist/requests/CollectionRequest.js +192 -11
  48. package/dist/requests/CollectionRequest.js.map +1 -1
  49. package/dist/requests/ResourceRequest.d.ts.map +1 -1
  50. package/dist/requests/ResourceRequest.js +20 -0
  51. package/dist/requests/ResourceRequest.js.map +1 -1
  52. package/dist/requests/SpaceRequest.d.ts +1 -0
  53. package/dist/requests/SpaceRequest.d.ts.map +1 -1
  54. package/dist/requests/SpaceRequest.js +21 -1
  55. package/dist/requests/SpaceRequest.js.map +1 -1
  56. package/dist/routes.d.ts.map +1 -1
  57. package/dist/routes.js +21 -0
  58. package/dist/routes.js.map +1 -1
  59. package/dist/types.d.ts +165 -2
  60. package/dist/types.d.ts.map +1 -1
  61. package/package.json +19 -18
  62. package/src/backends/filesystem.ts +924 -75
  63. package/src/backends/postgres.ts +1114 -32
  64. package/src/backends/postgresSchema.ts +69 -57
  65. package/src/errors.ts +44 -12
  66. package/src/lib/equalityIndex.ts +769 -0
  67. package/src/lib/importTar.ts +107 -4
  68. package/src/lib/paths.ts +48 -0
  69. package/src/lib/resourceFileName.ts +69 -0
  70. package/src/plugin.ts +5 -2
  71. package/src/requests/ChunkRequest.ts +478 -0
  72. package/src/requests/CollectionRequest.ts +221 -12
  73. package/src/requests/ResourceRequest.ts +20 -0
  74. package/src/requests/SpaceRequest.ts +25 -1
  75. package/src/routes.ts +46 -0
  76. package/src/types.ts +175 -1
@@ -30,19 +30,21 @@ import * as tar from 'tar-stream';
30
30
  import YAML from 'yaml';
31
31
  import { StorageError, ResourceNotFoundError, SpaceNotFoundError, QuotaExceededError, CountQuotaExceededError, PayloadTooLargeError, PreconditionFailedError, KeystoreStateConflictError, KeyIdConflictError, DuplicateRevocationError } from '../errors.js';
32
32
  import { applyMigrations } from './postgresSchema.js';
33
- import { extractTarEntries, buildImportPlan } from '../lib/importTar.js';
33
+ import { extractTarEntries, buildImportPlan, metaSidecarFileId } from '../lib/importTar.js';
34
34
  import { collectionPath, resourcePath } from '../lib/paths.js';
35
- import { fileNameFor, parseResourceFileName } from '../lib/resourceFileName.js';
35
+ import { fileNameFor, parseResourceFileName, chunkDirName, parseChunkDirName, parseChunkIndexSegment } from '../lib/resourceFileName.js';
36
+ import { assertValidId } from '../lib/validateId.js';
36
37
  import { sanitizeBackendRecord } from '../lib/backends.js';
37
38
  import { backendUsageFields } from '../lib/backendUsage.js';
38
39
  import { assertEncryptedWriteConforms } from '../lib/encryption.js';
39
40
  import { encodeCursor, decodeCursor } from '../lib/cursor.js';
40
- import { buildExportManifest } from '../lib/exportManifest.js';
41
+ import { buildExportManifest, EXPORT_ENTRY_MTIME } from '../lib/exportManifest.js';
41
42
  import { revocationFileName } from '../lib/revocations.js';
42
43
  import { isJson } from '../lib/isJson.js';
43
44
  import { normalizeDescriptionWrite } from '../lib/collectionDescription.js';
44
45
  import { DEFAULT_PAGE_SIZE, clampPageSize } from '../lib/pagination.js';
45
46
  import { runBlindedIndexQuery, collectUniqueBlindedTerms, assertNoUniqueBlindedConflict } from '../lib/blindedIndex.js';
47
+ import { runEqualityQuery, assertNoUniqueEqualityConflict, findEqualityUniqueViolation } from '../lib/equalityIndex.js';
46
48
  import { DEFAULT_MAX_UPLOAD_BYTES, DEFAULT_MAX_SPACES_PER_CONTROLLER, DEFAULT_MAX_COLLECTIONS_PER_SPACE, DEFAULT_MAX_RESOURCES_PER_SPACE, normalizeCountLimit } from '../config.default.js';
47
49
  import { assertWritePrecondition, assertMetaWritePrecondition, assertCollectionWritePrecondition } from '../lib/preconditions.js';
48
50
  /** Pool sizing and per-connection statement timeout (operational defaults). */
@@ -277,11 +279,18 @@ export class PostgresBackend {
277
279
  // `key-epochs`: multi-recipient encrypted Collections -- per-epoch wrapped
278
280
  // keys on the `encryption` marker, a client-declared `epoch` stamp on
279
281
  // Resources, and conditional (`If-Match`) Collection Description writes.
282
+ // `chunked-streams`: chunk addressing at `/{resourceId}/chunks/{n}`, opaque
283
+ // per-chunk raw-bytes storage in the `chunks` table.
284
+ // `equality-query`: serves the `equality` profile -- server-extracted
285
+ // plaintext attribute equality over a Collection's declared `indexes`
286
+ // (`queryByEquality`), plus the GET `filter[attr]=value` equality filter.
280
287
  features: [
281
288
  'conditional-writes',
282
289
  'changes-query',
283
290
  'blinded-index-query',
284
- 'key-epochs'
291
+ 'equality-query',
292
+ 'key-epochs',
293
+ 'chunked-streams'
285
294
  ]
286
295
  };
287
296
  }
@@ -340,6 +349,30 @@ export class PostgresBackend {
340
349
  });
341
350
  }
342
351
  }
352
+ /**
353
+ * Serializes concurrent CREATORS of one not-yet-existing row (a Resource or
354
+ * a chunk) on a transaction-scoped advisory lock keyed by the row's
355
+ * identity. Under READ COMMITTED a `SELECT ... FOR UPDATE` on an absent row
356
+ * locks nothing (no gap locks), so two concurrent creators would both read
357
+ * "no prior row" and both apply their full byte size as the usage delta --
358
+ * and while the `ON CONFLICT` arm recomputes `version` from the conflicting
359
+ * row, it cannot retroactively fix the loser's delta, leaving `usage_bytes`
360
+ * inflated by one write's size. The caller takes this lock when its
361
+ * lock-nothing SELECT found no row, then RE-reads the row: the second
362
+ * creator blocks here until the first commits, and its re-read sees the
363
+ * committed row, so its precondition, version, and usage delta are computed
364
+ * from accurate state. Held to commit (advisory xact lock). The `create:`
365
+ * prefix keeps this key domain distinct from the unique-blinded-term
366
+ * advisory lock, which hashes the bare `(spaceId, collectionId)`.
367
+ * @param options {object}
368
+ * @param options.client {pg.PoolClient}
369
+ * @param options.spaceId {string}
370
+ * @param options.rowKey {string} the row's identity within the Space
371
+ * @returns {Promise<void>}
372
+ */
373
+ async _lockSameKeyCreate({ client, spaceId, rowKey }) {
374
+ await client.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [spaceId, `create:${rowKey}`]);
375
+ }
343
376
  // Quotas
344
377
  /**
345
378
  * Reports the Space's usage from the transactional counter (no measurement
@@ -356,9 +389,16 @@ export class PostgresBackend {
356
389
  const usageBytes = rows[0] ? Number(rows[0].usage_bytes) : 0;
357
390
  let usageByCollection;
358
391
  if (includeCollections) {
359
- const { rows: collectionRows } = await this._pool.query(`SELECT collection_id, COALESCE(SUM(size_bytes), 0) AS usage
360
- FROM resources
361
- WHERE space_id = $1
392
+ // Per-Collection usage sums both Resource content bytes and chunk bytes
393
+ // (the `chunked-streams` feature) so the breakdown agrees with the
394
+ // Space total in the transactional counter.
395
+ const { rows: collectionRows } = await this._pool.query(`SELECT collection_id, COALESCE(SUM(bytes), 0) AS usage FROM (
396
+ SELECT collection_id, size_bytes AS bytes FROM resources
397
+ WHERE space_id = $1
398
+ UNION ALL
399
+ SELECT collection_id, size AS bytes FROM chunks
400
+ WHERE space_id = $1
401
+ ) usage_rows
362
402
  GROUP BY collection_id
363
403
  ORDER BY collection_id`, [spaceId]);
364
404
  usageByCollection = collectionRows.map(row => ({
@@ -387,6 +427,8 @@ export class PostgresBackend {
387
427
  (SELECT COALESCE(usage_bytes, 0) FROM spaces WHERE space_id = $1)
388
428
  AS space_total,
389
429
  (SELECT COALESCE(SUM(size_bytes), 0) FROM resources
430
+ WHERE space_id = $1 AND collection_id = $2)
431
+ + (SELECT COALESCE(SUM(size), 0) FROM chunks
390
432
  WHERE space_id = $1 AND collection_id = $2) AS collection_total`, [spaceId, collectionId]);
391
433
  const spaceTotalBytes = Number(rows[0]?.space_total ?? 0);
392
434
  const usageBytes = Number(rows[0]?.collection_total ?? 0);
@@ -698,8 +740,14 @@ export class PostgresBackend {
698
740
  */
699
741
  async deleteCollection({ spaceId, collectionId }) {
700
742
  await this._withTransaction(async (client) => {
701
- const { rows } = await client.query(`SELECT COALESCE(SUM(size_bytes), 0) AS total FROM resources
702
- WHERE space_id = $1 AND collection_id = $2`, [spaceId, collectionId]);
743
+ // The Collection's freed bytes are its Resource content plus its chunk
744
+ // bytes (the `chunked-streams` feature); both cascade away with the
745
+ // Collection row, so both leave the quota counter.
746
+ const { rows } = await client.query(`SELECT
747
+ (SELECT COALESCE(SUM(size_bytes), 0) FROM resources
748
+ WHERE space_id = $1 AND collection_id = $2)
749
+ + (SELECT COALESCE(SUM(size), 0) FROM chunks
750
+ WHERE space_id = $1 AND collection_id = $2) AS total`, [spaceId, collectionId]);
703
751
  const freedBytes = Number(rows[0]?.total ?? 0);
704
752
  await client.query(`DELETE FROM collections WHERE space_id = $1 AND collection_id = $2`, [spaceId, collectionId]);
705
753
  // Collection- and Resource-level policies live under the Collection (the
@@ -816,7 +864,7 @@ export class PostgresBackend {
816
864
  * @param [options.ifNoneMatch] {boolean}
817
865
  * @returns {Promise<{ version: number }>}
818
866
  */
819
- async writeResource({ spaceId, collectionId, resourceId, input, createdBy, epoch, ifMatch, ifNoneMatch }) {
867
+ async writeResource({ spaceId, collectionId, resourceId, input, createdBy, epoch, uniqueIndexes, ifMatch, ifNoneMatch }) {
820
868
  const { maxUploadBytes } = this;
821
869
  let content;
822
870
  if (input.kind === 'json') {
@@ -849,18 +897,24 @@ export class PostgresBackend {
849
897
  }
850
898
  return this._withTransaction(async (client) => {
851
899
  await this._ensureCollectionRow({ client, spaceId, collectionId });
852
- // The EDV unique-attribute invariant (`unique: true` blinded attributes;
853
- // the `blinded-index-query` feature): a write claiming a unique blinded
854
- // triple must not collide with another live document's claim. Only a
855
- // unique-carrying JSON write can create a new claim, so only such writes
856
- // pay for it: a per-Collection transaction-scoped advisory lock
900
+ // Two unique-attribute invariants can force a JSON content write to
901
+ // serialize before it upserts its row: the EDV blinded one (`unique: true`
902
+ // blinded attributes; the `blinded-index-query` feature) and the plaintext
903
+ // equality one (a Collection's `unique`-declared `indexes`; the
904
+ // `equality-query` feature). Only a unique-carrying JSON write can create
905
+ // either claim. A per-Collection transaction-scoped advisory lock
857
906
  // serializes the claimants (held to commit, so the loser's scan sees the
858
- // winner's committed row) without entering the row-lock ordering of
859
- // plain writes, then the conflict is evaluated against the Collection's
860
- // other live JSON documents.
861
- if (input.kind === 'json' &&
862
- collectUniqueBlindedTerms({ document: input.data }).length > 0) {
907
+ // winner's committed row) without entering the row-lock ordering of plain
908
+ // writes; it is taken once and shared by both checks.
909
+ const blindedUnique = input.kind === 'json' &&
910
+ collectUniqueBlindedTerms({ document: input.data }).length > 0;
911
+ const equalityUnique = input.kind === 'json' &&
912
+ uniqueIndexes !== undefined &&
913
+ uniqueIndexes.length > 0;
914
+ if (blindedUnique || equalityUnique) {
863
915
  await client.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [spaceId, collectionId]);
916
+ }
917
+ if (blindedUnique) {
864
918
  const { rows: candidateRows } = await client.query(`SELECT resource_id, content FROM resources
865
919
  WHERE space_id = $1 AND collection_id = $2 AND is_json
866
920
  AND NOT deleted AND resource_id <> $3`, [spaceId, collectionId, resourceId]);
@@ -879,7 +933,27 @@ export class PostgresBackend {
879
933
  // skip an unparsable body
880
934
  }
881
935
  }
882
- assertNoUniqueBlindedConflict({ document: input.data, candidates });
936
+ assertNoUniqueBlindedConflict({
937
+ document: input.kind === 'json' ? input.data : undefined,
938
+ candidates
939
+ });
940
+ }
941
+ if (equalityUnique) {
942
+ // A content write does not change the Resource's `custom`, so the custom
943
+ // side of the claim comes from the CURRENT stored row.
944
+ const { rows: selfRows } = await client.query(`SELECT custom FROM resources
945
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
946
+ AND NOT deleted`, [spaceId, collectionId, resourceId]);
947
+ assertNoUniqueEqualityConflict({
948
+ indexes: uniqueIndexes,
949
+ content: input.kind === 'json' ? input.data : undefined,
950
+ custom: selfRows[0]?.custom ?? undefined,
951
+ candidates: await this._readEqualityCandidates(client, {
952
+ spaceId,
953
+ collectionId,
954
+ excludeResourceId: resourceId
955
+ })
956
+ });
883
957
  }
884
958
  // Narrow projection: the lock needs the row, not its (possibly multi-MB)
885
959
  // `content` bytea, which this path never reads.
@@ -887,7 +961,23 @@ export class PostgresBackend {
887
961
  FROM resources
888
962
  WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
889
963
  FOR UPDATE`, [spaceId, collectionId, resourceId]);
890
- const prior = rows[0];
964
+ let prior = rows[0];
965
+ if (prior === undefined) {
966
+ // The lock-nothing case: serialize with any concurrent creator of the
967
+ // same id and re-read, so `exists` / `priorSize` below reflect the row
968
+ // it committed (see `_lockSameKeyCreate` -- without this, both
969
+ // creators would count their full byte size as the usage delta).
970
+ await this._lockSameKeyCreate({
971
+ client,
972
+ spaceId,
973
+ rowKey: `${collectionId}/${resourceId}`
974
+ });
975
+ const { rows: reread } = await client.query(`SELECT version, size_bytes, deleted, created_at, created_by
976
+ FROM resources
977
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
978
+ FOR UPDATE`, [spaceId, collectionId, resourceId]);
979
+ prior = reread[0];
980
+ }
891
981
  const exists = prior !== undefined && !prior.deleted;
892
982
  if (ifMatch !== undefined || ifNoneMatch) {
893
983
  assertWritePrecondition({
@@ -914,7 +1004,7 @@ export class PostgresBackend {
914
1004
  }
915
1005
  const now = new Date().toISOString();
916
1006
  const version = (prior?.version ?? 0) + 1;
917
- const priorSize = exists ? Number(prior.size_bytes) : 0;
1007
+ const priorSize = exists ? Number(prior?.size_bytes ?? 0) : 0;
918
1008
  const delta = content.length - priorSize;
919
1009
  if (delta !== 0) {
920
1010
  await this._applyUsageDelta({ client, spaceId, delta });
@@ -924,10 +1014,11 @@ export class PostgresBackend {
924
1014
  // dropped both (the metadata went with the deleted Resource).
925
1015
  //
926
1016
  // Create-if-absent atomicity: when `If-None-Match: *` found NO prior row
927
- // (a tombstone is a real row and stays lock-serialized), the SELECT FOR
928
- // UPDATE locked nothing -- READ COMMITTED has no gap locks -- so a
929
- // concurrent creator may have raced past the same precondition. A plain
930
- // INSERT (no ON CONFLICT) makes the primary key the arbiter: the loser's
1017
+ // (a tombstone is a real row and stays lock-serialized), concurrent
1018
+ // creators through this method are already serialized by
1019
+ // `_lockSameKeyCreate` above -- but a writer that does not take that
1020
+ // lock (`importSpace`'s plain INSERTs) can still race. A plain INSERT
1021
+ // (no ON CONFLICT) keeps the primary key as the arbiter: the loser's
931
1022
  // unique violation maps to the 412 the precondition would have thrown.
932
1023
  // `createdBy` names the Resource's creator, not its last writer: taken
933
1024
  // from this write's invoker only when there is no prior row at all
@@ -1062,7 +1153,19 @@ export class PostgresBackend {
1062
1153
  // no-op, keeping an existing tombstone's change-feed entry stable.
1063
1154
  return;
1064
1155
  }
1065
- const freedBytes = Number(prior.size_bytes);
1156
+ // A soft delete is an UPDATE, not a row removal, so the chunk foreign
1157
+ // key's ON DELETE CASCADE does not fire -- remove the Resource's chunks
1158
+ // (the `chunked-streams` feature) explicitly in this same transaction so
1159
+ // they never outlive their parent, and return their bytes to the quota
1160
+ // counter alongside the Resource's content bytes.
1161
+ const { rows: chunkRows } = await client.query(`SELECT COALESCE(SUM(size), 0) AS total FROM chunks
1162
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
1163
+ const freedChunkBytes = Number(chunkRows[0]?.total ?? 0);
1164
+ if (freedChunkBytes > 0) {
1165
+ await client.query(`DELETE FROM chunks
1166
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
1167
+ }
1168
+ const freedBytes = Number(prior.size_bytes) + freedChunkBytes;
1066
1169
  if (freedBytes > 0) {
1067
1170
  await this._applyUsageDelta({ client, spaceId, delta: -freedBytes });
1068
1171
  }
@@ -1129,8 +1232,17 @@ export class PostgresBackend {
1129
1232
  * @param [options.ifNoneMatch] {boolean}
1130
1233
  * @returns {Promise<{ metaVersion: number } | undefined>}
1131
1234
  */
1132
- async writeResourceMetadata({ spaceId, collectionId, resourceId, custom, epoch, ifMatch, ifNoneMatch }) {
1235
+ async writeResourceMetadata({ spaceId, collectionId, resourceId, custom, epoch, uniqueIndexes, ifMatch, ifNoneMatch }) {
1133
1236
  return this._withTransaction(async (client) => {
1237
+ // A metadata write can create a plaintext equality unique claim for a
1238
+ // `custom`-sourced attribute (the `equality-query` feature). When the
1239
+ // Collection declares any unique index, take the per-Collection advisory
1240
+ // lock first (held to commit, serializing concurrent claimants) so the
1241
+ // conflict scan below is atomic with the write.
1242
+ const equalityUnique = uniqueIndexes !== undefined && uniqueIndexes.length > 0;
1243
+ if (equalityUnique) {
1244
+ await client.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [spaceId, collectionId]);
1245
+ }
1134
1246
  const { rows } = await client.query(`SELECT meta_version, deleted FROM resources
1135
1247
  WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1136
1248
  FOR UPDATE`, [spaceId, collectionId, resourceId]);
@@ -1144,6 +1256,34 @@ export class PostgresBackend {
1144
1256
  ifMatch,
1145
1257
  ifNoneMatch
1146
1258
  });
1259
+ if (equalityUnique) {
1260
+ // Content is the Resource's stored JSON content (unchanged by a
1261
+ // metadata write); custom is the incoming value this write sets. Fetch
1262
+ // the (possibly multi-MB) content only on this uniqueness path -- the
1263
+ // row is already `FOR UPDATE`-locked above.
1264
+ const { rows: selfRows } = await client.query(`SELECT content, is_json FROM resources
1265
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
1266
+ let content;
1267
+ const selfRow = selfRows[0];
1268
+ if (selfRow?.is_json && selfRow.content) {
1269
+ try {
1270
+ content = JSON.parse(selfRow.content.toString('utf8'));
1271
+ }
1272
+ catch {
1273
+ content = undefined;
1274
+ }
1275
+ }
1276
+ assertNoUniqueEqualityConflict({
1277
+ indexes: uniqueIndexes,
1278
+ content,
1279
+ custom,
1280
+ candidates: await this._readEqualityCandidates(client, {
1281
+ spaceId,
1282
+ collectionId,
1283
+ excludeResourceId: resourceId
1284
+ })
1285
+ });
1286
+ }
1147
1287
  const metaVersion = (prior.meta_version ?? 0) + 1;
1148
1288
  const hasCustom = Object.keys(custom).length > 0;
1149
1289
  const now = new Date().toISOString();
@@ -1168,6 +1308,276 @@ export class PostgresBackend {
1168
1308
  return { metaVersion };
1169
1309
  });
1170
1310
  }
1311
+ // Chunks (the `chunked-streams` feature)
1312
+ /**
1313
+ * Writes one chunk of a chunked Resource as one transaction: the parent
1314
+ * Resource must exist (checked atomically -- a `FOR SHARE` lock on it also
1315
+ * blocks a concurrent delete of the parent for the duration of the write, so
1316
+ * a chunk can never be orphaned by a racing `deleteResource`), the chunk row
1317
+ * is locked, its precondition evaluated, its monotonic `version` bumped, and
1318
+ * the transactional quota delta applied. The chunk body is stored opaquely as
1319
+ * a single `bytea`, the buffered-blob path (bounded by `maxUploadBytes`),
1320
+ * exactly like a binary Resource representation.
1321
+ * @param options {object}
1322
+ * @param options.spaceId {string}
1323
+ * @param options.collectionId {string}
1324
+ * @param options.resourceId {string}
1325
+ * @param options.chunkIndex {number} a non-negative safe integer
1326
+ * @param options.input {ResourceInput}
1327
+ * @param [options.ifMatch] {string}
1328
+ * @param [options.ifNoneMatch] {boolean}
1329
+ * @returns {Promise<{ version: number }>} the chunk's new version
1330
+ */
1331
+ async writeChunk({ spaceId, collectionId, resourceId, chunkIndex, input, ifMatch, ifNoneMatch }) {
1332
+ const { maxUploadBytes } = this;
1333
+ let bytes;
1334
+ if (input.kind === 'json') {
1335
+ bytes = Buffer.from(JSON.stringify(input.data));
1336
+ if (bytes.length > maxUploadBytes) {
1337
+ throw new PayloadTooLargeError({
1338
+ maxUploadBytes,
1339
+ backendId: this.describe().id,
1340
+ uploadBytes: bytes.length
1341
+ });
1342
+ }
1343
+ }
1344
+ else {
1345
+ // Pre-flight a declared size, then buffer through the counting guard that
1346
+ // hard-caps a body whose size is omitted or understated. Buffering
1347
+ // happens BEFORE the transaction so a slow upload holds no row lock.
1348
+ if (input.declaredBytes !== undefined &&
1349
+ input.declaredBytes > maxUploadBytes) {
1350
+ throw new PayloadTooLargeError({
1351
+ maxUploadBytes,
1352
+ backendId: this.describe().id,
1353
+ uploadBytes: input.declaredBytes
1354
+ });
1355
+ }
1356
+ bytes = await bufferStreamCapped({
1357
+ stream: input.stream,
1358
+ maxUploadBytes,
1359
+ backendId: this.describe().id
1360
+ });
1361
+ }
1362
+ return this._withTransaction(async (client) => {
1363
+ // Parent Resource must exist (and not be a tombstone). `FOR SHARE`
1364
+ // conflicts with the `FOR UPDATE` a concurrent `deleteResource` takes, so
1365
+ // the two serialize on the parent row -- the parent cannot be deleted
1366
+ // between this check and the chunk write.
1367
+ const { rows: parentRows } = await client.query(`SELECT deleted FROM resources
1368
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1369
+ FOR SHARE`, [spaceId, collectionId, resourceId]);
1370
+ const parent = parentRows[0];
1371
+ if (!parent || parent.deleted) {
1372
+ throw new ResourceNotFoundError({ requestName: 'Write Chunk' });
1373
+ }
1374
+ // Lock the chunk row (if any) and read its current version/size, so the
1375
+ // precondition, the monotonic bump, and the usage delta are all atomic
1376
+ // with the write.
1377
+ const { rows } = await client.query(`SELECT version, size FROM chunks
1378
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1379
+ AND chunk_index = $4
1380
+ FOR UPDATE`, [spaceId, collectionId, resourceId, chunkIndex]);
1381
+ let prior = rows[0];
1382
+ const chunkLabel = `${resourceId}/chunks/${chunkIndex}`;
1383
+ if (prior === undefined) {
1384
+ // The lock-nothing case: serialize with any concurrent creator of the
1385
+ // same chunk and re-read, so the usage delta below reflects the row it
1386
+ // committed (see `_lockSameKeyCreate`).
1387
+ await this._lockSameKeyCreate({
1388
+ client,
1389
+ spaceId,
1390
+ rowKey: `${collectionId}/${chunkLabel}`
1391
+ });
1392
+ const { rows: reread } = await client.query(`SELECT version, size FROM chunks
1393
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1394
+ AND chunk_index = $4
1395
+ FOR UPDATE`, [spaceId, collectionId, resourceId, chunkIndex]);
1396
+ prior = reread[0];
1397
+ }
1398
+ const exists = prior !== undefined;
1399
+ if (ifMatch !== undefined || ifNoneMatch) {
1400
+ assertWritePrecondition({
1401
+ resourceId: chunkLabel,
1402
+ exists,
1403
+ currentVersion: prior?.version ?? 0,
1404
+ ifMatch,
1405
+ ifNoneMatch
1406
+ });
1407
+ }
1408
+ const version = (prior?.version ?? 0) + 1;
1409
+ const priorSize = exists ? Number(prior?.size ?? 0) : 0;
1410
+ const delta = bytes.length - priorSize;
1411
+ if (delta !== 0) {
1412
+ await this._applyUsageDelta({ client, spaceId, delta });
1413
+ }
1414
+ const values = [
1415
+ spaceId,
1416
+ collectionId,
1417
+ resourceId,
1418
+ chunkIndex,
1419
+ input.contentType,
1420
+ bytes,
1421
+ bytes.length,
1422
+ version
1423
+ ];
1424
+ const insertSql = `
1425
+ INSERT INTO chunks (
1426
+ space_id, collection_id, resource_id, chunk_index,
1427
+ content_type, bytes, size, version
1428
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`;
1429
+ // Create-if-absent atomicity mirrors `writeResource`: concurrent
1430
+ // creators through this method are serialized by `_lockSameKeyCreate`
1431
+ // above, and against a writer that does not take that lock
1432
+ // (`importSpace`'s plain INSERTs) the plain INSERT keeps the primary key
1433
+ // as the arbiter -- a racing creator's unique violation maps to the 412
1434
+ // the precondition would have thrown.
1435
+ if (ifNoneMatch && prior === undefined) {
1436
+ try {
1437
+ await client.query(insertSql, values);
1438
+ }
1439
+ catch (err) {
1440
+ if (err.code === '23505') {
1441
+ throw new PreconditionFailedError({
1442
+ detail: `Chunk '${chunkLabel}' already exists (If-None-Match: *).`
1443
+ });
1444
+ }
1445
+ throw err;
1446
+ }
1447
+ return { version };
1448
+ }
1449
+ // The conflict update derives `version` from the row (`chunks.version +
1450
+ // 1`), not from the pre-read, so a concurrent creator that slipped in
1451
+ // after our lock-nothing SELECT still advances the counter monotonically.
1452
+ // RETURNING reports the version that actually landed.
1453
+ const { rows: written } = await client.query(`${insertSql}
1454
+ ON CONFLICT (space_id, collection_id, resource_id, chunk_index)
1455
+ DO UPDATE SET
1456
+ content_type = EXCLUDED.content_type,
1457
+ bytes = EXCLUDED.bytes,
1458
+ size = EXCLUDED.size,
1459
+ version = chunks.version + 1
1460
+ RETURNING version`, values);
1461
+ return { version: written[0].version };
1462
+ });
1463
+ }
1464
+ /**
1465
+ * Reads a chunk's bytes. Rejects with `ResourceNotFoundError` (404) when no
1466
+ * chunk is stored at that index.
1467
+ * @param options {object}
1468
+ * @param options.spaceId {string}
1469
+ * @param options.collectionId {string}
1470
+ * @param options.resourceId {string}
1471
+ * @param options.chunkIndex {number}
1472
+ * @returns {Promise<ResourceResult>}
1473
+ */
1474
+ async getChunk({ spaceId, collectionId, resourceId, chunkIndex }) {
1475
+ const { rows } = await this._pool.query(`SELECT content_type, bytes, version FROM chunks
1476
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1477
+ AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
1478
+ const row = rows[0];
1479
+ if (!row) {
1480
+ throw new ResourceNotFoundError({ requestName: 'Get Chunk' });
1481
+ }
1482
+ return {
1483
+ resourceStream: Readable.from(row.bytes),
1484
+ storedResourceType: row.content_type,
1485
+ version: row.version
1486
+ };
1487
+ }
1488
+ /**
1489
+ * Reads a chunk's stored content-type / size / version (the HEAD payload
1490
+ * headers). Resolves `undefined` when the chunk is absent.
1491
+ * @param options {object}
1492
+ * @param options.spaceId {string}
1493
+ * @param options.collectionId {string}
1494
+ * @param options.resourceId {string}
1495
+ * @param options.chunkIndex {number}
1496
+ * @returns {Promise<{ contentType: string, size: number, version?: number }
1497
+ * | undefined>}
1498
+ */
1499
+ async getChunkMetadata({ spaceId, collectionId, resourceId, chunkIndex }) {
1500
+ const { rows } = await this._pool.query(`SELECT content_type, size, version FROM chunks
1501
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1502
+ AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
1503
+ const row = rows[0];
1504
+ if (!row) {
1505
+ return undefined;
1506
+ }
1507
+ return {
1508
+ contentType: row.content_type,
1509
+ size: Number(row.size),
1510
+ version: row.version
1511
+ };
1512
+ }
1513
+ /**
1514
+ * Deletes one chunk as one transaction (the chunk row is locked, its
1515
+ * `ifMatch` precondition evaluated atomically, and its bytes returned to the
1516
+ * quota counter). Resolves `true` when a chunk was removed, `false` when none
1517
+ * was stored at that index (the handler 404s on `false` -- chunk deletes are
1518
+ * not silently idempotent, unlike `deleteResource`).
1519
+ * @param options {object}
1520
+ * @param options.spaceId {string}
1521
+ * @param options.collectionId {string}
1522
+ * @param options.resourceId {string}
1523
+ * @param options.chunkIndex {number}
1524
+ * @param [options.ifMatch] {string}
1525
+ * @returns {Promise<boolean>}
1526
+ */
1527
+ async deleteChunk({ spaceId, collectionId, resourceId, chunkIndex, ifMatch }) {
1528
+ return this._withTransaction(async (client) => {
1529
+ const { rows } = await client.query(`SELECT version, size FROM chunks
1530
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1531
+ AND chunk_index = $4
1532
+ FOR UPDATE`, [spaceId, collectionId, resourceId, chunkIndex]);
1533
+ const prior = rows[0];
1534
+ if (prior === undefined) {
1535
+ return false;
1536
+ }
1537
+ if (ifMatch !== undefined) {
1538
+ assertWritePrecondition({
1539
+ resourceId: `${resourceId}/chunks/${chunkIndex}`,
1540
+ exists: true,
1541
+ currentVersion: prior.version,
1542
+ ifMatch
1543
+ });
1544
+ }
1545
+ await client.query(`DELETE FROM chunks
1546
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1547
+ AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
1548
+ const freedBytes = Number(prior.size);
1549
+ if (freedBytes > 0) {
1550
+ await this._applyUsageDelta({ client, spaceId, delta: -freedBytes });
1551
+ }
1552
+ return true;
1553
+ });
1554
+ }
1555
+ /**
1556
+ * Lists a Resource's stored chunks in ascending `chunk_index` order -- the
1557
+ * discovery/reassembly listing. The opaque `bytes` column is deliberately not
1558
+ * selected. An empty listing (Resource with no chunks, or an absent Resource)
1559
+ * resolves `{ count: 0, chunks: [] }`.
1560
+ * @param options {object}
1561
+ * @param options.spaceId {string}
1562
+ * @param options.collectionId {string}
1563
+ * @param options.resourceId {string}
1564
+ * @returns {Promise<{ count: number, chunks: Array<{ index: number, size:
1565
+ * number, contentType: string, version?: number }> }>}
1566
+ */
1567
+ async listChunks({ spaceId, collectionId, resourceId }) {
1568
+ const { rows } = await this._pool.query(`SELECT chunk_index, size, content_type, version FROM chunks
1569
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1570
+ ORDER BY chunk_index`, [spaceId, collectionId, resourceId]);
1571
+ return {
1572
+ count: rows.length,
1573
+ chunks: rows.map(row => ({
1574
+ index: row.chunk_index,
1575
+ size: Number(row.size),
1576
+ contentType: row.content_type,
1577
+ version: row.version
1578
+ }))
1579
+ };
1580
+ }
1171
1581
  /**
1172
1582
  * Replication change feed (the `changes` query profile): one indexed keyset
1173
1583
  * query over `(updatedAt, resourceId)`, tombstones included, JSON documents
@@ -1279,6 +1689,100 @@ export class PostgresBackend {
1279
1689
  }
1280
1690
  return runBlindedIndexQuery({ candidates, query, count, limit, cursor });
1281
1691
  }
1692
+ /**
1693
+ * Plaintext equality query (the `equality` query profile; see the
1694
+ * `StorageBackend.queryByEquality` contract). Reads the Collection's live
1695
+ * Resources -- JSON rows carrying parsed `content`, all rows carrying their
1696
+ * `custom` jsonb -- and hands the candidates to the shared evaluator
1697
+ * (`lib/equalityIndex.ts`) for extraction, matching, ordering, and cursor
1698
+ * pagination -- identical semantics to the filesystem backend. A full scan of
1699
+ * the Collection per call, deliberate for this teaching backend; a
1700
+ * materialized variant would answer from a JSONB expression index.
1701
+ * @param options {object}
1702
+ * @param options.spaceId {string}
1703
+ * @param options.collectionId {string}
1704
+ * @param options.query {EqualityQuery}
1705
+ * @param options.indexes {NormalizedIndexDeclaration[]}
1706
+ * @param [options.count] {boolean} return only the match count
1707
+ * @param [options.limit] {number} requested page size
1708
+ * @param [options.cursor] {string} opaque cursor from a prior page
1709
+ * @returns {Promise<{ count: number } | EqualityQueryPage>}
1710
+ */
1711
+ async queryByEquality({ spaceId, collectionId, query, indexes, count, limit, cursor }) {
1712
+ const candidates = await this._readEqualityCandidates(this._pool, {
1713
+ spaceId,
1714
+ collectionId
1715
+ });
1716
+ return runEqualityQuery({
1717
+ candidates,
1718
+ query,
1719
+ indexes,
1720
+ count,
1721
+ limit,
1722
+ cursor
1723
+ });
1724
+ }
1725
+ /**
1726
+ * Declare-time uniqueness scan for the `equality` profile (see the
1727
+ * `StorageBackend.findEqualityUniqueViolation` contract): reads the
1728
+ * Collection's live Resources and delegates to the shared scan, which reports
1729
+ * the first `(name, value)` claimed by two different Resources under the given
1730
+ * `unique` declarations (or `undefined` when none is).
1731
+ * @param options {object}
1732
+ * @param options.spaceId {string}
1733
+ * @param options.collectionId {string}
1734
+ * @param options.indexes {NormalizedIndexDeclaration[]}
1735
+ * @returns {Promise<{ name: string, value: EqualityValue } | undefined>}
1736
+ */
1737
+ async findEqualityUniqueViolation({ spaceId, collectionId, indexes }) {
1738
+ const candidates = await this._readEqualityCandidates(this._pool, {
1739
+ spaceId,
1740
+ collectionId
1741
+ });
1742
+ return findEqualityUniqueViolation({ indexes, candidates });
1743
+ }
1744
+ /**
1745
+ * Reads every live Resource of a Collection as an equality candidate -- the
1746
+ * candidate set for the equality query and the plaintext unique-attribute
1747
+ * conflict scans. Includes blob Resources (queryable through their
1748
+ * `custom`-sourced attributes): each row resolves `{ resourceId, content?,
1749
+ * custom? }`, where `content` is the parsed JSON of a JSON row (a blob and
1750
+ * unparsable JSON contribute none) and `custom` is the row's jsonb `custom`
1751
+ * when set. Tombstones are excluded (`NOT deleted`); an optional excluded
1752
+ * Resource is skipped. Runs on the given executor -- the pool for a read-only
1753
+ * query, or the write transaction's client for a uniqueness scan (so the scan
1754
+ * shares the advisory lock and sees a consistent snapshot).
1755
+ * @param executor {pg.Pool | pg.PoolClient}
1756
+ * @param options {object}
1757
+ * @param options.spaceId {string}
1758
+ * @param options.collectionId {string}
1759
+ * @param [options.excludeResourceId] {string}
1760
+ * @returns {Promise<EqualityCandidate[]>}
1761
+ */
1762
+ async _readEqualityCandidates(executor, { spaceId, collectionId, excludeResourceId }) {
1763
+ const { rows } = await executor.query(`SELECT resource_id, content, is_json, custom FROM resources
1764
+ WHERE space_id = $1 AND collection_id = $2 AND NOT deleted
1765
+ AND ($3::text IS NULL OR resource_id <> $3)
1766
+ ORDER BY resource_id`, [spaceId, collectionId, excludeResourceId ?? null]);
1767
+ const candidates = [];
1768
+ for (const row of rows) {
1769
+ let content;
1770
+ if (row.is_json && row.content) {
1771
+ try {
1772
+ content = JSON.parse(row.content.toString('utf8'));
1773
+ }
1774
+ catch {
1775
+ // skip an unparsable body -- it contributes no content attributes
1776
+ }
1777
+ }
1778
+ candidates.push({
1779
+ resourceId: row.resource_id,
1780
+ ...(content !== undefined && { content }),
1781
+ ...(row.custom !== null && { custom: row.custom })
1782
+ });
1783
+ }
1784
+ return candidates;
1785
+ }
1282
1786
  // Policies
1283
1787
  /**
1284
1788
  * Maps the optional-id policy addressing onto the sentinel-column primary
@@ -1678,7 +2182,7 @@ export class PostgresBackend {
1678
2182
  if (!spaceDescription) {
1679
2183
  throw new SpaceNotFoundError({ requestName: 'Export Space' });
1680
2184
  }
1681
- const [{ rows: policyRows }, { rows: collectionRows }, { rows: resourceRows }, { rows: revocationRows }] = await Promise.all([
2185
+ const [{ rows: policyRows }, { rows: collectionRows }, { rows: resourceRows }, { rows: revocationRows }, { rows: chunkRows }] = await Promise.all([
1682
2186
  this._pool.query(`SELECT collection_id, resource_id, policy FROM policies
1683
2187
  WHERE space_id = $1`, [spaceId]),
1684
2188
  this._pool.query(`SELECT collection_id, description, description_version FROM collections
@@ -1690,7 +2194,12 @@ export class PostgresBackend {
1690
2194
  created_at, updated_at, created_by
1691
2195
  FROM resources WHERE space_id = $1`, [spaceId]),
1692
2196
  this._pool.query(`SELECT delegator, capability_id, record FROM space_revocations
1693
- WHERE space_id = $1`, [spaceId])
2197
+ WHERE space_id = $1`, [spaceId]),
2198
+ // Chunk metadata only -- bytes are fetched one chunk at a time while
2199
+ // packing, so an export never holds a chunked Resource whole in memory.
2200
+ this._pool.query(`SELECT collection_id, resource_id, chunk_index, content_type, version
2201
+ FROM chunks WHERE space_id = $1
2202
+ ORDER BY collection_id, resource_id, chunk_index`, [spaceId])
1694
2203
  ]);
1695
2204
  // Assemble the per-entry file lists in the filesystem's shapes: files are
1696
2205
  // named by the shared codecs and sorted with localeCompare, matching the
@@ -1765,6 +2274,46 @@ export class PostgresBackend {
1765
2274
  });
1766
2275
  }
1767
2276
  }
2277
+ // Chunks (the `chunked-streams` feature): each chunked Resource contributes
2278
+ // one `.chunks.<encResourceId>/` subdirectory in its Collection dir, in the
2279
+ // exact filesystem-backend layout so an archive imports into either backend.
2280
+ // A chunk is stored there as a Resource keyed by its stringified index: an
2281
+ // `r.<index>.<encContentType>.<ext>` representation file (`fileNameFor`)
2282
+ // plus a `.meta.<index>.json` version sidecar. Files within a chunk dir are
2283
+ // sorted by name (the filesystem's readdir sort). Rows arrive ordered by
2284
+ // `(collection, resource, index)`.
2285
+ const chunkDirsByResource = new Map();
2286
+ for (const row of chunkRows) {
2287
+ const dirKey = `${row.collection_id}/${row.resource_id}`;
2288
+ let chunkFiles = chunkDirsByResource.get(dirKey);
2289
+ if (!chunkFiles) {
2290
+ chunkFiles = [];
2291
+ chunkDirsByResource.set(dirKey, chunkFiles);
2292
+ filesFor(row.collection_id).push({
2293
+ name: chunkDirName(row.resource_id),
2294
+ chunkDir: chunkFiles
2295
+ });
2296
+ }
2297
+ const chunkId = String(row.chunk_index);
2298
+ chunkFiles.push({
2299
+ name: fileNameFor({
2300
+ resourceId: chunkId,
2301
+ contentType: row.content_type
2302
+ }),
2303
+ chunk: {
2304
+ collectionId: row.collection_id,
2305
+ resourceId: row.resource_id,
2306
+ chunkIndex: row.chunk_index
2307
+ }
2308
+ });
2309
+ chunkFiles.push({
2310
+ name: `.meta.${chunkId}.json`,
2311
+ bytes: Buffer.from(JSON.stringify({ version: row.version }))
2312
+ });
2313
+ }
2314
+ for (const chunkFiles of chunkDirsByResource.values()) {
2315
+ chunkFiles.sort((left, right) => left.name.localeCompare(right.name));
2316
+ }
1768
2317
  // Top-level order: space-level files and collection dirs interleaved,
1769
2318
  // sorted by name -- the same order the filesystem's readdir+sort yields.
1770
2319
  const topLevel = [
@@ -1791,33 +2340,59 @@ export class PostgresBackend {
1791
2340
  const manifest = buildExportManifest({
1792
2341
  spaceId,
1793
2342
  entries: topLevel.map(entry => entry.kind === 'collection'
1794
- ? { name: entry.name, files: entry.files.map(file => file.name) }
2343
+ ? {
2344
+ name: entry.name,
2345
+ // A chunk directory expands to its `.chunks.<encId>/<file>`
2346
+ // relative paths (mirroring the pack order), so the manifest
2347
+ // matches the filesystem backend's for the same Space.
2348
+ files: entry.files.flatMap(file => 'chunkDir' in file
2349
+ ? file.chunkDir.map(child => `${file.name}/${child.name}`)
2350
+ : [file.name])
2351
+ }
1795
2352
  : { name: entry.name }),
1796
2353
  revocationFiles: revocationFiles.map(file => file.name)
1797
2354
  });
2355
+ // Fixed mtime on every entry so the archive is byte-reproducible (see
2356
+ // EXPORT_ENTRY_MTIME).
2357
+ const mtime = EXPORT_ENTRY_MTIME;
1798
2358
  const pack = tar.pack();
1799
- pack.entry({ name: 'manifest.yml' }, YAML.stringify(manifest));
1800
- pack.entry({ name: 'space/', type: 'directory' });
1801
- pack.entry({ name: `space/${spaceId}/`, type: 'directory' });
2359
+ pack.entry({ name: 'manifest.yml', mtime }, YAML.stringify(manifest));
2360
+ pack.entry({ name: 'space/', type: 'directory', mtime });
2361
+ pack.entry({ name: `space/${spaceId}/`, type: 'directory', mtime });
1802
2362
  for (const entry of topLevel) {
1803
2363
  const entryTarget = `space/${spaceId}/${entry.name}`;
1804
2364
  if (entry.kind === 'collection') {
1805
- pack.entry({ name: `${entryTarget}/`, type: 'directory' });
2365
+ pack.entry({ name: `${entryTarget}/`, type: 'directory', mtime });
1806
2366
  for (const file of entry.files) {
2367
+ if ('chunkDir' in file) {
2368
+ // A chunked Resource's `.chunks.<encId>/` subdirectory: emit the
2369
+ // directory entry, then each file -- an inline `.meta.<index>.json`
2370
+ // sidecar, or a chunk representation whose bytes are fetched one at
2371
+ // a time.
2372
+ const dirTarget = `${entryTarget}/${file.name}`;
2373
+ pack.entry({ name: `${dirTarget}/`, type: 'directory', mtime });
2374
+ for (const chunkFile of file.chunkDir) {
2375
+ const bytes = 'bytes' in chunkFile
2376
+ ? chunkFile.bytes
2377
+ : await this._chunkContent({ spaceId, ...chunkFile.chunk });
2378
+ pack.entry({ name: `${dirTarget}/${chunkFile.name}`, mtime }, bytes);
2379
+ }
2380
+ continue;
2381
+ }
1807
2382
  const bytes = 'bytes' in file
1808
2383
  ? file.bytes
1809
2384
  : await this._resourceContent({ spaceId, ...file.resource });
1810
- pack.entry({ name: `${entryTarget}/${file.name}` }, bytes);
2385
+ pack.entry({ name: `${entryTarget}/${file.name}`, mtime }, bytes);
1811
2386
  }
1812
2387
  }
1813
2388
  else {
1814
- pack.entry({ name: entryTarget }, entry.bytes);
2389
+ pack.entry({ name: entryTarget, mtime }, entry.bytes);
1815
2390
  }
1816
2391
  }
1817
2392
  if (revocationFiles.length > 0) {
1818
- pack.entry({ name: 'revocations/', type: 'directory' });
2393
+ pack.entry({ name: 'revocations/', type: 'directory', mtime });
1819
2394
  for (const file of revocationFiles) {
1820
- pack.entry({ name: `revocations/${file.name}` }, file.bytes);
2395
+ pack.entry({ name: `revocations/${file.name}`, mtime }, file.bytes);
1821
2396
  }
1822
2397
  }
1823
2398
  pack.finalize();
@@ -1839,6 +2414,24 @@ export class PostgresBackend {
1839
2414
  WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
1840
2415
  return rows[0]?.content ?? Buffer.alloc(0);
1841
2416
  }
2417
+ /**
2418
+ * Fetches one chunk's bytes for the export pack loop (the `chunked-streams`
2419
+ * feature). A chunk removed between the metadata pass and this read yields an
2420
+ * empty body rather than failing the whole archive, matching
2421
+ * `_resourceContent`.
2422
+ * @param options {object}
2423
+ * @param options.spaceId {string}
2424
+ * @param options.collectionId {string}
2425
+ * @param options.resourceId {string}
2426
+ * @param options.chunkIndex {number}
2427
+ * @returns {Promise<Buffer>}
2428
+ */
2429
+ async _chunkContent({ spaceId, collectionId, resourceId, chunkIndex }) {
2430
+ const { rows } = await this._pool.query(`SELECT bytes FROM chunks
2431
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2432
+ AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
2433
+ return rows[0]?.bytes ?? Buffer.alloc(0);
2434
+ }
1842
2435
  /**
1843
2436
  * Merges a WAS space-export tarball into an existing Space with the same
1844
2437
  * three-invariant pre-flight (per-entry 413, fail-closed 422 encryption
@@ -1855,6 +2448,10 @@ export class PostgresBackend {
1855
2448
  async importSpace({ spaceId, tarStream }) {
1856
2449
  const entries = await extractTarEntries(tarStream);
1857
2450
  const { spacePolicy, collections, revocations } = buildImportPlan(entries);
2451
+ // Chunk entries (the `chunked-streams` feature) live in per-Resource
2452
+ // `.chunks.<encId>/` subdirectories, which `buildImportPlan` skips (it
2453
+ // ignores nested files); parse them straight off the raw tar entries.
2454
+ const chunkEntries = this._chunkEntriesFromArchive(entries);
1858
2455
  const { capacityBytes, maxUploadBytes, maxCollectionsPerSpace, maxResourcesPerSpace } = this;
1859
2456
  return this._withTransaction(async (client) => {
1860
2457
  await this._ensureSpaceRow({ client, spaceId });
@@ -1918,6 +2515,19 @@ export class PostgresBackend {
1918
2515
  incomingBytes += body.length;
1919
2516
  }
1920
2517
  }
2518
+ // Chunks (the `chunked-streams` feature) are opaque bytes -- no
2519
+ // encryption-conformance check applies -- but they count the same per-body
2520
+ // 413 and the (conservative) capacity pre-flight as Resource bodies.
2521
+ for (const chunk of chunkEntries) {
2522
+ if (chunk.body.length > maxUploadBytes) {
2523
+ throw new PayloadTooLargeError({
2524
+ maxUploadBytes,
2525
+ backendId: this.describe().id,
2526
+ uploadBytes: chunk.body.length
2527
+ });
2528
+ }
2529
+ incomingBytes += chunk.body.length;
2530
+ }
1921
2531
  if (capacityBytes !== undefined &&
1922
2532
  currentUsage + incomingBytes > capacityBytes) {
1923
2533
  throw new QuotaExceededError({ spaceId, capacityBytes });
@@ -2081,13 +2691,53 @@ export class PostgresBackend {
2081
2691
  stats.resourcesCreated++;
2082
2692
  }
2083
2693
  }
2084
- if (createdBytes > 0) {
2694
+ // Chunks (the `chunked-streams` feature): restore each archived chunk
2695
+ // skip-not-overwrite, after the Resource apply loop so a chunk's parent
2696
+ // Resource row already exists in this transaction (the foreign key
2697
+ // requires it). An orphan chunk -- one whose parent is absent or a
2698
+ // tombstone -- is skipped rather than resurrected. Existing chunk rows are
2699
+ // left untouched.
2700
+ let createdChunkBytes = 0;
2701
+ for (const chunk of chunkEntries) {
2702
+ const { rows: parentRows } = await client.query(`SELECT deleted FROM resources
2703
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, chunk.collectionId, chunk.resourceId]);
2704
+ const parent = parentRows[0];
2705
+ if (!parent || parent.deleted) {
2706
+ continue;
2707
+ }
2708
+ const { rows: existingChunkRows } = await client.query(`SELECT 1 FROM chunks
2709
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2710
+ AND chunk_index = $4`, [spaceId, chunk.collectionId, chunk.resourceId, chunk.chunkIndex]);
2711
+ if (existingChunkRows.length > 0) {
2712
+ continue;
2713
+ }
2714
+ // The chunk's `version` comes from its archived `.meta.<index>.json`
2715
+ // sidecar; an archive without one (or a chunk written before sidecars)
2716
+ // starts at version 1, the same fresh-write default as a Resource
2717
+ // restored without a sidecar.
2718
+ await client.query(`INSERT INTO chunks (
2719
+ space_id, collection_id, resource_id, chunk_index,
2720
+ content_type, bytes, size, version
2721
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
2722
+ spaceId,
2723
+ chunk.collectionId,
2724
+ chunk.resourceId,
2725
+ chunk.chunkIndex,
2726
+ chunk.contentType,
2727
+ chunk.body,
2728
+ chunk.body.length,
2729
+ chunk.version ?? 1
2730
+ ]);
2731
+ createdChunkBytes += chunk.body.length;
2732
+ }
2733
+ const createdTotalBytes = createdBytes + createdChunkBytes;
2734
+ if (createdTotalBytes > 0) {
2085
2735
  // The pre-flight was conservative (it counted skips too), so the
2086
2736
  // actual created total always fits; apply it unguarded.
2087
2737
  await this._applyUsageDelta({
2088
2738
  client,
2089
2739
  spaceId,
2090
- delta: createdBytes
2740
+ delta: createdTotalBytes
2091
2741
  });
2092
2742
  }
2093
2743
  // Restore the archive's Space-scoped zcap revocations under this
@@ -2160,5 +2810,113 @@ export class PostgresBackend {
2160
2810
  sidecar?.epoch ?? null
2161
2811
  ]);
2162
2812
  }
2813
+ /**
2814
+ * Parses a Space-export archive's chunk entries (the `chunked-streams`
2815
+ * feature) out of the raw tar entry map, in the filesystem backend's on-disk
2816
+ * layout so archives round-trip between the two backends. Chunk files live at
2817
+ * `space/<sourceSpaceId>/<collectionId>/.chunks.<encResourceId>/<file>` --
2818
+ * where a chunk is a Resource keyed by its stringified index: an
2819
+ * `r.<index>.<encContentType>.<ext>` representation paired with an optional
2820
+ * `.meta.<index>.json` version sidecar. `buildImportPlan` skips these (it
2821
+ * ignores nested Collection-dir files), so they are parsed here. Ids parsed
2822
+ * from the archive are validated (the path-traversal guard); a file whose
2823
+ * raw index segment is not canonical (`parseChunkIndexSegment`) is ignored,
2824
+ * and a sidecar with no paired representation is dropped (a chunk keeps no
2825
+ * tombstone).
2826
+ * @param entries {Map<string, TarEntry>}
2827
+ * @returns {Array<{ collectionId: string, resourceId: string, chunkIndex:
2828
+ * number, contentType: string, body: Buffer, version?: number }>}
2829
+ */
2830
+ _chunkEntriesFromArchive(entries) {
2831
+ // Accumulate the representation and the sidecar of each chunk under one
2832
+ // key, then emit only the chunks that carry a representation.
2833
+ const staged = new Map();
2834
+ const pattern = /^space\/[^/]+\/([^/]+)\/(\.chunks\.[^/]+)\/([^/]+)$/;
2835
+ for (const [entryName, entry] of entries) {
2836
+ if (entry.type !== 'file' || !entry.body) {
2837
+ continue;
2838
+ }
2839
+ const match = entryName.match(pattern);
2840
+ if (!match) {
2841
+ continue;
2842
+ }
2843
+ const collectionId = match[1];
2844
+ const resourceId = parseChunkDirName(match[2]);
2845
+ const fileName = match[3];
2846
+ if (resourceId === undefined) {
2847
+ continue;
2848
+ }
2849
+ // Reject a path-traversal / non-URL-safe id parsed from the archive
2850
+ // before it keys a destination row.
2851
+ assertValidId(collectionId, {
2852
+ kind: 'collection',
2853
+ requestName: 'Import Space'
2854
+ });
2855
+ assertValidId(resourceId, {
2856
+ kind: 'resource',
2857
+ requestName: 'Import Space'
2858
+ });
2859
+ // A chunk's index is the "resource id" of its file within the chunk dir:
2860
+ // `r.<index>.<encType>.<ext>` for the representation, `.meta.<index>.json`
2861
+ // for the version sidecar. The RAW index segment must pass the same
2862
+ // canonical predicate as the live route (`parseChunkIndexSegment`), so a
2863
+ // non-canonical spelling (`r.01.*`, `r.%31.*`) is dropped rather than
2864
+ // coerced onto -- and colliding with -- the canonical chunk's row.
2865
+ let chunkIdSegment;
2866
+ let representation;
2867
+ let sidecarVersion;
2868
+ const metaId = metaSidecarFileId(fileName);
2869
+ if (metaId !== undefined) {
2870
+ chunkIdSegment = metaId;
2871
+ try {
2872
+ const sidecar = JSON.parse(entry.body.toString('utf8'));
2873
+ sidecarVersion = sidecar.version;
2874
+ }
2875
+ catch {
2876
+ sidecarVersion = undefined;
2877
+ }
2878
+ }
2879
+ else if (fileName.startsWith('r.')) {
2880
+ const { contentType } = parseResourceFileName(fileName);
2881
+ chunkIdSegment = fileName.split('.')[1];
2882
+ representation = { contentType, body: entry.body };
2883
+ }
2884
+ else {
2885
+ continue;
2886
+ }
2887
+ const chunkIndex = chunkIdSegment === undefined
2888
+ ? undefined
2889
+ : parseChunkIndexSegment(chunkIdSegment);
2890
+ if (chunkIndex === undefined) {
2891
+ continue;
2892
+ }
2893
+ const key = `${collectionId}/${resourceId}/${chunkIndex}`;
2894
+ const slot = staged.get(key) ?? { collectionId, resourceId, chunkIndex };
2895
+ if (representation !== undefined) {
2896
+ slot.contentType = representation.contentType;
2897
+ slot.body = representation.body;
2898
+ }
2899
+ if (sidecarVersion !== undefined) {
2900
+ slot.version = sidecarVersion;
2901
+ }
2902
+ staged.set(key, slot);
2903
+ }
2904
+ const parsed = [];
2905
+ for (const slot of staged.values()) {
2906
+ if (slot.body === undefined) {
2907
+ // A sidecar with no paired representation is not a valid chunk.
2908
+ continue;
2909
+ }
2910
+ parsed.push({
2911
+ collectionId: slot.collectionId,
2912
+ resourceId: slot.resourceId,
2913
+ chunkIndex: slot.chunkIndex,
2914
+ contentType: slot.contentType ?? 'application/octet-stream',
2915
+ body: slot.body,
2916
+ version: slot.version
2917
+ });
2918
+ }
2919
+ return parsed;
2920
+ }
2163
2921
  }
2164
2922
  //# sourceMappingURL=postgres.js.map