was-teaching-server 0.9.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +118 -1
  2. package/README.md +23 -11
  3. package/dist/backends/filesystem.d.ts +247 -4
  4. package/dist/backends/filesystem.d.ts.map +1 -1
  5. package/dist/backends/filesystem.js +711 -81
  6. package/dist/backends/filesystem.js.map +1 -1
  7. package/dist/backends/postgres.d.ts +240 -2
  8. package/dist/backends/postgres.d.ts.map +1 -1
  9. package/dist/backends/postgres.js +800 -42
  10. package/dist/backends/postgres.js.map +1 -1
  11. package/dist/backends/postgresSchema.d.ts.map +1 -1
  12. package/dist/backends/postgresSchema.js +69 -57
  13. package/dist/backends/postgresSchema.js.map +1 -1
  14. package/dist/errors.d.ts +26 -9
  15. package/dist/errors.d.ts.map +1 -1
  16. package/dist/errors.js +40 -12
  17. package/dist/errors.js.map +1 -1
  18. package/dist/lib/equalityIndex.d.ts +334 -0
  19. package/dist/lib/equalityIndex.d.ts.map +1 -0
  20. package/dist/lib/equalityIndex.js +552 -0
  21. package/dist/lib/equalityIndex.js.map +1 -0
  22. package/dist/lib/exportManifest.d.ts +9 -0
  23. package/dist/lib/exportManifest.d.ts.map +1 -1
  24. package/dist/lib/exportManifest.js +9 -0
  25. package/dist/lib/exportManifest.js.map +1 -1
  26. package/dist/lib/importTar.d.ts +18 -2
  27. package/dist/lib/importTar.d.ts.map +1 -1
  28. package/dist/lib/importTar.js +77 -4
  29. package/dist/lib/importTar.js.map +1 -1
  30. package/dist/lib/paths.d.ts +33 -0
  31. package/dist/lib/paths.d.ts.map +1 -1
  32. package/dist/lib/paths.js +28 -0
  33. package/dist/lib/paths.js.map +1 -1
  34. package/dist/lib/resourceFileName.d.ts +43 -0
  35. package/dist/lib/resourceFileName.d.ts.map +1 -1
  36. package/dist/lib/resourceFileName.js +63 -0
  37. package/dist/lib/resourceFileName.js.map +1 -1
  38. package/dist/plugin.d.ts.map +1 -1
  39. package/dist/plugin.js +5 -2
  40. package/dist/plugin.js.map +1 -1
  41. package/dist/requests/ChunkRequest.d.ts +104 -0
  42. package/dist/requests/ChunkRequest.d.ts.map +1 -0
  43. package/dist/requests/ChunkRequest.js +396 -0
  44. package/dist/requests/ChunkRequest.js.map +1 -0
  45. package/dist/requests/CollectionRequest.d.ts +22 -5
  46. package/dist/requests/CollectionRequest.d.ts.map +1 -1
  47. package/dist/requests/CollectionRequest.js +192 -11
  48. package/dist/requests/CollectionRequest.js.map +1 -1
  49. package/dist/requests/ResourceRequest.d.ts.map +1 -1
  50. package/dist/requests/ResourceRequest.js +20 -0
  51. package/dist/requests/ResourceRequest.js.map +1 -1
  52. package/dist/requests/SpaceRequest.d.ts +1 -0
  53. package/dist/requests/SpaceRequest.d.ts.map +1 -1
  54. package/dist/requests/SpaceRequest.js +21 -1
  55. package/dist/requests/SpaceRequest.js.map +1 -1
  56. package/dist/routes.d.ts.map +1 -1
  57. package/dist/routes.js +21 -0
  58. package/dist/routes.js.map +1 -1
  59. package/dist/types.d.ts +165 -2
  60. package/dist/types.d.ts.map +1 -1
  61. package/package.json +19 -18
  62. package/src/backends/filesystem.ts +924 -75
  63. package/src/backends/postgres.ts +1114 -32
  64. package/src/backends/postgresSchema.ts +69 -57
  65. package/src/errors.ts +44 -12
  66. package/src/lib/equalityIndex.ts +769 -0
  67. package/src/lib/importTar.ts +107 -4
  68. package/src/lib/paths.ts +48 -0
  69. package/src/lib/resourceFileName.ts +69 -0
  70. package/src/plugin.ts +5 -2
  71. package/src/requests/ChunkRequest.ts +478 -0
  72. package/src/requests/CollectionRequest.ts +221 -12
  73. package/src/requests/ResourceRequest.ts +20 -0
  74. package/src/requests/SpaceRequest.ts +25 -1
  75. package/src/routes.ts +46 -0
  76. package/src/types.ts +175 -1
@@ -42,9 +42,21 @@ import {
42
42
  DuplicateRevocationError
43
43
  } from '../errors.js'
44
44
  import { applyMigrations } from './postgresSchema.js'
45
- import { extractTarEntries, buildImportPlan } from '../lib/importTar.js'
45
+ import {
46
+ extractTarEntries,
47
+ buildImportPlan,
48
+ metaSidecarFileId
49
+ } from '../lib/importTar.js'
50
+ import type { TarEntry } from '../lib/importTar.js'
46
51
  import { collectionPath, resourcePath } from '../lib/paths.js'
47
- import { fileNameFor, parseResourceFileName } from '../lib/resourceFileName.js'
52
+ import {
53
+ fileNameFor,
54
+ parseResourceFileName,
55
+ chunkDirName,
56
+ parseChunkDirName,
57
+ parseChunkIndexSegment
58
+ } from '../lib/resourceFileName.js'
59
+ import { assertValidId } from '../lib/validateId.js'
48
60
  import { sanitizeBackendRecord } from '../lib/backends.js'
49
61
  import { backendUsageFields } from '../lib/backendUsage.js'
50
62
  import { assertEncryptedWriteConforms } from '../lib/encryption.js'
@@ -66,6 +78,18 @@ import type {
66
78
  BlindedIndexQuery,
67
79
  BlindedIndexQueryPage
68
80
  } from '../lib/blindedIndex.js'
81
+ import {
82
+ runEqualityQuery,
83
+ assertNoUniqueEqualityConflict,
84
+ findEqualityUniqueViolation
85
+ } from '../lib/equalityIndex.js'
86
+ import type {
87
+ EqualityQuery,
88
+ EqualityQueryPage,
89
+ EqualityCandidate,
90
+ EqualityValue,
91
+ NormalizedIndexDeclaration
92
+ } from '../lib/equalityIndex.js'
69
93
  import {
70
94
  DEFAULT_MAX_UPLOAD_BYTES,
71
95
  DEFAULT_MAX_SPACES_PER_CONTROLLER,
@@ -206,6 +230,17 @@ function parseSidecar(bytes: Buffer | undefined): SidecarShape | undefined {
206
230
  }
207
231
  }
208
232
 
233
+ /**
234
+ * The chunk-metadata sidecar shape (`.chunks.<encId>/.meta.<index>.json`) the
235
+ * filesystem backend writes per chunk. Only the monotonic `version` (the
236
+ * chunk's ETag validator) is carried across export/import; the filesystem
237
+ * writes `createdAt` / `updatedAt` too, but this backend's `chunks` table holds
238
+ * no chunk timestamps, so it emits and reads only `version`.
239
+ */
240
+ interface ChunkSidecar {
241
+ version?: number
242
+ }
243
+
209
244
  export class PostgresBackend implements StorageBackend {
210
245
  logger: FastifyBaseLogger
211
246
  /**
@@ -421,11 +456,18 @@ export class PostgresBackend implements StorageBackend {
421
456
  // `key-epochs`: multi-recipient encrypted Collections -- per-epoch wrapped
422
457
  // keys on the `encryption` marker, a client-declared `epoch` stamp on
423
458
  // Resources, and conditional (`If-Match`) Collection Description writes.
459
+ // `chunked-streams`: chunk addressing at `/{resourceId}/chunks/{n}`, opaque
460
+ // per-chunk raw-bytes storage in the `chunks` table.
461
+ // `equality-query`: serves the `equality` profile -- server-extracted
462
+ // plaintext attribute equality over a Collection's declared `indexes`
463
+ // (`queryByEquality`), plus the GET `filter[attr]=value` equality filter.
424
464
  features: [
425
465
  'conditional-writes',
426
466
  'changes-query',
427
467
  'blinded-index-query',
428
- 'key-epochs'
468
+ 'equality-query',
469
+ 'key-epochs',
470
+ 'chunked-streams'
429
471
  ]
430
472
  }
431
473
  }
@@ -519,6 +561,42 @@ export class PostgresBackend implements StorageBackend {
519
561
  }
520
562
  }
521
563
 
564
+ /**
565
+ * Serializes concurrent CREATORS of one not-yet-existing row (a Resource or
566
+ * a chunk) on a transaction-scoped advisory lock keyed by the row's
567
+ * identity. Under READ COMMITTED a `SELECT ... FOR UPDATE` on an absent row
568
+ * locks nothing (no gap locks), so two concurrent creators would both read
569
+ * "no prior row" and both apply their full byte size as the usage delta --
570
+ * and while the `ON CONFLICT` arm recomputes `version` from the conflicting
571
+ * row, it cannot retroactively fix the loser's delta, leaving `usage_bytes`
572
+ * inflated by one write's size. The caller takes this lock when its
573
+ * lock-nothing SELECT found no row, then RE-reads the row: the second
574
+ * creator blocks here until the first commits, and its re-read sees the
575
+ * committed row, so its precondition, version, and usage delta are computed
576
+ * from accurate state. Held to commit (advisory xact lock). The `create:`
577
+ * prefix keeps this key domain distinct from the unique-blinded-term
578
+ * advisory lock, which hashes the bare `(spaceId, collectionId)`.
579
+ * @param options {object}
580
+ * @param options.client {pg.PoolClient}
581
+ * @param options.spaceId {string}
582
+ * @param options.rowKey {string} the row's identity within the Space
583
+ * @returns {Promise<void>}
584
+ */
585
+ private async _lockSameKeyCreate({
586
+ client,
587
+ spaceId,
588
+ rowKey
589
+ }: {
590
+ client: pg.PoolClient
591
+ spaceId: string
592
+ rowKey: string
593
+ }): Promise<void> {
594
+ await client.query(
595
+ 'SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))',
596
+ [spaceId, `create:${rowKey}`]
597
+ )
598
+ }
599
+
522
600
  // Quotas
523
601
 
524
602
  /**
@@ -546,13 +624,20 @@ export class PostgresBackend implements StorageBackend {
546
624
 
547
625
  let usageByCollection: CollectionUsage[] | undefined
548
626
  if (includeCollections) {
627
+ // Per-Collection usage sums both Resource content bytes and chunk bytes
628
+ // (the `chunked-streams` feature) so the breakdown agrees with the
629
+ // Space total in the transactional counter.
549
630
  const { rows: collectionRows } = await this._pool.query<{
550
631
  collection_id: string
551
632
  usage: string
552
633
  }>(
553
- `SELECT collection_id, COALESCE(SUM(size_bytes), 0) AS usage
554
- FROM resources
555
- WHERE space_id = $1
634
+ `SELECT collection_id, COALESCE(SUM(bytes), 0) AS usage FROM (
635
+ SELECT collection_id, size_bytes AS bytes FROM resources
636
+ WHERE space_id = $1
637
+ UNION ALL
638
+ SELECT collection_id, size AS bytes FROM chunks
639
+ WHERE space_id = $1
640
+ ) usage_rows
556
641
  GROUP BY collection_id
557
642
  ORDER BY collection_id`,
558
643
  [spaceId]
@@ -595,6 +680,8 @@ export class PostgresBackend implements StorageBackend {
595
680
  (SELECT COALESCE(usage_bytes, 0) FROM spaces WHERE space_id = $1)
596
681
  AS space_total,
597
682
  (SELECT COALESCE(SUM(size_bytes), 0) FROM resources
683
+ WHERE space_id = $1 AND collection_id = $2)
684
+ + (SELECT COALESCE(SUM(size), 0) FROM chunks
598
685
  WHERE space_id = $1 AND collection_id = $2) AS collection_total`,
599
686
  [spaceId, collectionId]
600
687
  )
@@ -1024,9 +1111,15 @@ export class PostgresBackend implements StorageBackend {
1024
1111
  collectionId: string
1025
1112
  }): Promise<void> {
1026
1113
  await this._withTransaction(async client => {
1114
+ // The Collection's freed bytes are its Resource content plus its chunk
1115
+ // bytes (the `chunked-streams` feature); both cascade away with the
1116
+ // Collection row, so both leave the quota counter.
1027
1117
  const { rows } = await client.query<{ total: string }>(
1028
- `SELECT COALESCE(SUM(size_bytes), 0) AS total FROM resources
1029
- WHERE space_id = $1 AND collection_id = $2`,
1118
+ `SELECT
1119
+ (SELECT COALESCE(SUM(size_bytes), 0) FROM resources
1120
+ WHERE space_id = $1 AND collection_id = $2)
1121
+ + (SELECT COALESCE(SUM(size), 0) FROM chunks
1122
+ WHERE space_id = $1 AND collection_id = $2) AS total`,
1030
1123
  [spaceId, collectionId]
1031
1124
  )
1032
1125
  const freedBytes = Number(rows[0]?.total ?? 0)
@@ -1203,6 +1296,7 @@ export class PostgresBackend implements StorageBackend {
1203
1296
  input,
1204
1297
  createdBy,
1205
1298
  epoch,
1299
+ uniqueIndexes,
1206
1300
  ifMatch,
1207
1301
  ifNoneMatch
1208
1302
  }: {
@@ -1212,6 +1306,7 @@ export class PostgresBackend implements StorageBackend {
1212
1306
  input: ResourceInput
1213
1307
  createdBy?: IDID
1214
1308
  epoch?: string
1309
+ uniqueIndexes?: NormalizedIndexDeclaration[]
1215
1310
  ifMatch?: string
1216
1311
  ifNoneMatch?: boolean
1217
1312
  }): Promise<{ version: number }> {
@@ -1250,23 +1345,29 @@ export class PostgresBackend implements StorageBackend {
1250
1345
  return this._withTransaction(async client => {
1251
1346
  await this._ensureCollectionRow({ client, spaceId, collectionId })
1252
1347
 
1253
- // The EDV unique-attribute invariant (`unique: true` blinded attributes;
1254
- // the `blinded-index-query` feature): a write claiming a unique blinded
1255
- // triple must not collide with another live document's claim. Only a
1256
- // unique-carrying JSON write can create a new claim, so only such writes
1257
- // pay for it: a per-Collection transaction-scoped advisory lock
1348
+ // Two unique-attribute invariants can force a JSON content write to
1349
+ // serialize before it upserts its row: the EDV blinded one (`unique: true`
1350
+ // blinded attributes; the `blinded-index-query` feature) and the plaintext
1351
+ // equality one (a Collection's `unique`-declared `indexes`; the
1352
+ // `equality-query` feature). Only a unique-carrying JSON write can create
1353
+ // either claim. A per-Collection transaction-scoped advisory lock
1258
1354
  // serializes the claimants (held to commit, so the loser's scan sees the
1259
- // winner's committed row) without entering the row-lock ordering of
1260
- // plain writes, then the conflict is evaluated against the Collection's
1261
- // other live JSON documents.
1262
- if (
1355
+ // winner's committed row) without entering the row-lock ordering of plain
1356
+ // writes; it is taken once and shared by both checks.
1357
+ const blindedUnique =
1263
1358
  input.kind === 'json' &&
1264
1359
  collectUniqueBlindedTerms({ document: input.data }).length > 0
1265
- ) {
1360
+ const equalityUnique =
1361
+ input.kind === 'json' &&
1362
+ uniqueIndexes !== undefined &&
1363
+ uniqueIndexes.length > 0
1364
+ if (blindedUnique || equalityUnique) {
1266
1365
  await client.query(
1267
1366
  'SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))',
1268
1367
  [spaceId, collectionId]
1269
1368
  )
1369
+ }
1370
+ if (blindedUnique) {
1270
1371
  const { rows: candidateRows } = await client.query<{
1271
1372
  resource_id: string
1272
1373
  content: Buffer | null
@@ -1290,7 +1391,32 @@ export class PostgresBackend implements StorageBackend {
1290
1391
  // skip an unparsable body
1291
1392
  }
1292
1393
  }
1293
- assertNoUniqueBlindedConflict({ document: input.data, candidates })
1394
+ assertNoUniqueBlindedConflict({
1395
+ document: input.kind === 'json' ? input.data : undefined,
1396
+ candidates
1397
+ })
1398
+ }
1399
+ if (equalityUnique) {
1400
+ // A content write does not change the Resource's `custom`, so the custom
1401
+ // side of the claim comes from the CURRENT stored row.
1402
+ const { rows: selfRows } = await client.query<{
1403
+ custom: ResourceMetadataCustom | Record<string, unknown> | null
1404
+ }>(
1405
+ `SELECT custom FROM resources
1406
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1407
+ AND NOT deleted`,
1408
+ [spaceId, collectionId, resourceId]
1409
+ )
1410
+ assertNoUniqueEqualityConflict({
1411
+ indexes: uniqueIndexes!,
1412
+ content: input.kind === 'json' ? input.data : undefined,
1413
+ custom: selfRows[0]?.custom ?? undefined,
1414
+ candidates: await this._readEqualityCandidates(client, {
1415
+ spaceId,
1416
+ collectionId,
1417
+ excludeResourceId: resourceId
1418
+ })
1419
+ })
1294
1420
  }
1295
1421
 
1296
1422
  // Narrow projection: the lock needs the row, not its (possibly multi-MB)
@@ -1307,7 +1433,31 @@ export class PostgresBackend implements StorageBackend {
1307
1433
  FOR UPDATE`,
1308
1434
  [spaceId, collectionId, resourceId]
1309
1435
  )
1310
- const prior = rows[0]
1436
+ let prior = rows[0]
1437
+ if (prior === undefined) {
1438
+ // The lock-nothing case: serialize with any concurrent creator of the
1439
+ // same id and re-read, so `exists` / `priorSize` below reflect the row
1440
+ // it committed (see `_lockSameKeyCreate` -- without this, both
1441
+ // creators would count their full byte size as the usage delta).
1442
+ await this._lockSameKeyCreate({
1443
+ client,
1444
+ spaceId,
1445
+ rowKey: `${collectionId}/${resourceId}`
1446
+ })
1447
+ const { rows: reread } = await client.query<
1448
+ Pick<
1449
+ ResourceRow,
1450
+ 'version' | 'size_bytes' | 'deleted' | 'created_at' | 'created_by'
1451
+ >
1452
+ >(
1453
+ `SELECT version, size_bytes, deleted, created_at, created_by
1454
+ FROM resources
1455
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1456
+ FOR UPDATE`,
1457
+ [spaceId, collectionId, resourceId]
1458
+ )
1459
+ prior = reread[0]
1460
+ }
1311
1461
  const exists = prior !== undefined && !prior.deleted
1312
1462
  if (ifMatch !== undefined || ifNoneMatch) {
1313
1463
  assertWritePrecondition({
@@ -1339,7 +1489,7 @@ export class PostgresBackend implements StorageBackend {
1339
1489
 
1340
1490
  const now = new Date().toISOString()
1341
1491
  const version = (prior?.version ?? 0) + 1
1342
- const priorSize = exists ? Number(prior.size_bytes) : 0
1492
+ const priorSize = exists ? Number(prior?.size_bytes ?? 0) : 0
1343
1493
  const delta = content.length - priorSize
1344
1494
  if (delta !== 0) {
1345
1495
  await this._applyUsageDelta({ client, spaceId, delta })
@@ -1350,10 +1500,11 @@ export class PostgresBackend implements StorageBackend {
1350
1500
  // dropped both (the metadata went with the deleted Resource).
1351
1501
  //
1352
1502
  // Create-if-absent atomicity: when `If-None-Match: *` found NO prior row
1353
- // (a tombstone is a real row and stays lock-serialized), the SELECT FOR
1354
- // UPDATE locked nothing -- READ COMMITTED has no gap locks -- so a
1355
- // concurrent creator may have raced past the same precondition. A plain
1356
- // INSERT (no ON CONFLICT) makes the primary key the arbiter: the loser's
1503
+ // (a tombstone is a real row and stays lock-serialized), concurrent
1504
+ // creators through this method are already serialized by
1505
+ // `_lockSameKeyCreate` above -- but a writer that does not take that
1506
+ // lock (`importSpace`'s plain INSERTs) can still race. A plain INSERT
1507
+ // (no ON CONFLICT) keeps the primary key as the arbiter: the loser's
1357
1508
  // unique violation maps to the 412 the precondition would have thrown.
1358
1509
  // `createdBy` names the Resource's creator, not its last writer: taken
1359
1510
  // from this write's invoker only when there is no prior row at all
@@ -1519,7 +1670,25 @@ export class PostgresBackend implements StorageBackend {
1519
1670
  // no-op, keeping an existing tombstone's change-feed entry stable.
1520
1671
  return
1521
1672
  }
1522
- const freedBytes = Number(prior.size_bytes)
1673
+ // A soft delete is an UPDATE, not a row removal, so the chunk foreign
1674
+ // key's ON DELETE CASCADE does not fire -- remove the Resource's chunks
1675
+ // (the `chunked-streams` feature) explicitly in this same transaction so
1676
+ // they never outlive their parent, and return their bytes to the quota
1677
+ // counter alongside the Resource's content bytes.
1678
+ const { rows: chunkRows } = await client.query<{ total: string }>(
1679
+ `SELECT COALESCE(SUM(size), 0) AS total FROM chunks
1680
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
1681
+ [spaceId, collectionId, resourceId]
1682
+ )
1683
+ const freedChunkBytes = Number(chunkRows[0]?.total ?? 0)
1684
+ if (freedChunkBytes > 0) {
1685
+ await client.query(
1686
+ `DELETE FROM chunks
1687
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
1688
+ [spaceId, collectionId, resourceId]
1689
+ )
1690
+ }
1691
+ const freedBytes = Number(prior.size_bytes) + freedChunkBytes
1523
1692
  if (freedBytes > 0) {
1524
1693
  await this._applyUsageDelta({ client, spaceId, delta: -freedBytes })
1525
1694
  }
@@ -1615,6 +1784,7 @@ export class PostgresBackend implements StorageBackend {
1615
1784
  resourceId,
1616
1785
  custom,
1617
1786
  epoch,
1787
+ uniqueIndexes,
1618
1788
  ifMatch,
1619
1789
  ifNoneMatch
1620
1790
  }: {
@@ -1623,10 +1793,24 @@ export class PostgresBackend implements StorageBackend {
1623
1793
  resourceId: string
1624
1794
  custom: ResourceMetadataCustom | Record<string, unknown>
1625
1795
  epoch?: string
1796
+ uniqueIndexes?: NormalizedIndexDeclaration[]
1626
1797
  ifMatch?: string
1627
1798
  ifNoneMatch?: boolean
1628
1799
  }): Promise<{ metaVersion: number } | undefined> {
1629
1800
  return this._withTransaction(async client => {
1801
+ // A metadata write can create a plaintext equality unique claim for a
1802
+ // `custom`-sourced attribute (the `equality-query` feature). When the
1803
+ // Collection declares any unique index, take the per-Collection advisory
1804
+ // lock first (held to commit, serializing concurrent claimants) so the
1805
+ // conflict scan below is atomic with the write.
1806
+ const equalityUnique =
1807
+ uniqueIndexes !== undefined && uniqueIndexes.length > 0
1808
+ if (equalityUnique) {
1809
+ await client.query(
1810
+ 'SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))',
1811
+ [spaceId, collectionId]
1812
+ )
1813
+ }
1630
1814
  const { rows } = await client.query<ResourceRow>(
1631
1815
  `SELECT meta_version, deleted FROM resources
1632
1816
  WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
@@ -1643,6 +1827,39 @@ export class PostgresBackend implements StorageBackend {
1643
1827
  ifMatch,
1644
1828
  ifNoneMatch
1645
1829
  })
1830
+ if (equalityUnique) {
1831
+ // Content is the Resource's stored JSON content (unchanged by a
1832
+ // metadata write); custom is the incoming value this write sets. Fetch
1833
+ // the (possibly multi-MB) content only on this uniqueness path -- the
1834
+ // row is already `FOR UPDATE`-locked above.
1835
+ const { rows: selfRows } = await client.query<{
1836
+ content: Buffer | null
1837
+ is_json: boolean
1838
+ }>(
1839
+ `SELECT content, is_json FROM resources
1840
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
1841
+ [spaceId, collectionId, resourceId]
1842
+ )
1843
+ let content: unknown
1844
+ const selfRow = selfRows[0]
1845
+ if (selfRow?.is_json && selfRow.content) {
1846
+ try {
1847
+ content = JSON.parse(selfRow.content.toString('utf8')) as unknown
1848
+ } catch {
1849
+ content = undefined
1850
+ }
1851
+ }
1852
+ assertNoUniqueEqualityConflict({
1853
+ indexes: uniqueIndexes!,
1854
+ content,
1855
+ custom,
1856
+ candidates: await this._readEqualityCandidates(client, {
1857
+ spaceId,
1858
+ collectionId,
1859
+ excludeResourceId: resourceId
1860
+ })
1861
+ })
1862
+ }
1646
1863
  const metaVersion = (prior.meta_version ?? 0) + 1
1647
1864
  const hasCustom = Object.keys(custom).length > 0
1648
1865
  const now = new Date().toISOString()
@@ -1671,6 +1888,395 @@ export class PostgresBackend implements StorageBackend {
1671
1888
  })
1672
1889
  }
1673
1890
 
1891
+ // Chunks (the `chunked-streams` feature)
1892
+
1893
+ /**
1894
+ * Writes one chunk of a chunked Resource as one transaction: the parent
1895
+ * Resource must exist (checked atomically -- a `FOR SHARE` lock on it also
1896
+ * blocks a concurrent delete of the parent for the duration of the write, so
1897
+ * a chunk can never be orphaned by a racing `deleteResource`), the chunk row
1898
+ * is locked, its precondition evaluated, its monotonic `version` bumped, and
1899
+ * the transactional quota delta applied. The chunk body is stored opaquely as
1900
+ * a single `bytea`, the buffered-blob path (bounded by `maxUploadBytes`),
1901
+ * exactly like a binary Resource representation.
1902
+ * @param options {object}
1903
+ * @param options.spaceId {string}
1904
+ * @param options.collectionId {string}
1905
+ * @param options.resourceId {string}
1906
+ * @param options.chunkIndex {number} a non-negative safe integer
1907
+ * @param options.input {ResourceInput}
1908
+ * @param [options.ifMatch] {string}
1909
+ * @param [options.ifNoneMatch] {boolean}
1910
+ * @returns {Promise<{ version: number }>} the chunk's new version
1911
+ */
1912
+ async writeChunk({
1913
+ spaceId,
1914
+ collectionId,
1915
+ resourceId,
1916
+ chunkIndex,
1917
+ input,
1918
+ ifMatch,
1919
+ ifNoneMatch
1920
+ }: {
1921
+ spaceId: string
1922
+ collectionId: string
1923
+ resourceId: string
1924
+ chunkIndex: number
1925
+ input: ResourceInput
1926
+ ifMatch?: string
1927
+ ifNoneMatch?: boolean
1928
+ }): Promise<{ version: number }> {
1929
+ const { maxUploadBytes } = this
1930
+ let bytes: Buffer
1931
+ if (input.kind === 'json') {
1932
+ bytes = Buffer.from(JSON.stringify(input.data))
1933
+ if (bytes.length > maxUploadBytes) {
1934
+ throw new PayloadTooLargeError({
1935
+ maxUploadBytes,
1936
+ backendId: this.describe().id,
1937
+ uploadBytes: bytes.length
1938
+ })
1939
+ }
1940
+ } else {
1941
+ // Pre-flight a declared size, then buffer through the counting guard that
1942
+ // hard-caps a body whose size is omitted or understated. Buffering
1943
+ // happens BEFORE the transaction so a slow upload holds no row lock.
1944
+ if (
1945
+ input.declaredBytes !== undefined &&
1946
+ input.declaredBytes > maxUploadBytes
1947
+ ) {
1948
+ throw new PayloadTooLargeError({
1949
+ maxUploadBytes,
1950
+ backendId: this.describe().id,
1951
+ uploadBytes: input.declaredBytes
1952
+ })
1953
+ }
1954
+ bytes = await bufferStreamCapped({
1955
+ stream: input.stream,
1956
+ maxUploadBytes,
1957
+ backendId: this.describe().id
1958
+ })
1959
+ }
1960
+
1961
+ return this._withTransaction(async client => {
1962
+ // Parent Resource must exist (and not be a tombstone). `FOR SHARE`
1963
+ // conflicts with the `FOR UPDATE` a concurrent `deleteResource` takes, so
1964
+ // the two serialize on the parent row -- the parent cannot be deleted
1965
+ // between this check and the chunk write.
1966
+ const { rows: parentRows } = await client.query<{ deleted: boolean }>(
1967
+ `SELECT deleted FROM resources
1968
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1969
+ FOR SHARE`,
1970
+ [spaceId, collectionId, resourceId]
1971
+ )
1972
+ const parent = parentRows[0]
1973
+ if (!parent || parent.deleted) {
1974
+ throw new ResourceNotFoundError({ requestName: 'Write Chunk' })
1975
+ }
1976
+
1977
+ // Lock the chunk row (if any) and read its current version/size, so the
1978
+ // precondition, the monotonic bump, and the usage delta are all atomic
1979
+ // with the write.
1980
+ const { rows } = await client.query<{ version: number; size: string }>(
1981
+ `SELECT version, size FROM chunks
1982
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
1983
+ AND chunk_index = $4
1984
+ FOR UPDATE`,
1985
+ [spaceId, collectionId, resourceId, chunkIndex]
1986
+ )
1987
+ let prior = rows[0]
1988
+ const chunkLabel = `${resourceId}/chunks/${chunkIndex}`
1989
+ if (prior === undefined) {
1990
+ // The lock-nothing case: serialize with any concurrent creator of the
1991
+ // same chunk and re-read, so the usage delta below reflects the row it
1992
+ // committed (see `_lockSameKeyCreate`).
1993
+ await this._lockSameKeyCreate({
1994
+ client,
1995
+ spaceId,
1996
+ rowKey: `${collectionId}/${chunkLabel}`
1997
+ })
1998
+ const { rows: reread } = await client.query<{
1999
+ version: number
2000
+ size: string
2001
+ }>(
2002
+ `SELECT version, size FROM chunks
2003
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2004
+ AND chunk_index = $4
2005
+ FOR UPDATE`,
2006
+ [spaceId, collectionId, resourceId, chunkIndex]
2007
+ )
2008
+ prior = reread[0]
2009
+ }
2010
+ const exists = prior !== undefined
2011
+ if (ifMatch !== undefined || ifNoneMatch) {
2012
+ assertWritePrecondition({
2013
+ resourceId: chunkLabel,
2014
+ exists,
2015
+ currentVersion: prior?.version ?? 0,
2016
+ ifMatch,
2017
+ ifNoneMatch
2018
+ })
2019
+ }
2020
+
2021
+ const version = (prior?.version ?? 0) + 1
2022
+ const priorSize = exists ? Number(prior?.size ?? 0) : 0
2023
+ const delta = bytes.length - priorSize
2024
+ if (delta !== 0) {
2025
+ await this._applyUsageDelta({ client, spaceId, delta })
2026
+ }
2027
+
2028
+ const values = [
2029
+ spaceId,
2030
+ collectionId,
2031
+ resourceId,
2032
+ chunkIndex,
2033
+ input.contentType,
2034
+ bytes,
2035
+ bytes.length,
2036
+ version
2037
+ ]
2038
+ const insertSql = `
2039
+ INSERT INTO chunks (
2040
+ space_id, collection_id, resource_id, chunk_index,
2041
+ content_type, bytes, size, version
2042
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`
2043
+ // Create-if-absent atomicity mirrors `writeResource`: concurrent
2044
+ // creators through this method are serialized by `_lockSameKeyCreate`
2045
+ // above, and against a writer that does not take that lock
2046
+ // (`importSpace`'s plain INSERTs) the plain INSERT keeps the primary key
2047
+ // as the arbiter -- a racing creator's unique violation maps to the 412
2048
+ // the precondition would have thrown.
2049
+ if (ifNoneMatch && prior === undefined) {
2050
+ try {
2051
+ await client.query(insertSql, values)
2052
+ } catch (err) {
2053
+ if ((err as { code?: string }).code === '23505') {
2054
+ throw new PreconditionFailedError({
2055
+ detail: `Chunk '${chunkLabel}' already exists (If-None-Match: *).`
2056
+ })
2057
+ }
2058
+ throw err
2059
+ }
2060
+ return { version }
2061
+ }
2062
+ // The conflict update derives `version` from the row (`chunks.version +
2063
+ // 1`), not from the pre-read, so a concurrent creator that slipped in
2064
+ // after our lock-nothing SELECT still advances the counter monotonically.
2065
+ // RETURNING reports the version that actually landed.
2066
+ const { rows: written } = await client.query<{ version: number }>(
2067
+ `${insertSql}
2068
+ ON CONFLICT (space_id, collection_id, resource_id, chunk_index)
2069
+ DO UPDATE SET
2070
+ content_type = EXCLUDED.content_type,
2071
+ bytes = EXCLUDED.bytes,
2072
+ size = EXCLUDED.size,
2073
+ version = chunks.version + 1
2074
+ RETURNING version`,
2075
+ values
2076
+ )
2077
+ return { version: written[0]!.version }
2078
+ })
2079
+ }
2080
+
2081
+ /**
2082
+ * Reads a chunk's bytes. Rejects with `ResourceNotFoundError` (404) when no
2083
+ * chunk is stored at that index.
2084
+ * @param options {object}
2085
+ * @param options.spaceId {string}
2086
+ * @param options.collectionId {string}
2087
+ * @param options.resourceId {string}
2088
+ * @param options.chunkIndex {number}
2089
+ * @returns {Promise<ResourceResult>}
2090
+ */
2091
+ async getChunk({
2092
+ spaceId,
2093
+ collectionId,
2094
+ resourceId,
2095
+ chunkIndex
2096
+ }: {
2097
+ spaceId: string
2098
+ collectionId: string
2099
+ resourceId: string
2100
+ chunkIndex: number
2101
+ }): Promise<ResourceResult> {
2102
+ const { rows } = await this._pool.query<{
2103
+ content_type: string
2104
+ bytes: Buffer
2105
+ version: number
2106
+ }>(
2107
+ `SELECT content_type, bytes, version FROM chunks
2108
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2109
+ AND chunk_index = $4`,
2110
+ [spaceId, collectionId, resourceId, chunkIndex]
2111
+ )
2112
+ const row = rows[0]
2113
+ if (!row) {
2114
+ throw new ResourceNotFoundError({ requestName: 'Get Chunk' })
2115
+ }
2116
+ return {
2117
+ resourceStream: Readable.from(row.bytes),
2118
+ storedResourceType: row.content_type,
2119
+ version: row.version
2120
+ }
2121
+ }
2122
+
2123
+ /**
2124
+ * Reads a chunk's stored content-type / size / version (the HEAD payload
2125
+ * headers). Resolves `undefined` when the chunk is absent.
2126
+ * @param options {object}
2127
+ * @param options.spaceId {string}
2128
+ * @param options.collectionId {string}
2129
+ * @param options.resourceId {string}
2130
+ * @param options.chunkIndex {number}
2131
+ * @returns {Promise<{ contentType: string, size: number, version?: number }
2132
+ * | undefined>}
2133
+ */
2134
+ async getChunkMetadata({
2135
+ spaceId,
2136
+ collectionId,
2137
+ resourceId,
2138
+ chunkIndex
2139
+ }: {
2140
+ spaceId: string
2141
+ collectionId: string
2142
+ resourceId: string
2143
+ chunkIndex: number
2144
+ }): Promise<
2145
+ { contentType: string; size: number; version?: number } | undefined
2146
+ > {
2147
+ const { rows } = await this._pool.query<{
2148
+ content_type: string
2149
+ size: string
2150
+ version: number
2151
+ }>(
2152
+ `SELECT content_type, size, version FROM chunks
2153
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2154
+ AND chunk_index = $4`,
2155
+ [spaceId, collectionId, resourceId, chunkIndex]
2156
+ )
2157
+ const row = rows[0]
2158
+ if (!row) {
2159
+ return undefined
2160
+ }
2161
+ return {
2162
+ contentType: row.content_type,
2163
+ size: Number(row.size),
2164
+ version: row.version
2165
+ }
2166
+ }
2167
+
2168
+ /**
2169
+ * Deletes one chunk as one transaction (the chunk row is locked, its
2170
+ * `ifMatch` precondition evaluated atomically, and its bytes returned to the
2171
+ * quota counter). Resolves `true` when a chunk was removed, `false` when none
2172
+ * was stored at that index (the handler 404s on `false` -- chunk deletes are
2173
+ * not silently idempotent, unlike `deleteResource`).
2174
+ * @param options {object}
2175
+ * @param options.spaceId {string}
2176
+ * @param options.collectionId {string}
2177
+ * @param options.resourceId {string}
2178
+ * @param options.chunkIndex {number}
2179
+ * @param [options.ifMatch] {string}
2180
+ * @returns {Promise<boolean>}
2181
+ */
2182
+ async deleteChunk({
2183
+ spaceId,
2184
+ collectionId,
2185
+ resourceId,
2186
+ chunkIndex,
2187
+ ifMatch
2188
+ }: {
2189
+ spaceId: string
2190
+ collectionId: string
2191
+ resourceId: string
2192
+ chunkIndex: number
2193
+ ifMatch?: string
2194
+ }): Promise<boolean> {
2195
+ return this._withTransaction(async client => {
2196
+ const { rows } = await client.query<{ version: number; size: string }>(
2197
+ `SELECT version, size FROM chunks
2198
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2199
+ AND chunk_index = $4
2200
+ FOR UPDATE`,
2201
+ [spaceId, collectionId, resourceId, chunkIndex]
2202
+ )
2203
+ const prior = rows[0]
2204
+ if (prior === undefined) {
2205
+ return false
2206
+ }
2207
+ if (ifMatch !== undefined) {
2208
+ assertWritePrecondition({
2209
+ resourceId: `${resourceId}/chunks/${chunkIndex}`,
2210
+ exists: true,
2211
+ currentVersion: prior.version,
2212
+ ifMatch
2213
+ })
2214
+ }
2215
+ await client.query(
2216
+ `DELETE FROM chunks
2217
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2218
+ AND chunk_index = $4`,
2219
+ [spaceId, collectionId, resourceId, chunkIndex]
2220
+ )
2221
+ const freedBytes = Number(prior.size)
2222
+ if (freedBytes > 0) {
2223
+ await this._applyUsageDelta({ client, spaceId, delta: -freedBytes })
2224
+ }
2225
+ return true
2226
+ })
2227
+ }
2228
+
2229
+ /**
2230
+ * Lists a Resource's stored chunks in ascending `chunk_index` order -- the
2231
+ * discovery/reassembly listing. The opaque `bytes` column is deliberately not
2232
+ * selected. An empty listing (Resource with no chunks, or an absent Resource)
2233
+ * resolves `{ count: 0, chunks: [] }`.
2234
+ * @param options {object}
2235
+ * @param options.spaceId {string}
2236
+ * @param options.collectionId {string}
2237
+ * @param options.resourceId {string}
2238
+ * @returns {Promise<{ count: number, chunks: Array<{ index: number, size:
2239
+ * number, contentType: string, version?: number }> }>}
2240
+ */
2241
+ async listChunks({
2242
+ spaceId,
2243
+ collectionId,
2244
+ resourceId
2245
+ }: {
2246
+ spaceId: string
2247
+ collectionId: string
2248
+ resourceId: string
2249
+ }): Promise<{
2250
+ count: number
2251
+ chunks: Array<{
2252
+ index: number
2253
+ size: number
2254
+ contentType: string
2255
+ version?: number
2256
+ }>
2257
+ }> {
2258
+ const { rows } = await this._pool.query<{
2259
+ chunk_index: number
2260
+ size: string
2261
+ content_type: string
2262
+ version: number
2263
+ }>(
2264
+ `SELECT chunk_index, size, content_type, version FROM chunks
2265
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
2266
+ ORDER BY chunk_index`,
2267
+ [spaceId, collectionId, resourceId]
2268
+ )
2269
+ return {
2270
+ count: rows.length,
2271
+ chunks: rows.map(row => ({
2272
+ index: row.chunk_index,
2273
+ size: Number(row.size),
2274
+ contentType: row.content_type,
2275
+ version: row.version
2276
+ }))
2277
+ }
2278
+ }
2279
+
1674
2280
  /**
1675
2281
  * Replication change feed (the `changes` query profile): one indexed keyset
1676
2282
  * query over `(updatedAt, resourceId)`, tombstones included, JSON documents
@@ -1834,6 +2440,145 @@ export class PostgresBackend implements StorageBackend {
1834
2440
  return runBlindedIndexQuery({ candidates, query, count, limit, cursor })
1835
2441
  }
1836
2442
 
2443
+ /**
2444
+ * Plaintext equality query (the `equality` query profile; see the
2445
+ * `StorageBackend.queryByEquality` contract). Reads the Collection's live
2446
+ * Resources -- JSON rows carrying parsed `content`, all rows carrying their
2447
+ * `custom` jsonb -- and hands the candidates to the shared evaluator
2448
+ * (`lib/equalityIndex.ts`) for extraction, matching, ordering, and cursor
2449
+ * pagination -- identical semantics to the filesystem backend. A full scan of
2450
+ * the Collection per call, deliberate for this teaching backend; a
2451
+ * materialized variant would answer from a JSONB expression index.
2452
+ * @param options {object}
2453
+ * @param options.spaceId {string}
2454
+ * @param options.collectionId {string}
2455
+ * @param options.query {EqualityQuery}
2456
+ * @param options.indexes {NormalizedIndexDeclaration[]}
2457
+ * @param [options.count] {boolean} return only the match count
2458
+ * @param [options.limit] {number} requested page size
2459
+ * @param [options.cursor] {string} opaque cursor from a prior page
2460
+ * @returns {Promise<{ count: number } | EqualityQueryPage>}
2461
+ */
2462
+ async queryByEquality({
2463
+ spaceId,
2464
+ collectionId,
2465
+ query,
2466
+ indexes,
2467
+ count,
2468
+ limit,
2469
+ cursor
2470
+ }: {
2471
+ spaceId: string
2472
+ collectionId: string
2473
+ query: EqualityQuery
2474
+ indexes: NormalizedIndexDeclaration[]
2475
+ count?: boolean
2476
+ limit?: number
2477
+ cursor?: string
2478
+ }): Promise<{ count: number } | EqualityQueryPage> {
2479
+ const candidates = await this._readEqualityCandidates(this._pool, {
2480
+ spaceId,
2481
+ collectionId
2482
+ })
2483
+ return runEqualityQuery({
2484
+ candidates,
2485
+ query,
2486
+ indexes,
2487
+ count,
2488
+ limit,
2489
+ cursor
2490
+ })
2491
+ }
2492
+
2493
+ /**
2494
+ * Declare-time uniqueness scan for the `equality` profile (see the
2495
+ * `StorageBackend.findEqualityUniqueViolation` contract): reads the
2496
+ * Collection's live Resources and delegates to the shared scan, which reports
2497
+ * the first `(name, value)` claimed by two different Resources under the given
2498
+ * `unique` declarations (or `undefined` when none is).
2499
+ * @param options {object}
2500
+ * @param options.spaceId {string}
2501
+ * @param options.collectionId {string}
2502
+ * @param options.indexes {NormalizedIndexDeclaration[]}
2503
+ * @returns {Promise<{ name: string, value: EqualityValue } | undefined>}
2504
+ */
2505
+ async findEqualityUniqueViolation({
2506
+ spaceId,
2507
+ collectionId,
2508
+ indexes
2509
+ }: {
2510
+ spaceId: string
2511
+ collectionId: string
2512
+ indexes: NormalizedIndexDeclaration[]
2513
+ }): Promise<{ name: string; value: EqualityValue } | undefined> {
2514
+ const candidates = await this._readEqualityCandidates(this._pool, {
2515
+ spaceId,
2516
+ collectionId
2517
+ })
2518
+ return findEqualityUniqueViolation({ indexes, candidates })
2519
+ }
2520
+
2521
+ /**
2522
+ * Reads every live Resource of a Collection as an equality candidate -- the
2523
+ * candidate set for the equality query and the plaintext unique-attribute
2524
+ * conflict scans. Includes blob Resources (queryable through their
2525
+ * `custom`-sourced attributes): each row resolves `{ resourceId, content?,
2526
+ * custom? }`, where `content` is the parsed JSON of a JSON row (a blob and
2527
+ * unparsable JSON contribute none) and `custom` is the row's jsonb `custom`
2528
+ * when set. Tombstones are excluded (`NOT deleted`); an optional excluded
2529
+ * Resource is skipped. Runs on the given executor -- the pool for a read-only
2530
+ * query, or the write transaction's client for a uniqueness scan (so the scan
2531
+ * shares the advisory lock and sees a consistent snapshot).
2532
+ * @param executor {pg.Pool | pg.PoolClient}
2533
+ * @param options {object}
2534
+ * @param options.spaceId {string}
2535
+ * @param options.collectionId {string}
2536
+ * @param [options.excludeResourceId] {string}
2537
+ * @returns {Promise<EqualityCandidate[]>}
2538
+ */
2539
+ private async _readEqualityCandidates(
2540
+ executor: pg.Pool | pg.PoolClient,
2541
+ {
2542
+ spaceId,
2543
+ collectionId,
2544
+ excludeResourceId
2545
+ }: {
2546
+ spaceId: string
2547
+ collectionId: string
2548
+ excludeResourceId?: string
2549
+ }
2550
+ ): Promise<EqualityCandidate[]> {
2551
+ const { rows } = await executor.query<{
2552
+ resource_id: string
2553
+ content: Buffer | null
2554
+ is_json: boolean
2555
+ custom: ResourceMetadataCustom | Record<string, unknown> | null
2556
+ }>(
2557
+ `SELECT resource_id, content, is_json, custom FROM resources
2558
+ WHERE space_id = $1 AND collection_id = $2 AND NOT deleted
2559
+ AND ($3::text IS NULL OR resource_id <> $3)
2560
+ ORDER BY resource_id`,
2561
+ [spaceId, collectionId, excludeResourceId ?? null]
2562
+ )
2563
+ const candidates: EqualityCandidate[] = []
2564
+ for (const row of rows) {
2565
+ let content: unknown
2566
+ if (row.is_json && row.content) {
2567
+ try {
2568
+ content = JSON.parse(row.content.toString('utf8')) as unknown
2569
+ } catch {
2570
+ // skip an unparsable body -- it contributes no content attributes
2571
+ }
2572
+ }
2573
+ candidates.push({
2574
+ resourceId: row.resource_id,
2575
+ ...(content !== undefined && { content }),
2576
+ ...(row.custom !== null && { custom: row.custom })
2577
+ })
2578
+ }
2579
+ return candidates
2580
+ }
2581
+
1837
2582
  // Policies
1838
2583
 
1839
2584
  /**
@@ -2435,7 +3180,8 @@ export class PostgresBackend implements StorageBackend {
2435
3180
  { rows: policyRows },
2436
3181
  { rows: collectionRows },
2437
3182
  { rows: resourceRows },
2438
- { rows: revocationRows }
3183
+ { rows: revocationRows },
3184
+ { rows: chunkRows }
2439
3185
  ] = await Promise.all([
2440
3186
  this._pool.query<{
2441
3187
  collection_id: string
@@ -2477,6 +3223,20 @@ export class PostgresBackend implements StorageBackend {
2477
3223
  `SELECT delegator, capability_id, record FROM space_revocations
2478
3224
  WHERE space_id = $1`,
2479
3225
  [spaceId]
3226
+ ),
3227
+ // Chunk metadata only -- bytes are fetched one chunk at a time while
3228
+ // packing, so an export never holds a chunked Resource whole in memory.
3229
+ this._pool.query<{
3230
+ collection_id: string
3231
+ resource_id: string
3232
+ chunk_index: number
3233
+ content_type: string
3234
+ version: number
3235
+ }>(
3236
+ `SELECT collection_id, resource_id, chunk_index, content_type, version
3237
+ FROM chunks WHERE space_id = $1
3238
+ ORDER BY collection_id, resource_id, chunk_index`,
3239
+ [spaceId]
2480
3240
  )
2481
3241
  ])
2482
3242
 
@@ -2487,11 +3247,26 @@ export class PostgresBackend implements StorageBackend {
2487
3247
  row => row.collection_id === '' && row.resource_id === ''
2488
3248
  )?.policy
2489
3249
 
2490
- // A file entry carries its bytes inline (the small JSON dot-files) or a
2491
- // lazy reference to a resource representation, resolved at pack time.
3250
+ // A file entry carries its bytes inline (the small JSON dot-files), a lazy
3251
+ // reference to a resource representation, or a chunk directory (the
3252
+ // `chunked-streams` feature) whose files are likewise inline (the per-chunk
3253
+ // `.meta.<index>.json` sidecar) or lazy (the chunk `r.<index>...`
3254
+ // representation, resolved at pack time). `name` is the entry's sort key
3255
+ // within its dir; a chunk directory sorts by its `.chunks.<encId>` dir name.
3256
+ type ChunkFile =
3257
+ | { name: string; bytes: Buffer }
3258
+ | {
3259
+ name: string
3260
+ chunk: {
3261
+ collectionId: string
3262
+ resourceId: string
3263
+ chunkIndex: number
3264
+ }
3265
+ }
2492
3266
  type FileEntry =
2493
3267
  | { name: string; bytes: Buffer }
2494
3268
  | { name: string; resource: { collectionId: string; resourceId: string } }
3269
+ | { name: string; chunkDir: ChunkFile[] }
2495
3270
  // Space-level dot-files are always small JSON, carried inline.
2496
3271
  const spaceFiles: Array<{ name: string; bytes: Buffer }> = [
2497
3272
  {
@@ -2564,6 +3339,48 @@ export class PostgresBackend implements StorageBackend {
2564
3339
  })
2565
3340
  }
2566
3341
  }
3342
+ // Chunks (the `chunked-streams` feature): each chunked Resource contributes
3343
+ // one `.chunks.<encResourceId>/` subdirectory in its Collection dir, in the
3344
+ // exact filesystem-backend layout so an archive imports into either backend.
3345
+ // A chunk is stored there as a Resource keyed by its stringified index: an
3346
+ // `r.<index>.<encContentType>.<ext>` representation file (`fileNameFor`)
3347
+ // plus a `.meta.<index>.json` version sidecar. Files within a chunk dir are
3348
+ // sorted by name (the filesystem's readdir sort). Rows arrive ordered by
3349
+ // `(collection, resource, index)`.
3350
+ const chunkDirsByResource = new Map<string, ChunkFile[]>()
3351
+ for (const row of chunkRows) {
3352
+ const dirKey = `${row.collection_id}/${row.resource_id}`
3353
+ let chunkFiles = chunkDirsByResource.get(dirKey)
3354
+ if (!chunkFiles) {
3355
+ chunkFiles = []
3356
+ chunkDirsByResource.set(dirKey, chunkFiles)
3357
+ filesFor(row.collection_id).push({
3358
+ name: chunkDirName(row.resource_id),
3359
+ chunkDir: chunkFiles
3360
+ })
3361
+ }
3362
+ const chunkId = String(row.chunk_index)
3363
+ chunkFiles.push({
3364
+ name: fileNameFor({
3365
+ resourceId: chunkId,
3366
+ contentType: row.content_type
3367
+ }),
3368
+ chunk: {
3369
+ collectionId: row.collection_id,
3370
+ resourceId: row.resource_id,
3371
+ chunkIndex: row.chunk_index
3372
+ }
3373
+ })
3374
+ chunkFiles.push({
3375
+ name: `.meta.${chunkId}.json`,
3376
+ bytes: Buffer.from(
3377
+ JSON.stringify({ version: row.version } satisfies ChunkSidecar)
3378
+ )
3379
+ })
3380
+ }
3381
+ for (const chunkFiles of chunkDirsByResource.values()) {
3382
+ chunkFiles.sort((left, right) => left.name.localeCompare(right.name))
3383
+ }
2567
3384
 
2568
3385
  // Top-level order: space-level files and collection dirs interleaved,
2569
3386
  // sorted by name -- the same order the filesystem's readdir+sort yields.
@@ -2597,7 +3414,17 @@ export class PostgresBackend implements StorageBackend {
2597
3414
  spaceId,
2598
3415
  entries: topLevel.map(entry =>
2599
3416
  entry.kind === 'collection'
2600
- ? { name: entry.name, files: entry.files.map(file => file.name) }
3417
+ ? {
3418
+ name: entry.name,
3419
+ // A chunk directory expands to its `.chunks.<encId>/<file>`
3420
+ // relative paths (mirroring the pack order), so the manifest
3421
+ // matches the filesystem backend's for the same Space.
3422
+ files: entry.files.flatMap(file =>
3423
+ 'chunkDir' in file
3424
+ ? file.chunkDir.map(child => `${file.name}/${child.name}`)
3425
+ : [file.name]
3426
+ )
3427
+ }
2601
3428
  : { name: entry.name }
2602
3429
  ),
2603
3430
  revocationFiles: revocationFiles.map(file => file.name)
@@ -2615,6 +3442,25 @@ export class PostgresBackend implements StorageBackend {
2615
3442
  if (entry.kind === 'collection') {
2616
3443
  pack.entry({ name: `${entryTarget}/`, type: 'directory', mtime })
2617
3444
  for (const file of entry.files) {
3445
+ if ('chunkDir' in file) {
3446
+ // A chunked Resource's `.chunks.<encId>/` subdirectory: emit the
3447
+ // directory entry, then each file -- an inline `.meta.<index>.json`
3448
+ // sidecar, or a chunk representation whose bytes are fetched one at
3449
+ // a time.
3450
+ const dirTarget = `${entryTarget}/${file.name}`
3451
+ pack.entry({ name: `${dirTarget}/`, type: 'directory', mtime })
3452
+ for (const chunkFile of file.chunkDir) {
3453
+ const bytes =
3454
+ 'bytes' in chunkFile
3455
+ ? chunkFile.bytes
3456
+ : await this._chunkContent({ spaceId, ...chunkFile.chunk })
3457
+ pack.entry(
3458
+ { name: `${dirTarget}/${chunkFile.name}`, mtime },
3459
+ bytes
3460
+ )
3461
+ }
3462
+ continue
3463
+ }
2618
3464
  const bytes =
2619
3465
  'bytes' in file
2620
3466
  ? file.bytes
@@ -2663,6 +3509,38 @@ export class PostgresBackend implements StorageBackend {
2663
3509
  return rows[0]?.content ?? Buffer.alloc(0)
2664
3510
  }
2665
3511
 
3512
+ /**
3513
+ * Fetches one chunk's bytes for the export pack loop (the `chunked-streams`
3514
+ * feature). A chunk removed between the metadata pass and this read yields an
3515
+ * empty body rather than failing the whole archive, matching
3516
+ * `_resourceContent`.
3517
+ * @param options {object}
3518
+ * @param options.spaceId {string}
3519
+ * @param options.collectionId {string}
3520
+ * @param options.resourceId {string}
3521
+ * @param options.chunkIndex {number}
3522
+ * @returns {Promise<Buffer>}
3523
+ */
3524
+ private async _chunkContent({
3525
+ spaceId,
3526
+ collectionId,
3527
+ resourceId,
3528
+ chunkIndex
3529
+ }: {
3530
+ spaceId: string
3531
+ collectionId: string
3532
+ resourceId: string
3533
+ chunkIndex: number
3534
+ }): Promise<Buffer> {
3535
+ const { rows } = await this._pool.query<{ bytes: Buffer | null }>(
3536
+ `SELECT bytes FROM chunks
3537
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
3538
+ AND chunk_index = $4`,
3539
+ [spaceId, collectionId, resourceId, chunkIndex]
3540
+ )
3541
+ return rows[0]?.bytes ?? Buffer.alloc(0)
3542
+ }
3543
+
2666
3544
  /**
2667
3545
  * Merges a WAS space-export tarball into an existing Space with the same
2668
3546
  * three-invariant pre-flight (per-entry 413, fail-closed 422 encryption
@@ -2685,6 +3563,10 @@ export class PostgresBackend implements StorageBackend {
2685
3563
  }): Promise<ImportStats> {
2686
3564
  const entries = await extractTarEntries(tarStream)
2687
3565
  const { spacePolicy, collections, revocations } = buildImportPlan(entries)
3566
+ // Chunk entries (the `chunked-streams` feature) live in per-Resource
3567
+ // `.chunks.<encId>/` subdirectories, which `buildImportPlan` skips (it
3568
+ // ignores nested files); parse them straight off the raw tar entries.
3569
+ const chunkEntries = this._chunkEntriesFromArchive(entries)
2688
3570
  const {
2689
3571
  capacityBytes,
2690
3572
  maxUploadBytes,
@@ -2774,6 +3656,19 @@ export class PostgresBackend implements StorageBackend {
2774
3656
  incomingBytes += body.length
2775
3657
  }
2776
3658
  }
3659
+ // Chunks (the `chunked-streams` feature) are opaque bytes -- no
3660
+ // encryption-conformance check applies -- but they count the same per-body
3661
+ // 413 and the (conservative) capacity pre-flight as Resource bodies.
3662
+ for (const chunk of chunkEntries) {
3663
+ if (chunk.body.length > maxUploadBytes) {
3664
+ throw new PayloadTooLargeError({
3665
+ maxUploadBytes,
3666
+ backendId: this.describe().id,
3667
+ uploadBytes: chunk.body.length
3668
+ })
3669
+ }
3670
+ incomingBytes += chunk.body.length
3671
+ }
2777
3672
  if (
2778
3673
  capacityBytes !== undefined &&
2779
3674
  currentUsage + incomingBytes > capacityBytes
@@ -2966,13 +3861,63 @@ export class PostgresBackend implements StorageBackend {
2966
3861
  }
2967
3862
  }
2968
3863
 
2969
- if (createdBytes > 0) {
3864
+ // Chunks (the `chunked-streams` feature): restore each archived chunk
3865
+ // skip-not-overwrite, after the Resource apply loop so a chunk's parent
3866
+ // Resource row already exists in this transaction (the foreign key
3867
+ // requires it). An orphan chunk -- one whose parent is absent or a
3868
+ // tombstone -- is skipped rather than resurrected. Existing chunk rows are
3869
+ // left untouched.
3870
+ let createdChunkBytes = 0
3871
+ for (const chunk of chunkEntries) {
3872
+ const { rows: parentRows } = await client.query<{ deleted: boolean }>(
3873
+ `SELECT deleted FROM resources
3874
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
3875
+ [spaceId, chunk.collectionId, chunk.resourceId]
3876
+ )
3877
+ const parent = parentRows[0]
3878
+ if (!parent || parent.deleted) {
3879
+ continue
3880
+ }
3881
+ const { rows: existingChunkRows } = await client.query(
3882
+ `SELECT 1 FROM chunks
3883
+ WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
3884
+ AND chunk_index = $4`,
3885
+ [spaceId, chunk.collectionId, chunk.resourceId, chunk.chunkIndex]
3886
+ )
3887
+ if (existingChunkRows.length > 0) {
3888
+ continue
3889
+ }
3890
+ // The chunk's `version` comes from its archived `.meta.<index>.json`
3891
+ // sidecar; an archive without one (or a chunk written before sidecars)
3892
+ // starts at version 1, the same fresh-write default as a Resource
3893
+ // restored without a sidecar.
3894
+ await client.query(
3895
+ `INSERT INTO chunks (
3896
+ space_id, collection_id, resource_id, chunk_index,
3897
+ content_type, bytes, size, version
3898
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
3899
+ [
3900
+ spaceId,
3901
+ chunk.collectionId,
3902
+ chunk.resourceId,
3903
+ chunk.chunkIndex,
3904
+ chunk.contentType,
3905
+ chunk.body,
3906
+ chunk.body.length,
3907
+ chunk.version ?? 1
3908
+ ]
3909
+ )
3910
+ createdChunkBytes += chunk.body.length
3911
+ }
3912
+
3913
+ const createdTotalBytes = createdBytes + createdChunkBytes
3914
+ if (createdTotalBytes > 0) {
2970
3915
  // The pre-flight was conservative (it counted skips too), so the
2971
3916
  // actual created total always fits; apply it unguarded.
2972
3917
  await this._applyUsageDelta({
2973
3918
  client,
2974
3919
  spaceId,
2975
- delta: createdBytes
3920
+ delta: createdTotalBytes
2976
3921
  })
2977
3922
  }
2978
3923
 
@@ -3070,4 +4015,141 @@ export class PostgresBackend implements StorageBackend {
3070
4015
  ]
3071
4016
  )
3072
4017
  }
4018
+
4019
+ /**
4020
+ * Parses a Space-export archive's chunk entries (the `chunked-streams`
4021
+ * feature) out of the raw tar entry map, in the filesystem backend's on-disk
4022
+ * layout so archives round-trip between the two backends. Chunk files live at
4023
+ * `space/<sourceSpaceId>/<collectionId>/.chunks.<encResourceId>/<file>` --
4024
+ * where a chunk is a Resource keyed by its stringified index: an
4025
+ * `r.<index>.<encContentType>.<ext>` representation paired with an optional
4026
+ * `.meta.<index>.json` version sidecar. `buildImportPlan` skips these (it
4027
+ * ignores nested Collection-dir files), so they are parsed here. Ids parsed
4028
+ * from the archive are validated (the path-traversal guard); a file whose
4029
+ * raw index segment is not canonical (`parseChunkIndexSegment`) is ignored,
4030
+ * and a sidecar with no paired representation is dropped (a chunk keeps no
4031
+ * tombstone).
4032
+ * @param entries {Map<string, TarEntry>}
4033
+ * @returns {Array<{ collectionId: string, resourceId: string, chunkIndex:
4034
+ * number, contentType: string, body: Buffer, version?: number }>}
4035
+ */
4036
+ private _chunkEntriesFromArchive(entries: Map<string, TarEntry>): Array<{
4037
+ collectionId: string
4038
+ resourceId: string
4039
+ chunkIndex: number
4040
+ contentType: string
4041
+ body: Buffer
4042
+ version?: number
4043
+ }> {
4044
+ // Accumulate the representation and the sidecar of each chunk under one
4045
+ // key, then emit only the chunks that carry a representation.
4046
+ const staged = new Map<
4047
+ string,
4048
+ {
4049
+ collectionId: string
4050
+ resourceId: string
4051
+ chunkIndex: number
4052
+ contentType?: string
4053
+ body?: Buffer
4054
+ version?: number
4055
+ }
4056
+ >()
4057
+ const pattern = /^space\/[^/]+\/([^/]+)\/(\.chunks\.[^/]+)\/([^/]+)$/
4058
+ for (const [entryName, entry] of entries) {
4059
+ if (entry.type !== 'file' || !entry.body) {
4060
+ continue
4061
+ }
4062
+ const match = entryName.match(pattern)
4063
+ if (!match) {
4064
+ continue
4065
+ }
4066
+ const collectionId = match[1]!
4067
+ const resourceId = parseChunkDirName(match[2]!)
4068
+ const fileName = match[3]!
4069
+ if (resourceId === undefined) {
4070
+ continue
4071
+ }
4072
+ // Reject a path-traversal / non-URL-safe id parsed from the archive
4073
+ // before it keys a destination row.
4074
+ assertValidId(collectionId, {
4075
+ kind: 'collection',
4076
+ requestName: 'Import Space'
4077
+ })
4078
+ assertValidId(resourceId, {
4079
+ kind: 'resource',
4080
+ requestName: 'Import Space'
4081
+ })
4082
+
4083
+ // A chunk's index is the "resource id" of its file within the chunk dir:
4084
+ // `r.<index>.<encType>.<ext>` for the representation, `.meta.<index>.json`
4085
+ // for the version sidecar. The RAW index segment must pass the same
4086
+ // canonical predicate as the live route (`parseChunkIndexSegment`), so a
4087
+ // non-canonical spelling (`r.01.*`, `r.%31.*`) is dropped rather than
4088
+ // coerced onto -- and colliding with -- the canonical chunk's row.
4089
+ let chunkIdSegment: string | undefined
4090
+ let representation: { contentType: string; body: Buffer } | undefined
4091
+ let sidecarVersion: number | undefined
4092
+ const metaId = metaSidecarFileId(fileName)
4093
+ if (metaId !== undefined) {
4094
+ chunkIdSegment = metaId
4095
+ try {
4096
+ const sidecar = JSON.parse(
4097
+ entry.body.toString('utf8')
4098
+ ) as ChunkSidecar
4099
+ sidecarVersion = sidecar.version
4100
+ } catch {
4101
+ sidecarVersion = undefined
4102
+ }
4103
+ } else if (fileName.startsWith('r.')) {
4104
+ const { contentType } = parseResourceFileName(fileName)
4105
+ chunkIdSegment = fileName.split('.')[1]
4106
+ representation = { contentType, body: entry.body }
4107
+ } else {
4108
+ continue
4109
+ }
4110
+
4111
+ const chunkIndex =
4112
+ chunkIdSegment === undefined
4113
+ ? undefined
4114
+ : parseChunkIndexSegment(chunkIdSegment)
4115
+ if (chunkIndex === undefined) {
4116
+ continue
4117
+ }
4118
+
4119
+ const key = `${collectionId}/${resourceId}/${chunkIndex}`
4120
+ const slot = staged.get(key) ?? { collectionId, resourceId, chunkIndex }
4121
+ if (representation !== undefined) {
4122
+ slot.contentType = representation.contentType
4123
+ slot.body = representation.body
4124
+ }
4125
+ if (sidecarVersion !== undefined) {
4126
+ slot.version = sidecarVersion
4127
+ }
4128
+ staged.set(key, slot)
4129
+ }
4130
+
4131
+ const parsed: Array<{
4132
+ collectionId: string
4133
+ resourceId: string
4134
+ chunkIndex: number
4135
+ contentType: string
4136
+ body: Buffer
4137
+ version?: number
4138
+ }> = []
4139
+ for (const slot of staged.values()) {
4140
+ if (slot.body === undefined) {
4141
+ // A sidecar with no paired representation is not a valid chunk.
4142
+ continue
4143
+ }
4144
+ parsed.push({
4145
+ collectionId: slot.collectionId,
4146
+ resourceId: slot.resourceId,
4147
+ chunkIndex: slot.chunkIndex,
4148
+ contentType: slot.contentType ?? 'application/octet-stream',
4149
+ body: slot.body,
4150
+ version: slot.version
4151
+ })
4152
+ }
4153
+ return parsed
4154
+ }
3073
4155
  }