was-teaching-server 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +118 -1
- package/README.md +23 -11
- package/dist/backends/filesystem.d.ts +247 -4
- package/dist/backends/filesystem.d.ts.map +1 -1
- package/dist/backends/filesystem.js +711 -81
- package/dist/backends/filesystem.js.map +1 -1
- package/dist/backends/postgres.d.ts +240 -2
- package/dist/backends/postgres.d.ts.map +1 -1
- package/dist/backends/postgres.js +800 -42
- package/dist/backends/postgres.js.map +1 -1
- package/dist/backends/postgresSchema.d.ts.map +1 -1
- package/dist/backends/postgresSchema.js +69 -57
- package/dist/backends/postgresSchema.js.map +1 -1
- package/dist/errors.d.ts +26 -9
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +40 -12
- package/dist/errors.js.map +1 -1
- package/dist/lib/equalityIndex.d.ts +334 -0
- package/dist/lib/equalityIndex.d.ts.map +1 -0
- package/dist/lib/equalityIndex.js +552 -0
- package/dist/lib/equalityIndex.js.map +1 -0
- package/dist/lib/exportManifest.d.ts +9 -0
- package/dist/lib/exportManifest.d.ts.map +1 -1
- package/dist/lib/exportManifest.js +9 -0
- package/dist/lib/exportManifest.js.map +1 -1
- package/dist/lib/importTar.d.ts +18 -2
- package/dist/lib/importTar.d.ts.map +1 -1
- package/dist/lib/importTar.js +77 -4
- package/dist/lib/importTar.js.map +1 -1
- package/dist/lib/paths.d.ts +33 -0
- package/dist/lib/paths.d.ts.map +1 -1
- package/dist/lib/paths.js +28 -0
- package/dist/lib/paths.js.map +1 -1
- package/dist/lib/resourceFileName.d.ts +43 -0
- package/dist/lib/resourceFileName.d.ts.map +1 -1
- package/dist/lib/resourceFileName.js +63 -0
- package/dist/lib/resourceFileName.js.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +5 -2
- package/dist/plugin.js.map +1 -1
- package/dist/requests/ChunkRequest.d.ts +104 -0
- package/dist/requests/ChunkRequest.d.ts.map +1 -0
- package/dist/requests/ChunkRequest.js +396 -0
- package/dist/requests/ChunkRequest.js.map +1 -0
- package/dist/requests/CollectionRequest.d.ts +22 -5
- package/dist/requests/CollectionRequest.d.ts.map +1 -1
- package/dist/requests/CollectionRequest.js +192 -11
- package/dist/requests/CollectionRequest.js.map +1 -1
- package/dist/requests/ResourceRequest.d.ts.map +1 -1
- package/dist/requests/ResourceRequest.js +20 -0
- package/dist/requests/ResourceRequest.js.map +1 -1
- package/dist/requests/SpaceRequest.d.ts +1 -0
- package/dist/requests/SpaceRequest.d.ts.map +1 -1
- package/dist/requests/SpaceRequest.js +21 -1
- package/dist/requests/SpaceRequest.js.map +1 -1
- package/dist/routes.d.ts.map +1 -1
- package/dist/routes.js +21 -0
- package/dist/routes.js.map +1 -1
- package/dist/types.d.ts +165 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +19 -18
- package/src/backends/filesystem.ts +924 -75
- package/src/backends/postgres.ts +1114 -32
- package/src/backends/postgresSchema.ts +69 -57
- package/src/errors.ts +44 -12
- package/src/lib/equalityIndex.ts +769 -0
- package/src/lib/importTar.ts +107 -4
- package/src/lib/paths.ts +48 -0
- package/src/lib/resourceFileName.ts +69 -0
- package/src/plugin.ts +5 -2
- package/src/requests/ChunkRequest.ts +478 -0
- package/src/requests/CollectionRequest.ts +221 -12
- package/src/requests/ResourceRequest.ts +20 -0
- package/src/requests/SpaceRequest.ts +25 -1
- package/src/routes.ts +46 -0
- package/src/types.ts +175 -1
package/src/backends/postgres.ts
CHANGED
|
@@ -42,9 +42,21 @@ import {
|
|
|
42
42
|
DuplicateRevocationError
|
|
43
43
|
} from '../errors.js'
|
|
44
44
|
import { applyMigrations } from './postgresSchema.js'
|
|
45
|
-
import {
|
|
45
|
+
import {
|
|
46
|
+
extractTarEntries,
|
|
47
|
+
buildImportPlan,
|
|
48
|
+
metaSidecarFileId
|
|
49
|
+
} from '../lib/importTar.js'
|
|
50
|
+
import type { TarEntry } from '../lib/importTar.js'
|
|
46
51
|
import { collectionPath, resourcePath } from '../lib/paths.js'
|
|
47
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
fileNameFor,
|
|
54
|
+
parseResourceFileName,
|
|
55
|
+
chunkDirName,
|
|
56
|
+
parseChunkDirName,
|
|
57
|
+
parseChunkIndexSegment
|
|
58
|
+
} from '../lib/resourceFileName.js'
|
|
59
|
+
import { assertValidId } from '../lib/validateId.js'
|
|
48
60
|
import { sanitizeBackendRecord } from '../lib/backends.js'
|
|
49
61
|
import { backendUsageFields } from '../lib/backendUsage.js'
|
|
50
62
|
import { assertEncryptedWriteConforms } from '../lib/encryption.js'
|
|
@@ -66,6 +78,18 @@ import type {
|
|
|
66
78
|
BlindedIndexQuery,
|
|
67
79
|
BlindedIndexQueryPage
|
|
68
80
|
} from '../lib/blindedIndex.js'
|
|
81
|
+
import {
|
|
82
|
+
runEqualityQuery,
|
|
83
|
+
assertNoUniqueEqualityConflict,
|
|
84
|
+
findEqualityUniqueViolation
|
|
85
|
+
} from '../lib/equalityIndex.js'
|
|
86
|
+
import type {
|
|
87
|
+
EqualityQuery,
|
|
88
|
+
EqualityQueryPage,
|
|
89
|
+
EqualityCandidate,
|
|
90
|
+
EqualityValue,
|
|
91
|
+
NormalizedIndexDeclaration
|
|
92
|
+
} from '../lib/equalityIndex.js'
|
|
69
93
|
import {
|
|
70
94
|
DEFAULT_MAX_UPLOAD_BYTES,
|
|
71
95
|
DEFAULT_MAX_SPACES_PER_CONTROLLER,
|
|
@@ -206,6 +230,17 @@ function parseSidecar(bytes: Buffer | undefined): SidecarShape | undefined {
|
|
|
206
230
|
}
|
|
207
231
|
}
|
|
208
232
|
|
|
233
|
+
/**
|
|
234
|
+
* The chunk-metadata sidecar shape (`.chunks.<encId>/.meta.<index>.json`) the
|
|
235
|
+
* filesystem backend writes per chunk. Only the monotonic `version` (the
|
|
236
|
+
* chunk's ETag validator) is carried across export/import; the filesystem
|
|
237
|
+
* writes `createdAt` / `updatedAt` too, but this backend's `chunks` table holds
|
|
238
|
+
* no chunk timestamps, so it emits and reads only `version`.
|
|
239
|
+
*/
|
|
240
|
+
interface ChunkSidecar {
|
|
241
|
+
version?: number
|
|
242
|
+
}
|
|
243
|
+
|
|
209
244
|
export class PostgresBackend implements StorageBackend {
|
|
210
245
|
logger: FastifyBaseLogger
|
|
211
246
|
/**
|
|
@@ -421,11 +456,18 @@ export class PostgresBackend implements StorageBackend {
|
|
|
421
456
|
// `key-epochs`: multi-recipient encrypted Collections -- per-epoch wrapped
|
|
422
457
|
// keys on the `encryption` marker, a client-declared `epoch` stamp on
|
|
423
458
|
// Resources, and conditional (`If-Match`) Collection Description writes.
|
|
459
|
+
// `chunked-streams`: chunk addressing at `/{resourceId}/chunks/{n}`, opaque
|
|
460
|
+
// per-chunk raw-bytes storage in the `chunks` table.
|
|
461
|
+
// `equality-query`: serves the `equality` profile -- server-extracted
|
|
462
|
+
// plaintext attribute equality over a Collection's declared `indexes`
|
|
463
|
+
// (`queryByEquality`), plus the GET `filter[attr]=value` equality filter.
|
|
424
464
|
features: [
|
|
425
465
|
'conditional-writes',
|
|
426
466
|
'changes-query',
|
|
427
467
|
'blinded-index-query',
|
|
428
|
-
'
|
|
468
|
+
'equality-query',
|
|
469
|
+
'key-epochs',
|
|
470
|
+
'chunked-streams'
|
|
429
471
|
]
|
|
430
472
|
}
|
|
431
473
|
}
|
|
@@ -519,6 +561,42 @@ export class PostgresBackend implements StorageBackend {
|
|
|
519
561
|
}
|
|
520
562
|
}
|
|
521
563
|
|
|
564
|
+
/**
|
|
565
|
+
* Serializes concurrent CREATORS of one not-yet-existing row (a Resource or
|
|
566
|
+
* a chunk) on a transaction-scoped advisory lock keyed by the row's
|
|
567
|
+
* identity. Under READ COMMITTED a `SELECT ... FOR UPDATE` on an absent row
|
|
568
|
+
* locks nothing (no gap locks), so two concurrent creators would both read
|
|
569
|
+
* "no prior row" and both apply their full byte size as the usage delta --
|
|
570
|
+
* and while the `ON CONFLICT` arm recomputes `version` from the conflicting
|
|
571
|
+
* row, it cannot retroactively fix the loser's delta, leaving `usage_bytes`
|
|
572
|
+
* inflated by one write's size. The caller takes this lock when its
|
|
573
|
+
* lock-nothing SELECT found no row, then RE-reads the row: the second
|
|
574
|
+
* creator blocks here until the first commits, and its re-read sees the
|
|
575
|
+
* committed row, so its precondition, version, and usage delta are computed
|
|
576
|
+
* from accurate state. Held to commit (advisory xact lock). The `create:`
|
|
577
|
+
* prefix keeps this key domain distinct from the unique-blinded-term
|
|
578
|
+
* advisory lock, which hashes the bare `(spaceId, collectionId)`.
|
|
579
|
+
* @param options {object}
|
|
580
|
+
* @param options.client {pg.PoolClient}
|
|
581
|
+
* @param options.spaceId {string}
|
|
582
|
+
* @param options.rowKey {string} the row's identity within the Space
|
|
583
|
+
* @returns {Promise<void>}
|
|
584
|
+
*/
|
|
585
|
+
private async _lockSameKeyCreate({
|
|
586
|
+
client,
|
|
587
|
+
spaceId,
|
|
588
|
+
rowKey
|
|
589
|
+
}: {
|
|
590
|
+
client: pg.PoolClient
|
|
591
|
+
spaceId: string
|
|
592
|
+
rowKey: string
|
|
593
|
+
}): Promise<void> {
|
|
594
|
+
await client.query(
|
|
595
|
+
'SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))',
|
|
596
|
+
[spaceId, `create:${rowKey}`]
|
|
597
|
+
)
|
|
598
|
+
}
|
|
599
|
+
|
|
522
600
|
// Quotas
|
|
523
601
|
|
|
524
602
|
/**
|
|
@@ -546,13 +624,20 @@ export class PostgresBackend implements StorageBackend {
|
|
|
546
624
|
|
|
547
625
|
let usageByCollection: CollectionUsage[] | undefined
|
|
548
626
|
if (includeCollections) {
|
|
627
|
+
// Per-Collection usage sums both Resource content bytes and chunk bytes
|
|
628
|
+
// (the `chunked-streams` feature) so the breakdown agrees with the
|
|
629
|
+
// Space total in the transactional counter.
|
|
549
630
|
const { rows: collectionRows } = await this._pool.query<{
|
|
550
631
|
collection_id: string
|
|
551
632
|
usage: string
|
|
552
633
|
}>(
|
|
553
|
-
`SELECT collection_id, COALESCE(SUM(
|
|
554
|
-
FROM resources
|
|
555
|
-
|
|
634
|
+
`SELECT collection_id, COALESCE(SUM(bytes), 0) AS usage FROM (
|
|
635
|
+
SELECT collection_id, size_bytes AS bytes FROM resources
|
|
636
|
+
WHERE space_id = $1
|
|
637
|
+
UNION ALL
|
|
638
|
+
SELECT collection_id, size AS bytes FROM chunks
|
|
639
|
+
WHERE space_id = $1
|
|
640
|
+
) usage_rows
|
|
556
641
|
GROUP BY collection_id
|
|
557
642
|
ORDER BY collection_id`,
|
|
558
643
|
[spaceId]
|
|
@@ -595,6 +680,8 @@ export class PostgresBackend implements StorageBackend {
|
|
|
595
680
|
(SELECT COALESCE(usage_bytes, 0) FROM spaces WHERE space_id = $1)
|
|
596
681
|
AS space_total,
|
|
597
682
|
(SELECT COALESCE(SUM(size_bytes), 0) FROM resources
|
|
683
|
+
WHERE space_id = $1 AND collection_id = $2)
|
|
684
|
+
+ (SELECT COALESCE(SUM(size), 0) FROM chunks
|
|
598
685
|
WHERE space_id = $1 AND collection_id = $2) AS collection_total`,
|
|
599
686
|
[spaceId, collectionId]
|
|
600
687
|
)
|
|
@@ -1024,9 +1111,15 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1024
1111
|
collectionId: string
|
|
1025
1112
|
}): Promise<void> {
|
|
1026
1113
|
await this._withTransaction(async client => {
|
|
1114
|
+
// The Collection's freed bytes are its Resource content plus its chunk
|
|
1115
|
+
// bytes (the `chunked-streams` feature); both cascade away with the
|
|
1116
|
+
// Collection row, so both leave the quota counter.
|
|
1027
1117
|
const { rows } = await client.query<{ total: string }>(
|
|
1028
|
-
`SELECT
|
|
1029
|
-
|
|
1118
|
+
`SELECT
|
|
1119
|
+
(SELECT COALESCE(SUM(size_bytes), 0) FROM resources
|
|
1120
|
+
WHERE space_id = $1 AND collection_id = $2)
|
|
1121
|
+
+ (SELECT COALESCE(SUM(size), 0) FROM chunks
|
|
1122
|
+
WHERE space_id = $1 AND collection_id = $2) AS total`,
|
|
1030
1123
|
[spaceId, collectionId]
|
|
1031
1124
|
)
|
|
1032
1125
|
const freedBytes = Number(rows[0]?.total ?? 0)
|
|
@@ -1203,6 +1296,7 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1203
1296
|
input,
|
|
1204
1297
|
createdBy,
|
|
1205
1298
|
epoch,
|
|
1299
|
+
uniqueIndexes,
|
|
1206
1300
|
ifMatch,
|
|
1207
1301
|
ifNoneMatch
|
|
1208
1302
|
}: {
|
|
@@ -1212,6 +1306,7 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1212
1306
|
input: ResourceInput
|
|
1213
1307
|
createdBy?: IDID
|
|
1214
1308
|
epoch?: string
|
|
1309
|
+
uniqueIndexes?: NormalizedIndexDeclaration[]
|
|
1215
1310
|
ifMatch?: string
|
|
1216
1311
|
ifNoneMatch?: boolean
|
|
1217
1312
|
}): Promise<{ version: number }> {
|
|
@@ -1250,23 +1345,29 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1250
1345
|
return this._withTransaction(async client => {
|
|
1251
1346
|
await this._ensureCollectionRow({ client, spaceId, collectionId })
|
|
1252
1347
|
|
|
1253
|
-
//
|
|
1254
|
-
//
|
|
1255
|
-
//
|
|
1256
|
-
//
|
|
1257
|
-
//
|
|
1348
|
+
// Two unique-attribute invariants can force a JSON content write to
|
|
1349
|
+
// serialize before it upserts its row: the EDV blinded one (`unique: true`
|
|
1350
|
+
// blinded attributes; the `blinded-index-query` feature) and the plaintext
|
|
1351
|
+
// equality one (a Collection's `unique`-declared `indexes`; the
|
|
1352
|
+
// `equality-query` feature). Only a unique-carrying JSON write can create
|
|
1353
|
+
// either claim. A per-Collection transaction-scoped advisory lock
|
|
1258
1354
|
// serializes the claimants (held to commit, so the loser's scan sees the
|
|
1259
|
-
// winner's committed row) without entering the row-lock ordering of
|
|
1260
|
-
//
|
|
1261
|
-
|
|
1262
|
-
if (
|
|
1355
|
+
// winner's committed row) without entering the row-lock ordering of plain
|
|
1356
|
+
// writes; it is taken once and shared by both checks.
|
|
1357
|
+
const blindedUnique =
|
|
1263
1358
|
input.kind === 'json' &&
|
|
1264
1359
|
collectUniqueBlindedTerms({ document: input.data }).length > 0
|
|
1265
|
-
|
|
1360
|
+
const equalityUnique =
|
|
1361
|
+
input.kind === 'json' &&
|
|
1362
|
+
uniqueIndexes !== undefined &&
|
|
1363
|
+
uniqueIndexes.length > 0
|
|
1364
|
+
if (blindedUnique || equalityUnique) {
|
|
1266
1365
|
await client.query(
|
|
1267
1366
|
'SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))',
|
|
1268
1367
|
[spaceId, collectionId]
|
|
1269
1368
|
)
|
|
1369
|
+
}
|
|
1370
|
+
if (blindedUnique) {
|
|
1270
1371
|
const { rows: candidateRows } = await client.query<{
|
|
1271
1372
|
resource_id: string
|
|
1272
1373
|
content: Buffer | null
|
|
@@ -1290,7 +1391,32 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1290
1391
|
// skip an unparsable body
|
|
1291
1392
|
}
|
|
1292
1393
|
}
|
|
1293
|
-
assertNoUniqueBlindedConflict({
|
|
1394
|
+
assertNoUniqueBlindedConflict({
|
|
1395
|
+
document: input.kind === 'json' ? input.data : undefined,
|
|
1396
|
+
candidates
|
|
1397
|
+
})
|
|
1398
|
+
}
|
|
1399
|
+
if (equalityUnique) {
|
|
1400
|
+
// A content write does not change the Resource's `custom`, so the custom
|
|
1401
|
+
// side of the claim comes from the CURRENT stored row.
|
|
1402
|
+
const { rows: selfRows } = await client.query<{
|
|
1403
|
+
custom: ResourceMetadataCustom | Record<string, unknown> | null
|
|
1404
|
+
}>(
|
|
1405
|
+
`SELECT custom FROM resources
|
|
1406
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1407
|
+
AND NOT deleted`,
|
|
1408
|
+
[spaceId, collectionId, resourceId]
|
|
1409
|
+
)
|
|
1410
|
+
assertNoUniqueEqualityConflict({
|
|
1411
|
+
indexes: uniqueIndexes!,
|
|
1412
|
+
content: input.kind === 'json' ? input.data : undefined,
|
|
1413
|
+
custom: selfRows[0]?.custom ?? undefined,
|
|
1414
|
+
candidates: await this._readEqualityCandidates(client, {
|
|
1415
|
+
spaceId,
|
|
1416
|
+
collectionId,
|
|
1417
|
+
excludeResourceId: resourceId
|
|
1418
|
+
})
|
|
1419
|
+
})
|
|
1294
1420
|
}
|
|
1295
1421
|
|
|
1296
1422
|
// Narrow projection: the lock needs the row, not its (possibly multi-MB)
|
|
@@ -1307,7 +1433,31 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1307
1433
|
FOR UPDATE`,
|
|
1308
1434
|
[spaceId, collectionId, resourceId]
|
|
1309
1435
|
)
|
|
1310
|
-
|
|
1436
|
+
let prior = rows[0]
|
|
1437
|
+
if (prior === undefined) {
|
|
1438
|
+
// The lock-nothing case: serialize with any concurrent creator of the
|
|
1439
|
+
// same id and re-read, so `exists` / `priorSize` below reflect the row
|
|
1440
|
+
// it committed (see `_lockSameKeyCreate` -- without this, both
|
|
1441
|
+
// creators would count their full byte size as the usage delta).
|
|
1442
|
+
await this._lockSameKeyCreate({
|
|
1443
|
+
client,
|
|
1444
|
+
spaceId,
|
|
1445
|
+
rowKey: `${collectionId}/${resourceId}`
|
|
1446
|
+
})
|
|
1447
|
+
const { rows: reread } = await client.query<
|
|
1448
|
+
Pick<
|
|
1449
|
+
ResourceRow,
|
|
1450
|
+
'version' | 'size_bytes' | 'deleted' | 'created_at' | 'created_by'
|
|
1451
|
+
>
|
|
1452
|
+
>(
|
|
1453
|
+
`SELECT version, size_bytes, deleted, created_at, created_by
|
|
1454
|
+
FROM resources
|
|
1455
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1456
|
+
FOR UPDATE`,
|
|
1457
|
+
[spaceId, collectionId, resourceId]
|
|
1458
|
+
)
|
|
1459
|
+
prior = reread[0]
|
|
1460
|
+
}
|
|
1311
1461
|
const exists = prior !== undefined && !prior.deleted
|
|
1312
1462
|
if (ifMatch !== undefined || ifNoneMatch) {
|
|
1313
1463
|
assertWritePrecondition({
|
|
@@ -1339,7 +1489,7 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1339
1489
|
|
|
1340
1490
|
const now = new Date().toISOString()
|
|
1341
1491
|
const version = (prior?.version ?? 0) + 1
|
|
1342
|
-
const priorSize = exists ? Number(prior
|
|
1492
|
+
const priorSize = exists ? Number(prior?.size_bytes ?? 0) : 0
|
|
1343
1493
|
const delta = content.length - priorSize
|
|
1344
1494
|
if (delta !== 0) {
|
|
1345
1495
|
await this._applyUsageDelta({ client, spaceId, delta })
|
|
@@ -1350,10 +1500,11 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1350
1500
|
// dropped both (the metadata went with the deleted Resource).
|
|
1351
1501
|
//
|
|
1352
1502
|
// Create-if-absent atomicity: when `If-None-Match: *` found NO prior row
|
|
1353
|
-
// (a tombstone is a real row and stays lock-serialized),
|
|
1354
|
-
//
|
|
1355
|
-
//
|
|
1356
|
-
//
|
|
1503
|
+
// (a tombstone is a real row and stays lock-serialized), concurrent
|
|
1504
|
+
// creators through this method are already serialized by
|
|
1505
|
+
// `_lockSameKeyCreate` above -- but a writer that does not take that
|
|
1506
|
+
// lock (`importSpace`'s plain INSERTs) can still race. A plain INSERT
|
|
1507
|
+
// (no ON CONFLICT) keeps the primary key as the arbiter: the loser's
|
|
1357
1508
|
// unique violation maps to the 412 the precondition would have thrown.
|
|
1358
1509
|
// `createdBy` names the Resource's creator, not its last writer: taken
|
|
1359
1510
|
// from this write's invoker only when there is no prior row at all
|
|
@@ -1519,7 +1670,25 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1519
1670
|
// no-op, keeping an existing tombstone's change-feed entry stable.
|
|
1520
1671
|
return
|
|
1521
1672
|
}
|
|
1522
|
-
|
|
1673
|
+
// A soft delete is an UPDATE, not a row removal, so the chunk foreign
|
|
1674
|
+
// key's ON DELETE CASCADE does not fire -- remove the Resource's chunks
|
|
1675
|
+
// (the `chunked-streams` feature) explicitly in this same transaction so
|
|
1676
|
+
// they never outlive their parent, and return their bytes to the quota
|
|
1677
|
+
// counter alongside the Resource's content bytes.
|
|
1678
|
+
const { rows: chunkRows } = await client.query<{ total: string }>(
|
|
1679
|
+
`SELECT COALESCE(SUM(size), 0) AS total FROM chunks
|
|
1680
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
|
|
1681
|
+
[spaceId, collectionId, resourceId]
|
|
1682
|
+
)
|
|
1683
|
+
const freedChunkBytes = Number(chunkRows[0]?.total ?? 0)
|
|
1684
|
+
if (freedChunkBytes > 0) {
|
|
1685
|
+
await client.query(
|
|
1686
|
+
`DELETE FROM chunks
|
|
1687
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
|
|
1688
|
+
[spaceId, collectionId, resourceId]
|
|
1689
|
+
)
|
|
1690
|
+
}
|
|
1691
|
+
const freedBytes = Number(prior.size_bytes) + freedChunkBytes
|
|
1523
1692
|
if (freedBytes > 0) {
|
|
1524
1693
|
await this._applyUsageDelta({ client, spaceId, delta: -freedBytes })
|
|
1525
1694
|
}
|
|
@@ -1615,6 +1784,7 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1615
1784
|
resourceId,
|
|
1616
1785
|
custom,
|
|
1617
1786
|
epoch,
|
|
1787
|
+
uniqueIndexes,
|
|
1618
1788
|
ifMatch,
|
|
1619
1789
|
ifNoneMatch
|
|
1620
1790
|
}: {
|
|
@@ -1623,10 +1793,24 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1623
1793
|
resourceId: string
|
|
1624
1794
|
custom: ResourceMetadataCustom | Record<string, unknown>
|
|
1625
1795
|
epoch?: string
|
|
1796
|
+
uniqueIndexes?: NormalizedIndexDeclaration[]
|
|
1626
1797
|
ifMatch?: string
|
|
1627
1798
|
ifNoneMatch?: boolean
|
|
1628
1799
|
}): Promise<{ metaVersion: number } | undefined> {
|
|
1629
1800
|
return this._withTransaction(async client => {
|
|
1801
|
+
// A metadata write can create a plaintext equality unique claim for a
|
|
1802
|
+
// `custom`-sourced attribute (the `equality-query` feature). When the
|
|
1803
|
+
// Collection declares any unique index, take the per-Collection advisory
|
|
1804
|
+
// lock first (held to commit, serializing concurrent claimants) so the
|
|
1805
|
+
// conflict scan below is atomic with the write.
|
|
1806
|
+
const equalityUnique =
|
|
1807
|
+
uniqueIndexes !== undefined && uniqueIndexes.length > 0
|
|
1808
|
+
if (equalityUnique) {
|
|
1809
|
+
await client.query(
|
|
1810
|
+
'SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))',
|
|
1811
|
+
[spaceId, collectionId]
|
|
1812
|
+
)
|
|
1813
|
+
}
|
|
1630
1814
|
const { rows } = await client.query<ResourceRow>(
|
|
1631
1815
|
`SELECT meta_version, deleted FROM resources
|
|
1632
1816
|
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
@@ -1643,6 +1827,39 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1643
1827
|
ifMatch,
|
|
1644
1828
|
ifNoneMatch
|
|
1645
1829
|
})
|
|
1830
|
+
if (equalityUnique) {
|
|
1831
|
+
// Content is the Resource's stored JSON content (unchanged by a
|
|
1832
|
+
// metadata write); custom is the incoming value this write sets. Fetch
|
|
1833
|
+
// the (possibly multi-MB) content only on this uniqueness path -- the
|
|
1834
|
+
// row is already `FOR UPDATE`-locked above.
|
|
1835
|
+
const { rows: selfRows } = await client.query<{
|
|
1836
|
+
content: Buffer | null
|
|
1837
|
+
is_json: boolean
|
|
1838
|
+
}>(
|
|
1839
|
+
`SELECT content, is_json FROM resources
|
|
1840
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
|
|
1841
|
+
[spaceId, collectionId, resourceId]
|
|
1842
|
+
)
|
|
1843
|
+
let content: unknown
|
|
1844
|
+
const selfRow = selfRows[0]
|
|
1845
|
+
if (selfRow?.is_json && selfRow.content) {
|
|
1846
|
+
try {
|
|
1847
|
+
content = JSON.parse(selfRow.content.toString('utf8')) as unknown
|
|
1848
|
+
} catch {
|
|
1849
|
+
content = undefined
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
assertNoUniqueEqualityConflict({
|
|
1853
|
+
indexes: uniqueIndexes!,
|
|
1854
|
+
content,
|
|
1855
|
+
custom,
|
|
1856
|
+
candidates: await this._readEqualityCandidates(client, {
|
|
1857
|
+
spaceId,
|
|
1858
|
+
collectionId,
|
|
1859
|
+
excludeResourceId: resourceId
|
|
1860
|
+
})
|
|
1861
|
+
})
|
|
1862
|
+
}
|
|
1646
1863
|
const metaVersion = (prior.meta_version ?? 0) + 1
|
|
1647
1864
|
const hasCustom = Object.keys(custom).length > 0
|
|
1648
1865
|
const now = new Date().toISOString()
|
|
@@ -1671,6 +1888,395 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1671
1888
|
})
|
|
1672
1889
|
}
|
|
1673
1890
|
|
|
1891
|
+
// Chunks (the `chunked-streams` feature)
|
|
1892
|
+
|
|
1893
|
+
/**
|
|
1894
|
+
* Writes one chunk of a chunked Resource as one transaction: the parent
|
|
1895
|
+
* Resource must exist (checked atomically -- a `FOR SHARE` lock on it also
|
|
1896
|
+
* blocks a concurrent delete of the parent for the duration of the write, so
|
|
1897
|
+
* a chunk can never be orphaned by a racing `deleteResource`), the chunk row
|
|
1898
|
+
* is locked, its precondition evaluated, its monotonic `version` bumped, and
|
|
1899
|
+
* the transactional quota delta applied. The chunk body is stored opaquely as
|
|
1900
|
+
* a single `bytea`, the buffered-blob path (bounded by `maxUploadBytes`),
|
|
1901
|
+
* exactly like a binary Resource representation.
|
|
1902
|
+
* @param options {object}
|
|
1903
|
+
* @param options.spaceId {string}
|
|
1904
|
+
* @param options.collectionId {string}
|
|
1905
|
+
* @param options.resourceId {string}
|
|
1906
|
+
* @param options.chunkIndex {number} a non-negative safe integer
|
|
1907
|
+
* @param options.input {ResourceInput}
|
|
1908
|
+
* @param [options.ifMatch] {string}
|
|
1909
|
+
* @param [options.ifNoneMatch] {boolean}
|
|
1910
|
+
* @returns {Promise<{ version: number }>} the chunk's new version
|
|
1911
|
+
*/
|
|
1912
|
+
async writeChunk({
|
|
1913
|
+
spaceId,
|
|
1914
|
+
collectionId,
|
|
1915
|
+
resourceId,
|
|
1916
|
+
chunkIndex,
|
|
1917
|
+
input,
|
|
1918
|
+
ifMatch,
|
|
1919
|
+
ifNoneMatch
|
|
1920
|
+
}: {
|
|
1921
|
+
spaceId: string
|
|
1922
|
+
collectionId: string
|
|
1923
|
+
resourceId: string
|
|
1924
|
+
chunkIndex: number
|
|
1925
|
+
input: ResourceInput
|
|
1926
|
+
ifMatch?: string
|
|
1927
|
+
ifNoneMatch?: boolean
|
|
1928
|
+
}): Promise<{ version: number }> {
|
|
1929
|
+
const { maxUploadBytes } = this
|
|
1930
|
+
let bytes: Buffer
|
|
1931
|
+
if (input.kind === 'json') {
|
|
1932
|
+
bytes = Buffer.from(JSON.stringify(input.data))
|
|
1933
|
+
if (bytes.length > maxUploadBytes) {
|
|
1934
|
+
throw new PayloadTooLargeError({
|
|
1935
|
+
maxUploadBytes,
|
|
1936
|
+
backendId: this.describe().id,
|
|
1937
|
+
uploadBytes: bytes.length
|
|
1938
|
+
})
|
|
1939
|
+
}
|
|
1940
|
+
} else {
|
|
1941
|
+
// Pre-flight a declared size, then buffer through the counting guard that
|
|
1942
|
+
// hard-caps a body whose size is omitted or understated. Buffering
|
|
1943
|
+
// happens BEFORE the transaction so a slow upload holds no row lock.
|
|
1944
|
+
if (
|
|
1945
|
+
input.declaredBytes !== undefined &&
|
|
1946
|
+
input.declaredBytes > maxUploadBytes
|
|
1947
|
+
) {
|
|
1948
|
+
throw new PayloadTooLargeError({
|
|
1949
|
+
maxUploadBytes,
|
|
1950
|
+
backendId: this.describe().id,
|
|
1951
|
+
uploadBytes: input.declaredBytes
|
|
1952
|
+
})
|
|
1953
|
+
}
|
|
1954
|
+
bytes = await bufferStreamCapped({
|
|
1955
|
+
stream: input.stream,
|
|
1956
|
+
maxUploadBytes,
|
|
1957
|
+
backendId: this.describe().id
|
|
1958
|
+
})
|
|
1959
|
+
}
|
|
1960
|
+
|
|
1961
|
+
return this._withTransaction(async client => {
|
|
1962
|
+
// Parent Resource must exist (and not be a tombstone). `FOR SHARE`
|
|
1963
|
+
// conflicts with the `FOR UPDATE` a concurrent `deleteResource` takes, so
|
|
1964
|
+
// the two serialize on the parent row -- the parent cannot be deleted
|
|
1965
|
+
// between this check and the chunk write.
|
|
1966
|
+
const { rows: parentRows } = await client.query<{ deleted: boolean }>(
|
|
1967
|
+
`SELECT deleted FROM resources
|
|
1968
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1969
|
+
FOR SHARE`,
|
|
1970
|
+
[spaceId, collectionId, resourceId]
|
|
1971
|
+
)
|
|
1972
|
+
const parent = parentRows[0]
|
|
1973
|
+
if (!parent || parent.deleted) {
|
|
1974
|
+
throw new ResourceNotFoundError({ requestName: 'Write Chunk' })
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
// Lock the chunk row (if any) and read its current version/size, so the
|
|
1978
|
+
// precondition, the monotonic bump, and the usage delta are all atomic
|
|
1979
|
+
// with the write.
|
|
1980
|
+
const { rows } = await client.query<{ version: number; size: string }>(
|
|
1981
|
+
`SELECT version, size FROM chunks
|
|
1982
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1983
|
+
AND chunk_index = $4
|
|
1984
|
+
FOR UPDATE`,
|
|
1985
|
+
[spaceId, collectionId, resourceId, chunkIndex]
|
|
1986
|
+
)
|
|
1987
|
+
let prior = rows[0]
|
|
1988
|
+
const chunkLabel = `${resourceId}/chunks/${chunkIndex}`
|
|
1989
|
+
if (prior === undefined) {
|
|
1990
|
+
// The lock-nothing case: serialize with any concurrent creator of the
|
|
1991
|
+
// same chunk and re-read, so the usage delta below reflects the row it
|
|
1992
|
+
// committed (see `_lockSameKeyCreate`).
|
|
1993
|
+
await this._lockSameKeyCreate({
|
|
1994
|
+
client,
|
|
1995
|
+
spaceId,
|
|
1996
|
+
rowKey: `${collectionId}/${chunkLabel}`
|
|
1997
|
+
})
|
|
1998
|
+
const { rows: reread } = await client.query<{
|
|
1999
|
+
version: number
|
|
2000
|
+
size: string
|
|
2001
|
+
}>(
|
|
2002
|
+
`SELECT version, size FROM chunks
|
|
2003
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2004
|
+
AND chunk_index = $4
|
|
2005
|
+
FOR UPDATE`,
|
|
2006
|
+
[spaceId, collectionId, resourceId, chunkIndex]
|
|
2007
|
+
)
|
|
2008
|
+
prior = reread[0]
|
|
2009
|
+
}
|
|
2010
|
+
const exists = prior !== undefined
|
|
2011
|
+
if (ifMatch !== undefined || ifNoneMatch) {
|
|
2012
|
+
assertWritePrecondition({
|
|
2013
|
+
resourceId: chunkLabel,
|
|
2014
|
+
exists,
|
|
2015
|
+
currentVersion: prior?.version ?? 0,
|
|
2016
|
+
ifMatch,
|
|
2017
|
+
ifNoneMatch
|
|
2018
|
+
})
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
const version = (prior?.version ?? 0) + 1
|
|
2022
|
+
const priorSize = exists ? Number(prior?.size ?? 0) : 0
|
|
2023
|
+
const delta = bytes.length - priorSize
|
|
2024
|
+
if (delta !== 0) {
|
|
2025
|
+
await this._applyUsageDelta({ client, spaceId, delta })
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
const values = [
|
|
2029
|
+
spaceId,
|
|
2030
|
+
collectionId,
|
|
2031
|
+
resourceId,
|
|
2032
|
+
chunkIndex,
|
|
2033
|
+
input.contentType,
|
|
2034
|
+
bytes,
|
|
2035
|
+
bytes.length,
|
|
2036
|
+
version
|
|
2037
|
+
]
|
|
2038
|
+
const insertSql = `
|
|
2039
|
+
INSERT INTO chunks (
|
|
2040
|
+
space_id, collection_id, resource_id, chunk_index,
|
|
2041
|
+
content_type, bytes, size, version
|
|
2042
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`
|
|
2043
|
+
// Create-if-absent atomicity mirrors `writeResource`: concurrent
|
|
2044
|
+
// creators through this method are serialized by `_lockSameKeyCreate`
|
|
2045
|
+
// above, and against a writer that does not take that lock
|
|
2046
|
+
// (`importSpace`'s plain INSERTs) the plain INSERT keeps the primary key
|
|
2047
|
+
// as the arbiter -- a racing creator's unique violation maps to the 412
|
|
2048
|
+
// the precondition would have thrown.
|
|
2049
|
+
if (ifNoneMatch && prior === undefined) {
|
|
2050
|
+
try {
|
|
2051
|
+
await client.query(insertSql, values)
|
|
2052
|
+
} catch (err) {
|
|
2053
|
+
if ((err as { code?: string }).code === '23505') {
|
|
2054
|
+
throw new PreconditionFailedError({
|
|
2055
|
+
detail: `Chunk '${chunkLabel}' already exists (If-None-Match: *).`
|
|
2056
|
+
})
|
|
2057
|
+
}
|
|
2058
|
+
throw err
|
|
2059
|
+
}
|
|
2060
|
+
return { version }
|
|
2061
|
+
}
|
|
2062
|
+
// The conflict update derives `version` from the row (`chunks.version +
|
|
2063
|
+
// 1`), not from the pre-read, so a concurrent creator that slipped in
|
|
2064
|
+
// after our lock-nothing SELECT still advances the counter monotonically.
|
|
2065
|
+
// RETURNING reports the version that actually landed.
|
|
2066
|
+
const { rows: written } = await client.query<{ version: number }>(
|
|
2067
|
+
`${insertSql}
|
|
2068
|
+
ON CONFLICT (space_id, collection_id, resource_id, chunk_index)
|
|
2069
|
+
DO UPDATE SET
|
|
2070
|
+
content_type = EXCLUDED.content_type,
|
|
2071
|
+
bytes = EXCLUDED.bytes,
|
|
2072
|
+
size = EXCLUDED.size,
|
|
2073
|
+
version = chunks.version + 1
|
|
2074
|
+
RETURNING version`,
|
|
2075
|
+
values
|
|
2076
|
+
)
|
|
2077
|
+
return { version: written[0]!.version }
|
|
2078
|
+
})
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
/**
|
|
2082
|
+
* Reads a chunk's bytes. Rejects with `ResourceNotFoundError` (404) when no
|
|
2083
|
+
* chunk is stored at that index.
|
|
2084
|
+
* @param options {object}
|
|
2085
|
+
* @param options.spaceId {string}
|
|
2086
|
+
* @param options.collectionId {string}
|
|
2087
|
+
* @param options.resourceId {string}
|
|
2088
|
+
* @param options.chunkIndex {number}
|
|
2089
|
+
* @returns {Promise<ResourceResult>}
|
|
2090
|
+
*/
|
|
2091
|
+
async getChunk({
|
|
2092
|
+
spaceId,
|
|
2093
|
+
collectionId,
|
|
2094
|
+
resourceId,
|
|
2095
|
+
chunkIndex
|
|
2096
|
+
}: {
|
|
2097
|
+
spaceId: string
|
|
2098
|
+
collectionId: string
|
|
2099
|
+
resourceId: string
|
|
2100
|
+
chunkIndex: number
|
|
2101
|
+
}): Promise<ResourceResult> {
|
|
2102
|
+
const { rows } = await this._pool.query<{
|
|
2103
|
+
content_type: string
|
|
2104
|
+
bytes: Buffer
|
|
2105
|
+
version: number
|
|
2106
|
+
}>(
|
|
2107
|
+
`SELECT content_type, bytes, version FROM chunks
|
|
2108
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2109
|
+
AND chunk_index = $4`,
|
|
2110
|
+
[spaceId, collectionId, resourceId, chunkIndex]
|
|
2111
|
+
)
|
|
2112
|
+
const row = rows[0]
|
|
2113
|
+
if (!row) {
|
|
2114
|
+
throw new ResourceNotFoundError({ requestName: 'Get Chunk' })
|
|
2115
|
+
}
|
|
2116
|
+
return {
|
|
2117
|
+
resourceStream: Readable.from(row.bytes),
|
|
2118
|
+
storedResourceType: row.content_type,
|
|
2119
|
+
version: row.version
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
/**
|
|
2124
|
+
* Reads a chunk's stored content-type / size / version (the HEAD payload
|
|
2125
|
+
* headers). Resolves `undefined` when the chunk is absent.
|
|
2126
|
+
* @param options {object}
|
|
2127
|
+
* @param options.spaceId {string}
|
|
2128
|
+
* @param options.collectionId {string}
|
|
2129
|
+
* @param options.resourceId {string}
|
|
2130
|
+
* @param options.chunkIndex {number}
|
|
2131
|
+
* @returns {Promise<{ contentType: string, size: number, version?: number }
|
|
2132
|
+
* | undefined>}
|
|
2133
|
+
*/
|
|
2134
|
+
async getChunkMetadata({
|
|
2135
|
+
spaceId,
|
|
2136
|
+
collectionId,
|
|
2137
|
+
resourceId,
|
|
2138
|
+
chunkIndex
|
|
2139
|
+
}: {
|
|
2140
|
+
spaceId: string
|
|
2141
|
+
collectionId: string
|
|
2142
|
+
resourceId: string
|
|
2143
|
+
chunkIndex: number
|
|
2144
|
+
}): Promise<
|
|
2145
|
+
{ contentType: string; size: number; version?: number } | undefined
|
|
2146
|
+
> {
|
|
2147
|
+
const { rows } = await this._pool.query<{
|
|
2148
|
+
content_type: string
|
|
2149
|
+
size: string
|
|
2150
|
+
version: number
|
|
2151
|
+
}>(
|
|
2152
|
+
`SELECT content_type, size, version FROM chunks
|
|
2153
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2154
|
+
AND chunk_index = $4`,
|
|
2155
|
+
[spaceId, collectionId, resourceId, chunkIndex]
|
|
2156
|
+
)
|
|
2157
|
+
const row = rows[0]
|
|
2158
|
+
if (!row) {
|
|
2159
|
+
return undefined
|
|
2160
|
+
}
|
|
2161
|
+
return {
|
|
2162
|
+
contentType: row.content_type,
|
|
2163
|
+
size: Number(row.size),
|
|
2164
|
+
version: row.version
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
/**
|
|
2169
|
+
* Deletes one chunk as one transaction (the chunk row is locked, its
|
|
2170
|
+
* `ifMatch` precondition evaluated atomically, and its bytes returned to the
|
|
2171
|
+
* quota counter). Resolves `true` when a chunk was removed, `false` when none
|
|
2172
|
+
* was stored at that index (the handler 404s on `false` -- chunk deletes are
|
|
2173
|
+
* not silently idempotent, unlike `deleteResource`).
|
|
2174
|
+
* @param options {object}
|
|
2175
|
+
* @param options.spaceId {string}
|
|
2176
|
+
* @param options.collectionId {string}
|
|
2177
|
+
* @param options.resourceId {string}
|
|
2178
|
+
* @param options.chunkIndex {number}
|
|
2179
|
+
* @param [options.ifMatch] {string}
|
|
2180
|
+
* @returns {Promise<boolean>}
|
|
2181
|
+
*/
|
|
2182
|
+
async deleteChunk({
|
|
2183
|
+
spaceId,
|
|
2184
|
+
collectionId,
|
|
2185
|
+
resourceId,
|
|
2186
|
+
chunkIndex,
|
|
2187
|
+
ifMatch
|
|
2188
|
+
}: {
|
|
2189
|
+
spaceId: string
|
|
2190
|
+
collectionId: string
|
|
2191
|
+
resourceId: string
|
|
2192
|
+
chunkIndex: number
|
|
2193
|
+
ifMatch?: string
|
|
2194
|
+
}): Promise<boolean> {
|
|
2195
|
+
return this._withTransaction(async client => {
|
|
2196
|
+
const { rows } = await client.query<{ version: number; size: string }>(
|
|
2197
|
+
`SELECT version, size FROM chunks
|
|
2198
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2199
|
+
AND chunk_index = $4
|
|
2200
|
+
FOR UPDATE`,
|
|
2201
|
+
[spaceId, collectionId, resourceId, chunkIndex]
|
|
2202
|
+
)
|
|
2203
|
+
const prior = rows[0]
|
|
2204
|
+
if (prior === undefined) {
|
|
2205
|
+
return false
|
|
2206
|
+
}
|
|
2207
|
+
if (ifMatch !== undefined) {
|
|
2208
|
+
assertWritePrecondition({
|
|
2209
|
+
resourceId: `${resourceId}/chunks/${chunkIndex}`,
|
|
2210
|
+
exists: true,
|
|
2211
|
+
currentVersion: prior.version,
|
|
2212
|
+
ifMatch
|
|
2213
|
+
})
|
|
2214
|
+
}
|
|
2215
|
+
await client.query(
|
|
2216
|
+
`DELETE FROM chunks
|
|
2217
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2218
|
+
AND chunk_index = $4`,
|
|
2219
|
+
[spaceId, collectionId, resourceId, chunkIndex]
|
|
2220
|
+
)
|
|
2221
|
+
const freedBytes = Number(prior.size)
|
|
2222
|
+
if (freedBytes > 0) {
|
|
2223
|
+
await this._applyUsageDelta({ client, spaceId, delta: -freedBytes })
|
|
2224
|
+
}
|
|
2225
|
+
return true
|
|
2226
|
+
})
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
/**
|
|
2230
|
+
* Lists a Resource's stored chunks in ascending `chunk_index` order -- the
|
|
2231
|
+
* discovery/reassembly listing. The opaque `bytes` column is deliberately not
|
|
2232
|
+
* selected. An empty listing (Resource with no chunks, or an absent Resource)
|
|
2233
|
+
* resolves `{ count: 0, chunks: [] }`.
|
|
2234
|
+
* @param options {object}
|
|
2235
|
+
* @param options.spaceId {string}
|
|
2236
|
+
* @param options.collectionId {string}
|
|
2237
|
+
* @param options.resourceId {string}
|
|
2238
|
+
* @returns {Promise<{ count: number, chunks: Array<{ index: number, size:
|
|
2239
|
+
* number, contentType: string, version?: number }> }>}
|
|
2240
|
+
*/
|
|
2241
|
+
async listChunks({
|
|
2242
|
+
spaceId,
|
|
2243
|
+
collectionId,
|
|
2244
|
+
resourceId
|
|
2245
|
+
}: {
|
|
2246
|
+
spaceId: string
|
|
2247
|
+
collectionId: string
|
|
2248
|
+
resourceId: string
|
|
2249
|
+
}): Promise<{
|
|
2250
|
+
count: number
|
|
2251
|
+
chunks: Array<{
|
|
2252
|
+
index: number
|
|
2253
|
+
size: number
|
|
2254
|
+
contentType: string
|
|
2255
|
+
version?: number
|
|
2256
|
+
}>
|
|
2257
|
+
}> {
|
|
2258
|
+
const { rows } = await this._pool.query<{
|
|
2259
|
+
chunk_index: number
|
|
2260
|
+
size: string
|
|
2261
|
+
content_type: string
|
|
2262
|
+
version: number
|
|
2263
|
+
}>(
|
|
2264
|
+
`SELECT chunk_index, size, content_type, version FROM chunks
|
|
2265
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2266
|
+
ORDER BY chunk_index`,
|
|
2267
|
+
[spaceId, collectionId, resourceId]
|
|
2268
|
+
)
|
|
2269
|
+
return {
|
|
2270
|
+
count: rows.length,
|
|
2271
|
+
chunks: rows.map(row => ({
|
|
2272
|
+
index: row.chunk_index,
|
|
2273
|
+
size: Number(row.size),
|
|
2274
|
+
contentType: row.content_type,
|
|
2275
|
+
version: row.version
|
|
2276
|
+
}))
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
|
|
1674
2280
|
/**
|
|
1675
2281
|
* Replication change feed (the `changes` query profile): one indexed keyset
|
|
1676
2282
|
* query over `(updatedAt, resourceId)`, tombstones included, JSON documents
|
|
@@ -1834,6 +2440,145 @@ export class PostgresBackend implements StorageBackend {
|
|
|
1834
2440
|
return runBlindedIndexQuery({ candidates, query, count, limit, cursor })
|
|
1835
2441
|
}
|
|
1836
2442
|
|
|
2443
|
+
/**
|
|
2444
|
+
* Plaintext equality query (the `equality` query profile; see the
|
|
2445
|
+
* `StorageBackend.queryByEquality` contract). Reads the Collection's live
|
|
2446
|
+
* Resources -- JSON rows carrying parsed `content`, all rows carrying their
|
|
2447
|
+
* `custom` jsonb -- and hands the candidates to the shared evaluator
|
|
2448
|
+
* (`lib/equalityIndex.ts`) for extraction, matching, ordering, and cursor
|
|
2449
|
+
* pagination -- identical semantics to the filesystem backend. A full scan of
|
|
2450
|
+
* the Collection per call, deliberate for this teaching backend; a
|
|
2451
|
+
* materialized variant would answer from a JSONB expression index.
|
|
2452
|
+
* @param options {object}
|
|
2453
|
+
* @param options.spaceId {string}
|
|
2454
|
+
* @param options.collectionId {string}
|
|
2455
|
+
* @param options.query {EqualityQuery}
|
|
2456
|
+
* @param options.indexes {NormalizedIndexDeclaration[]}
|
|
2457
|
+
* @param [options.count] {boolean} return only the match count
|
|
2458
|
+
* @param [options.limit] {number} requested page size
|
|
2459
|
+
* @param [options.cursor] {string} opaque cursor from a prior page
|
|
2460
|
+
* @returns {Promise<{ count: number } | EqualityQueryPage>}
|
|
2461
|
+
*/
|
|
2462
|
+
async queryByEquality({
|
|
2463
|
+
spaceId,
|
|
2464
|
+
collectionId,
|
|
2465
|
+
query,
|
|
2466
|
+
indexes,
|
|
2467
|
+
count,
|
|
2468
|
+
limit,
|
|
2469
|
+
cursor
|
|
2470
|
+
}: {
|
|
2471
|
+
spaceId: string
|
|
2472
|
+
collectionId: string
|
|
2473
|
+
query: EqualityQuery
|
|
2474
|
+
indexes: NormalizedIndexDeclaration[]
|
|
2475
|
+
count?: boolean
|
|
2476
|
+
limit?: number
|
|
2477
|
+
cursor?: string
|
|
2478
|
+
}): Promise<{ count: number } | EqualityQueryPage> {
|
|
2479
|
+
const candidates = await this._readEqualityCandidates(this._pool, {
|
|
2480
|
+
spaceId,
|
|
2481
|
+
collectionId
|
|
2482
|
+
})
|
|
2483
|
+
return runEqualityQuery({
|
|
2484
|
+
candidates,
|
|
2485
|
+
query,
|
|
2486
|
+
indexes,
|
|
2487
|
+
count,
|
|
2488
|
+
limit,
|
|
2489
|
+
cursor
|
|
2490
|
+
})
|
|
2491
|
+
}
|
|
2492
|
+
|
|
2493
|
+
/**
|
|
2494
|
+
* Declare-time uniqueness scan for the `equality` profile (see the
|
|
2495
|
+
* `StorageBackend.findEqualityUniqueViolation` contract): reads the
|
|
2496
|
+
* Collection's live Resources and delegates to the shared scan, which reports
|
|
2497
|
+
* the first `(name, value)` claimed by two different Resources under the given
|
|
2498
|
+
* `unique` declarations (or `undefined` when none is).
|
|
2499
|
+
* @param options {object}
|
|
2500
|
+
* @param options.spaceId {string}
|
|
2501
|
+
* @param options.collectionId {string}
|
|
2502
|
+
* @param options.indexes {NormalizedIndexDeclaration[]}
|
|
2503
|
+
* @returns {Promise<{ name: string, value: EqualityValue } | undefined>}
|
|
2504
|
+
*/
|
|
2505
|
+
async findEqualityUniqueViolation({
|
|
2506
|
+
spaceId,
|
|
2507
|
+
collectionId,
|
|
2508
|
+
indexes
|
|
2509
|
+
}: {
|
|
2510
|
+
spaceId: string
|
|
2511
|
+
collectionId: string
|
|
2512
|
+
indexes: NormalizedIndexDeclaration[]
|
|
2513
|
+
}): Promise<{ name: string; value: EqualityValue } | undefined> {
|
|
2514
|
+
const candidates = await this._readEqualityCandidates(this._pool, {
|
|
2515
|
+
spaceId,
|
|
2516
|
+
collectionId
|
|
2517
|
+
})
|
|
2518
|
+
return findEqualityUniqueViolation({ indexes, candidates })
|
|
2519
|
+
}
|
|
2520
|
+
|
|
2521
|
+
/**
|
|
2522
|
+
* Reads every live Resource of a Collection as an equality candidate -- the
|
|
2523
|
+
* candidate set for the equality query and the plaintext unique-attribute
|
|
2524
|
+
* conflict scans. Includes blob Resources (queryable through their
|
|
2525
|
+
* `custom`-sourced attributes): each row resolves `{ resourceId, content?,
|
|
2526
|
+
* custom? }`, where `content` is the parsed JSON of a JSON row (a blob and
|
|
2527
|
+
* unparsable JSON contribute none) and `custom` is the row's jsonb `custom`
|
|
2528
|
+
* when set. Tombstones are excluded (`NOT deleted`); an optional excluded
|
|
2529
|
+
* Resource is skipped. Runs on the given executor -- the pool for a read-only
|
|
2530
|
+
* query, or the write transaction's client for a uniqueness scan (so the scan
|
|
2531
|
+
* shares the advisory lock and sees a consistent snapshot).
|
|
2532
|
+
* @param executor {pg.Pool | pg.PoolClient}
|
|
2533
|
+
* @param options {object}
|
|
2534
|
+
* @param options.spaceId {string}
|
|
2535
|
+
* @param options.collectionId {string}
|
|
2536
|
+
* @param [options.excludeResourceId] {string}
|
|
2537
|
+
* @returns {Promise<EqualityCandidate[]>}
|
|
2538
|
+
*/
|
|
2539
|
+
private async _readEqualityCandidates(
|
|
2540
|
+
executor: pg.Pool | pg.PoolClient,
|
|
2541
|
+
{
|
|
2542
|
+
spaceId,
|
|
2543
|
+
collectionId,
|
|
2544
|
+
excludeResourceId
|
|
2545
|
+
}: {
|
|
2546
|
+
spaceId: string
|
|
2547
|
+
collectionId: string
|
|
2548
|
+
excludeResourceId?: string
|
|
2549
|
+
}
|
|
2550
|
+
): Promise<EqualityCandidate[]> {
|
|
2551
|
+
const { rows } = await executor.query<{
|
|
2552
|
+
resource_id: string
|
|
2553
|
+
content: Buffer | null
|
|
2554
|
+
is_json: boolean
|
|
2555
|
+
custom: ResourceMetadataCustom | Record<string, unknown> | null
|
|
2556
|
+
}>(
|
|
2557
|
+
`SELECT resource_id, content, is_json, custom FROM resources
|
|
2558
|
+
WHERE space_id = $1 AND collection_id = $2 AND NOT deleted
|
|
2559
|
+
AND ($3::text IS NULL OR resource_id <> $3)
|
|
2560
|
+
ORDER BY resource_id`,
|
|
2561
|
+
[spaceId, collectionId, excludeResourceId ?? null]
|
|
2562
|
+
)
|
|
2563
|
+
const candidates: EqualityCandidate[] = []
|
|
2564
|
+
for (const row of rows) {
|
|
2565
|
+
let content: unknown
|
|
2566
|
+
if (row.is_json && row.content) {
|
|
2567
|
+
try {
|
|
2568
|
+
content = JSON.parse(row.content.toString('utf8')) as unknown
|
|
2569
|
+
} catch {
|
|
2570
|
+
// skip an unparsable body -- it contributes no content attributes
|
|
2571
|
+
}
|
|
2572
|
+
}
|
|
2573
|
+
candidates.push({
|
|
2574
|
+
resourceId: row.resource_id,
|
|
2575
|
+
...(content !== undefined && { content }),
|
|
2576
|
+
...(row.custom !== null && { custom: row.custom })
|
|
2577
|
+
})
|
|
2578
|
+
}
|
|
2579
|
+
return candidates
|
|
2580
|
+
}
|
|
2581
|
+
|
|
1837
2582
|
// Policies
|
|
1838
2583
|
|
|
1839
2584
|
/**
|
|
@@ -2435,7 +3180,8 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2435
3180
|
{ rows: policyRows },
|
|
2436
3181
|
{ rows: collectionRows },
|
|
2437
3182
|
{ rows: resourceRows },
|
|
2438
|
-
{ rows: revocationRows }
|
|
3183
|
+
{ rows: revocationRows },
|
|
3184
|
+
{ rows: chunkRows }
|
|
2439
3185
|
] = await Promise.all([
|
|
2440
3186
|
this._pool.query<{
|
|
2441
3187
|
collection_id: string
|
|
@@ -2477,6 +3223,20 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2477
3223
|
`SELECT delegator, capability_id, record FROM space_revocations
|
|
2478
3224
|
WHERE space_id = $1`,
|
|
2479
3225
|
[spaceId]
|
|
3226
|
+
),
|
|
3227
|
+
// Chunk metadata only -- bytes are fetched one chunk at a time while
|
|
3228
|
+
// packing, so an export never holds a chunked Resource whole in memory.
|
|
3229
|
+
this._pool.query<{
|
|
3230
|
+
collection_id: string
|
|
3231
|
+
resource_id: string
|
|
3232
|
+
chunk_index: number
|
|
3233
|
+
content_type: string
|
|
3234
|
+
version: number
|
|
3235
|
+
}>(
|
|
3236
|
+
`SELECT collection_id, resource_id, chunk_index, content_type, version
|
|
3237
|
+
FROM chunks WHERE space_id = $1
|
|
3238
|
+
ORDER BY collection_id, resource_id, chunk_index`,
|
|
3239
|
+
[spaceId]
|
|
2480
3240
|
)
|
|
2481
3241
|
])
|
|
2482
3242
|
|
|
@@ -2487,11 +3247,26 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2487
3247
|
row => row.collection_id === '' && row.resource_id === ''
|
|
2488
3248
|
)?.policy
|
|
2489
3249
|
|
|
2490
|
-
// A file entry carries its bytes inline (the small JSON dot-files)
|
|
2491
|
-
//
|
|
3250
|
+
// A file entry carries its bytes inline (the small JSON dot-files), a lazy
|
|
3251
|
+
// reference to a resource representation, or a chunk directory (the
|
|
3252
|
+
// `chunked-streams` feature) whose files are likewise inline (the per-chunk
|
|
3253
|
+
// `.meta.<index>.json` sidecar) or lazy (the chunk `r.<index>...`
|
|
3254
|
+
// representation, resolved at pack time). `name` is the entry's sort key
|
|
3255
|
+
// within its dir; a chunk directory sorts by its `.chunks.<encId>` dir name.
|
|
3256
|
+
type ChunkFile =
|
|
3257
|
+
| { name: string; bytes: Buffer }
|
|
3258
|
+
| {
|
|
3259
|
+
name: string
|
|
3260
|
+
chunk: {
|
|
3261
|
+
collectionId: string
|
|
3262
|
+
resourceId: string
|
|
3263
|
+
chunkIndex: number
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
2492
3266
|
type FileEntry =
|
|
2493
3267
|
| { name: string; bytes: Buffer }
|
|
2494
3268
|
| { name: string; resource: { collectionId: string; resourceId: string } }
|
|
3269
|
+
| { name: string; chunkDir: ChunkFile[] }
|
|
2495
3270
|
// Space-level dot-files are always small JSON, carried inline.
|
|
2496
3271
|
const spaceFiles: Array<{ name: string; bytes: Buffer }> = [
|
|
2497
3272
|
{
|
|
@@ -2564,6 +3339,48 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2564
3339
|
})
|
|
2565
3340
|
}
|
|
2566
3341
|
}
|
|
3342
|
+
// Chunks (the `chunked-streams` feature): each chunked Resource contributes
|
|
3343
|
+
// one `.chunks.<encResourceId>/` subdirectory in its Collection dir, in the
|
|
3344
|
+
// exact filesystem-backend layout so an archive imports into either backend.
|
|
3345
|
+
// A chunk is stored there as a Resource keyed by its stringified index: an
|
|
3346
|
+
// `r.<index>.<encContentType>.<ext>` representation file (`fileNameFor`)
|
|
3347
|
+
// plus a `.meta.<index>.json` version sidecar. Files within a chunk dir are
|
|
3348
|
+
// sorted by name (the filesystem's readdir sort). Rows arrive ordered by
|
|
3349
|
+
// `(collection, resource, index)`.
|
|
3350
|
+
const chunkDirsByResource = new Map<string, ChunkFile[]>()
|
|
3351
|
+
for (const row of chunkRows) {
|
|
3352
|
+
const dirKey = `${row.collection_id}/${row.resource_id}`
|
|
3353
|
+
let chunkFiles = chunkDirsByResource.get(dirKey)
|
|
3354
|
+
if (!chunkFiles) {
|
|
3355
|
+
chunkFiles = []
|
|
3356
|
+
chunkDirsByResource.set(dirKey, chunkFiles)
|
|
3357
|
+
filesFor(row.collection_id).push({
|
|
3358
|
+
name: chunkDirName(row.resource_id),
|
|
3359
|
+
chunkDir: chunkFiles
|
|
3360
|
+
})
|
|
3361
|
+
}
|
|
3362
|
+
const chunkId = String(row.chunk_index)
|
|
3363
|
+
chunkFiles.push({
|
|
3364
|
+
name: fileNameFor({
|
|
3365
|
+
resourceId: chunkId,
|
|
3366
|
+
contentType: row.content_type
|
|
3367
|
+
}),
|
|
3368
|
+
chunk: {
|
|
3369
|
+
collectionId: row.collection_id,
|
|
3370
|
+
resourceId: row.resource_id,
|
|
3371
|
+
chunkIndex: row.chunk_index
|
|
3372
|
+
}
|
|
3373
|
+
})
|
|
3374
|
+
chunkFiles.push({
|
|
3375
|
+
name: `.meta.${chunkId}.json`,
|
|
3376
|
+
bytes: Buffer.from(
|
|
3377
|
+
JSON.stringify({ version: row.version } satisfies ChunkSidecar)
|
|
3378
|
+
)
|
|
3379
|
+
})
|
|
3380
|
+
}
|
|
3381
|
+
for (const chunkFiles of chunkDirsByResource.values()) {
|
|
3382
|
+
chunkFiles.sort((left, right) => left.name.localeCompare(right.name))
|
|
3383
|
+
}
|
|
2567
3384
|
|
|
2568
3385
|
// Top-level order: space-level files and collection dirs interleaved,
|
|
2569
3386
|
// sorted by name -- the same order the filesystem's readdir+sort yields.
|
|
@@ -2597,7 +3414,17 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2597
3414
|
spaceId,
|
|
2598
3415
|
entries: topLevel.map(entry =>
|
|
2599
3416
|
entry.kind === 'collection'
|
|
2600
|
-
? {
|
|
3417
|
+
? {
|
|
3418
|
+
name: entry.name,
|
|
3419
|
+
// A chunk directory expands to its `.chunks.<encId>/<file>`
|
|
3420
|
+
// relative paths (mirroring the pack order), so the manifest
|
|
3421
|
+
// matches the filesystem backend's for the same Space.
|
|
3422
|
+
files: entry.files.flatMap(file =>
|
|
3423
|
+
'chunkDir' in file
|
|
3424
|
+
? file.chunkDir.map(child => `${file.name}/${child.name}`)
|
|
3425
|
+
: [file.name]
|
|
3426
|
+
)
|
|
3427
|
+
}
|
|
2601
3428
|
: { name: entry.name }
|
|
2602
3429
|
),
|
|
2603
3430
|
revocationFiles: revocationFiles.map(file => file.name)
|
|
@@ -2615,6 +3442,25 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2615
3442
|
if (entry.kind === 'collection') {
|
|
2616
3443
|
pack.entry({ name: `${entryTarget}/`, type: 'directory', mtime })
|
|
2617
3444
|
for (const file of entry.files) {
|
|
3445
|
+
if ('chunkDir' in file) {
|
|
3446
|
+
// A chunked Resource's `.chunks.<encId>/` subdirectory: emit the
|
|
3447
|
+
// directory entry, then each file -- an inline `.meta.<index>.json`
|
|
3448
|
+
// sidecar, or a chunk representation whose bytes are fetched one at
|
|
3449
|
+
// a time.
|
|
3450
|
+
const dirTarget = `${entryTarget}/${file.name}`
|
|
3451
|
+
pack.entry({ name: `${dirTarget}/`, type: 'directory', mtime })
|
|
3452
|
+
for (const chunkFile of file.chunkDir) {
|
|
3453
|
+
const bytes =
|
|
3454
|
+
'bytes' in chunkFile
|
|
3455
|
+
? chunkFile.bytes
|
|
3456
|
+
: await this._chunkContent({ spaceId, ...chunkFile.chunk })
|
|
3457
|
+
pack.entry(
|
|
3458
|
+
{ name: `${dirTarget}/${chunkFile.name}`, mtime },
|
|
3459
|
+
bytes
|
|
3460
|
+
)
|
|
3461
|
+
}
|
|
3462
|
+
continue
|
|
3463
|
+
}
|
|
2618
3464
|
const bytes =
|
|
2619
3465
|
'bytes' in file
|
|
2620
3466
|
? file.bytes
|
|
@@ -2663,6 +3509,38 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2663
3509
|
return rows[0]?.content ?? Buffer.alloc(0)
|
|
2664
3510
|
}
|
|
2665
3511
|
|
|
3512
|
+
/**
|
|
3513
|
+
* Fetches one chunk's bytes for the export pack loop (the `chunked-streams`
|
|
3514
|
+
* feature). A chunk removed between the metadata pass and this read yields an
|
|
3515
|
+
* empty body rather than failing the whole archive, matching
|
|
3516
|
+
* `_resourceContent`.
|
|
3517
|
+
* @param options {object}
|
|
3518
|
+
* @param options.spaceId {string}
|
|
3519
|
+
* @param options.collectionId {string}
|
|
3520
|
+
* @param options.resourceId {string}
|
|
3521
|
+
* @param options.chunkIndex {number}
|
|
3522
|
+
* @returns {Promise<Buffer>}
|
|
3523
|
+
*/
|
|
3524
|
+
private async _chunkContent({
|
|
3525
|
+
spaceId,
|
|
3526
|
+
collectionId,
|
|
3527
|
+
resourceId,
|
|
3528
|
+
chunkIndex
|
|
3529
|
+
}: {
|
|
3530
|
+
spaceId: string
|
|
3531
|
+
collectionId: string
|
|
3532
|
+
resourceId: string
|
|
3533
|
+
chunkIndex: number
|
|
3534
|
+
}): Promise<Buffer> {
|
|
3535
|
+
const { rows } = await this._pool.query<{ bytes: Buffer | null }>(
|
|
3536
|
+
`SELECT bytes FROM chunks
|
|
3537
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
3538
|
+
AND chunk_index = $4`,
|
|
3539
|
+
[spaceId, collectionId, resourceId, chunkIndex]
|
|
3540
|
+
)
|
|
3541
|
+
return rows[0]?.bytes ?? Buffer.alloc(0)
|
|
3542
|
+
}
|
|
3543
|
+
|
|
2666
3544
|
/**
|
|
2667
3545
|
* Merges a WAS space-export tarball into an existing Space with the same
|
|
2668
3546
|
* three-invariant pre-flight (per-entry 413, fail-closed 422 encryption
|
|
@@ -2685,6 +3563,10 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2685
3563
|
}): Promise<ImportStats> {
|
|
2686
3564
|
const entries = await extractTarEntries(tarStream)
|
|
2687
3565
|
const { spacePolicy, collections, revocations } = buildImportPlan(entries)
|
|
3566
|
+
// Chunk entries (the `chunked-streams` feature) live in per-Resource
|
|
3567
|
+
// `.chunks.<encId>/` subdirectories, which `buildImportPlan` skips (it
|
|
3568
|
+
// ignores nested files); parse them straight off the raw tar entries.
|
|
3569
|
+
const chunkEntries = this._chunkEntriesFromArchive(entries)
|
|
2688
3570
|
const {
|
|
2689
3571
|
capacityBytes,
|
|
2690
3572
|
maxUploadBytes,
|
|
@@ -2774,6 +3656,19 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2774
3656
|
incomingBytes += body.length
|
|
2775
3657
|
}
|
|
2776
3658
|
}
|
|
3659
|
+
// Chunks (the `chunked-streams` feature) are opaque bytes -- no
|
|
3660
|
+
// encryption-conformance check applies -- but they count the same per-body
|
|
3661
|
+
// 413 and the (conservative) capacity pre-flight as Resource bodies.
|
|
3662
|
+
for (const chunk of chunkEntries) {
|
|
3663
|
+
if (chunk.body.length > maxUploadBytes) {
|
|
3664
|
+
throw new PayloadTooLargeError({
|
|
3665
|
+
maxUploadBytes,
|
|
3666
|
+
backendId: this.describe().id,
|
|
3667
|
+
uploadBytes: chunk.body.length
|
|
3668
|
+
})
|
|
3669
|
+
}
|
|
3670
|
+
incomingBytes += chunk.body.length
|
|
3671
|
+
}
|
|
2777
3672
|
if (
|
|
2778
3673
|
capacityBytes !== undefined &&
|
|
2779
3674
|
currentUsage + incomingBytes > capacityBytes
|
|
@@ -2966,13 +3861,63 @@ export class PostgresBackend implements StorageBackend {
|
|
|
2966
3861
|
}
|
|
2967
3862
|
}
|
|
2968
3863
|
|
|
2969
|
-
|
|
3864
|
+
// Chunks (the `chunked-streams` feature): restore each archived chunk
|
|
3865
|
+
// skip-not-overwrite, after the Resource apply loop so a chunk's parent
|
|
3866
|
+
// Resource row already exists in this transaction (the foreign key
|
|
3867
|
+
// requires it). An orphan chunk -- one whose parent is absent or a
|
|
3868
|
+
// tombstone -- is skipped rather than resurrected. Existing chunk rows are
|
|
3869
|
+
// left untouched.
|
|
3870
|
+
let createdChunkBytes = 0
|
|
3871
|
+
for (const chunk of chunkEntries) {
|
|
3872
|
+
const { rows: parentRows } = await client.query<{ deleted: boolean }>(
|
|
3873
|
+
`SELECT deleted FROM resources
|
|
3874
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`,
|
|
3875
|
+
[spaceId, chunk.collectionId, chunk.resourceId]
|
|
3876
|
+
)
|
|
3877
|
+
const parent = parentRows[0]
|
|
3878
|
+
if (!parent || parent.deleted) {
|
|
3879
|
+
continue
|
|
3880
|
+
}
|
|
3881
|
+
const { rows: existingChunkRows } = await client.query(
|
|
3882
|
+
`SELECT 1 FROM chunks
|
|
3883
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
3884
|
+
AND chunk_index = $4`,
|
|
3885
|
+
[spaceId, chunk.collectionId, chunk.resourceId, chunk.chunkIndex]
|
|
3886
|
+
)
|
|
3887
|
+
if (existingChunkRows.length > 0) {
|
|
3888
|
+
continue
|
|
3889
|
+
}
|
|
3890
|
+
// The chunk's `version` comes from its archived `.meta.<index>.json`
|
|
3891
|
+
// sidecar; an archive without one (or a chunk written before sidecars)
|
|
3892
|
+
// starts at version 1, the same fresh-write default as a Resource
|
|
3893
|
+
// restored without a sidecar.
|
|
3894
|
+
await client.query(
|
|
3895
|
+
`INSERT INTO chunks (
|
|
3896
|
+
space_id, collection_id, resource_id, chunk_index,
|
|
3897
|
+
content_type, bytes, size, version
|
|
3898
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
|
3899
|
+
[
|
|
3900
|
+
spaceId,
|
|
3901
|
+
chunk.collectionId,
|
|
3902
|
+
chunk.resourceId,
|
|
3903
|
+
chunk.chunkIndex,
|
|
3904
|
+
chunk.contentType,
|
|
3905
|
+
chunk.body,
|
|
3906
|
+
chunk.body.length,
|
|
3907
|
+
chunk.version ?? 1
|
|
3908
|
+
]
|
|
3909
|
+
)
|
|
3910
|
+
createdChunkBytes += chunk.body.length
|
|
3911
|
+
}
|
|
3912
|
+
|
|
3913
|
+
const createdTotalBytes = createdBytes + createdChunkBytes
|
|
3914
|
+
if (createdTotalBytes > 0) {
|
|
2970
3915
|
// The pre-flight was conservative (it counted skips too), so the
|
|
2971
3916
|
// actual created total always fits; apply it unguarded.
|
|
2972
3917
|
await this._applyUsageDelta({
|
|
2973
3918
|
client,
|
|
2974
3919
|
spaceId,
|
|
2975
|
-
delta:
|
|
3920
|
+
delta: createdTotalBytes
|
|
2976
3921
|
})
|
|
2977
3922
|
}
|
|
2978
3923
|
|
|
@@ -3070,4 +4015,141 @@ export class PostgresBackend implements StorageBackend {
|
|
|
3070
4015
|
]
|
|
3071
4016
|
)
|
|
3072
4017
|
}
|
|
4018
|
+
|
|
4019
|
+
/**
|
|
4020
|
+
* Parses a Space-export archive's chunk entries (the `chunked-streams`
|
|
4021
|
+
* feature) out of the raw tar entry map, in the filesystem backend's on-disk
|
|
4022
|
+
* layout so archives round-trip between the two backends. Chunk files live at
|
|
4023
|
+
* `space/<sourceSpaceId>/<collectionId>/.chunks.<encResourceId>/<file>` --
|
|
4024
|
+
* where a chunk is a Resource keyed by its stringified index: an
|
|
4025
|
+
* `r.<index>.<encContentType>.<ext>` representation paired with an optional
|
|
4026
|
+
* `.meta.<index>.json` version sidecar. `buildImportPlan` skips these (it
|
|
4027
|
+
* ignores nested Collection-dir files), so they are parsed here. Ids parsed
|
|
4028
|
+
* from the archive are validated (the path-traversal guard); a file whose
|
|
4029
|
+
* raw index segment is not canonical (`parseChunkIndexSegment`) is ignored,
|
|
4030
|
+
* and a sidecar with no paired representation is dropped (a chunk keeps no
|
|
4031
|
+
* tombstone).
|
|
4032
|
+
* @param entries {Map<string, TarEntry>}
|
|
4033
|
+
* @returns {Array<{ collectionId: string, resourceId: string, chunkIndex:
|
|
4034
|
+
* number, contentType: string, body: Buffer, version?: number }>}
|
|
4035
|
+
*/
|
|
4036
|
+
private _chunkEntriesFromArchive(entries: Map<string, TarEntry>): Array<{
|
|
4037
|
+
collectionId: string
|
|
4038
|
+
resourceId: string
|
|
4039
|
+
chunkIndex: number
|
|
4040
|
+
contentType: string
|
|
4041
|
+
body: Buffer
|
|
4042
|
+
version?: number
|
|
4043
|
+
}> {
|
|
4044
|
+
// Accumulate the representation and the sidecar of each chunk under one
|
|
4045
|
+
// key, then emit only the chunks that carry a representation.
|
|
4046
|
+
const staged = new Map<
|
|
4047
|
+
string,
|
|
4048
|
+
{
|
|
4049
|
+
collectionId: string
|
|
4050
|
+
resourceId: string
|
|
4051
|
+
chunkIndex: number
|
|
4052
|
+
contentType?: string
|
|
4053
|
+
body?: Buffer
|
|
4054
|
+
version?: number
|
|
4055
|
+
}
|
|
4056
|
+
>()
|
|
4057
|
+
const pattern = /^space\/[^/]+\/([^/]+)\/(\.chunks\.[^/]+)\/([^/]+)$/
|
|
4058
|
+
for (const [entryName, entry] of entries) {
|
|
4059
|
+
if (entry.type !== 'file' || !entry.body) {
|
|
4060
|
+
continue
|
|
4061
|
+
}
|
|
4062
|
+
const match = entryName.match(pattern)
|
|
4063
|
+
if (!match) {
|
|
4064
|
+
continue
|
|
4065
|
+
}
|
|
4066
|
+
const collectionId = match[1]!
|
|
4067
|
+
const resourceId = parseChunkDirName(match[2]!)
|
|
4068
|
+
const fileName = match[3]!
|
|
4069
|
+
if (resourceId === undefined) {
|
|
4070
|
+
continue
|
|
4071
|
+
}
|
|
4072
|
+
// Reject a path-traversal / non-URL-safe id parsed from the archive
|
|
4073
|
+
// before it keys a destination row.
|
|
4074
|
+
assertValidId(collectionId, {
|
|
4075
|
+
kind: 'collection',
|
|
4076
|
+
requestName: 'Import Space'
|
|
4077
|
+
})
|
|
4078
|
+
assertValidId(resourceId, {
|
|
4079
|
+
kind: 'resource',
|
|
4080
|
+
requestName: 'Import Space'
|
|
4081
|
+
})
|
|
4082
|
+
|
|
4083
|
+
// A chunk's index is the "resource id" of its file within the chunk dir:
|
|
4084
|
+
// `r.<index>.<encType>.<ext>` for the representation, `.meta.<index>.json`
|
|
4085
|
+
// for the version sidecar. The RAW index segment must pass the same
|
|
4086
|
+
// canonical predicate as the live route (`parseChunkIndexSegment`), so a
|
|
4087
|
+
// non-canonical spelling (`r.01.*`, `r.%31.*`) is dropped rather than
|
|
4088
|
+
// coerced onto -- and colliding with -- the canonical chunk's row.
|
|
4089
|
+
let chunkIdSegment: string | undefined
|
|
4090
|
+
let representation: { contentType: string; body: Buffer } | undefined
|
|
4091
|
+
let sidecarVersion: number | undefined
|
|
4092
|
+
const metaId = metaSidecarFileId(fileName)
|
|
4093
|
+
if (metaId !== undefined) {
|
|
4094
|
+
chunkIdSegment = metaId
|
|
4095
|
+
try {
|
|
4096
|
+
const sidecar = JSON.parse(
|
|
4097
|
+
entry.body.toString('utf8')
|
|
4098
|
+
) as ChunkSidecar
|
|
4099
|
+
sidecarVersion = sidecar.version
|
|
4100
|
+
} catch {
|
|
4101
|
+
sidecarVersion = undefined
|
|
4102
|
+
}
|
|
4103
|
+
} else if (fileName.startsWith('r.')) {
|
|
4104
|
+
const { contentType } = parseResourceFileName(fileName)
|
|
4105
|
+
chunkIdSegment = fileName.split('.')[1]
|
|
4106
|
+
representation = { contentType, body: entry.body }
|
|
4107
|
+
} else {
|
|
4108
|
+
continue
|
|
4109
|
+
}
|
|
4110
|
+
|
|
4111
|
+
const chunkIndex =
|
|
4112
|
+
chunkIdSegment === undefined
|
|
4113
|
+
? undefined
|
|
4114
|
+
: parseChunkIndexSegment(chunkIdSegment)
|
|
4115
|
+
if (chunkIndex === undefined) {
|
|
4116
|
+
continue
|
|
4117
|
+
}
|
|
4118
|
+
|
|
4119
|
+
const key = `${collectionId}/${resourceId}/${chunkIndex}`
|
|
4120
|
+
const slot = staged.get(key) ?? { collectionId, resourceId, chunkIndex }
|
|
4121
|
+
if (representation !== undefined) {
|
|
4122
|
+
slot.contentType = representation.contentType
|
|
4123
|
+
slot.body = representation.body
|
|
4124
|
+
}
|
|
4125
|
+
if (sidecarVersion !== undefined) {
|
|
4126
|
+
slot.version = sidecarVersion
|
|
4127
|
+
}
|
|
4128
|
+
staged.set(key, slot)
|
|
4129
|
+
}
|
|
4130
|
+
|
|
4131
|
+
const parsed: Array<{
|
|
4132
|
+
collectionId: string
|
|
4133
|
+
resourceId: string
|
|
4134
|
+
chunkIndex: number
|
|
4135
|
+
contentType: string
|
|
4136
|
+
body: Buffer
|
|
4137
|
+
version?: number
|
|
4138
|
+
}> = []
|
|
4139
|
+
for (const slot of staged.values()) {
|
|
4140
|
+
if (slot.body === undefined) {
|
|
4141
|
+
// A sidecar with no paired representation is not a valid chunk.
|
|
4142
|
+
continue
|
|
4143
|
+
}
|
|
4144
|
+
parsed.push({
|
|
4145
|
+
collectionId: slot.collectionId,
|
|
4146
|
+
resourceId: slot.resourceId,
|
|
4147
|
+
chunkIndex: slot.chunkIndex,
|
|
4148
|
+
contentType: slot.contentType ?? 'application/octet-stream',
|
|
4149
|
+
body: slot.body,
|
|
4150
|
+
version: slot.version
|
|
4151
|
+
})
|
|
4152
|
+
}
|
|
4153
|
+
return parsed
|
|
4154
|
+
}
|
|
3073
4155
|
}
|