was-teaching-server 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +118 -1
- package/README.md +23 -11
- package/dist/backends/filesystem.d.ts +247 -4
- package/dist/backends/filesystem.d.ts.map +1 -1
- package/dist/backends/filesystem.js +711 -81
- package/dist/backends/filesystem.js.map +1 -1
- package/dist/backends/postgres.d.ts +240 -2
- package/dist/backends/postgres.d.ts.map +1 -1
- package/dist/backends/postgres.js +800 -42
- package/dist/backends/postgres.js.map +1 -1
- package/dist/backends/postgresSchema.d.ts.map +1 -1
- package/dist/backends/postgresSchema.js +69 -57
- package/dist/backends/postgresSchema.js.map +1 -1
- package/dist/errors.d.ts +26 -9
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +40 -12
- package/dist/errors.js.map +1 -1
- package/dist/lib/equalityIndex.d.ts +334 -0
- package/dist/lib/equalityIndex.d.ts.map +1 -0
- package/dist/lib/equalityIndex.js +552 -0
- package/dist/lib/equalityIndex.js.map +1 -0
- package/dist/lib/exportManifest.d.ts +9 -0
- package/dist/lib/exportManifest.d.ts.map +1 -1
- package/dist/lib/exportManifest.js +9 -0
- package/dist/lib/exportManifest.js.map +1 -1
- package/dist/lib/importTar.d.ts +18 -2
- package/dist/lib/importTar.d.ts.map +1 -1
- package/dist/lib/importTar.js +77 -4
- package/dist/lib/importTar.js.map +1 -1
- package/dist/lib/paths.d.ts +33 -0
- package/dist/lib/paths.d.ts.map +1 -1
- package/dist/lib/paths.js +28 -0
- package/dist/lib/paths.js.map +1 -1
- package/dist/lib/resourceFileName.d.ts +43 -0
- package/dist/lib/resourceFileName.d.ts.map +1 -1
- package/dist/lib/resourceFileName.js +63 -0
- package/dist/lib/resourceFileName.js.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +5 -2
- package/dist/plugin.js.map +1 -1
- package/dist/requests/ChunkRequest.d.ts +104 -0
- package/dist/requests/ChunkRequest.d.ts.map +1 -0
- package/dist/requests/ChunkRequest.js +396 -0
- package/dist/requests/ChunkRequest.js.map +1 -0
- package/dist/requests/CollectionRequest.d.ts +22 -5
- package/dist/requests/CollectionRequest.d.ts.map +1 -1
- package/dist/requests/CollectionRequest.js +192 -11
- package/dist/requests/CollectionRequest.js.map +1 -1
- package/dist/requests/ResourceRequest.d.ts.map +1 -1
- package/dist/requests/ResourceRequest.js +20 -0
- package/dist/requests/ResourceRequest.js.map +1 -1
- package/dist/requests/SpaceRequest.d.ts +1 -0
- package/dist/requests/SpaceRequest.d.ts.map +1 -1
- package/dist/requests/SpaceRequest.js +21 -1
- package/dist/requests/SpaceRequest.js.map +1 -1
- package/dist/routes.d.ts.map +1 -1
- package/dist/routes.js +21 -0
- package/dist/routes.js.map +1 -1
- package/dist/types.d.ts +165 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +19 -18
- package/src/backends/filesystem.ts +924 -75
- package/src/backends/postgres.ts +1114 -32
- package/src/backends/postgresSchema.ts +69 -57
- package/src/errors.ts +44 -12
- package/src/lib/equalityIndex.ts +769 -0
- package/src/lib/importTar.ts +107 -4
- package/src/lib/paths.ts +48 -0
- package/src/lib/resourceFileName.ts +69 -0
- package/src/plugin.ts +5 -2
- package/src/requests/ChunkRequest.ts +478 -0
- package/src/requests/CollectionRequest.ts +221 -12
- package/src/requests/ResourceRequest.ts +20 -0
- package/src/requests/SpaceRequest.ts +25 -1
- package/src/routes.ts +46 -0
- package/src/types.ts +175 -1
|
@@ -30,19 +30,21 @@ import * as tar from 'tar-stream';
|
|
|
30
30
|
import YAML from 'yaml';
|
|
31
31
|
import { StorageError, ResourceNotFoundError, SpaceNotFoundError, QuotaExceededError, CountQuotaExceededError, PayloadTooLargeError, PreconditionFailedError, KeystoreStateConflictError, KeyIdConflictError, DuplicateRevocationError } from '../errors.js';
|
|
32
32
|
import { applyMigrations } from './postgresSchema.js';
|
|
33
|
-
import { extractTarEntries, buildImportPlan } from '../lib/importTar.js';
|
|
33
|
+
import { extractTarEntries, buildImportPlan, metaSidecarFileId } from '../lib/importTar.js';
|
|
34
34
|
import { collectionPath, resourcePath } from '../lib/paths.js';
|
|
35
|
-
import { fileNameFor, parseResourceFileName } from '../lib/resourceFileName.js';
|
|
35
|
+
import { fileNameFor, parseResourceFileName, chunkDirName, parseChunkDirName, parseChunkIndexSegment } from '../lib/resourceFileName.js';
|
|
36
|
+
import { assertValidId } from '../lib/validateId.js';
|
|
36
37
|
import { sanitizeBackendRecord } from '../lib/backends.js';
|
|
37
38
|
import { backendUsageFields } from '../lib/backendUsage.js';
|
|
38
39
|
import { assertEncryptedWriteConforms } from '../lib/encryption.js';
|
|
39
40
|
import { encodeCursor, decodeCursor } from '../lib/cursor.js';
|
|
40
|
-
import { buildExportManifest } from '../lib/exportManifest.js';
|
|
41
|
+
import { buildExportManifest, EXPORT_ENTRY_MTIME } from '../lib/exportManifest.js';
|
|
41
42
|
import { revocationFileName } from '../lib/revocations.js';
|
|
42
43
|
import { isJson } from '../lib/isJson.js';
|
|
43
44
|
import { normalizeDescriptionWrite } from '../lib/collectionDescription.js';
|
|
44
45
|
import { DEFAULT_PAGE_SIZE, clampPageSize } from '../lib/pagination.js';
|
|
45
46
|
import { runBlindedIndexQuery, collectUniqueBlindedTerms, assertNoUniqueBlindedConflict } from '../lib/blindedIndex.js';
|
|
47
|
+
import { runEqualityQuery, assertNoUniqueEqualityConflict, findEqualityUniqueViolation } from '../lib/equalityIndex.js';
|
|
46
48
|
import { DEFAULT_MAX_UPLOAD_BYTES, DEFAULT_MAX_SPACES_PER_CONTROLLER, DEFAULT_MAX_COLLECTIONS_PER_SPACE, DEFAULT_MAX_RESOURCES_PER_SPACE, normalizeCountLimit } from '../config.default.js';
|
|
47
49
|
import { assertWritePrecondition, assertMetaWritePrecondition, assertCollectionWritePrecondition } from '../lib/preconditions.js';
|
|
48
50
|
/** Pool sizing and per-connection statement timeout (operational defaults). */
|
|
@@ -277,11 +279,18 @@ export class PostgresBackend {
|
|
|
277
279
|
// `key-epochs`: multi-recipient encrypted Collections -- per-epoch wrapped
|
|
278
280
|
// keys on the `encryption` marker, a client-declared `epoch` stamp on
|
|
279
281
|
// Resources, and conditional (`If-Match`) Collection Description writes.
|
|
282
|
+
// `chunked-streams`: chunk addressing at `/{resourceId}/chunks/{n}`, opaque
|
|
283
|
+
// per-chunk raw-bytes storage in the `chunks` table.
|
|
284
|
+
// `equality-query`: serves the `equality` profile -- server-extracted
|
|
285
|
+
// plaintext attribute equality over a Collection's declared `indexes`
|
|
286
|
+
// (`queryByEquality`), plus the GET `filter[attr]=value` equality filter.
|
|
280
287
|
features: [
|
|
281
288
|
'conditional-writes',
|
|
282
289
|
'changes-query',
|
|
283
290
|
'blinded-index-query',
|
|
284
|
-
'
|
|
291
|
+
'equality-query',
|
|
292
|
+
'key-epochs',
|
|
293
|
+
'chunked-streams'
|
|
285
294
|
]
|
|
286
295
|
};
|
|
287
296
|
}
|
|
@@ -340,6 +349,30 @@ export class PostgresBackend {
|
|
|
340
349
|
});
|
|
341
350
|
}
|
|
342
351
|
}
|
|
352
|
+
/**
|
|
353
|
+
* Serializes concurrent CREATORS of one not-yet-existing row (a Resource or
|
|
354
|
+
* a chunk) on a transaction-scoped advisory lock keyed by the row's
|
|
355
|
+
* identity. Under READ COMMITTED a `SELECT ... FOR UPDATE` on an absent row
|
|
356
|
+
* locks nothing (no gap locks), so two concurrent creators would both read
|
|
357
|
+
* "no prior row" and both apply their full byte size as the usage delta --
|
|
358
|
+
* and while the `ON CONFLICT` arm recomputes `version` from the conflicting
|
|
359
|
+
* row, it cannot retroactively fix the loser's delta, leaving `usage_bytes`
|
|
360
|
+
* inflated by one write's size. The caller takes this lock when its
|
|
361
|
+
* lock-nothing SELECT found no row, then RE-reads the row: the second
|
|
362
|
+
* creator blocks here until the first commits, and its re-read sees the
|
|
363
|
+
* committed row, so its precondition, version, and usage delta are computed
|
|
364
|
+
* from accurate state. Held to commit (advisory xact lock). The `create:`
|
|
365
|
+
* prefix keeps this key domain distinct from the unique-blinded-term
|
|
366
|
+
* advisory lock, which hashes the bare `(spaceId, collectionId)`.
|
|
367
|
+
* @param options {object}
|
|
368
|
+
* @param options.client {pg.PoolClient}
|
|
369
|
+
* @param options.spaceId {string}
|
|
370
|
+
* @param options.rowKey {string} the row's identity within the Space
|
|
371
|
+
* @returns {Promise<void>}
|
|
372
|
+
*/
|
|
373
|
+
async _lockSameKeyCreate({ client, spaceId, rowKey }) {
|
|
374
|
+
await client.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [spaceId, `create:${rowKey}`]);
|
|
375
|
+
}
|
|
343
376
|
// Quotas
|
|
344
377
|
/**
|
|
345
378
|
* Reports the Space's usage from the transactional counter (no measurement
|
|
@@ -356,9 +389,16 @@ export class PostgresBackend {
|
|
|
356
389
|
const usageBytes = rows[0] ? Number(rows[0].usage_bytes) : 0;
|
|
357
390
|
let usageByCollection;
|
|
358
391
|
if (includeCollections) {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
392
|
+
// Per-Collection usage sums both Resource content bytes and chunk bytes
|
|
393
|
+
// (the `chunked-streams` feature) so the breakdown agrees with the
|
|
394
|
+
// Space total in the transactional counter.
|
|
395
|
+
const { rows: collectionRows } = await this._pool.query(`SELECT collection_id, COALESCE(SUM(bytes), 0) AS usage FROM (
|
|
396
|
+
SELECT collection_id, size_bytes AS bytes FROM resources
|
|
397
|
+
WHERE space_id = $1
|
|
398
|
+
UNION ALL
|
|
399
|
+
SELECT collection_id, size AS bytes FROM chunks
|
|
400
|
+
WHERE space_id = $1
|
|
401
|
+
) usage_rows
|
|
362
402
|
GROUP BY collection_id
|
|
363
403
|
ORDER BY collection_id`, [spaceId]);
|
|
364
404
|
usageByCollection = collectionRows.map(row => ({
|
|
@@ -387,6 +427,8 @@ export class PostgresBackend {
|
|
|
387
427
|
(SELECT COALESCE(usage_bytes, 0) FROM spaces WHERE space_id = $1)
|
|
388
428
|
AS space_total,
|
|
389
429
|
(SELECT COALESCE(SUM(size_bytes), 0) FROM resources
|
|
430
|
+
WHERE space_id = $1 AND collection_id = $2)
|
|
431
|
+
+ (SELECT COALESCE(SUM(size), 0) FROM chunks
|
|
390
432
|
WHERE space_id = $1 AND collection_id = $2) AS collection_total`, [spaceId, collectionId]);
|
|
391
433
|
const spaceTotalBytes = Number(rows[0]?.space_total ?? 0);
|
|
392
434
|
const usageBytes = Number(rows[0]?.collection_total ?? 0);
|
|
@@ -698,8 +740,14 @@ export class PostgresBackend {
|
|
|
698
740
|
*/
|
|
699
741
|
async deleteCollection({ spaceId, collectionId }) {
|
|
700
742
|
await this._withTransaction(async (client) => {
|
|
701
|
-
|
|
702
|
-
|
|
743
|
+
// The Collection's freed bytes are its Resource content plus its chunk
|
|
744
|
+
// bytes (the `chunked-streams` feature); both cascade away with the
|
|
745
|
+
// Collection row, so both leave the quota counter.
|
|
746
|
+
const { rows } = await client.query(`SELECT
|
|
747
|
+
(SELECT COALESCE(SUM(size_bytes), 0) FROM resources
|
|
748
|
+
WHERE space_id = $1 AND collection_id = $2)
|
|
749
|
+
+ (SELECT COALESCE(SUM(size), 0) FROM chunks
|
|
750
|
+
WHERE space_id = $1 AND collection_id = $2) AS total`, [spaceId, collectionId]);
|
|
703
751
|
const freedBytes = Number(rows[0]?.total ?? 0);
|
|
704
752
|
await client.query(`DELETE FROM collections WHERE space_id = $1 AND collection_id = $2`, [spaceId, collectionId]);
|
|
705
753
|
// Collection- and Resource-level policies live under the Collection (the
|
|
@@ -816,7 +864,7 @@ export class PostgresBackend {
|
|
|
816
864
|
* @param [options.ifNoneMatch] {boolean}
|
|
817
865
|
* @returns {Promise<{ version: number }>}
|
|
818
866
|
*/
|
|
819
|
-
async writeResource({ spaceId, collectionId, resourceId, input, createdBy, epoch, ifMatch, ifNoneMatch }) {
|
|
867
|
+
async writeResource({ spaceId, collectionId, resourceId, input, createdBy, epoch, uniqueIndexes, ifMatch, ifNoneMatch }) {
|
|
820
868
|
const { maxUploadBytes } = this;
|
|
821
869
|
let content;
|
|
822
870
|
if (input.kind === 'json') {
|
|
@@ -849,18 +897,24 @@ export class PostgresBackend {
|
|
|
849
897
|
}
|
|
850
898
|
return this._withTransaction(async (client) => {
|
|
851
899
|
await this._ensureCollectionRow({ client, spaceId, collectionId });
|
|
852
|
-
//
|
|
853
|
-
//
|
|
854
|
-
//
|
|
855
|
-
//
|
|
856
|
-
//
|
|
900
|
+
// Two unique-attribute invariants can force a JSON content write to
|
|
901
|
+
// serialize before it upserts its row: the EDV blinded one (`unique: true`
|
|
902
|
+
// blinded attributes; the `blinded-index-query` feature) and the plaintext
|
|
903
|
+
// equality one (a Collection's `unique`-declared `indexes`; the
|
|
904
|
+
// `equality-query` feature). Only a unique-carrying JSON write can create
|
|
905
|
+
// either claim. A per-Collection transaction-scoped advisory lock
|
|
857
906
|
// serializes the claimants (held to commit, so the loser's scan sees the
|
|
858
|
-
// winner's committed row) without entering the row-lock ordering of
|
|
859
|
-
//
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
907
|
+
// winner's committed row) without entering the row-lock ordering of plain
|
|
908
|
+
// writes; it is taken once and shared by both checks.
|
|
909
|
+
const blindedUnique = input.kind === 'json' &&
|
|
910
|
+
collectUniqueBlindedTerms({ document: input.data }).length > 0;
|
|
911
|
+
const equalityUnique = input.kind === 'json' &&
|
|
912
|
+
uniqueIndexes !== undefined &&
|
|
913
|
+
uniqueIndexes.length > 0;
|
|
914
|
+
if (blindedUnique || equalityUnique) {
|
|
863
915
|
await client.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [spaceId, collectionId]);
|
|
916
|
+
}
|
|
917
|
+
if (blindedUnique) {
|
|
864
918
|
const { rows: candidateRows } = await client.query(`SELECT resource_id, content FROM resources
|
|
865
919
|
WHERE space_id = $1 AND collection_id = $2 AND is_json
|
|
866
920
|
AND NOT deleted AND resource_id <> $3`, [spaceId, collectionId, resourceId]);
|
|
@@ -879,7 +933,27 @@ export class PostgresBackend {
|
|
|
879
933
|
// skip an unparsable body
|
|
880
934
|
}
|
|
881
935
|
}
|
|
882
|
-
assertNoUniqueBlindedConflict({
|
|
936
|
+
assertNoUniqueBlindedConflict({
|
|
937
|
+
document: input.kind === 'json' ? input.data : undefined,
|
|
938
|
+
candidates
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
if (equalityUnique) {
|
|
942
|
+
// A content write does not change the Resource's `custom`, so the custom
|
|
943
|
+
// side of the claim comes from the CURRENT stored row.
|
|
944
|
+
const { rows: selfRows } = await client.query(`SELECT custom FROM resources
|
|
945
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
946
|
+
AND NOT deleted`, [spaceId, collectionId, resourceId]);
|
|
947
|
+
assertNoUniqueEqualityConflict({
|
|
948
|
+
indexes: uniqueIndexes,
|
|
949
|
+
content: input.kind === 'json' ? input.data : undefined,
|
|
950
|
+
custom: selfRows[0]?.custom ?? undefined,
|
|
951
|
+
candidates: await this._readEqualityCandidates(client, {
|
|
952
|
+
spaceId,
|
|
953
|
+
collectionId,
|
|
954
|
+
excludeResourceId: resourceId
|
|
955
|
+
})
|
|
956
|
+
});
|
|
883
957
|
}
|
|
884
958
|
// Narrow projection: the lock needs the row, not its (possibly multi-MB)
|
|
885
959
|
// `content` bytea, which this path never reads.
|
|
@@ -887,7 +961,23 @@ export class PostgresBackend {
|
|
|
887
961
|
FROM resources
|
|
888
962
|
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
889
963
|
FOR UPDATE`, [spaceId, collectionId, resourceId]);
|
|
890
|
-
|
|
964
|
+
let prior = rows[0];
|
|
965
|
+
if (prior === undefined) {
|
|
966
|
+
// The lock-nothing case: serialize with any concurrent creator of the
|
|
967
|
+
// same id and re-read, so `exists` / `priorSize` below reflect the row
|
|
968
|
+
// it committed (see `_lockSameKeyCreate` -- without this, both
|
|
969
|
+
// creators would count their full byte size as the usage delta).
|
|
970
|
+
await this._lockSameKeyCreate({
|
|
971
|
+
client,
|
|
972
|
+
spaceId,
|
|
973
|
+
rowKey: `${collectionId}/${resourceId}`
|
|
974
|
+
});
|
|
975
|
+
const { rows: reread } = await client.query(`SELECT version, size_bytes, deleted, created_at, created_by
|
|
976
|
+
FROM resources
|
|
977
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
978
|
+
FOR UPDATE`, [spaceId, collectionId, resourceId]);
|
|
979
|
+
prior = reread[0];
|
|
980
|
+
}
|
|
891
981
|
const exists = prior !== undefined && !prior.deleted;
|
|
892
982
|
if (ifMatch !== undefined || ifNoneMatch) {
|
|
893
983
|
assertWritePrecondition({
|
|
@@ -914,7 +1004,7 @@ export class PostgresBackend {
|
|
|
914
1004
|
}
|
|
915
1005
|
const now = new Date().toISOString();
|
|
916
1006
|
const version = (prior?.version ?? 0) + 1;
|
|
917
|
-
const priorSize = exists ? Number(prior
|
|
1007
|
+
const priorSize = exists ? Number(prior?.size_bytes ?? 0) : 0;
|
|
918
1008
|
const delta = content.length - priorSize;
|
|
919
1009
|
if (delta !== 0) {
|
|
920
1010
|
await this._applyUsageDelta({ client, spaceId, delta });
|
|
@@ -924,10 +1014,11 @@ export class PostgresBackend {
|
|
|
924
1014
|
// dropped both (the metadata went with the deleted Resource).
|
|
925
1015
|
//
|
|
926
1016
|
// Create-if-absent atomicity: when `If-None-Match: *` found NO prior row
|
|
927
|
-
// (a tombstone is a real row and stays lock-serialized),
|
|
928
|
-
//
|
|
929
|
-
//
|
|
930
|
-
//
|
|
1017
|
+
// (a tombstone is a real row and stays lock-serialized), concurrent
|
|
1018
|
+
// creators through this method are already serialized by
|
|
1019
|
+
// `_lockSameKeyCreate` above -- but a writer that does not take that
|
|
1020
|
+
// lock (`importSpace`'s plain INSERTs) can still race. A plain INSERT
|
|
1021
|
+
// (no ON CONFLICT) keeps the primary key as the arbiter: the loser's
|
|
931
1022
|
// unique violation maps to the 412 the precondition would have thrown.
|
|
932
1023
|
// `createdBy` names the Resource's creator, not its last writer: taken
|
|
933
1024
|
// from this write's invoker only when there is no prior row at all
|
|
@@ -1062,7 +1153,19 @@ export class PostgresBackend {
|
|
|
1062
1153
|
// no-op, keeping an existing tombstone's change-feed entry stable.
|
|
1063
1154
|
return;
|
|
1064
1155
|
}
|
|
1065
|
-
|
|
1156
|
+
// A soft delete is an UPDATE, not a row removal, so the chunk foreign
|
|
1157
|
+
// key's ON DELETE CASCADE does not fire -- remove the Resource's chunks
|
|
1158
|
+
// (the `chunked-streams` feature) explicitly in this same transaction so
|
|
1159
|
+
// they never outlive their parent, and return their bytes to the quota
|
|
1160
|
+
// counter alongside the Resource's content bytes.
|
|
1161
|
+
const { rows: chunkRows } = await client.query(`SELECT COALESCE(SUM(size), 0) AS total FROM chunks
|
|
1162
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
|
|
1163
|
+
const freedChunkBytes = Number(chunkRows[0]?.total ?? 0);
|
|
1164
|
+
if (freedChunkBytes > 0) {
|
|
1165
|
+
await client.query(`DELETE FROM chunks
|
|
1166
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
|
|
1167
|
+
}
|
|
1168
|
+
const freedBytes = Number(prior.size_bytes) + freedChunkBytes;
|
|
1066
1169
|
if (freedBytes > 0) {
|
|
1067
1170
|
await this._applyUsageDelta({ client, spaceId, delta: -freedBytes });
|
|
1068
1171
|
}
|
|
@@ -1129,8 +1232,17 @@ export class PostgresBackend {
|
|
|
1129
1232
|
* @param [options.ifNoneMatch] {boolean}
|
|
1130
1233
|
* @returns {Promise<{ metaVersion: number } | undefined>}
|
|
1131
1234
|
*/
|
|
1132
|
-
async writeResourceMetadata({ spaceId, collectionId, resourceId, custom, epoch, ifMatch, ifNoneMatch }) {
|
|
1235
|
+
async writeResourceMetadata({ spaceId, collectionId, resourceId, custom, epoch, uniqueIndexes, ifMatch, ifNoneMatch }) {
|
|
1133
1236
|
return this._withTransaction(async (client) => {
|
|
1237
|
+
// A metadata write can create a plaintext equality unique claim for a
|
|
1238
|
+
// `custom`-sourced attribute (the `equality-query` feature). When the
|
|
1239
|
+
// Collection declares any unique index, take the per-Collection advisory
|
|
1240
|
+
// lock first (held to commit, serializing concurrent claimants) so the
|
|
1241
|
+
// conflict scan below is atomic with the write.
|
|
1242
|
+
const equalityUnique = uniqueIndexes !== undefined && uniqueIndexes.length > 0;
|
|
1243
|
+
if (equalityUnique) {
|
|
1244
|
+
await client.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [spaceId, collectionId]);
|
|
1245
|
+
}
|
|
1134
1246
|
const { rows } = await client.query(`SELECT meta_version, deleted FROM resources
|
|
1135
1247
|
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1136
1248
|
FOR UPDATE`, [spaceId, collectionId, resourceId]);
|
|
@@ -1144,6 +1256,34 @@ export class PostgresBackend {
|
|
|
1144
1256
|
ifMatch,
|
|
1145
1257
|
ifNoneMatch
|
|
1146
1258
|
});
|
|
1259
|
+
if (equalityUnique) {
|
|
1260
|
+
// Content is the Resource's stored JSON content (unchanged by a
|
|
1261
|
+
// metadata write); custom is the incoming value this write sets. Fetch
|
|
1262
|
+
// the (possibly multi-MB) content only on this uniqueness path -- the
|
|
1263
|
+
// row is already `FOR UPDATE`-locked above.
|
|
1264
|
+
const { rows: selfRows } = await client.query(`SELECT content, is_json FROM resources
|
|
1265
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
|
|
1266
|
+
let content;
|
|
1267
|
+
const selfRow = selfRows[0];
|
|
1268
|
+
if (selfRow?.is_json && selfRow.content) {
|
|
1269
|
+
try {
|
|
1270
|
+
content = JSON.parse(selfRow.content.toString('utf8'));
|
|
1271
|
+
}
|
|
1272
|
+
catch {
|
|
1273
|
+
content = undefined;
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
assertNoUniqueEqualityConflict({
|
|
1277
|
+
indexes: uniqueIndexes,
|
|
1278
|
+
content,
|
|
1279
|
+
custom,
|
|
1280
|
+
candidates: await this._readEqualityCandidates(client, {
|
|
1281
|
+
spaceId,
|
|
1282
|
+
collectionId,
|
|
1283
|
+
excludeResourceId: resourceId
|
|
1284
|
+
})
|
|
1285
|
+
});
|
|
1286
|
+
}
|
|
1147
1287
|
const metaVersion = (prior.meta_version ?? 0) + 1;
|
|
1148
1288
|
const hasCustom = Object.keys(custom).length > 0;
|
|
1149
1289
|
const now = new Date().toISOString();
|
|
@@ -1168,6 +1308,276 @@ export class PostgresBackend {
|
|
|
1168
1308
|
return { metaVersion };
|
|
1169
1309
|
});
|
|
1170
1310
|
}
|
|
1311
|
+
// Chunks (the `chunked-streams` feature)
|
|
1312
|
+
/**
|
|
1313
|
+
* Writes one chunk of a chunked Resource as one transaction: the parent
|
|
1314
|
+
* Resource must exist (checked atomically -- a `FOR SHARE` lock on it also
|
|
1315
|
+
* blocks a concurrent delete of the parent for the duration of the write, so
|
|
1316
|
+
* a chunk can never be orphaned by a racing `deleteResource`), the chunk row
|
|
1317
|
+
* is locked, its precondition evaluated, its monotonic `version` bumped, and
|
|
1318
|
+
* the transactional quota delta applied. The chunk body is stored opaquely as
|
|
1319
|
+
* a single `bytea`, the buffered-blob path (bounded by `maxUploadBytes`),
|
|
1320
|
+
* exactly like a binary Resource representation.
|
|
1321
|
+
* @param options {object}
|
|
1322
|
+
* @param options.spaceId {string}
|
|
1323
|
+
* @param options.collectionId {string}
|
|
1324
|
+
* @param options.resourceId {string}
|
|
1325
|
+
* @param options.chunkIndex {number} a non-negative safe integer
|
|
1326
|
+
* @param options.input {ResourceInput}
|
|
1327
|
+
* @param [options.ifMatch] {string}
|
|
1328
|
+
* @param [options.ifNoneMatch] {boolean}
|
|
1329
|
+
* @returns {Promise<{ version: number }>} the chunk's new version
|
|
1330
|
+
*/
|
|
1331
|
+
async writeChunk({ spaceId, collectionId, resourceId, chunkIndex, input, ifMatch, ifNoneMatch }) {
|
|
1332
|
+
const { maxUploadBytes } = this;
|
|
1333
|
+
let bytes;
|
|
1334
|
+
if (input.kind === 'json') {
|
|
1335
|
+
bytes = Buffer.from(JSON.stringify(input.data));
|
|
1336
|
+
if (bytes.length > maxUploadBytes) {
|
|
1337
|
+
throw new PayloadTooLargeError({
|
|
1338
|
+
maxUploadBytes,
|
|
1339
|
+
backendId: this.describe().id,
|
|
1340
|
+
uploadBytes: bytes.length
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
else {
|
|
1345
|
+
// Pre-flight a declared size, then buffer through the counting guard that
|
|
1346
|
+
// hard-caps a body whose size is omitted or understated. Buffering
|
|
1347
|
+
// happens BEFORE the transaction so a slow upload holds no row lock.
|
|
1348
|
+
if (input.declaredBytes !== undefined &&
|
|
1349
|
+
input.declaredBytes > maxUploadBytes) {
|
|
1350
|
+
throw new PayloadTooLargeError({
|
|
1351
|
+
maxUploadBytes,
|
|
1352
|
+
backendId: this.describe().id,
|
|
1353
|
+
uploadBytes: input.declaredBytes
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
1356
|
+
bytes = await bufferStreamCapped({
|
|
1357
|
+
stream: input.stream,
|
|
1358
|
+
maxUploadBytes,
|
|
1359
|
+
backendId: this.describe().id
|
|
1360
|
+
});
|
|
1361
|
+
}
|
|
1362
|
+
return this._withTransaction(async (client) => {
|
|
1363
|
+
// Parent Resource must exist (and not be a tombstone). `FOR SHARE`
|
|
1364
|
+
// conflicts with the `FOR UPDATE` a concurrent `deleteResource` takes, so
|
|
1365
|
+
// the two serialize on the parent row -- the parent cannot be deleted
|
|
1366
|
+
// between this check and the chunk write.
|
|
1367
|
+
const { rows: parentRows } = await client.query(`SELECT deleted FROM resources
|
|
1368
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1369
|
+
FOR SHARE`, [spaceId, collectionId, resourceId]);
|
|
1370
|
+
const parent = parentRows[0];
|
|
1371
|
+
if (!parent || parent.deleted) {
|
|
1372
|
+
throw new ResourceNotFoundError({ requestName: 'Write Chunk' });
|
|
1373
|
+
}
|
|
1374
|
+
// Lock the chunk row (if any) and read its current version/size, so the
|
|
1375
|
+
// precondition, the monotonic bump, and the usage delta are all atomic
|
|
1376
|
+
// with the write.
|
|
1377
|
+
const { rows } = await client.query(`SELECT version, size FROM chunks
|
|
1378
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1379
|
+
AND chunk_index = $4
|
|
1380
|
+
FOR UPDATE`, [spaceId, collectionId, resourceId, chunkIndex]);
|
|
1381
|
+
let prior = rows[0];
|
|
1382
|
+
const chunkLabel = `${resourceId}/chunks/${chunkIndex}`;
|
|
1383
|
+
if (prior === undefined) {
|
|
1384
|
+
// The lock-nothing case: serialize with any concurrent creator of the
|
|
1385
|
+
// same chunk and re-read, so the usage delta below reflects the row it
|
|
1386
|
+
// committed (see `_lockSameKeyCreate`).
|
|
1387
|
+
await this._lockSameKeyCreate({
|
|
1388
|
+
client,
|
|
1389
|
+
spaceId,
|
|
1390
|
+
rowKey: `${collectionId}/${chunkLabel}`
|
|
1391
|
+
});
|
|
1392
|
+
const { rows: reread } = await client.query(`SELECT version, size FROM chunks
|
|
1393
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1394
|
+
AND chunk_index = $4
|
|
1395
|
+
FOR UPDATE`, [spaceId, collectionId, resourceId, chunkIndex]);
|
|
1396
|
+
prior = reread[0];
|
|
1397
|
+
}
|
|
1398
|
+
const exists = prior !== undefined;
|
|
1399
|
+
if (ifMatch !== undefined || ifNoneMatch) {
|
|
1400
|
+
assertWritePrecondition({
|
|
1401
|
+
resourceId: chunkLabel,
|
|
1402
|
+
exists,
|
|
1403
|
+
currentVersion: prior?.version ?? 0,
|
|
1404
|
+
ifMatch,
|
|
1405
|
+
ifNoneMatch
|
|
1406
|
+
});
|
|
1407
|
+
}
|
|
1408
|
+
const version = (prior?.version ?? 0) + 1;
|
|
1409
|
+
const priorSize = exists ? Number(prior?.size ?? 0) : 0;
|
|
1410
|
+
const delta = bytes.length - priorSize;
|
|
1411
|
+
if (delta !== 0) {
|
|
1412
|
+
await this._applyUsageDelta({ client, spaceId, delta });
|
|
1413
|
+
}
|
|
1414
|
+
const values = [
|
|
1415
|
+
spaceId,
|
|
1416
|
+
collectionId,
|
|
1417
|
+
resourceId,
|
|
1418
|
+
chunkIndex,
|
|
1419
|
+
input.contentType,
|
|
1420
|
+
bytes,
|
|
1421
|
+
bytes.length,
|
|
1422
|
+
version
|
|
1423
|
+
];
|
|
1424
|
+
const insertSql = `
|
|
1425
|
+
INSERT INTO chunks (
|
|
1426
|
+
space_id, collection_id, resource_id, chunk_index,
|
|
1427
|
+
content_type, bytes, size, version
|
|
1428
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`;
|
|
1429
|
+
// Create-if-absent atomicity mirrors `writeResource`: concurrent
|
|
1430
|
+
// creators through this method are serialized by `_lockSameKeyCreate`
|
|
1431
|
+
// above, and against a writer that does not take that lock
|
|
1432
|
+
// (`importSpace`'s plain INSERTs) the plain INSERT keeps the primary key
|
|
1433
|
+
// as the arbiter -- a racing creator's unique violation maps to the 412
|
|
1434
|
+
// the precondition would have thrown.
|
|
1435
|
+
if (ifNoneMatch && prior === undefined) {
|
|
1436
|
+
try {
|
|
1437
|
+
await client.query(insertSql, values);
|
|
1438
|
+
}
|
|
1439
|
+
catch (err) {
|
|
1440
|
+
if (err.code === '23505') {
|
|
1441
|
+
throw new PreconditionFailedError({
|
|
1442
|
+
detail: `Chunk '${chunkLabel}' already exists (If-None-Match: *).`
|
|
1443
|
+
});
|
|
1444
|
+
}
|
|
1445
|
+
throw err;
|
|
1446
|
+
}
|
|
1447
|
+
return { version };
|
|
1448
|
+
}
|
|
1449
|
+
// The conflict update derives `version` from the row (`chunks.version +
|
|
1450
|
+
// 1`), not from the pre-read, so a concurrent creator that slipped in
|
|
1451
|
+
// after our lock-nothing SELECT still advances the counter monotonically.
|
|
1452
|
+
// RETURNING reports the version that actually landed.
|
|
1453
|
+
const { rows: written } = await client.query(`${insertSql}
|
|
1454
|
+
ON CONFLICT (space_id, collection_id, resource_id, chunk_index)
|
|
1455
|
+
DO UPDATE SET
|
|
1456
|
+
content_type = EXCLUDED.content_type,
|
|
1457
|
+
bytes = EXCLUDED.bytes,
|
|
1458
|
+
size = EXCLUDED.size,
|
|
1459
|
+
version = chunks.version + 1
|
|
1460
|
+
RETURNING version`, values);
|
|
1461
|
+
return { version: written[0].version };
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
/**
|
|
1465
|
+
* Reads a chunk's bytes. Rejects with `ResourceNotFoundError` (404) when no
|
|
1466
|
+
* chunk is stored at that index.
|
|
1467
|
+
* @param options {object}
|
|
1468
|
+
* @param options.spaceId {string}
|
|
1469
|
+
* @param options.collectionId {string}
|
|
1470
|
+
* @param options.resourceId {string}
|
|
1471
|
+
* @param options.chunkIndex {number}
|
|
1472
|
+
* @returns {Promise<ResourceResult>}
|
|
1473
|
+
*/
|
|
1474
|
+
async getChunk({ spaceId, collectionId, resourceId, chunkIndex }) {
|
|
1475
|
+
const { rows } = await this._pool.query(`SELECT content_type, bytes, version FROM chunks
|
|
1476
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1477
|
+
AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
|
|
1478
|
+
const row = rows[0];
|
|
1479
|
+
if (!row) {
|
|
1480
|
+
throw new ResourceNotFoundError({ requestName: 'Get Chunk' });
|
|
1481
|
+
}
|
|
1482
|
+
return {
|
|
1483
|
+
resourceStream: Readable.from(row.bytes),
|
|
1484
|
+
storedResourceType: row.content_type,
|
|
1485
|
+
version: row.version
|
|
1486
|
+
};
|
|
1487
|
+
}
|
|
1488
|
+
/**
|
|
1489
|
+
* Reads a chunk's stored content-type / size / version (the HEAD payload
|
|
1490
|
+
* headers). Resolves `undefined` when the chunk is absent.
|
|
1491
|
+
* @param options {object}
|
|
1492
|
+
* @param options.spaceId {string}
|
|
1493
|
+
* @param options.collectionId {string}
|
|
1494
|
+
* @param options.resourceId {string}
|
|
1495
|
+
* @param options.chunkIndex {number}
|
|
1496
|
+
* @returns {Promise<{ contentType: string, size: number, version?: number }
|
|
1497
|
+
* | undefined>}
|
|
1498
|
+
*/
|
|
1499
|
+
async getChunkMetadata({ spaceId, collectionId, resourceId, chunkIndex }) {
|
|
1500
|
+
const { rows } = await this._pool.query(`SELECT content_type, size, version FROM chunks
|
|
1501
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1502
|
+
AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
|
|
1503
|
+
const row = rows[0];
|
|
1504
|
+
if (!row) {
|
|
1505
|
+
return undefined;
|
|
1506
|
+
}
|
|
1507
|
+
return {
|
|
1508
|
+
contentType: row.content_type,
|
|
1509
|
+
size: Number(row.size),
|
|
1510
|
+
version: row.version
|
|
1511
|
+
};
|
|
1512
|
+
}
|
|
1513
|
+
/**
|
|
1514
|
+
* Deletes one chunk as one transaction (the chunk row is locked, its
|
|
1515
|
+
* `ifMatch` precondition evaluated atomically, and its bytes returned to the
|
|
1516
|
+
* quota counter). Resolves `true` when a chunk was removed, `false` when none
|
|
1517
|
+
* was stored at that index (the handler 404s on `false` -- chunk deletes are
|
|
1518
|
+
* not silently idempotent, unlike `deleteResource`).
|
|
1519
|
+
* @param options {object}
|
|
1520
|
+
* @param options.spaceId {string}
|
|
1521
|
+
* @param options.collectionId {string}
|
|
1522
|
+
* @param options.resourceId {string}
|
|
1523
|
+
* @param options.chunkIndex {number}
|
|
1524
|
+
* @param [options.ifMatch] {string}
|
|
1525
|
+
* @returns {Promise<boolean>}
|
|
1526
|
+
*/
|
|
1527
|
+
async deleteChunk({ spaceId, collectionId, resourceId, chunkIndex, ifMatch }) {
|
|
1528
|
+
return this._withTransaction(async (client) => {
|
|
1529
|
+
const { rows } = await client.query(`SELECT version, size FROM chunks
|
|
1530
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1531
|
+
AND chunk_index = $4
|
|
1532
|
+
FOR UPDATE`, [spaceId, collectionId, resourceId, chunkIndex]);
|
|
1533
|
+
const prior = rows[0];
|
|
1534
|
+
if (prior === undefined) {
|
|
1535
|
+
return false;
|
|
1536
|
+
}
|
|
1537
|
+
if (ifMatch !== undefined) {
|
|
1538
|
+
assertWritePrecondition({
|
|
1539
|
+
resourceId: `${resourceId}/chunks/${chunkIndex}`,
|
|
1540
|
+
exists: true,
|
|
1541
|
+
currentVersion: prior.version,
|
|
1542
|
+
ifMatch
|
|
1543
|
+
});
|
|
1544
|
+
}
|
|
1545
|
+
await client.query(`DELETE FROM chunks
|
|
1546
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1547
|
+
AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
|
|
1548
|
+
const freedBytes = Number(prior.size);
|
|
1549
|
+
if (freedBytes > 0) {
|
|
1550
|
+
await this._applyUsageDelta({ client, spaceId, delta: -freedBytes });
|
|
1551
|
+
}
|
|
1552
|
+
return true;
|
|
1553
|
+
});
|
|
1554
|
+
}
|
|
1555
|
+
/**
|
|
1556
|
+
* Lists a Resource's stored chunks in ascending `chunk_index` order -- the
|
|
1557
|
+
* discovery/reassembly listing. The opaque `bytes` column is deliberately not
|
|
1558
|
+
* selected. An empty listing (Resource with no chunks, or an absent Resource)
|
|
1559
|
+
* resolves `{ count: 0, chunks: [] }`.
|
|
1560
|
+
* @param options {object}
|
|
1561
|
+
* @param options.spaceId {string}
|
|
1562
|
+
* @param options.collectionId {string}
|
|
1563
|
+
* @param options.resourceId {string}
|
|
1564
|
+
* @returns {Promise<{ count: number, chunks: Array<{ index: number, size:
|
|
1565
|
+
* number, contentType: string, version?: number }> }>}
|
|
1566
|
+
*/
|
|
1567
|
+
async listChunks({ spaceId, collectionId, resourceId }) {
|
|
1568
|
+
const { rows } = await this._pool.query(`SELECT chunk_index, size, content_type, version FROM chunks
|
|
1569
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
1570
|
+
ORDER BY chunk_index`, [spaceId, collectionId, resourceId]);
|
|
1571
|
+
return {
|
|
1572
|
+
count: rows.length,
|
|
1573
|
+
chunks: rows.map(row => ({
|
|
1574
|
+
index: row.chunk_index,
|
|
1575
|
+
size: Number(row.size),
|
|
1576
|
+
contentType: row.content_type,
|
|
1577
|
+
version: row.version
|
|
1578
|
+
}))
|
|
1579
|
+
};
|
|
1580
|
+
}
|
|
1171
1581
|
/**
|
|
1172
1582
|
* Replication change feed (the `changes` query profile): one indexed keyset
|
|
1173
1583
|
* query over `(updatedAt, resourceId)`, tombstones included, JSON documents
|
|
@@ -1279,6 +1689,100 @@ export class PostgresBackend {
|
|
|
1279
1689
|
}
|
|
1280
1690
|
return runBlindedIndexQuery({ candidates, query, count, limit, cursor });
|
|
1281
1691
|
}
|
|
1692
|
+
/**
|
|
1693
|
+
* Plaintext equality query (the `equality` query profile; see the
|
|
1694
|
+
* `StorageBackend.queryByEquality` contract). Reads the Collection's live
|
|
1695
|
+
* Resources -- JSON rows carrying parsed `content`, all rows carrying their
|
|
1696
|
+
* `custom` jsonb -- and hands the candidates to the shared evaluator
|
|
1697
|
+
* (`lib/equalityIndex.ts`) for extraction, matching, ordering, and cursor
|
|
1698
|
+
* pagination -- identical semantics to the filesystem backend. A full scan of
|
|
1699
|
+
* the Collection per call, deliberate for this teaching backend; a
|
|
1700
|
+
* materialized variant would answer from a JSONB expression index.
|
|
1701
|
+
* @param options {object}
|
|
1702
|
+
* @param options.spaceId {string}
|
|
1703
|
+
* @param options.collectionId {string}
|
|
1704
|
+
* @param options.query {EqualityQuery}
|
|
1705
|
+
* @param options.indexes {NormalizedIndexDeclaration[]}
|
|
1706
|
+
* @param [options.count] {boolean} return only the match count
|
|
1707
|
+
* @param [options.limit] {number} requested page size
|
|
1708
|
+
* @param [options.cursor] {string} opaque cursor from a prior page
|
|
1709
|
+
* @returns {Promise<{ count: number } | EqualityQueryPage>}
|
|
1710
|
+
*/
|
|
1711
|
+
async queryByEquality({ spaceId, collectionId, query, indexes, count, limit, cursor }) {
|
|
1712
|
+
const candidates = await this._readEqualityCandidates(this._pool, {
|
|
1713
|
+
spaceId,
|
|
1714
|
+
collectionId
|
|
1715
|
+
});
|
|
1716
|
+
return runEqualityQuery({
|
|
1717
|
+
candidates,
|
|
1718
|
+
query,
|
|
1719
|
+
indexes,
|
|
1720
|
+
count,
|
|
1721
|
+
limit,
|
|
1722
|
+
cursor
|
|
1723
|
+
});
|
|
1724
|
+
}
|
|
1725
|
+
/**
|
|
1726
|
+
* Declare-time uniqueness scan for the `equality` profile (see the
|
|
1727
|
+
* `StorageBackend.findEqualityUniqueViolation` contract): reads the
|
|
1728
|
+
* Collection's live Resources and delegates to the shared scan, which reports
|
|
1729
|
+
* the first `(name, value)` claimed by two different Resources under the given
|
|
1730
|
+
* `unique` declarations (or `undefined` when none is).
|
|
1731
|
+
* @param options {object}
|
|
1732
|
+
* @param options.spaceId {string}
|
|
1733
|
+
* @param options.collectionId {string}
|
|
1734
|
+
* @param options.indexes {NormalizedIndexDeclaration[]}
|
|
1735
|
+
* @returns {Promise<{ name: string, value: EqualityValue } | undefined>}
|
|
1736
|
+
*/
|
|
1737
|
+
async findEqualityUniqueViolation({ spaceId, collectionId, indexes }) {
|
|
1738
|
+
const candidates = await this._readEqualityCandidates(this._pool, {
|
|
1739
|
+
spaceId,
|
|
1740
|
+
collectionId
|
|
1741
|
+
});
|
|
1742
|
+
return findEqualityUniqueViolation({ indexes, candidates });
|
|
1743
|
+
}
|
|
1744
|
+
/**
|
|
1745
|
+
* Reads every live Resource of a Collection as an equality candidate -- the
|
|
1746
|
+
* candidate set for the equality query and the plaintext unique-attribute
|
|
1747
|
+
* conflict scans. Includes blob Resources (queryable through their
|
|
1748
|
+
* `custom`-sourced attributes): each row resolves `{ resourceId, content?,
|
|
1749
|
+
* custom? }`, where `content` is the parsed JSON of a JSON row (a blob and
|
|
1750
|
+
* unparsable JSON contribute none) and `custom` is the row's jsonb `custom`
|
|
1751
|
+
* when set. Tombstones are excluded (`NOT deleted`); an optional excluded
|
|
1752
|
+
* Resource is skipped. Runs on the given executor -- the pool for a read-only
|
|
1753
|
+
* query, or the write transaction's client for a uniqueness scan (so the scan
|
|
1754
|
+
* shares the advisory lock and sees a consistent snapshot).
|
|
1755
|
+
* @param executor {pg.Pool | pg.PoolClient}
|
|
1756
|
+
* @param options {object}
|
|
1757
|
+
* @param options.spaceId {string}
|
|
1758
|
+
* @param options.collectionId {string}
|
|
1759
|
+
* @param [options.excludeResourceId] {string}
|
|
1760
|
+
* @returns {Promise<EqualityCandidate[]>}
|
|
1761
|
+
*/
|
|
1762
|
+
async _readEqualityCandidates(executor, { spaceId, collectionId, excludeResourceId }) {
|
|
1763
|
+
const { rows } = await executor.query(`SELECT resource_id, content, is_json, custom FROM resources
|
|
1764
|
+
WHERE space_id = $1 AND collection_id = $2 AND NOT deleted
|
|
1765
|
+
AND ($3::text IS NULL OR resource_id <> $3)
|
|
1766
|
+
ORDER BY resource_id`, [spaceId, collectionId, excludeResourceId ?? null]);
|
|
1767
|
+
const candidates = [];
|
|
1768
|
+
for (const row of rows) {
|
|
1769
|
+
let content;
|
|
1770
|
+
if (row.is_json && row.content) {
|
|
1771
|
+
try {
|
|
1772
|
+
content = JSON.parse(row.content.toString('utf8'));
|
|
1773
|
+
}
|
|
1774
|
+
catch {
|
|
1775
|
+
// skip an unparsable body -- it contributes no content attributes
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
candidates.push({
|
|
1779
|
+
resourceId: row.resource_id,
|
|
1780
|
+
...(content !== undefined && { content }),
|
|
1781
|
+
...(row.custom !== null && { custom: row.custom })
|
|
1782
|
+
});
|
|
1783
|
+
}
|
|
1784
|
+
return candidates;
|
|
1785
|
+
}
|
|
1282
1786
|
// Policies
|
|
1283
1787
|
/**
|
|
1284
1788
|
* Maps the optional-id policy addressing onto the sentinel-column primary
|
|
@@ -1678,7 +2182,7 @@ export class PostgresBackend {
|
|
|
1678
2182
|
if (!spaceDescription) {
|
|
1679
2183
|
throw new SpaceNotFoundError({ requestName: 'Export Space' });
|
|
1680
2184
|
}
|
|
1681
|
-
const [{ rows: policyRows }, { rows: collectionRows }, { rows: resourceRows }, { rows: revocationRows }] = await Promise.all([
|
|
2185
|
+
const [{ rows: policyRows }, { rows: collectionRows }, { rows: resourceRows }, { rows: revocationRows }, { rows: chunkRows }] = await Promise.all([
|
|
1682
2186
|
this._pool.query(`SELECT collection_id, resource_id, policy FROM policies
|
|
1683
2187
|
WHERE space_id = $1`, [spaceId]),
|
|
1684
2188
|
this._pool.query(`SELECT collection_id, description, description_version FROM collections
|
|
@@ -1690,7 +2194,12 @@ export class PostgresBackend {
|
|
|
1690
2194
|
created_at, updated_at, created_by
|
|
1691
2195
|
FROM resources WHERE space_id = $1`, [spaceId]),
|
|
1692
2196
|
this._pool.query(`SELECT delegator, capability_id, record FROM space_revocations
|
|
1693
|
-
WHERE space_id = $1`, [spaceId])
|
|
2197
|
+
WHERE space_id = $1`, [spaceId]),
|
|
2198
|
+
// Chunk metadata only -- bytes are fetched one chunk at a time while
|
|
2199
|
+
// packing, so an export never holds a chunked Resource whole in memory.
|
|
2200
|
+
this._pool.query(`SELECT collection_id, resource_id, chunk_index, content_type, version
|
|
2201
|
+
FROM chunks WHERE space_id = $1
|
|
2202
|
+
ORDER BY collection_id, resource_id, chunk_index`, [spaceId])
|
|
1694
2203
|
]);
|
|
1695
2204
|
// Assemble the per-entry file lists in the filesystem's shapes: files are
|
|
1696
2205
|
// named by the shared codecs and sorted with localeCompare, matching the
|
|
@@ -1765,6 +2274,46 @@ export class PostgresBackend {
|
|
|
1765
2274
|
});
|
|
1766
2275
|
}
|
|
1767
2276
|
}
|
|
2277
|
+
// Chunks (the `chunked-streams` feature): each chunked Resource contributes
|
|
2278
|
+
// one `.chunks.<encResourceId>/` subdirectory in its Collection dir, in the
|
|
2279
|
+
// exact filesystem-backend layout so an archive imports into either backend.
|
|
2280
|
+
// A chunk is stored there as a Resource keyed by its stringified index: an
|
|
2281
|
+
// `r.<index>.<encContentType>.<ext>` representation file (`fileNameFor`)
|
|
2282
|
+
// plus a `.meta.<index>.json` version sidecar. Files within a chunk dir are
|
|
2283
|
+
// sorted by name (the filesystem's readdir sort). Rows arrive ordered by
|
|
2284
|
+
// `(collection, resource, index)`.
|
|
2285
|
+
const chunkDirsByResource = new Map();
|
|
2286
|
+
for (const row of chunkRows) {
|
|
2287
|
+
const dirKey = `${row.collection_id}/${row.resource_id}`;
|
|
2288
|
+
let chunkFiles = chunkDirsByResource.get(dirKey);
|
|
2289
|
+
if (!chunkFiles) {
|
|
2290
|
+
chunkFiles = [];
|
|
2291
|
+
chunkDirsByResource.set(dirKey, chunkFiles);
|
|
2292
|
+
filesFor(row.collection_id).push({
|
|
2293
|
+
name: chunkDirName(row.resource_id),
|
|
2294
|
+
chunkDir: chunkFiles
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
const chunkId = String(row.chunk_index);
|
|
2298
|
+
chunkFiles.push({
|
|
2299
|
+
name: fileNameFor({
|
|
2300
|
+
resourceId: chunkId,
|
|
2301
|
+
contentType: row.content_type
|
|
2302
|
+
}),
|
|
2303
|
+
chunk: {
|
|
2304
|
+
collectionId: row.collection_id,
|
|
2305
|
+
resourceId: row.resource_id,
|
|
2306
|
+
chunkIndex: row.chunk_index
|
|
2307
|
+
}
|
|
2308
|
+
});
|
|
2309
|
+
chunkFiles.push({
|
|
2310
|
+
name: `.meta.${chunkId}.json`,
|
|
2311
|
+
bytes: Buffer.from(JSON.stringify({ version: row.version }))
|
|
2312
|
+
});
|
|
2313
|
+
}
|
|
2314
|
+
for (const chunkFiles of chunkDirsByResource.values()) {
|
|
2315
|
+
chunkFiles.sort((left, right) => left.name.localeCompare(right.name));
|
|
2316
|
+
}
|
|
1768
2317
|
// Top-level order: space-level files and collection dirs interleaved,
|
|
1769
2318
|
// sorted by name -- the same order the filesystem's readdir+sort yields.
|
|
1770
2319
|
const topLevel = [
|
|
@@ -1791,33 +2340,59 @@ export class PostgresBackend {
|
|
|
1791
2340
|
const manifest = buildExportManifest({
|
|
1792
2341
|
spaceId,
|
|
1793
2342
|
entries: topLevel.map(entry => entry.kind === 'collection'
|
|
1794
|
-
? {
|
|
2343
|
+
? {
|
|
2344
|
+
name: entry.name,
|
|
2345
|
+
// A chunk directory expands to its `.chunks.<encId>/<file>`
|
|
2346
|
+
// relative paths (mirroring the pack order), so the manifest
|
|
2347
|
+
// matches the filesystem backend's for the same Space.
|
|
2348
|
+
files: entry.files.flatMap(file => 'chunkDir' in file
|
|
2349
|
+
? file.chunkDir.map(child => `${file.name}/${child.name}`)
|
|
2350
|
+
: [file.name])
|
|
2351
|
+
}
|
|
1795
2352
|
: { name: entry.name }),
|
|
1796
2353
|
revocationFiles: revocationFiles.map(file => file.name)
|
|
1797
2354
|
});
|
|
2355
|
+
// Fixed mtime on every entry so the archive is byte-reproducible (see
|
|
2356
|
+
// EXPORT_ENTRY_MTIME).
|
|
2357
|
+
const mtime = EXPORT_ENTRY_MTIME;
|
|
1798
2358
|
const pack = tar.pack();
|
|
1799
|
-
pack.entry({ name: 'manifest.yml' }, YAML.stringify(manifest));
|
|
1800
|
-
pack.entry({ name: 'space/', type: 'directory' });
|
|
1801
|
-
pack.entry({ name: `space/${spaceId}/`, type: 'directory' });
|
|
2359
|
+
pack.entry({ name: 'manifest.yml', mtime }, YAML.stringify(manifest));
|
|
2360
|
+
pack.entry({ name: 'space/', type: 'directory', mtime });
|
|
2361
|
+
pack.entry({ name: `space/${spaceId}/`, type: 'directory', mtime });
|
|
1802
2362
|
for (const entry of topLevel) {
|
|
1803
2363
|
const entryTarget = `space/${spaceId}/${entry.name}`;
|
|
1804
2364
|
if (entry.kind === 'collection') {
|
|
1805
|
-
pack.entry({ name: `${entryTarget}/`, type: 'directory' });
|
|
2365
|
+
pack.entry({ name: `${entryTarget}/`, type: 'directory', mtime });
|
|
1806
2366
|
for (const file of entry.files) {
|
|
2367
|
+
if ('chunkDir' in file) {
|
|
2368
|
+
// A chunked Resource's `.chunks.<encId>/` subdirectory: emit the
|
|
2369
|
+
// directory entry, then each file -- an inline `.meta.<index>.json`
|
|
2370
|
+
// sidecar, or a chunk representation whose bytes are fetched one at
|
|
2371
|
+
// a time.
|
|
2372
|
+
const dirTarget = `${entryTarget}/${file.name}`;
|
|
2373
|
+
pack.entry({ name: `${dirTarget}/`, type: 'directory', mtime });
|
|
2374
|
+
for (const chunkFile of file.chunkDir) {
|
|
2375
|
+
const bytes = 'bytes' in chunkFile
|
|
2376
|
+
? chunkFile.bytes
|
|
2377
|
+
: await this._chunkContent({ spaceId, ...chunkFile.chunk });
|
|
2378
|
+
pack.entry({ name: `${dirTarget}/${chunkFile.name}`, mtime }, bytes);
|
|
2379
|
+
}
|
|
2380
|
+
continue;
|
|
2381
|
+
}
|
|
1807
2382
|
const bytes = 'bytes' in file
|
|
1808
2383
|
? file.bytes
|
|
1809
2384
|
: await this._resourceContent({ spaceId, ...file.resource });
|
|
1810
|
-
pack.entry({ name: `${entryTarget}/${file.name}
|
|
2385
|
+
pack.entry({ name: `${entryTarget}/${file.name}`, mtime }, bytes);
|
|
1811
2386
|
}
|
|
1812
2387
|
}
|
|
1813
2388
|
else {
|
|
1814
|
-
pack.entry({ name: entryTarget }, entry.bytes);
|
|
2389
|
+
pack.entry({ name: entryTarget, mtime }, entry.bytes);
|
|
1815
2390
|
}
|
|
1816
2391
|
}
|
|
1817
2392
|
if (revocationFiles.length > 0) {
|
|
1818
|
-
pack.entry({ name: 'revocations/', type: 'directory' });
|
|
2393
|
+
pack.entry({ name: 'revocations/', type: 'directory', mtime });
|
|
1819
2394
|
for (const file of revocationFiles) {
|
|
1820
|
-
pack.entry({ name: `revocations/${file.name}
|
|
2395
|
+
pack.entry({ name: `revocations/${file.name}`, mtime }, file.bytes);
|
|
1821
2396
|
}
|
|
1822
2397
|
}
|
|
1823
2398
|
pack.finalize();
|
|
@@ -1839,6 +2414,24 @@ export class PostgresBackend {
|
|
|
1839
2414
|
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, collectionId, resourceId]);
|
|
1840
2415
|
return rows[0]?.content ?? Buffer.alloc(0);
|
|
1841
2416
|
}
|
|
2417
|
+
/**
|
|
2418
|
+
* Fetches one chunk's bytes for the export pack loop (the `chunked-streams`
|
|
2419
|
+
* feature). A chunk removed between the metadata pass and this read yields an
|
|
2420
|
+
* empty body rather than failing the whole archive, matching
|
|
2421
|
+
* `_resourceContent`.
|
|
2422
|
+
* @param options {object}
|
|
2423
|
+
* @param options.spaceId {string}
|
|
2424
|
+
* @param options.collectionId {string}
|
|
2425
|
+
* @param options.resourceId {string}
|
|
2426
|
+
* @param options.chunkIndex {number}
|
|
2427
|
+
* @returns {Promise<Buffer>}
|
|
2428
|
+
*/
|
|
2429
|
+
async _chunkContent({ spaceId, collectionId, resourceId, chunkIndex }) {
|
|
2430
|
+
const { rows } = await this._pool.query(`SELECT bytes FROM chunks
|
|
2431
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2432
|
+
AND chunk_index = $4`, [spaceId, collectionId, resourceId, chunkIndex]);
|
|
2433
|
+
return rows[0]?.bytes ?? Buffer.alloc(0);
|
|
2434
|
+
}
|
|
1842
2435
|
/**
|
|
1843
2436
|
* Merges a WAS space-export tarball into an existing Space with the same
|
|
1844
2437
|
* three-invariant pre-flight (per-entry 413, fail-closed 422 encryption
|
|
@@ -1855,6 +2448,10 @@ export class PostgresBackend {
|
|
|
1855
2448
|
async importSpace({ spaceId, tarStream }) {
|
|
1856
2449
|
const entries = await extractTarEntries(tarStream);
|
|
1857
2450
|
const { spacePolicy, collections, revocations } = buildImportPlan(entries);
|
|
2451
|
+
// Chunk entries (the `chunked-streams` feature) live in per-Resource
|
|
2452
|
+
// `.chunks.<encId>/` subdirectories, which `buildImportPlan` skips (it
|
|
2453
|
+
// ignores nested files); parse them straight off the raw tar entries.
|
|
2454
|
+
const chunkEntries = this._chunkEntriesFromArchive(entries);
|
|
1858
2455
|
const { capacityBytes, maxUploadBytes, maxCollectionsPerSpace, maxResourcesPerSpace } = this;
|
|
1859
2456
|
return this._withTransaction(async (client) => {
|
|
1860
2457
|
await this._ensureSpaceRow({ client, spaceId });
|
|
@@ -1918,6 +2515,19 @@ export class PostgresBackend {
|
|
|
1918
2515
|
incomingBytes += body.length;
|
|
1919
2516
|
}
|
|
1920
2517
|
}
|
|
2518
|
+
// Chunks (the `chunked-streams` feature) are opaque bytes -- no
|
|
2519
|
+
// encryption-conformance check applies -- but they count the same per-body
|
|
2520
|
+
// 413 and the (conservative) capacity pre-flight as Resource bodies.
|
|
2521
|
+
for (const chunk of chunkEntries) {
|
|
2522
|
+
if (chunk.body.length > maxUploadBytes) {
|
|
2523
|
+
throw new PayloadTooLargeError({
|
|
2524
|
+
maxUploadBytes,
|
|
2525
|
+
backendId: this.describe().id,
|
|
2526
|
+
uploadBytes: chunk.body.length
|
|
2527
|
+
});
|
|
2528
|
+
}
|
|
2529
|
+
incomingBytes += chunk.body.length;
|
|
2530
|
+
}
|
|
1921
2531
|
if (capacityBytes !== undefined &&
|
|
1922
2532
|
currentUsage + incomingBytes > capacityBytes) {
|
|
1923
2533
|
throw new QuotaExceededError({ spaceId, capacityBytes });
|
|
@@ -2081,13 +2691,53 @@ export class PostgresBackend {
|
|
|
2081
2691
|
stats.resourcesCreated++;
|
|
2082
2692
|
}
|
|
2083
2693
|
}
|
|
2084
|
-
|
|
2694
|
+
// Chunks (the `chunked-streams` feature): restore each archived chunk
|
|
2695
|
+
// skip-not-overwrite, after the Resource apply loop so a chunk's parent
|
|
2696
|
+
// Resource row already exists in this transaction (the foreign key
|
|
2697
|
+
// requires it). An orphan chunk -- one whose parent is absent or a
|
|
2698
|
+
// tombstone -- is skipped rather than resurrected. Existing chunk rows are
|
|
2699
|
+
// left untouched.
|
|
2700
|
+
let createdChunkBytes = 0;
|
|
2701
|
+
for (const chunk of chunkEntries) {
|
|
2702
|
+
const { rows: parentRows } = await client.query(`SELECT deleted FROM resources
|
|
2703
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3`, [spaceId, chunk.collectionId, chunk.resourceId]);
|
|
2704
|
+
const parent = parentRows[0];
|
|
2705
|
+
if (!parent || parent.deleted) {
|
|
2706
|
+
continue;
|
|
2707
|
+
}
|
|
2708
|
+
const { rows: existingChunkRows } = await client.query(`SELECT 1 FROM chunks
|
|
2709
|
+
WHERE space_id = $1 AND collection_id = $2 AND resource_id = $3
|
|
2710
|
+
AND chunk_index = $4`, [spaceId, chunk.collectionId, chunk.resourceId, chunk.chunkIndex]);
|
|
2711
|
+
if (existingChunkRows.length > 0) {
|
|
2712
|
+
continue;
|
|
2713
|
+
}
|
|
2714
|
+
// The chunk's `version` comes from its archived `.meta.<index>.json`
|
|
2715
|
+
// sidecar; an archive without one (or a chunk written before sidecars)
|
|
2716
|
+
// starts at version 1, the same fresh-write default as a Resource
|
|
2717
|
+
// restored without a sidecar.
|
|
2718
|
+
await client.query(`INSERT INTO chunks (
|
|
2719
|
+
space_id, collection_id, resource_id, chunk_index,
|
|
2720
|
+
content_type, bytes, size, version
|
|
2721
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
|
|
2722
|
+
spaceId,
|
|
2723
|
+
chunk.collectionId,
|
|
2724
|
+
chunk.resourceId,
|
|
2725
|
+
chunk.chunkIndex,
|
|
2726
|
+
chunk.contentType,
|
|
2727
|
+
chunk.body,
|
|
2728
|
+
chunk.body.length,
|
|
2729
|
+
chunk.version ?? 1
|
|
2730
|
+
]);
|
|
2731
|
+
createdChunkBytes += chunk.body.length;
|
|
2732
|
+
}
|
|
2733
|
+
const createdTotalBytes = createdBytes + createdChunkBytes;
|
|
2734
|
+
if (createdTotalBytes > 0) {
|
|
2085
2735
|
// The pre-flight was conservative (it counted skips too), so the
|
|
2086
2736
|
// actual created total always fits; apply it unguarded.
|
|
2087
2737
|
await this._applyUsageDelta({
|
|
2088
2738
|
client,
|
|
2089
2739
|
spaceId,
|
|
2090
|
-
delta:
|
|
2740
|
+
delta: createdTotalBytes
|
|
2091
2741
|
});
|
|
2092
2742
|
}
|
|
2093
2743
|
// Restore the archive's Space-scoped zcap revocations under this
|
|
@@ -2160,5 +2810,113 @@ export class PostgresBackend {
|
|
|
2160
2810
|
sidecar?.epoch ?? null
|
|
2161
2811
|
]);
|
|
2162
2812
|
}
|
|
2813
|
+
/**
|
|
2814
|
+
* Parses a Space-export archive's chunk entries (the `chunked-streams`
|
|
2815
|
+
* feature) out of the raw tar entry map, in the filesystem backend's on-disk
|
|
2816
|
+
* layout so archives round-trip between the two backends. Chunk files live at
|
|
2817
|
+
* `space/<sourceSpaceId>/<collectionId>/.chunks.<encResourceId>/<file>` --
|
|
2818
|
+
* where a chunk is a Resource keyed by its stringified index: an
|
|
2819
|
+
* `r.<index>.<encContentType>.<ext>` representation paired with an optional
|
|
2820
|
+
* `.meta.<index>.json` version sidecar. `buildImportPlan` skips these (it
|
|
2821
|
+
* ignores nested Collection-dir files), so they are parsed here. Ids parsed
|
|
2822
|
+
* from the archive are validated (the path-traversal guard); a file whose
|
|
2823
|
+
* raw index segment is not canonical (`parseChunkIndexSegment`) is ignored,
|
|
2824
|
+
* and a sidecar with no paired representation is dropped (a chunk keeps no
|
|
2825
|
+
* tombstone).
|
|
2826
|
+
* @param entries {Map<string, TarEntry>}
|
|
2827
|
+
* @returns {Array<{ collectionId: string, resourceId: string, chunkIndex:
|
|
2828
|
+
* number, contentType: string, body: Buffer, version?: number }>}
|
|
2829
|
+
*/
|
|
2830
|
+
_chunkEntriesFromArchive(entries) {
|
|
2831
|
+
// Accumulate the representation and the sidecar of each chunk under one
|
|
2832
|
+
// key, then emit only the chunks that carry a representation.
|
|
2833
|
+
const staged = new Map();
|
|
2834
|
+
const pattern = /^space\/[^/]+\/([^/]+)\/(\.chunks\.[^/]+)\/([^/]+)$/;
|
|
2835
|
+
for (const [entryName, entry] of entries) {
|
|
2836
|
+
if (entry.type !== 'file' || !entry.body) {
|
|
2837
|
+
continue;
|
|
2838
|
+
}
|
|
2839
|
+
const match = entryName.match(pattern);
|
|
2840
|
+
if (!match) {
|
|
2841
|
+
continue;
|
|
2842
|
+
}
|
|
2843
|
+
const collectionId = match[1];
|
|
2844
|
+
const resourceId = parseChunkDirName(match[2]);
|
|
2845
|
+
const fileName = match[3];
|
|
2846
|
+
if (resourceId === undefined) {
|
|
2847
|
+
continue;
|
|
2848
|
+
}
|
|
2849
|
+
// Reject a path-traversal / non-URL-safe id parsed from the archive
|
|
2850
|
+
// before it keys a destination row.
|
|
2851
|
+
assertValidId(collectionId, {
|
|
2852
|
+
kind: 'collection',
|
|
2853
|
+
requestName: 'Import Space'
|
|
2854
|
+
});
|
|
2855
|
+
assertValidId(resourceId, {
|
|
2856
|
+
kind: 'resource',
|
|
2857
|
+
requestName: 'Import Space'
|
|
2858
|
+
});
|
|
2859
|
+
// A chunk's index is the "resource id" of its file within the chunk dir:
|
|
2860
|
+
// `r.<index>.<encType>.<ext>` for the representation, `.meta.<index>.json`
|
|
2861
|
+
// for the version sidecar. The RAW index segment must pass the same
|
|
2862
|
+
// canonical predicate as the live route (`parseChunkIndexSegment`), so a
|
|
2863
|
+
// non-canonical spelling (`r.01.*`, `r.%31.*`) is dropped rather than
|
|
2864
|
+
// coerced onto -- and colliding with -- the canonical chunk's row.
|
|
2865
|
+
let chunkIdSegment;
|
|
2866
|
+
let representation;
|
|
2867
|
+
let sidecarVersion;
|
|
2868
|
+
const metaId = metaSidecarFileId(fileName);
|
|
2869
|
+
if (metaId !== undefined) {
|
|
2870
|
+
chunkIdSegment = metaId;
|
|
2871
|
+
try {
|
|
2872
|
+
const sidecar = JSON.parse(entry.body.toString('utf8'));
|
|
2873
|
+
sidecarVersion = sidecar.version;
|
|
2874
|
+
}
|
|
2875
|
+
catch {
|
|
2876
|
+
sidecarVersion = undefined;
|
|
2877
|
+
}
|
|
2878
|
+
}
|
|
2879
|
+
else if (fileName.startsWith('r.')) {
|
|
2880
|
+
const { contentType } = parseResourceFileName(fileName);
|
|
2881
|
+
chunkIdSegment = fileName.split('.')[1];
|
|
2882
|
+
representation = { contentType, body: entry.body };
|
|
2883
|
+
}
|
|
2884
|
+
else {
|
|
2885
|
+
continue;
|
|
2886
|
+
}
|
|
2887
|
+
const chunkIndex = chunkIdSegment === undefined
|
|
2888
|
+
? undefined
|
|
2889
|
+
: parseChunkIndexSegment(chunkIdSegment);
|
|
2890
|
+
if (chunkIndex === undefined) {
|
|
2891
|
+
continue;
|
|
2892
|
+
}
|
|
2893
|
+
const key = `${collectionId}/${resourceId}/${chunkIndex}`;
|
|
2894
|
+
const slot = staged.get(key) ?? { collectionId, resourceId, chunkIndex };
|
|
2895
|
+
if (representation !== undefined) {
|
|
2896
|
+
slot.contentType = representation.contentType;
|
|
2897
|
+
slot.body = representation.body;
|
|
2898
|
+
}
|
|
2899
|
+
if (sidecarVersion !== undefined) {
|
|
2900
|
+
slot.version = sidecarVersion;
|
|
2901
|
+
}
|
|
2902
|
+
staged.set(key, slot);
|
|
2903
|
+
}
|
|
2904
|
+
const parsed = [];
|
|
2905
|
+
for (const slot of staged.values()) {
|
|
2906
|
+
if (slot.body === undefined) {
|
|
2907
|
+
// A sidecar with no paired representation is not a valid chunk.
|
|
2908
|
+
continue;
|
|
2909
|
+
}
|
|
2910
|
+
parsed.push({
|
|
2911
|
+
collectionId: slot.collectionId,
|
|
2912
|
+
resourceId: slot.resourceId,
|
|
2913
|
+
chunkIndex: slot.chunkIndex,
|
|
2914
|
+
contentType: slot.contentType ?? 'application/octet-stream',
|
|
2915
|
+
body: slot.body,
|
|
2916
|
+
version: slot.version
|
|
2917
|
+
});
|
|
2918
|
+
}
|
|
2919
|
+
return parsed;
|
|
2920
|
+
}
|
|
2163
2921
|
}
|
|
2164
2922
|
//# sourceMappingURL=postgres.js.map
|