was-teaching-server 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -1
- package/dist/backends/filesystem.d.ts +161 -2
- package/dist/backends/filesystem.d.ts.map +1 -1
- package/dist/backends/filesystem.js +491 -60
- package/dist/backends/filesystem.js.map +1 -1
- package/dist/backends/postgres.d.ts +166 -0
- package/dist/backends/postgres.d.ts.map +1 -1
- package/dist/backends/postgres.js +625 -29
- package/dist/backends/postgres.js.map +1 -1
- package/dist/backends/postgresSchema.d.ts.map +1 -1
- package/dist/backends/postgresSchema.js +25 -0
- package/dist/backends/postgresSchema.js.map +1 -1
- package/dist/errors.d.ts +13 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +18 -0
- package/dist/errors.js.map +1 -1
- package/dist/lib/exportManifest.d.ts +9 -0
- package/dist/lib/exportManifest.d.ts.map +1 -1
- package/dist/lib/exportManifest.js +9 -0
- package/dist/lib/exportManifest.js.map +1 -1
- package/dist/lib/importTar.d.ts +18 -2
- package/dist/lib/importTar.d.ts.map +1 -1
- package/dist/lib/importTar.js +77 -4
- package/dist/lib/importTar.js.map +1 -1
- package/dist/lib/paths.d.ts +33 -0
- package/dist/lib/paths.d.ts.map +1 -1
- package/dist/lib/paths.js +28 -0
- package/dist/lib/paths.js.map +1 -1
- package/dist/lib/resourceFileName.d.ts +43 -0
- package/dist/lib/resourceFileName.d.ts.map +1 -1
- package/dist/lib/resourceFileName.js +63 -0
- package/dist/lib/resourceFileName.js.map +1 -1
- package/dist/requests/ChunkRequest.d.ts +104 -0
- package/dist/requests/ChunkRequest.d.ts.map +1 -0
- package/dist/requests/ChunkRequest.js +396 -0
- package/dist/requests/ChunkRequest.js.map +1 -0
- package/dist/routes.d.ts.map +1 -1
- package/dist/routes.js +21 -0
- package/dist/routes.js.map +1 -1
- package/dist/types.d.ts +81 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/backends/filesystem.ts +638 -57
- package/src/backends/postgres.ts +867 -21
- package/src/backends/postgresSchema.ts +69 -57
- package/src/errors.ts +19 -0
- package/src/lib/importTar.ts +107 -4
- package/src/lib/paths.ts +48 -0
- package/src/lib/resourceFileName.ts +69 -0
- package/src/requests/ChunkRequest.ts +478 -0
- package/src/routes.ts +46 -0
- package/src/types.ts +78 -1
|
@@ -19,12 +19,12 @@ import YAML from 'yaml';
|
|
|
19
19
|
import { DEFAULT_MAX_UPLOAD_BYTES, DEFAULT_MAX_SPACES_PER_CONTROLLER, DEFAULT_MAX_COLLECTIONS_PER_SPACE, DEFAULT_MAX_RESOURCES_PER_SPACE, QUOTA_USAGE_CACHE_TTL, normalizeCountLimit } from '../config.default.js';
|
|
20
20
|
import { extractTarEntries, buildImportPlan, metaSidecarFileId } from '../lib/importTar.js';
|
|
21
21
|
import { collectionPath, resourcePath } from '../lib/paths.js';
|
|
22
|
-
import { encodeFilenameSegment, fileNameFor, parseResourceFileName } from '../lib/resourceFileName.js';
|
|
22
|
+
import { encodeFilenameSegment, fileNameFor, parseResourceFileName, chunkDirName, CHUNK_DIR_PREFIX } from '../lib/resourceFileName.js';
|
|
23
23
|
import { sanitizeBackendRecord } from '../lib/backends.js';
|
|
24
24
|
import { backendUsageFields } from '../lib/backendUsage.js';
|
|
25
25
|
import { assertEncryptedWriteConforms } from '../lib/encryption.js';
|
|
26
26
|
import { encodeCursor, decodeCursor } from '../lib/cursor.js';
|
|
27
|
-
import { buildExportManifest } from '../lib/exportManifest.js';
|
|
27
|
+
import { buildExportManifest, EXPORT_ENTRY_MTIME } from '../lib/exportManifest.js';
|
|
28
28
|
import { revocationFileName } from '../lib/revocations.js';
|
|
29
29
|
import { KeyedMutex } from '../lib/keyedMutex.js';
|
|
30
30
|
import { isJson } from '../lib/isJson.js';
|
|
@@ -203,8 +203,9 @@ export class FileSystemBackend {
|
|
|
203
203
|
* It advertises the `conditional-writes` affordance: it exposes a per-Resource
|
|
204
204
|
* `version` as an HTTP `ETag` validator and honors `If-Match` / `If-None-Match`
|
|
205
205
|
* write preconditions atomically (returning `412 precondition-failed` on a
|
|
206
|
-
* mismatch).
|
|
207
|
-
*
|
|
206
|
+
* mismatch). It also advertises `chunked-streams`: chunk addressing for large
|
|
207
|
+
* Resources (`/{resourceId}/chunks/{n}`), each chunk stored opaquely like a
|
|
208
|
+
* binary Resource representation.
|
|
208
209
|
* (Client-side encryption is deliberately not a backend feature: encrypted
|
|
209
210
|
* documents are opaque client-encrypted JSON this backend already stores
|
|
210
211
|
* faithfully, with no server cooperation.)
|
|
@@ -229,11 +230,14 @@ export class FileSystemBackend {
|
|
|
229
230
|
// `key-epochs`: multi-recipient encrypted Collections -- per-epoch wrapped
|
|
230
231
|
// keys on the `encryption` marker, a client-declared `epoch` stamp on
|
|
231
232
|
// Resources, and conditional (`If-Match`) Collection Description writes.
|
|
233
|
+
// `chunked-streams`: chunk addressing (`/{resourceId}/chunks/{n}`) for a
|
|
234
|
+
// large Resource, each chunk stored opaquely (bytes + content-type).
|
|
232
235
|
features: [
|
|
233
236
|
'conditional-writes',
|
|
234
237
|
'changes-query',
|
|
235
238
|
'blinded-index-query',
|
|
236
|
-
'key-epochs'
|
|
239
|
+
'key-epochs',
|
|
240
|
+
'chunked-streams'
|
|
237
241
|
]
|
|
238
242
|
};
|
|
239
243
|
}
|
|
@@ -812,6 +816,11 @@ export class FileSystemBackend {
|
|
|
812
816
|
entry => !(entry.isFile() && entry.name.startsWith('.backend.')));
|
|
813
817
|
spaceEntries.sort((a, b) => a.name.localeCompare(b.name));
|
|
814
818
|
const collectionEntriesByDir = {};
|
|
819
|
+
// Per-Resource chunk directories (`.chunks.<encId>/`; the `chunked-streams`
|
|
820
|
+
// feature) are subdirectories of a Collection dir, so the file filter below
|
|
821
|
+
// skips them. Gather each one's files here so a chunked Resource's chunks
|
|
822
|
+
// travel in the export, keyed by Collection then by chunk-directory name.
|
|
823
|
+
const chunkDirsByCollection = {};
|
|
815
824
|
for (const entry of spaceEntries) {
|
|
816
825
|
if (!entry.isDirectory()) {
|
|
817
826
|
continue;
|
|
@@ -820,6 +829,17 @@ export class FileSystemBackend {
|
|
|
820
829
|
collectionEntriesByDir[entry.name] = entries
|
|
821
830
|
.filter(e => e.isFile())
|
|
822
831
|
.sort((a, b) => a.name.localeCompare(b.name));
|
|
832
|
+
const chunkDirs = [];
|
|
833
|
+
for (const sub of entries
|
|
834
|
+
.filter(e => e.isDirectory() && e.name.startsWith(CHUNK_DIR_PREFIX))
|
|
835
|
+
.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
836
|
+
const files = (await fs.promises.readdir(path.join(sourceSpaceDir, entry.name, sub.name), { withFileTypes: true }))
|
|
837
|
+
.filter(e => e.isFile())
|
|
838
|
+
.map(e => e.name)
|
|
839
|
+
.sort((a, b) => a.localeCompare(b));
|
|
840
|
+
chunkDirs.push({ dirName: sub.name, files });
|
|
841
|
+
}
|
|
842
|
+
chunkDirsByCollection[entry.name] = chunkDirs;
|
|
823
843
|
}
|
|
824
844
|
// Space-scoped zcap revocations travel with the export. They live in the
|
|
825
845
|
// sibling `spaceRevocationsDir` root (see that property's doc), so the
|
|
@@ -844,35 +864,58 @@ export class FileSystemBackend {
|
|
|
844
864
|
entries: spaceEntries.map(entry => entry.isDirectory()
|
|
845
865
|
? {
|
|
846
866
|
name: entry.name,
|
|
847
|
-
files
|
|
867
|
+
// The Collection's own files, then each chunked Resource's chunk
|
|
868
|
+
// files listed by their `.chunks.<encId>/<file>` relative path
|
|
869
|
+
// (the manifest mirrors the pack order below).
|
|
870
|
+
files: [
|
|
871
|
+
...(collectionEntriesByDir[entry.name] ?? []).map(file => file.name),
|
|
872
|
+
...(chunkDirsByCollection[entry.name] ?? []).flatMap(({ dirName, files }) => files.map(file => `${dirName}/${file}`))
|
|
873
|
+
]
|
|
848
874
|
}
|
|
849
875
|
: // top-level files in space (e.g. .space.<spaceId>.json)
|
|
850
876
|
{ name: entry.name }),
|
|
851
877
|
revocationFiles
|
|
852
878
|
});
|
|
879
|
+
// Fixed mtime on every entry so the archive is byte-reproducible (see
|
|
880
|
+
// EXPORT_ENTRY_MTIME).
|
|
881
|
+
const mtime = EXPORT_ENTRY_MTIME;
|
|
853
882
|
const pack = tar.pack();
|
|
854
|
-
pack.entry({ name: 'manifest.yml' }, YAML.stringify(manifest));
|
|
855
|
-
pack.entry({ name: 'space/', type: 'directory' });
|
|
856
|
-
pack.entry({ name: `space/${spaceId}/`, type: 'directory' });
|
|
883
|
+
pack.entry({ name: 'manifest.yml', mtime }, YAML.stringify(manifest));
|
|
884
|
+
pack.entry({ name: 'space/', type: 'directory', mtime });
|
|
885
|
+
pack.entry({ name: `space/${spaceId}/`, type: 'directory', mtime });
|
|
857
886
|
for (const entry of spaceEntries) {
|
|
858
887
|
const entryTarget = `space/${spaceId}/${entry.name}`;
|
|
859
888
|
if (entry.isDirectory()) {
|
|
860
|
-
pack.entry({ name: `${entryTarget}/`, type: 'directory' });
|
|
889
|
+
pack.entry({ name: `${entryTarget}/`, type: 'directory', mtime });
|
|
861
890
|
for (const file of collectionEntriesByDir[entry.name] ?? []) {
|
|
862
891
|
const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name, file.name));
|
|
863
|
-
pack.entry({ name: `${entryTarget}/${file.name}
|
|
892
|
+
pack.entry({ name: `${entryTarget}/${file.name}`, mtime }, bytes);
|
|
893
|
+
}
|
|
894
|
+
// A chunked Resource's chunk directory and its files, packed under the
|
|
895
|
+
// Collection so `.chunks.<encId>/<file>` round-trips on import.
|
|
896
|
+
for (const { dirName, files } of chunkDirsByCollection[entry.name] ??
|
|
897
|
+
[]) {
|
|
898
|
+
pack.entry({
|
|
899
|
+
name: `${entryTarget}/${dirName}/`,
|
|
900
|
+
type: 'directory',
|
|
901
|
+
mtime
|
|
902
|
+
});
|
|
903
|
+
for (const file of files) {
|
|
904
|
+
const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name, dirName, file));
|
|
905
|
+
pack.entry({ name: `${entryTarget}/${dirName}/${file}`, mtime }, bytes);
|
|
906
|
+
}
|
|
864
907
|
}
|
|
865
908
|
}
|
|
866
909
|
else if (entry.isFile()) {
|
|
867
910
|
const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name));
|
|
868
|
-
pack.entry({ name: entryTarget }, bytes);
|
|
911
|
+
pack.entry({ name: entryTarget, mtime }, bytes);
|
|
869
912
|
}
|
|
870
913
|
}
|
|
871
914
|
if (revocationFiles.length > 0) {
|
|
872
|
-
pack.entry({ name: 'revocations/', type: 'directory' });
|
|
915
|
+
pack.entry({ name: 'revocations/', type: 'directory', mtime });
|
|
873
916
|
for (const name of revocationFiles) {
|
|
874
917
|
const bytes = await fs.promises.readFile(path.join(revocationsDir, name));
|
|
875
|
-
pack.entry({ name: `revocations/${name}
|
|
918
|
+
pack.entry({ name: `revocations/${name}`, mtime }, bytes);
|
|
876
919
|
}
|
|
877
920
|
}
|
|
878
921
|
pack.finalize();
|
|
@@ -903,7 +946,7 @@ export class FileSystemBackend {
|
|
|
903
946
|
// (existing ids) are counted conservatively, as for the quota estimate.
|
|
904
947
|
const { capacityBytes, maxUploadBytes, maxCollectionsPerSpace, maxResourcesPerSpace } = this;
|
|
905
948
|
let incomingBytes = 0;
|
|
906
|
-
for (const { collectionId, collectionDescription, resources } of collections) {
|
|
949
|
+
for (const { collectionId, collectionDescription, resources, chunkFiles } of collections) {
|
|
907
950
|
const existing = await this.getCollectionDescription({
|
|
908
951
|
spaceId,
|
|
909
952
|
collectionId
|
|
@@ -936,6 +979,19 @@ export class FileSystemBackend {
|
|
|
936
979
|
}
|
|
937
980
|
incomingBytes += body.length;
|
|
938
981
|
}
|
|
982
|
+
// Chunk files (the `chunked-streams` feature) inherit the per-upload cap
|
|
983
|
+
// and quota estimate, but NOT the encryption-conformance check: a chunk is
|
|
984
|
+
// opaque bytes, not a JSON envelope of the Collection's scheme.
|
|
985
|
+
for (const { body } of chunkFiles) {
|
|
986
|
+
if (maxUploadBytes !== undefined && body.length > maxUploadBytes) {
|
|
987
|
+
throw new PayloadTooLargeError({
|
|
988
|
+
maxUploadBytes,
|
|
989
|
+
backendId: this.describe().id,
|
|
990
|
+
uploadBytes: body.length
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
incomingBytes += body.length;
|
|
994
|
+
}
|
|
939
995
|
}
|
|
940
996
|
if (capacityBytes !== undefined) {
|
|
941
997
|
await this._assertSpaceHeadroom({
|
|
@@ -974,7 +1030,7 @@ export class FileSystemBackend {
|
|
|
974
1030
|
let liveResourceCount = maxResourcesPerSpace !== undefined
|
|
975
1031
|
? await this._countLiveResources({ spaceId })
|
|
976
1032
|
: 0;
|
|
977
|
-
for (const { collectionId, collectionDescription, collectionPolicy, resources, resourcePolicies, resourceMetadata } of collections) {
|
|
1033
|
+
for (const { collectionId, collectionDescription, collectionPolicy, resources, resourcePolicies, resourceMetadata, chunkFiles } of collections) {
|
|
978
1034
|
// check if collection already exists
|
|
979
1035
|
const collectionExisted = Boolean(await this.getCollectionDescription({ spaceId, collectionId }));
|
|
980
1036
|
if (collectionExisted) {
|
|
@@ -1104,6 +1160,43 @@ export class FileSystemBackend {
|
|
|
1104
1160
|
});
|
|
1105
1161
|
stats.resourcesCreated++;
|
|
1106
1162
|
}
|
|
1163
|
+
// Restore chunk files of chunked Resources (the `chunked-streams` feature)
|
|
1164
|
+
// into their per-Resource chunk directories. Skip-not-overwrite, per chunk
|
|
1165
|
+
// file: an existing chunk file is left untouched, so a re-import never
|
|
1166
|
+
// clobbers stored chunk bytes (or their version sidecar). An ORPHAN chunk
|
|
1167
|
+
// file -- one whose parent Resource is absent or a tombstone (no live
|
|
1168
|
+
// representation on the destination after the Resource apply loop above)
|
|
1169
|
+
// -- is skipped rather than resurrected, matching the live write path's
|
|
1170
|
+
// parent-exists rule (and the Postgres import).
|
|
1171
|
+
const parentIsLive = new Map();
|
|
1172
|
+
for (const { resourceId, fileName, body } of chunkFiles) {
|
|
1173
|
+
let live = parentIsLive.get(resourceId);
|
|
1174
|
+
if (live === undefined) {
|
|
1175
|
+
live = Boolean(await this._findFile({ collectionDir, resourceId }));
|
|
1176
|
+
parentIsLive.set(resourceId, live);
|
|
1177
|
+
}
|
|
1178
|
+
if (!live) {
|
|
1179
|
+
continue;
|
|
1180
|
+
}
|
|
1181
|
+
const chunkDir = this._chunkDir({ collectionDir, resourceId });
|
|
1182
|
+
const target = path.join(chunkDir, fileName);
|
|
1183
|
+
this._assertContained(target);
|
|
1184
|
+
let present = false;
|
|
1185
|
+
try {
|
|
1186
|
+
await fsStat(target);
|
|
1187
|
+
present = true;
|
|
1188
|
+
}
|
|
1189
|
+
catch (err) {
|
|
1190
|
+
if (err.code !== 'ENOENT') {
|
|
1191
|
+
throw err;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
if (present) {
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
await mkdir(chunkDir, { recursive: true });
|
|
1198
|
+
await atomicWriteFile({ filePath: target, data: body });
|
|
1199
|
+
}
|
|
1107
1200
|
}
|
|
1108
1201
|
// Restore the archive's Space-scoped zcap revocations under this Space's
|
|
1109
1202
|
// scope: a capability revoked before the export must stay revoked after
|
|
@@ -1515,6 +1608,70 @@ export class FileSystemBackend {
|
|
|
1515
1608
|
}
|
|
1516
1609
|
}
|
|
1517
1610
|
}
|
|
1611
|
+
await this._writeRepresentationBytes({ spaceId, filePath, input });
|
|
1612
|
+
// A Resource has a single current representation: remove any prior
|
|
1613
|
+
// representation stored under a different content-type (its filename
|
|
1614
|
+
// differs). Write-new-then-prune (not delete-then-write) so the resource is
|
|
1615
|
+
// never momentarily absent.
|
|
1616
|
+
const existing = await this._resourceFilesFor({ collectionDir, resourceId });
|
|
1617
|
+
await Promise.all(existing
|
|
1618
|
+
.filter(name => path.resolve(name) !== path.resolve(filePath))
|
|
1619
|
+
.map(name => rm(name)));
|
|
1620
|
+
// Maintain the server-managed timestamps and the monotonic `version`: a
|
|
1621
|
+
// content write sets `createdAt` on first write, bumps `updatedAt`, and
|
|
1622
|
+
// increments `version` (the ETag validator) from its prior value, preserving
|
|
1623
|
+
// any user-writable `custom` and the independent `metaVersion` already stored
|
|
1624
|
+
// in the sidecar (a content write does not touch the metadata sub-resource).
|
|
1625
|
+
//
|
|
1626
|
+
// `createdBy` pairs with `createdAt`: taken from this write's invoker only
|
|
1627
|
+
// when this write creates the sidecar, so it names the creator rather than
|
|
1628
|
+
// the last writer, and is preserved verbatim afterward -- including
|
|
1629
|
+
// preserved-as-absent, so a Resource created with no invoker never has a
|
|
1630
|
+
// later writer backfilled into it. A tombstone keeps both, so re-creating a
|
|
1631
|
+
// deleted id under a different invoker preserves the original creator, as
|
|
1632
|
+
// it does the original `createdAt`.
|
|
1633
|
+
const now = new Date().toISOString();
|
|
1634
|
+
const prior = await this._readMetaSidecar({ collectionDir, resourceId });
|
|
1635
|
+
const version = (prior?.version ?? 0) + 1;
|
|
1636
|
+
const creator = prior ? prior.createdBy : createdBy;
|
|
1637
|
+
await this._writeMetaSidecar({
|
|
1638
|
+
collectionDir,
|
|
1639
|
+
resourceId,
|
|
1640
|
+
sidecar: {
|
|
1641
|
+
createdAt: prior?.createdAt ?? now,
|
|
1642
|
+
updatedAt: now,
|
|
1643
|
+
...(creator !== undefined && { createdBy: creator }),
|
|
1644
|
+
version,
|
|
1645
|
+
...(prior?.metaVersion !== undefined && {
|
|
1646
|
+
metaVersion: prior.metaVersion
|
|
1647
|
+
}),
|
|
1648
|
+
...(prior?.custom && { custom: prior.custom }),
|
|
1649
|
+
// The key-epoch stamp is set from this write's declaration and CLEARED
|
|
1650
|
+
// when absent (the new ciphertext's epoch is unknown -- a stale stamp is
|
|
1651
|
+
// worse than none), so it is NOT preserved from `prior` like `custom`.
|
|
1652
|
+
...(epoch !== undefined && { epoch })
|
|
1653
|
+
}
|
|
1654
|
+
});
|
|
1655
|
+
return { version };
|
|
1656
|
+
}
|
|
1657
|
+
/**
|
|
1658
|
+
* Writes a representation body (JSON value or byte stream) to `filePath`,
|
|
1659
|
+
* applying the same size guards every write path shares: the per-upload cap
|
|
1660
|
+
* (413 `PayloadTooLargeError`) and, when a byte quota is configured, the
|
|
1661
|
+
* Space headroom (507). A JSON body is fully in memory, so its size is checked
|
|
1662
|
+
* up front and written atomically; a binary body is streamed through the cap /
|
|
1663
|
+
* quota guards into a temp file and durably committed (fsync + rename + dir
|
|
1664
|
+
* fsync), removing the partial file on any failure. Shared by the Resource
|
|
1665
|
+
* write path (`_writeResourceLocked`) and the chunk write path
|
|
1666
|
+
* (`_writeChunkLocked`); the caller has already resolved `filePath` and ensured
|
|
1667
|
+
* its parent directory exists for the streamed case.
|
|
1668
|
+
* @param options {object}
|
|
1669
|
+
* @param options.spaceId {string}
|
|
1670
|
+
* @param options.filePath {string} absolute path of the representation file
|
|
1671
|
+
* @param options.input {ResourceInput}
|
|
1672
|
+
* @returns {Promise<void>}
|
|
1673
|
+
*/
|
|
1674
|
+
async _writeRepresentationBytes({ spaceId, filePath, input }) {
|
|
1518
1675
|
const { capacityBytes, maxUploadBytes } = this;
|
|
1519
1676
|
if (input.kind === 'json') {
|
|
1520
1677
|
// JSON bodies are fully in memory, so their serialized size is known up
|
|
@@ -1603,50 +1760,6 @@ export class FileSystemBackend {
|
|
|
1603
1760
|
throw err;
|
|
1604
1761
|
}
|
|
1605
1762
|
}
|
|
1606
|
-
// A Resource has a single current representation: remove any prior
|
|
1607
|
-
// representation stored under a different content-type (its filename
|
|
1608
|
-
// differs). Write-new-then-prune (not delete-then-write) so the resource is
|
|
1609
|
-
// never momentarily absent.
|
|
1610
|
-
const existing = await this._resourceFilesFor({ collectionDir, resourceId });
|
|
1611
|
-
await Promise.all(existing
|
|
1612
|
-
.filter(name => path.resolve(name) !== path.resolve(filePath))
|
|
1613
|
-
.map(name => rm(name)));
|
|
1614
|
-
// Maintain the server-managed timestamps and the monotonic `version`: a
|
|
1615
|
-
// content write sets `createdAt` on first write, bumps `updatedAt`, and
|
|
1616
|
-
// increments `version` (the ETag validator) from its prior value, preserving
|
|
1617
|
-
// any user-writable `custom` and the independent `metaVersion` already stored
|
|
1618
|
-
// in the sidecar (a content write does not touch the metadata sub-resource).
|
|
1619
|
-
//
|
|
1620
|
-
// `createdBy` pairs with `createdAt`: taken from this write's invoker only
|
|
1621
|
-
// when this write creates the sidecar, so it names the creator rather than
|
|
1622
|
-
// the last writer, and is preserved verbatim afterward -- including
|
|
1623
|
-
// preserved-as-absent, so a Resource created with no invoker never has a
|
|
1624
|
-
// later writer backfilled into it. A tombstone keeps both, so re-creating a
|
|
1625
|
-
// deleted id under a different invoker preserves the original creator, as
|
|
1626
|
-
// it does the original `createdAt`.
|
|
1627
|
-
const now = new Date().toISOString();
|
|
1628
|
-
const prior = await this._readMetaSidecar({ collectionDir, resourceId });
|
|
1629
|
-
const version = (prior?.version ?? 0) + 1;
|
|
1630
|
-
const creator = prior ? prior.createdBy : createdBy;
|
|
1631
|
-
await this._writeMetaSidecar({
|
|
1632
|
-
collectionDir,
|
|
1633
|
-
resourceId,
|
|
1634
|
-
sidecar: {
|
|
1635
|
-
createdAt: prior?.createdAt ?? now,
|
|
1636
|
-
updatedAt: now,
|
|
1637
|
-
...(creator !== undefined && { createdBy: creator }),
|
|
1638
|
-
version,
|
|
1639
|
-
...(prior?.metaVersion !== undefined && {
|
|
1640
|
-
metaVersion: prior.metaVersion
|
|
1641
|
-
}),
|
|
1642
|
-
...(prior?.custom && { custom: prior.custom }),
|
|
1643
|
-
// The key-epoch stamp is set from this write's declaration and CLEARED
|
|
1644
|
-
// when absent (the new ciphertext's epoch is unknown -- a stale stamp is
|
|
1645
|
-
// worse than none), so it is NOT preserved from `prior` like `custom`.
|
|
1646
|
-
...(epoch !== undefined && { epoch })
|
|
1647
|
-
}
|
|
1648
|
-
});
|
|
1649
|
-
return { version };
|
|
1650
1763
|
}
|
|
1651
1764
|
/**
|
|
1652
1765
|
* Builds the per-Resource serialization key for `_writeMutex`
|
|
@@ -1979,6 +2092,15 @@ export class FileSystemBackend {
|
|
|
1979
2092
|
const { contentType } = parseResourceFileName(path.basename(filesForResource[0]));
|
|
1980
2093
|
// Drop the content representation(s) but KEEP the sidecar as the tombstone.
|
|
1981
2094
|
await Promise.all(filesForResource.map(filename => rm(filename)));
|
|
2095
|
+
// Cascade-delete the Resource's chunks (the `chunked-streams` feature): a
|
|
2096
|
+
// chunk must never outlive its parent Resource, so its whole chunk
|
|
2097
|
+
// directory goes with the content. Runs under the same per-Resource lock a
|
|
2098
|
+
// `writeChunk` takes, so a chunk write racing this delete cannot re-create
|
|
2099
|
+
// an orphan directory. `force` makes an absent chunk directory a no-op.
|
|
2100
|
+
await rm(this._chunkDir({ collectionDir, resourceId }), {
|
|
2101
|
+
recursive: true,
|
|
2102
|
+
force: true
|
|
2103
|
+
});
|
|
1982
2104
|
// Bump `version` / `updatedAt` so the tombstone sorts after the Resource's
|
|
1983
2105
|
// prior state in the change feed, and continues the monotonic version (a
|
|
1984
2106
|
// later re-create reads this sidecar and keeps counting up). `custom` is
|
|
@@ -2007,6 +2129,315 @@ export class FileSystemBackend {
|
|
|
2007
2129
|
// for a conditional delete, as the old unconditional removal did).
|
|
2008
2130
|
return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), softDelete);
|
|
2009
2131
|
}
|
|
2132
|
+
// Chunks (the `chunked-streams` feature)
|
|
2133
|
+
/**
|
|
2134
|
+
* Builds the on-disk path for a Resource's chunk directory
|
|
2135
|
+
* (`.chunks.<encodedResourceId>/`) inside its Collection dir. A hidden
|
|
2136
|
+
* subdirectory (leading `.`, so it is invisible to the `r.`-prefixed Collection
|
|
2137
|
+
* listing and the live-Resource count) that holds the Resource's chunk
|
|
2138
|
+
* representations; inside it a chunk is stored exactly like a Resource keyed by
|
|
2139
|
+
* its index (`r.<index>.<encodedContentType>.<ext>` plus a `.meta.<index>.json`
|
|
2140
|
+
* version sidecar), so the Resource file / sidecar helpers are reused verbatim
|
|
2141
|
+
* with the chunk directory as the `collectionDir`.
|
|
2142
|
+
* @param options {object}
|
|
2143
|
+
* @param options.collectionDir {string}
|
|
2144
|
+
* @param options.resourceId {string}
|
|
2145
|
+
* @returns {string}
|
|
2146
|
+
*/
|
|
2147
|
+
_chunkDir({ collectionDir, resourceId }) {
|
|
2148
|
+
const chunkDir = path.join(collectionDir, chunkDirName(resourceId));
|
|
2149
|
+
this._assertContained(chunkDir);
|
|
2150
|
+
return chunkDir;
|
|
2151
|
+
}
|
|
2152
|
+
/**
|
|
2153
|
+
* Writes one chunk of a chunked Resource, keyed by
|
|
2154
|
+
* `(spaceId, collectionId, resourceId, chunkIndex)`. The parent Resource MUST
|
|
2155
|
+
* already exist (else `ResourceNotFoundError`, 404), checked under the same
|
|
2156
|
+
* per-Resource lock a `deleteResource` cascade takes so a chunk can never be
|
|
2157
|
+
* orphaned by a racing delete. The body is stored opaquely (bytes +
|
|
2158
|
+
* content-type) through the shared upload-cap / quota guards, and the chunk's
|
|
2159
|
+
* own monotonic `version` (its ETag validator, independent of the parent's) is
|
|
2160
|
+
* bumped; any `If-Match` / `If-None-Match` precondition is evaluated on that
|
|
2161
|
+
* version atomically with the write (`PreconditionFailedError`, 412).
|
|
2162
|
+
* @param options {object}
|
|
2163
|
+
* @param options.spaceId {string}
|
|
2164
|
+
* @param options.collectionId {string}
|
|
2165
|
+
* @param options.resourceId {string}
|
|
2166
|
+
* @param options.chunkIndex {number} non-negative integer chunk position
|
|
2167
|
+
* @param options.input {ResourceInput}
|
|
2168
|
+
* @param [options.ifMatch] {string} `If-Match` precondition (a quoted ETag)
|
|
2169
|
+
* @param [options.ifNoneMatch] {boolean} `If-None-Match: *` (create-if-absent)
|
|
2170
|
+
* @returns {Promise<{ version: number }>} the chunk's new version
|
|
2171
|
+
*/
|
|
2172
|
+
async writeChunk({ spaceId, collectionId, resourceId, chunkIndex, input, ifMatch, ifNoneMatch }) {
|
|
2173
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
2174
|
+
// Serialize on the parent Resource's lock key -- the same key
|
|
2175
|
+
// `deleteResource` takes -- so the parent-exists check, the write, and the
|
|
2176
|
+
// cascade delete cannot interleave (no orphan chunk).
|
|
2177
|
+
return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), () => this._writeChunkLocked({
|
|
2178
|
+
spaceId,
|
|
2179
|
+
collectionDir,
|
|
2180
|
+
resourceId,
|
|
2181
|
+
chunkIndex,
|
|
2182
|
+
input,
|
|
2183
|
+
ifMatch,
|
|
2184
|
+
ifNoneMatch
|
|
2185
|
+
}));
|
|
2186
|
+
}
|
|
2187
|
+
/**
|
|
2188
|
+
* The critical section of `writeChunk`, run under the per-Resource lock. See
|
|
2189
|
+
* `writeChunk` for the parameters.
|
|
2190
|
+
* @returns {Promise<{ version: number }>}
|
|
2191
|
+
*/
|
|
2192
|
+
async _writeChunkLocked({ spaceId, collectionDir, resourceId, chunkIndex, input, ifMatch, ifNoneMatch }) {
|
|
2193
|
+
// The parent Resource must exist: writing a chunk of an absent Resource
|
|
2194
|
+
// rejects, so orphan chunks cannot accumulate.
|
|
2195
|
+
const parentExists = (await this._findFile({ collectionDir, resourceId })) !== undefined;
|
|
2196
|
+
if (!parentExists) {
|
|
2197
|
+
throw new ResourceNotFoundError({ requestName: 'Write Chunk' });
|
|
2198
|
+
}
|
|
2199
|
+
// Inside the chunk directory a chunk is a Resource keyed by its index, so the
|
|
2200
|
+
// Resource file / sidecar helpers apply with `chunkDir` as the collectionDir
|
|
2201
|
+
// and the stringified index as the resourceId.
|
|
2202
|
+
const chunkDir = this._chunkDir({ collectionDir, resourceId });
|
|
2203
|
+
const chunkId = String(chunkIndex);
|
|
2204
|
+
const filename = fileNameFor({
|
|
2205
|
+
resourceId: chunkId,
|
|
2206
|
+
contentType: input.contentType
|
|
2207
|
+
});
|
|
2208
|
+
const filePath = path.join(chunkDir, filename);
|
|
2209
|
+
this._assertContained(filePath);
|
|
2210
|
+
// Evaluate any precondition against the chunk's current version before
|
|
2211
|
+
// writing (still inside the lock, so check and write are atomic).
|
|
2212
|
+
if (ifMatch !== undefined || ifNoneMatch) {
|
|
2213
|
+
await this._assertWritePrecondition({
|
|
2214
|
+
collectionDir: chunkDir,
|
|
2215
|
+
resourceId: chunkId,
|
|
2216
|
+
ifMatch,
|
|
2217
|
+
ifNoneMatch
|
|
2218
|
+
});
|
|
2219
|
+
}
|
|
2220
|
+
// The chunk directory is created lazily on first write (a binary body is
|
|
2221
|
+
// streamed into a temp file in it, so it must exist first).
|
|
2222
|
+
await mkdir(chunkDir, { recursive: true });
|
|
2223
|
+
await this._writeRepresentationBytes({ spaceId, filePath, input });
|
|
2224
|
+
// A chunk has a single current representation: remove any prior one stored
|
|
2225
|
+
// under a different content-type (write-new-then-prune).
|
|
2226
|
+
const existing = await this._resourceFilesFor({
|
|
2227
|
+
collectionDir: chunkDir,
|
|
2228
|
+
resourceId: chunkId
|
|
2229
|
+
});
|
|
2230
|
+
await Promise.all(existing
|
|
2231
|
+
.filter(name => path.resolve(name) !== path.resolve(filePath))
|
|
2232
|
+
.map(name => rm(name)));
|
|
2233
|
+
// Bump the chunk's monotonic `version` (its ETag validator), preserving its
|
|
2234
|
+
// `createdAt`. A chunk carries no user Metadata / `createdBy` / epoch stamp.
|
|
2235
|
+
const now = new Date().toISOString();
|
|
2236
|
+
const prior = await this._readMetaSidecar({
|
|
2237
|
+
collectionDir: chunkDir,
|
|
2238
|
+
resourceId: chunkId
|
|
2239
|
+
});
|
|
2240
|
+
const version = (prior?.version ?? 0) + 1;
|
|
2241
|
+
await this._writeMetaSidecar({
|
|
2242
|
+
collectionDir: chunkDir,
|
|
2243
|
+
resourceId: chunkId,
|
|
2244
|
+
sidecar: {
|
|
2245
|
+
createdAt: prior?.createdAt ?? now,
|
|
2246
|
+
updatedAt: now,
|
|
2247
|
+
version
|
|
2248
|
+
}
|
|
2249
|
+
});
|
|
2250
|
+
return { version };
|
|
2251
|
+
}
|
|
2252
|
+
/**
|
|
2253
|
+
* Reads a chunk's bytes, resolving a `ResourceResult` (stream + resolved
|
|
2254
|
+
* content-type + the chunk's `version`). Throws `ResourceNotFoundError` (404)
|
|
2255
|
+
* when the chunk is absent.
|
|
2256
|
+
* @param options {object}
|
|
2257
|
+
* @param options.spaceId {string}
|
|
2258
|
+
* @param options.collectionId {string}
|
|
2259
|
+
* @param options.resourceId {string}
|
|
2260
|
+
* @param options.chunkIndex {number}
|
|
2261
|
+
* @returns {Promise<ResourceResult>}
|
|
2262
|
+
*/
|
|
2263
|
+
async getChunk({ spaceId, collectionId, resourceId, chunkIndex }) {
|
|
2264
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
2265
|
+
const chunkDir = this._chunkDir({ collectionDir, resourceId });
|
|
2266
|
+
const chunkId = String(chunkIndex);
|
|
2267
|
+
const filePath = await this._findFile({
|
|
2268
|
+
collectionDir: chunkDir,
|
|
2269
|
+
resourceId: chunkId
|
|
2270
|
+
});
|
|
2271
|
+
if (!filePath) {
|
|
2272
|
+
throw new ResourceNotFoundError({ requestName: 'Get Chunk' });
|
|
2273
|
+
}
|
|
2274
|
+
try {
|
|
2275
|
+
await fsStat(filePath);
|
|
2276
|
+
}
|
|
2277
|
+
catch (err) {
|
|
2278
|
+
if (err.code === 'ENOENT') {
|
|
2279
|
+
throw new ResourceNotFoundError({ requestName: 'Get Chunk' });
|
|
2280
|
+
}
|
|
2281
|
+
throw err;
|
|
2282
|
+
}
|
|
2283
|
+
const { contentType: storedResourceType } = parseResourceFileName(path.basename(filePath));
|
|
2284
|
+
const sidecar = await this._readMetaSidecar({
|
|
2285
|
+
collectionDir: chunkDir,
|
|
2286
|
+
resourceId: chunkId
|
|
2287
|
+
});
|
|
2288
|
+
return {
|
|
2289
|
+
resourceStream: await openFileStream(filePath, this.logger),
|
|
2290
|
+
storedResourceType,
|
|
2291
|
+
...(sidecar?.version !== undefined && { version: sidecar.version })
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
2294
|
+
/**
|
|
2295
|
+
* Reads a chunk's stored content-type / size / version (the HEAD payload
|
|
2296
|
+
* headers). Resolves `undefined` when the chunk is absent.
|
|
2297
|
+
* @param options {object}
|
|
2298
|
+
* @param options.spaceId {string}
|
|
2299
|
+
* @param options.collectionId {string}
|
|
2300
|
+
* @param options.resourceId {string}
|
|
2301
|
+
* @param options.chunkIndex {number}
|
|
2302
|
+
* @returns {Promise<{ contentType: string, size: number, version?: number } |
|
|
2303
|
+
* undefined>}
|
|
2304
|
+
*/
|
|
2305
|
+
async getChunkMetadata({ spaceId, collectionId, resourceId, chunkIndex }) {
|
|
2306
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
2307
|
+
const chunkDir = this._chunkDir({ collectionDir, resourceId });
|
|
2308
|
+
const chunkId = String(chunkIndex);
|
|
2309
|
+
const filePath = await this._findFile({
|
|
2310
|
+
collectionDir: chunkDir,
|
|
2311
|
+
resourceId: chunkId
|
|
2312
|
+
});
|
|
2313
|
+
if (!filePath) {
|
|
2314
|
+
return undefined;
|
|
2315
|
+
}
|
|
2316
|
+
let stats;
|
|
2317
|
+
try {
|
|
2318
|
+
stats = await fsStat(filePath);
|
|
2319
|
+
}
|
|
2320
|
+
catch (err) {
|
|
2321
|
+
if (err.code === 'ENOENT') {
|
|
2322
|
+
return undefined;
|
|
2323
|
+
}
|
|
2324
|
+
throw err;
|
|
2325
|
+
}
|
|
2326
|
+
const { contentType } = parseResourceFileName(path.basename(filePath));
|
|
2327
|
+
const sidecar = await this._readMetaSidecar({
|
|
2328
|
+
collectionDir: chunkDir,
|
|
2329
|
+
resourceId: chunkId
|
|
2330
|
+
});
|
|
2331
|
+
return {
|
|
2332
|
+
contentType,
|
|
2333
|
+
size: stats.size,
|
|
2334
|
+
...(sidecar?.version !== undefined && { version: sidecar.version })
|
|
2335
|
+
};
|
|
2336
|
+
}
|
|
2337
|
+
/**
|
|
2338
|
+
* Deletes one chunk (a hard delete: its bytes and version sidecar both go, and
|
|
2339
|
+
* -- unlike a Resource -- it leaves no tombstone, since chunks are not part of
|
|
2340
|
+
* the change feed). Resolves `true` when a chunk was removed and `false` when
|
|
2341
|
+
* none was stored at that index. When `ifMatch` is supplied it is evaluated on
|
|
2342
|
+
* the chunk's current version atomically with the removal (under the same
|
|
2343
|
+
* per-Resource lock), throwing `PreconditionFailedError` (412) on a mismatch.
|
|
2344
|
+
* @param options {object}
|
|
2345
|
+
* @param options.spaceId {string}
|
|
2346
|
+
* @param options.collectionId {string}
|
|
2347
|
+
* @param options.resourceId {string}
|
|
2348
|
+
* @param options.chunkIndex {number}
|
|
2349
|
+
* @param [options.ifMatch] {string} `If-Match` precondition (a quoted ETag)
|
|
2350
|
+
* @returns {Promise<boolean>}
|
|
2351
|
+
*/
|
|
2352
|
+
async deleteChunk({ spaceId, collectionId, resourceId, chunkIndex, ifMatch }) {
|
|
2353
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
2354
|
+
const chunkDir = this._chunkDir({ collectionDir, resourceId });
|
|
2355
|
+
const chunkId = String(chunkIndex);
|
|
2356
|
+
return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), async () => {
|
|
2357
|
+
const files = await this._resourceFilesFor({
|
|
2358
|
+
collectionDir: chunkDir,
|
|
2359
|
+
resourceId: chunkId
|
|
2360
|
+
});
|
|
2361
|
+
if (files.length === 0) {
|
|
2362
|
+
// Absent: the handler 404s on `false` (chunk deletes are not silently
|
|
2363
|
+
// idempotent, mirroring the EDV chunk contract).
|
|
2364
|
+
return false;
|
|
2365
|
+
}
|
|
2366
|
+
if (ifMatch !== undefined) {
|
|
2367
|
+
await this._assertWritePrecondition({
|
|
2368
|
+
collectionDir: chunkDir,
|
|
2369
|
+
resourceId: chunkId,
|
|
2370
|
+
ifMatch
|
|
2371
|
+
});
|
|
2372
|
+
}
|
|
2373
|
+
await Promise.all(files.map(name => rm(name)));
|
|
2374
|
+
// Remove the version sidecar too: a chunk keeps no tombstone.
|
|
2375
|
+
await rm(this._metaSidecarPath({
|
|
2376
|
+
collectionDir: chunkDir,
|
|
2377
|
+
resourceId: chunkId
|
|
2378
|
+
}), { force: true });
|
|
2379
|
+
// When that was the last chunk, remove the now-empty chunk directory
|
|
2380
|
+
// itself: a lingering empty `.chunks.<encId>/` would otherwise appear
|
|
2381
|
+
// in the export walk (diverging from a Postgres export of the same
|
|
2382
|
+
// logical state) and count its allocated block toward the du-based
|
|
2383
|
+
// quota measurement. Safe under the per-Resource lock (`writeChunk`
|
|
2384
|
+
// serializes on the same key, so nothing lands in the directory
|
|
2385
|
+
// between the check and the rmdir).
|
|
2386
|
+
const remaining = await fs.promises.readdir(chunkDir);
|
|
2387
|
+
if (remaining.length === 0) {
|
|
2388
|
+
await fs.promises.rmdir(chunkDir);
|
|
2389
|
+
}
|
|
2390
|
+
// Freed bytes: drop the cached quota usage so the next write re-measures.
|
|
2391
|
+
this._usageCache.delete(spaceId);
|
|
2392
|
+
return true;
|
|
2393
|
+
});
|
|
2394
|
+
}
|
|
2395
|
+
/**
|
|
2396
|
+
* Lists a Resource's stored chunks in ascending `index` order -- the
|
|
2397
|
+
* discovery/reassembly listing (the server never reassembles). Resolves an
|
|
2398
|
+
* empty listing when the Resource has no chunk directory (including when the
|
|
2399
|
+
* Resource itself is absent -- existence is the parent routes' concern).
|
|
2400
|
+
* @param options {object}
|
|
2401
|
+
* @param options.spaceId {string}
|
|
2402
|
+
* @param options.collectionId {string}
|
|
2403
|
+
* @param options.resourceId {string}
|
|
2404
|
+
* @returns {Promise<{ count: number, chunks: Array<{ index: number, size:
|
|
2405
|
+
* number, contentType: string, version?: number }> }>}
|
|
2406
|
+
*/
|
|
2407
|
+
async listChunks({ spaceId, collectionId, resourceId }) {
|
|
2408
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
2409
|
+
const chunkDir = this._chunkDir({ collectionDir, resourceId });
|
|
2410
|
+
let entries;
|
|
2411
|
+
try {
|
|
2412
|
+
entries = await fs.promises.readdir(chunkDir, { withFileTypes: true });
|
|
2413
|
+
}
|
|
2414
|
+
catch (err) {
|
|
2415
|
+
if (err.code === 'ENOENT') {
|
|
2416
|
+
return { count: 0, chunks: [] };
|
|
2417
|
+
}
|
|
2418
|
+
throw err;
|
|
2419
|
+
}
|
|
2420
|
+
// Keep only chunk representations (`r.<index>.<type>.<ext>`), dropping the
|
|
2421
|
+
// `.meta.<index>.json` version sidecars.
|
|
2422
|
+
const chunkEntries = entries.filter(entry => entry.isFile() && entry.name.startsWith('r.'));
|
|
2423
|
+
const chunks = await Promise.all(chunkEntries.map(async (entry) => {
|
|
2424
|
+
const { resourceId: indexStr, contentType } = parseResourceFileName(entry.name);
|
|
2425
|
+
const filePath = path.join(chunkDir, entry.name);
|
|
2426
|
+
const stats = await fsStat(filePath);
|
|
2427
|
+
const sidecar = await this._readMetaSidecar({
|
|
2428
|
+
collectionDir: chunkDir,
|
|
2429
|
+
resourceId: indexStr
|
|
2430
|
+
});
|
|
2431
|
+
return {
|
|
2432
|
+
index: Number(indexStr),
|
|
2433
|
+
size: stats.size,
|
|
2434
|
+
contentType,
|
|
2435
|
+
...(sidecar?.version !== undefined && { version: sidecar.version })
|
|
2436
|
+
};
|
|
2437
|
+
}));
|
|
2438
|
+
chunks.sort((left, right) => left.index - right.index);
|
|
2439
|
+
return { count: chunks.length, chunks };
|
|
2440
|
+
}
|
|
2010
2441
|
/**
|
|
2011
2442
|
* Replication change feed (the `changes` query profile; see the
|
|
2012
2443
|
* `StorageBackend.changesSince` contract.
|