was-teaching-server 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1432 -0
- package/LICENSE +661 -0
- package/README.md +305 -0
- package/common/css/materialize.1.0.0.min.css +13 -0
- package/dist/auth-header-hooks.d.ts +84 -0
- package/dist/auth-header-hooks.d.ts.map +1 -0
- package/dist/auth-header-hooks.js +136 -0
- package/dist/auth-header-hooks.js.map +1 -0
- package/dist/authorize.d.ts +45 -0
- package/dist/authorize.d.ts.map +1 -0
- package/dist/authorize.js +76 -0
- package/dist/authorize.js.map +1 -0
- package/dist/backends/filesystem.d.ts +1179 -0
- package/dist/backends/filesystem.d.ts.map +1 -0
- package/dist/backends/filesystem.js +2727 -0
- package/dist/backends/filesystem.js.map +1 -0
- package/dist/backends/postgres.d.ts +833 -0
- package/dist/backends/postgres.d.ts.map +1 -0
- package/dist/backends/postgres.js +2164 -0
- package/dist/backends/postgres.js.map +1 -0
- package/dist/backends/postgresSchema.d.ts +36 -0
- package/dist/backends/postgresSchema.d.ts.map +1 -0
- package/dist/backends/postgresSchema.js +203 -0
- package/dist/backends/postgresSchema.js.map +1 -0
- package/dist/config.default.d.ts +360 -0
- package/dist/config.default.d.ts.map +1 -0
- package/dist/config.default.js +493 -0
- package/dist/config.default.js.map +1 -0
- package/dist/corsProxy.d.ts +45 -0
- package/dist/corsProxy.d.ts.map +1 -0
- package/dist/corsProxy.js +278 -0
- package/dist/corsProxy.js.map +1 -0
- package/dist/digest.d.ts +45 -0
- package/dist/digest.d.ts.map +1 -0
- package/dist/digest.js +174 -0
- package/dist/digest.js.map +1 -0
- package/dist/errors.d.ts +654 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +960 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/atomicFile.d.ts +61 -0
- package/dist/lib/atomicFile.d.ts.map +1 -0
- package/dist/lib/atomicFile.js +150 -0
- package/dist/lib/atomicFile.js.map +1 -0
- package/dist/lib/backendRegistry.d.ts +59 -0
- package/dist/lib/backendRegistry.d.ts.map +1 -0
- package/dist/lib/backendRegistry.js +92 -0
- package/dist/lib/backendRegistry.js.map +1 -0
- package/dist/lib/backendUsage.d.ts +28 -0
- package/dist/lib/backendUsage.d.ts.map +1 -0
- package/dist/lib/backendUsage.js +55 -0
- package/dist/lib/backendUsage.js.map +1 -0
- package/dist/lib/backends.d.ts +142 -0
- package/dist/lib/backends.d.ts.map +1 -0
- package/dist/lib/backends.js +272 -0
- package/dist/lib/backends.js.map +1 -0
- package/dist/lib/blindedIndex.d.ts +156 -0
- package/dist/lib/blindedIndex.d.ts.map +1 -0
- package/dist/lib/blindedIndex.js +338 -0
- package/dist/lib/blindedIndex.js.map +1 -0
- package/dist/lib/collectionDescription.d.ts +33 -0
- package/dist/lib/collectionDescription.d.ts.map +1 -0
- package/dist/lib/collectionDescription.js +21 -0
- package/dist/lib/collectionDescription.js.map +1 -0
- package/dist/lib/cursor.d.ts +18 -0
- package/dist/lib/cursor.d.ts.map +1 -0
- package/dist/lib/cursor.js +50 -0
- package/dist/lib/cursor.js.map +1 -0
- package/dist/lib/edvEnvelope.d.ts +50 -0
- package/dist/lib/edvEnvelope.d.ts.map +1 -0
- package/dist/lib/edvEnvelope.js +106 -0
- package/dist/lib/edvEnvelope.js.map +1 -0
- package/dist/lib/encryption.d.ts +192 -0
- package/dist/lib/encryption.d.ts.map +1 -0
- package/dist/lib/encryption.js +383 -0
- package/dist/lib/encryption.js.map +1 -0
- package/dist/lib/etag.d.ts +33 -0
- package/dist/lib/etag.d.ts.map +1 -0
- package/dist/lib/etag.js +36 -0
- package/dist/lib/etag.js.map +1 -0
- package/dist/lib/exportManifest.d.ts +29 -0
- package/dist/lib/exportManifest.d.ts.map +1 -0
- package/dist/lib/exportManifest.js +77 -0
- package/dist/lib/exportManifest.js.map +1 -0
- package/dist/lib/importTar.d.ts +81 -0
- package/dist/lib/importTar.d.ts.map +1 -0
- package/dist/lib/importTar.js +301 -0
- package/dist/lib/importTar.js.map +1 -0
- package/dist/lib/isJson.d.ts +16 -0
- package/dist/lib/isJson.d.ts.map +1 -0
- package/dist/lib/isJson.js +17 -0
- package/dist/lib/isJson.js.map +1 -0
- package/dist/lib/keyEpoch.d.ts +38 -0
- package/dist/lib/keyEpoch.d.ts.map +1 -0
- package/dist/lib/keyEpoch.js +75 -0
- package/dist/lib/keyEpoch.js.map +1 -0
- package/dist/lib/keyedMutex.d.ts +28 -0
- package/dist/lib/keyedMutex.d.ts.map +1 -0
- package/dist/lib/keyedMutex.js +45 -0
- package/dist/lib/keyedMutex.js.map +1 -0
- package/dist/lib/kmsModule.d.ts +62 -0
- package/dist/lib/kmsModule.d.ts.map +1 -0
- package/dist/lib/kmsModule.js +327 -0
- package/dist/lib/kmsModule.js.map +1 -0
- package/dist/lib/kmsRecordCipher.d.ts +82 -0
- package/dist/lib/kmsRecordCipher.d.ts.map +1 -0
- package/dist/lib/kmsRecordCipher.js +226 -0
- package/dist/lib/kmsRecordCipher.js.map +1 -0
- package/dist/lib/pagination.d.ts +19 -0
- package/dist/lib/pagination.d.ts.map +1 -0
- package/dist/lib/pagination.js +21 -0
- package/dist/lib/pagination.js.map +1 -0
- package/dist/lib/paths.d.ts +273 -0
- package/dist/lib/paths.d.ts.map +1 -0
- package/dist/lib/paths.js +270 -0
- package/dist/lib/paths.js.map +1 -0
- package/dist/lib/preconditions.d.ts +65 -0
- package/dist/lib/preconditions.d.ts.map +1 -0
- package/dist/lib/preconditions.js +112 -0
- package/dist/lib/preconditions.js.map +1 -0
- package/dist/lib/resourceFileName.d.ts +41 -0
- package/dist/lib/resourceFileName.d.ts.map +1 -0
- package/dist/lib/resourceFileName.js +58 -0
- package/dist/lib/resourceFileName.js.map +1 -0
- package/dist/lib/revocations.d.ts +49 -0
- package/dist/lib/revocations.d.ts.map +1 -0
- package/dist/lib/revocations.js +86 -0
- package/dist/lib/revocations.js.map +1 -0
- package/dist/lib/validateDid.d.ts +19 -0
- package/dist/lib/validateDid.d.ts.map +1 -0
- package/dist/lib/validateDid.js +35 -0
- package/dist/lib/validateDid.js.map +1 -0
- package/dist/lib/validateId.d.ts +66 -0
- package/dist/lib/validateId.d.ts.map +1 -0
- package/dist/lib/validateId.js +126 -0
- package/dist/lib/validateId.js.map +1 -0
- package/dist/plugin.d.ts +115 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/plugin.js +134 -0
- package/dist/plugin.js.map +1 -0
- package/dist/policy.d.ts +78 -0
- package/dist/policy.d.ts.map +1 -0
- package/dist/policy.js +115 -0
- package/dist/policy.js.map +1 -0
- package/dist/provisioning.d.ts +38 -0
- package/dist/provisioning.d.ts.map +1 -0
- package/dist/provisioning.js +72 -0
- package/dist/provisioning.js.map +1 -0
- package/dist/requests/BackendRequest.d.ts +61 -0
- package/dist/requests/BackendRequest.d.ts.map +1 -0
- package/dist/requests/BackendRequest.js +152 -0
- package/dist/requests/BackendRequest.js.map +1 -0
- package/dist/requests/CollectionRequest.d.ts +234 -0
- package/dist/requests/CollectionRequest.d.ts.map +1 -0
- package/dist/requests/CollectionRequest.js +705 -0
- package/dist/requests/CollectionRequest.js.map +1 -0
- package/dist/requests/KeyRequest.d.ts +115 -0
- package/dist/requests/KeyRequest.d.ts.map +1 -0
- package/dist/requests/KeyRequest.js +468 -0
- package/dist/requests/KeyRequest.js.map +1 -0
- package/dist/requests/KeystoreRequest.d.ts +99 -0
- package/dist/requests/KeystoreRequest.d.ts.map +1 -0
- package/dist/requests/KeystoreRequest.js +247 -0
- package/dist/requests/KeystoreRequest.js.map +1 -0
- package/dist/requests/PolicyRequest.d.ts +53 -0
- package/dist/requests/PolicyRequest.d.ts.map +1 -0
- package/dist/requests/PolicyRequest.js +124 -0
- package/dist/requests/PolicyRequest.js.map +1 -0
- package/dist/requests/ResourceRequest.d.ts +134 -0
- package/dist/requests/ResourceRequest.d.ts.map +1 -0
- package/dist/requests/ResourceRequest.js +548 -0
- package/dist/requests/ResourceRequest.js.map +1 -0
- package/dist/requests/RevocationRequest.d.ts +67 -0
- package/dist/requests/RevocationRequest.d.ts.map +1 -0
- package/dist/requests/RevocationRequest.js +187 -0
- package/dist/requests/RevocationRequest.js.map +1 -0
- package/dist/requests/SpaceRequest.d.ts +204 -0
- package/dist/requests/SpaceRequest.d.ts.map +1 -0
- package/dist/requests/SpaceRequest.js +520 -0
- package/dist/requests/SpaceRequest.js.map +1 -0
- package/dist/requests/SpacesRepositoryRequest.d.ts +50 -0
- package/dist/requests/SpacesRepositoryRequest.d.ts.map +1 -0
- package/dist/requests/SpacesRepositoryRequest.js +159 -0
- package/dist/requests/SpacesRepositoryRequest.js.map +1 -0
- package/dist/requests/collectionContext.d.ts +21 -0
- package/dist/requests/collectionContext.d.ts.map +1 -0
- package/dist/requests/collectionContext.js +30 -0
- package/dist/requests/collectionContext.js.map +1 -0
- package/dist/requests/controllerConsent.d.ts +58 -0
- package/dist/requests/controllerConsent.d.ts.map +1 -0
- package/dist/requests/controllerConsent.js +73 -0
- package/dist/requests/controllerConsent.js.map +1 -0
- package/dist/requests/keystoreContext.d.ts +68 -0
- package/dist/requests/keystoreContext.d.ts.map +1 -0
- package/dist/requests/keystoreContext.js +87 -0
- package/dist/requests/keystoreContext.js.map +1 -0
- package/dist/requests/resourceInput.d.ts +19 -0
- package/dist/requests/resourceInput.d.ts.map +1 -0
- package/dist/requests/resourceInput.js +102 -0
- package/dist/requests/resourceInput.js.map +1 -0
- package/dist/requests/spaceContext.d.ts +127 -0
- package/dist/requests/spaceContext.d.ts.map +1 -0
- package/dist/requests/spaceContext.js +201 -0
- package/dist/requests/spaceContext.js.map +1 -0
- package/dist/routes.d.ts +55 -0
- package/dist/routes.d.ts.map +1 -0
- package/dist/routes.js +306 -0
- package/dist/routes.js.map +1 -0
- package/dist/server.d.ts +20 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +60 -0
- package/dist/server.js.map +1 -0
- package/dist/start.d.ts +7 -0
- package/dist/start.d.ts.map +1 -0
- package/dist/start.js +61 -0
- package/dist/start.js.map +1 -0
- package/dist/storage.d.ts +29 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/storage.js +47 -0
- package/dist/storage.js.map +1 -0
- package/dist/types.d.ts +915 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/views/home.hbs +9 -0
- package/dist/views/templates/main.hbs +12 -0
- package/dist/zcap.d.ts +267 -0
- package/dist/zcap.d.ts.map +1 -0
- package/dist/zcap.js +478 -0
- package/dist/zcap.js.map +1 -0
- package/package.json +109 -0
- package/src/auth-header-hooks.ts +163 -0
- package/src/authorize.ts +117 -0
- package/src/backends/filesystem.ts +3626 -0
- package/src/backends/postgres.ts +3073 -0
- package/src/backends/postgresSchema.ts +231 -0
- package/src/config.default.ts +657 -0
- package/src/corsProxy.ts +326 -0
- package/src/declarations.d.ts +28 -0
- package/src/digest.ts +223 -0
- package/src/errors.ts +1122 -0
- package/src/index.ts +21 -0
- package/src/lib/atomicFile.ts +176 -0
- package/src/lib/backendRegistry.ts +139 -0
- package/src/lib/backendUsage.ts +79 -0
- package/src/lib/backends.ts +351 -0
- package/src/lib/blindedIndex.ts +466 -0
- package/src/lib/collectionDescription.ts +43 -0
- package/src/lib/cursor.ts +52 -0
- package/src/lib/edvEnvelope.ts +120 -0
- package/src/lib/encryption.ts +472 -0
- package/src/lib/etag.ts +40 -0
- package/src/lib/exportManifest.ts +115 -0
- package/src/lib/importTar.ts +381 -0
- package/src/lib/isJson.ts +18 -0
- package/src/lib/keyEpoch.ts +89 -0
- package/src/lib/keyedMutex.ts +45 -0
- package/src/lib/kmsModule.ts +451 -0
- package/src/lib/kmsRecordCipher.ts +292 -0
- package/src/lib/pagination.ts +22 -0
- package/src/lib/paths.ts +382 -0
- package/src/lib/preconditions.ts +145 -0
- package/src/lib/resourceFileName.ts +69 -0
- package/src/lib/revocations.ts +116 -0
- package/src/lib/validateDid.ts +41 -0
- package/src/lib/validateId.ts +148 -0
- package/src/plugin.ts +288 -0
- package/src/policy.ts +176 -0
- package/src/provisioning.ts +98 -0
- package/src/requests/BackendRequest.ts +219 -0
- package/src/requests/CollectionRequest.ts +899 -0
- package/src/requests/KeyRequest.ts +617 -0
- package/src/requests/KeystoreRequest.ts +325 -0
- package/src/requests/PolicyRequest.ts +169 -0
- package/src/requests/ResourceRequest.ts +649 -0
- package/src/requests/RevocationRequest.ts +267 -0
- package/src/requests/SpaceRequest.ts +686 -0
- package/src/requests/SpacesRepositoryRequest.ts +194 -0
- package/src/requests/collectionContext.ts +41 -0
- package/src/requests/controllerConsent.ts +108 -0
- package/src/requests/keystoreContext.ts +124 -0
- package/src/requests/resourceInput.ts +118 -0
- package/src/requests/spaceContext.ts +311 -0
- package/src/routes.ts +420 -0
- package/src/server.ts +69 -0
- package/src/start.ts +71 -0
- package/src/storage.ts +60 -0
- package/src/types.ts +1016 -0
- package/src/views/home.hbs +9 -0
- package/src/views/templates/main.hbs +12 -0
- package/src/zcap.ts +657 -0
|
@@ -0,0 +1,2727 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filesystem persistence backend: stores Spaces, Collections, and Resources as
|
|
3
|
+
* directories and files under `data/spaces/`, and WebKMS keystores under the
|
|
4
|
+
* sibling `data/keystores/` tree. The default (and currently only) adapter
|
|
5
|
+
* implementing the StorageBackend contract documented in types.ts.
|
|
6
|
+
*/
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
import { mkdir, rm, stat as fsStat } from 'node:fs/promises';
|
|
9
|
+
import { pipeline } from 'node:stream/promises';
|
|
10
|
+
import { execFile } from 'node:child_process';
|
|
11
|
+
import { promisify } from 'node:util';
|
|
12
|
+
import { Transform } from 'node:stream';
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import jsonfs from 'fs-json-store';
|
|
15
|
+
import pino from 'pino';
|
|
16
|
+
import { StorageError, ResourceNotFoundError, SpaceNotFoundError, QuotaExceededError, CountQuotaExceededError, PayloadTooLargeError, KeystoreStateConflictError, KeyIdConflictError, DuplicateRevocationError } from '../errors.js';
|
|
17
|
+
import * as tar from 'tar-stream';
|
|
18
|
+
import YAML from 'yaml';
|
|
19
|
+
import { DEFAULT_MAX_UPLOAD_BYTES, DEFAULT_MAX_SPACES_PER_CONTROLLER, DEFAULT_MAX_COLLECTIONS_PER_SPACE, DEFAULT_MAX_RESOURCES_PER_SPACE, QUOTA_USAGE_CACHE_TTL, normalizeCountLimit } from '../config.default.js';
|
|
20
|
+
import { extractTarEntries, buildImportPlan, metaSidecarFileId } from '../lib/importTar.js';
|
|
21
|
+
import { collectionPath, resourcePath } from '../lib/paths.js';
|
|
22
|
+
import { encodeFilenameSegment, fileNameFor, parseResourceFileName } from '../lib/resourceFileName.js';
|
|
23
|
+
import { sanitizeBackendRecord } from '../lib/backends.js';
|
|
24
|
+
import { backendUsageFields } from '../lib/backendUsage.js';
|
|
25
|
+
import { assertEncryptedWriteConforms } from '../lib/encryption.js';
|
|
26
|
+
import { encodeCursor, decodeCursor } from '../lib/cursor.js';
|
|
27
|
+
import { buildExportManifest } from '../lib/exportManifest.js';
|
|
28
|
+
import { revocationFileName } from '../lib/revocations.js';
|
|
29
|
+
import { KeyedMutex } from '../lib/keyedMutex.js';
|
|
30
|
+
import { isJson } from '../lib/isJson.js';
|
|
31
|
+
import { normalizeDescriptionWrite } from '../lib/collectionDescription.js';
|
|
32
|
+
import { atomicWriteFile, atomicCreateFile, tempPathFor, commitTempFile } from '../lib/atomicFile.js';
|
|
33
|
+
import { DEFAULT_PAGE_SIZE, clampPageSize } from '../lib/pagination.js';
|
|
34
|
+
import { runBlindedIndexQuery, collectUniqueBlindedTerms, assertNoUniqueBlindedConflict } from '../lib/blindedIndex.js';
|
|
35
|
+
import { assertWritePrecondition, assertMetaWritePrecondition, assertCollectionWritePrecondition } from '../lib/preconditions.js';
|
|
36
|
+
const { Store: MetadataJsonStore } = jsonfs;
|
|
37
|
+
const execFileAsync = promisify(execFile);
|
|
38
|
+
/**
|
|
39
|
+
* Compares two strings in code-unit order (the order the `<` / `>` operators
|
|
40
|
+
* use), returning -1 / 0 / 1. Keyset pagination sorts and seeks with the same
|
|
41
|
+
* operator, so the comparator must agree with `>` -- `localeCompare` can not.
|
|
42
|
+
*/
|
|
43
|
+
function compareCodeUnits(left, right) {
|
|
44
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Silent logger used when no logger is injected into the backend, so the backend
|
|
48
|
+
* stays quiet by default (e.g. in `defaultBackend()` before `createApp` wires
|
|
49
|
+
* `fastify.log` in, or in tests).
|
|
50
|
+
*/
|
|
51
|
+
const silentLogger = pino({ level: 'silent' });
|
|
52
|
+
/**
|
|
53
|
+
* Builds the on-disk filename for a Resource's metadata sidecar:
|
|
54
|
+
* `.meta.<resourceId>.json`. A dot-file kept alongside the resource
|
|
55
|
+
* representation in the Collection dir (the same convention as `.policy.` /
|
|
56
|
+
* `.collection.`), holding the timestamps and user-writable `custom` object.
|
|
57
|
+
* @param resourceId {string}
|
|
58
|
+
* @returns {string}
|
|
59
|
+
*/
|
|
60
|
+
export function metaSidecarFileName(resourceId) {
|
|
61
|
+
return `.meta.${resourceId}.json`;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Opens a read stream for a file, resolving once the stream has opened (and
|
|
65
|
+
* rejecting if it errors first).
|
|
66
|
+
* @param filePath {string}
|
|
67
|
+
* @param logger {FastifyBaseLogger}
|
|
68
|
+
* @returns {Promise<import('node:fs').ReadStream>}
|
|
69
|
+
*/
|
|
70
|
+
async function openFileStream(filePath, logger) {
|
|
71
|
+
const resourceStream = fs.createReadStream(filePath);
|
|
72
|
+
return new Promise((resolve, reject) => {
|
|
73
|
+
resourceStream
|
|
74
|
+
.on('error', error => {
|
|
75
|
+
reject(new Error(`Error creating a read stream: ${error}`));
|
|
76
|
+
})
|
|
77
|
+
.on('open', () => {
|
|
78
|
+
logger.info(`GET -- Reading ${filePath}`);
|
|
79
|
+
resolve(resourceStream);
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
export class FileSystemBackend {
|
|
84
|
+
spacesDir;
|
|
85
|
+
/**
|
|
86
|
+
* Root of the WebKMS keystore tree (`data/keystores/<localId>/`), a sibling
|
|
87
|
+
* of `spacesDir` -- the `/kms` facet is deliberately separable from Spaces
|
|
88
|
+
* (own route family, own storage tree).
|
|
89
|
+
*/
|
|
90
|
+
keystoresDir;
|
|
91
|
+
/**
|
|
92
|
+
* Root of the Space zcap revocation tree (`data/space-revocations/<spaceId>/`),
|
|
93
|
+
* a sibling of `spacesDir` rather than a subdirectory of each Space. Space
|
|
94
|
+
* revocations deliberately live OUTSIDE the Space's own directory because
|
|
95
|
+
* `listCollections` and `_countLiveResources` treat every subdirectory of a
|
|
96
|
+
* Space dir as a Collection -- a `revocations/` dir nested inside a Space would
|
|
97
|
+
* surface as a phantom Collection (and could collide with a real one), so it
|
|
98
|
+
* gets its own root.
|
|
99
|
+
*/
|
|
100
|
+
spaceRevocationsDir;
|
|
101
|
+
logger;
|
|
102
|
+
/**
|
|
103
|
+
* Per-Space storage capacity, in bytes (spec "Quotas"). `undefined` means no
|
|
104
|
+
* configured limit -- the backend reports an unlimited quota (state always
|
|
105
|
+
* `ok`) and skips write-path enforcement. A finite value drives the
|
|
106
|
+
* `near-limit` / `over-quota` state thresholds (see `reportUsage`) and is
|
|
107
|
+
* enforced on the write path: `writeResource` and `importSpace` reject writes
|
|
108
|
+
* that would push a Space over capacity with `QuotaExceededError` (507). The
|
|
109
|
+
* constructor normalizes a non-finite ctor option (`Infinity`) to `undefined`.
|
|
110
|
+
*/
|
|
111
|
+
capacityBytes;
|
|
112
|
+
/**
|
|
113
|
+
* Largest single upload the backend accepts, in bytes (spec "Quotas", the
|
|
114
|
+
* `maxUploadBytes` constraint). `undefined` means no per-upload cap. Distinct
|
|
115
|
+
* from `capacityBytes` (the cumulative per-Space quota): a write larger than
|
|
116
|
+
* this cap is rejected with `PayloadTooLargeError` (413) even when the Space
|
|
117
|
+
* has ample headroom, while smaller writes still succeed. Advertised in quota
|
|
118
|
+
* reports under `constraints.maxUploadBytes` and enforced on `writeResource`.
|
|
119
|
+
* The constructor normalizes an unset ctor option to
|
|
120
|
+
* {@link DEFAULT_MAX_UPLOAD_BYTES} (a default-on cap) and a non-finite option
|
|
121
|
+
* (`Infinity`) to `undefined` (explicitly no cap).
|
|
122
|
+
*/
|
|
123
|
+
maxUploadBytes;
|
|
124
|
+
/**
|
|
125
|
+
* Max Spaces a single controller may create (spec "Quotas", a default-on
|
|
126
|
+
* count quota). `undefined` means no cap. Enforced on the Space create path
|
|
127
|
+
* (`writeSpace`): a new Space whose `controller` already owns this many
|
|
128
|
+
* Spaces is rejected with `CountQuotaExceededError` (507); overwriting an
|
|
129
|
+
* existing Space never trips it. The constructor normalizes an unset ctor
|
|
130
|
+
* option to {@link DEFAULT_MAX_SPACES_PER_CONTROLLER} and a non-finite option
|
|
131
|
+
* (`Infinity`) to `undefined` (explicitly no cap). Soft under concurrency,
|
|
132
|
+
* like the byte quota.
|
|
133
|
+
*/
|
|
134
|
+
maxSpacesPerController;
|
|
135
|
+
/**
|
|
136
|
+
* Max Collections a single Space may hold (spec "Quotas", a default-on count
|
|
137
|
+
* quota). `undefined` means no cap. Enforced on the Collection create path
|
|
138
|
+
* (`writeCollection`); overwriting an existing Collection description never
|
|
139
|
+
* trips it. Normalized like {@link maxSpacesPerController}.
|
|
140
|
+
*/
|
|
141
|
+
maxCollectionsPerSpace;
|
|
142
|
+
/**
|
|
143
|
+
* Max live Resources a single Space may hold across all its Collections (spec
|
|
144
|
+
* "Quotas", a default-on count quota). `undefined` means no cap. Enforced on
|
|
145
|
+
* the Resource create path (`writeResource`); a tombstone does not count, and
|
|
146
|
+
* a write over an existing live Resource never trips it. Normalized like
|
|
147
|
+
* {@link maxSpacesPerController}.
|
|
148
|
+
*/
|
|
149
|
+
maxResourcesPerSpace;
|
|
150
|
+
/**
|
|
151
|
+
* Per-Resource write serialization (the `conditional-writes` feature). A
|
|
152
|
+
* content write or delete that carries a precondition reads the current
|
|
153
|
+
* `version`, evaluates `If-Match` / `If-None-Match`, and writes -- all under
|
|
154
|
+
* this lock, keyed per Resource -- so two concurrent writers cannot both
|
|
155
|
+
* observe the same prior version and both succeed. Single-instance only.
|
|
156
|
+
*/
|
|
157
|
+
_writeMutex = new KeyedMutex();
|
|
158
|
+
/**
|
|
159
|
+
* Per-Space usage totals for the write-path quota pre-flight, so
|
|
160
|
+
* `_assertSpaceHeadroom` does not spawn `du` (a whole-Space tree walk) on
|
|
161
|
+
* every resource write. Entries live `QUOTA_USAGE_CACHE_TTL` ms; each
|
|
162
|
+
* accepted write adds its incoming bytes to the cached total, and deletes
|
|
163
|
+
* invalidate the Space's entry. Quota reports (`reportUsage`) always
|
|
164
|
+
* re-measure. Single-instance only, like `_writeMutex`.
|
|
165
|
+
*/
|
|
166
|
+
_usageCache = new Map();
|
|
167
|
+
constructor({ dataDir, logger, capacityBytes, maxUploadBytes, maxSpacesPerController, maxCollectionsPerSpace, maxResourcesPerSpace }) {
|
|
168
|
+
this.spacesDir = path.join(dataDir, 'spaces');
|
|
169
|
+
this.keystoresDir = path.join(dataDir, 'keystores');
|
|
170
|
+
// A sibling of spacesDir, NOT nested under each Space: a `revocations/` dir
|
|
171
|
+
// inside a Space dir would be mistaken for a Collection (see the
|
|
172
|
+
// `spaceRevocationsDir` property doc).
|
|
173
|
+
this.spaceRevocationsDir = path.join(dataDir, 'space-revocations');
|
|
174
|
+
this.logger = logger ?? silentLogger;
|
|
175
|
+
// A non-finite `capacityBytes` (`Infinity` from an explicit `unlimited`)
|
|
176
|
+
// behaves exactly like unset inside the backend: no configured limit.
|
|
177
|
+
this.capacityBytes =
|
|
178
|
+
capacityBytes !== undefined && Number.isFinite(capacityBytes)
|
|
179
|
+
? capacityBytes
|
|
180
|
+
: undefined;
|
|
181
|
+
// Normalize the per-upload cap so every downstream guard keeps its plain
|
|
182
|
+
// `!== undefined` test: an unset option applies the default-on cap; a
|
|
183
|
+
// non-finite option (`Infinity`) means explicitly no cap (the streaming
|
|
184
|
+
// write path this backend uses makes an unbounded upload safe).
|
|
185
|
+
this.maxUploadBytes =
|
|
186
|
+
maxUploadBytes === undefined
|
|
187
|
+
? DEFAULT_MAX_UPLOAD_BYTES
|
|
188
|
+
: Number.isFinite(maxUploadBytes)
|
|
189
|
+
? maxUploadBytes
|
|
190
|
+
: undefined;
|
|
191
|
+
// Count quotas normalize like `maxUploadBytes`: an unset option applies the
|
|
192
|
+
// default-on limit, a non-finite option (`Infinity`) means explicitly no
|
|
193
|
+
// cap, so every guard keeps its plain `!== undefined` test.
|
|
194
|
+
this.maxSpacesPerController = normalizeCountLimit(maxSpacesPerController, DEFAULT_MAX_SPACES_PER_CONTROLLER);
|
|
195
|
+
this.maxCollectionsPerSpace = normalizeCountLimit(maxCollectionsPerSpace, DEFAULT_MAX_COLLECTIONS_PER_SPACE);
|
|
196
|
+
this.maxResourcesPerSpace = normalizeCountLimit(maxResourcesPerSpace, DEFAULT_MAX_RESOURCES_PER_SPACE);
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Self-description advertised at `GET /space/:spaceId/backends`. The
|
|
200
|
+
* filesystem backend is the single server-configured default: it stores both
|
|
201
|
+
* JSON documents and binary blobs on disk, so its data survives restarts.
|
|
202
|
+
*
|
|
203
|
+
* It advertises the `conditional-writes` affordance: it exposes a per-Resource
|
|
204
|
+
* `version` as an HTTP `ETag` validator and honors `If-Match` / `If-None-Match`
|
|
205
|
+
* write preconditions atomically (returning `412 precondition-failed` on a
|
|
206
|
+
* mismatch). The remaining `features` vocabulary token (`chunked-streams`)
|
|
207
|
+
* is not implemented yet and is added when it lands.
|
|
208
|
+
* (Client-side encryption is deliberately not a backend feature: encrypted
|
|
209
|
+
* documents are opaque client-encrypted JSON this backend already stores
|
|
210
|
+
* faithfully, with no server cooperation.)
|
|
211
|
+
* @returns {Required<Omit<BackendDescriptor, 'provider' | 'connection'>>}
|
|
212
|
+
*/
|
|
213
|
+
describe() {
|
|
214
|
+
// The wire type only REQUIRES `id`; this backend always populates every
|
|
215
|
+
// field except the `external`-only `provider` / `connection` (the default
|
|
216
|
+
// backend is server-managed), so its return is the stricter
|
|
217
|
+
// `Required<Omit<..., 'provider' | 'connection'>>` (which also lets
|
|
218
|
+
// `reportUsage` read a non-optional `managedBy` off `describe()`).
|
|
219
|
+
return {
|
|
220
|
+
id: 'default',
|
|
221
|
+
name: 'Server Filesystem',
|
|
222
|
+
managedBy: 'server',
|
|
223
|
+
storageMode: ['document', 'blob'],
|
|
224
|
+
persistence: 'durable',
|
|
225
|
+
// `changes-query`: serves the `changes` profile of the reserved `query`
|
|
226
|
+
// endpoint -- the replication change feed (`changesSince`).
|
|
227
|
+
// `blinded-index-query`: serves the `blinded-index` profile -- EDV
|
|
228
|
+
// blinded-attribute queries (`queryByBlindedIndex`).
|
|
229
|
+
// `key-epochs`: multi-recipient encrypted Collections -- per-epoch wrapped
|
|
230
|
+
// keys on the `encryption` marker, a client-declared `epoch` stamp on
|
|
231
|
+
// Resources, and conditional (`If-Match`) Collection Description writes.
|
|
232
|
+
features: [
|
|
233
|
+
'conditional-writes',
|
|
234
|
+
'changes-query',
|
|
235
|
+
'blinded-index-query',
|
|
236
|
+
'key-epochs'
|
|
237
|
+
]
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Measures disk usage under the Space dir with `du`, returning the grand total
|
|
242
|
+
* and a per-Collection breakdown in one pass. `du -d 1 -B 1` (GNU coreutils)
|
|
243
|
+
* reports each immediate subdirectory (one per Collection) plus the Space dir
|
|
244
|
+
* itself (the total, which also covers top-level Space files such as the
|
|
245
|
+
* `.space.` / `.policy.` documents), all in bytes. An absent Space dir (ENOENT
|
|
246
|
+
* before the dir is provisioned) reports zero usage rather than throwing.
|
|
247
|
+
* @param spaceDir {string}
|
|
248
|
+
* @returns {Promise<{ total: number, byCollection: CollectionUsage[] }>}
|
|
249
|
+
*/
|
|
250
|
+
async _diskUsage(spaceDir) {
|
|
251
|
+
let stdout;
|
|
252
|
+
try {
|
|
253
|
+
;
|
|
254
|
+
({ stdout } = await execFileAsync('du', [
|
|
255
|
+
'-d',
|
|
256
|
+
'1',
|
|
257
|
+
'-B',
|
|
258
|
+
'1',
|
|
259
|
+
spaceDir
|
|
260
|
+
]));
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
// `du` exits non-zero (with an ENOENT-style stderr) when the dir is
|
|
264
|
+
// absent; treat that as zero usage. Anything else is a real failure.
|
|
265
|
+
if (err.code === 'ENOENT' ||
|
|
266
|
+
/No such file or directory/.test(err.message)) {
|
|
267
|
+
return { total: 0, byCollection: [] };
|
|
268
|
+
}
|
|
269
|
+
throw new StorageError({ cause: err });
|
|
270
|
+
}
|
|
271
|
+
const rootResolved = path.resolve(spaceDir);
|
|
272
|
+
let total = 0;
|
|
273
|
+
const byCollection = [];
|
|
274
|
+
for (const line of stdout.split('\n')) {
|
|
275
|
+
if (!line) {
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
const tab = line.indexOf('\t');
|
|
279
|
+
const usageBytes = Number(line.slice(0, tab));
|
|
280
|
+
const entryPath = line.slice(tab + 1);
|
|
281
|
+
if (path.resolve(entryPath) === rootResolved) {
|
|
282
|
+
// The summary line for the Space dir itself is the grand total.
|
|
283
|
+
total = usageBytes;
|
|
284
|
+
}
|
|
285
|
+
else {
|
|
286
|
+
// Every immediate subdirectory is a Collection (see `listCollections`).
|
|
287
|
+
byCollection.push({ id: path.basename(entryPath), usageBytes });
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
byCollection.sort((a, b) => a.id.localeCompare(b.id));
|
|
291
|
+
return { total, byCollection };
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Measures the bytes this Space consumes on disk for the Space Quota report
|
|
295
|
+
* (spec "Quotas"). `usageBytes` is the `du` total under the Space dir
|
|
296
|
+
* (Collection dirs plus top-level Space files); `usageByCollection` breaks the
|
|
297
|
+
* per-Collection totals out (they sum to slightly less than `usageBytes`,
|
|
298
|
+
* since the Space-level files belong to no Collection).
|
|
299
|
+
*
|
|
300
|
+
* The per-Collection `usageByCollection` breakdown is included only when
|
|
301
|
+
* `includeCollections` is set -- the spec's `?include=collections` opt-in (see
|
|
302
|
+
* the `quotas` handler, which now tolerates the query string via the
|
|
303
|
+
* `allowTargetQuery` ZCap path). On the filesystem the breakdown is free (the
|
|
304
|
+
* one `du -d 1` pass yields it alongside the total), but it is still omitted by
|
|
305
|
+
* default to keep the hot-path payload lean and match the wire contract.
|
|
306
|
+
*
|
|
307
|
+
* `state` / `restrictedActions` derive from usage vs `capacityBytes`: an
|
|
308
|
+
* unlimited backend is always `ok`; a finite capacity yields `near-limit` at
|
|
309
|
+
* `QUOTA_NEAR_LIMIT_FRACTION` of capacity and `over-quota` (with reads/deletes
|
|
310
|
+
* still allowed, but `POST`/`PUT` restricted) at or above full.
|
|
311
|
+
* @param options {object}
|
|
312
|
+
* @param options.spaceId {string}
|
|
313
|
+
* @param [options.includeCollections] {boolean} include the per-Collection
|
|
314
|
+
* breakdown (spec `?include=collections`)
|
|
315
|
+
* @returns {Promise<BackendUsage>}
|
|
316
|
+
*/
|
|
317
|
+
async reportUsage({ spaceId, includeCollections = false }) {
|
|
318
|
+
const spaceDir = this._spaceDir(spaceId);
|
|
319
|
+
const measuredAt = new Date().toISOString();
|
|
320
|
+
const { total: usageBytes, byCollection: usageByCollection } = await this._diskUsage(spaceDir);
|
|
321
|
+
return {
|
|
322
|
+
...this._backendUsageFields({ usageBytes, spaceTotalBytes: usageBytes }),
|
|
323
|
+
measuredAt,
|
|
324
|
+
...(includeCollections && { usageByCollection })
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Measures the bytes a single Collection consumes on disk for the
|
|
329
|
+
* per-Collection Quota report (spec "Quotas", `GET /space/{id}/{cid}/quota`).
|
|
330
|
+
* `usageBytes` is scoped to the Collection (its slice of the one-pass
|
|
331
|
+
* `_diskUsage` breakdown; zero if the Collection dir is empty or absent),
|
|
332
|
+
* while `state` / `limit` / `restrictedActions` describe the backend's overall
|
|
333
|
+
* condition (derived from the Space total -- the quota is a per-backend limit,
|
|
334
|
+
* not per-Collection). The per-Collection `usageByCollection` breakdown is
|
|
335
|
+
* omitted (a single Collection is the whole report).
|
|
336
|
+
* @param options {object}
|
|
337
|
+
* @param options.spaceId {string}
|
|
338
|
+
* @param options.collectionId {string}
|
|
339
|
+
* @returns {Promise<BackendUsage>}
|
|
340
|
+
*/
|
|
341
|
+
async reportCollectionUsage({ spaceId, collectionId }) {
|
|
342
|
+
const spaceDir = this._spaceDir(spaceId);
|
|
343
|
+
const measuredAt = new Date().toISOString();
|
|
344
|
+
const { total: spaceTotalBytes, byCollection } = await this._diskUsage(spaceDir);
|
|
345
|
+
const usageBytes = byCollection.find(entry => entry.id === collectionId)?.usageBytes ?? 0;
|
|
346
|
+
return {
|
|
347
|
+
...this._backendUsageFields({ usageBytes, spaceTotalBytes }),
|
|
348
|
+
measuredAt
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Builds the backend-identity and condition fields shared by the Space and
|
|
353
|
+
* per-Collection quota reports. `usageBytes` is what the report shows (the
|
|
354
|
+
* Space total or a single Collection's slice); `spaceTotalBytes` drives the
|
|
355
|
+
* `state` / `restrictedActions`, which are backend-wide (the quota is a
|
|
356
|
+
* per-Space limit) and so always measured against the Space total. The
|
|
357
|
+
* `constraints.maxUploadBytes` cap is advertised when configured.
|
|
358
|
+
* @param options {object}
|
|
359
|
+
* @param options.usageBytes {number} the usage figure to report
|
|
360
|
+
* @param options.spaceTotalBytes {number} the Space total, for state
|
|
361
|
+
* @returns {Omit<BackendUsage, 'measuredAt' | 'usageByCollection'>}
|
|
362
|
+
*/
|
|
363
|
+
_backendUsageFields({ usageBytes, spaceTotalBytes }) {
|
|
364
|
+
const { id, name, managedBy } = this.describe();
|
|
365
|
+
return backendUsageFields({
|
|
366
|
+
usageBytes,
|
|
367
|
+
spaceTotalBytes,
|
|
368
|
+
capacityBytes: this.capacityBytes,
|
|
369
|
+
maxUploadBytes: this.maxUploadBytes,
|
|
370
|
+
id,
|
|
371
|
+
name,
|
|
372
|
+
managedBy
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Quota pre-flight for the write path (spec "Quotas"): measures the Space's
|
|
377
|
+
* current on-disk usage and returns the remaining headroom in bytes against
|
|
378
|
+
* `capacityBytes`. Throws `QuotaExceededError` (507) when the Space is already
|
|
379
|
+
* at or over capacity, or when a known `incomingBytes` would not fit. Callers
|
|
380
|
+
* pass the configured `capacityBytes` explicitly (an unlimited backend skips
|
|
381
|
+
* enforcement entirely and never calls this).
|
|
382
|
+
*
|
|
383
|
+
* This is a soft limit under concurrency: two simultaneous writes can each pass
|
|
384
|
+
* against the same usage snapshot and jointly overshoot. The per-write
|
|
385
|
+
* streaming guard (`_quotaGuard`) still bounds each individual write.
|
|
386
|
+
*
|
|
387
|
+
* The `du` measurement (a whole-Space tree walk) is cached per Space for
|
|
388
|
+
* `QUOTA_USAGE_CACHE_TTL` ms (see `_usageCache`): between re-measurements
|
|
389
|
+
* each accepted write's `incomingBytes` is added to the cached total, so a
|
|
390
|
+
* burst of writes costs one tree walk, not one per write.
|
|
391
|
+
* @param options {object}
|
|
392
|
+
* @param options.spaceId {string}
|
|
393
|
+
* @param options.capacityBytes {number} the configured per-Space limit
|
|
394
|
+
* @param [options.incomingBytes] {number} known size of the pending write
|
|
395
|
+
* @returns {Promise<number>} remaining headroom in bytes
|
|
396
|
+
*/
|
|
397
|
+
async _assertSpaceHeadroom({ spaceId, capacityBytes, incomingBytes = 0 }) {
|
|
398
|
+
let cached = this._usageCache.get(spaceId);
|
|
399
|
+
if (!cached || cached.expiresAt <= Date.now()) {
|
|
400
|
+
const { total } = await this._diskUsage(this._spaceDir(spaceId));
|
|
401
|
+
cached = {
|
|
402
|
+
usageBytes: total,
|
|
403
|
+
expiresAt: Date.now() + QUOTA_USAGE_CACHE_TTL
|
|
404
|
+
};
|
|
405
|
+
this._usageCache.set(spaceId, cached);
|
|
406
|
+
}
|
|
407
|
+
const headroom = capacityBytes - cached.usageBytes;
|
|
408
|
+
if (headroom <= 0 || incomingBytes > headroom) {
|
|
409
|
+
throw new QuotaExceededError({ spaceId, capacityBytes });
|
|
410
|
+
}
|
|
411
|
+
// Count the accepted write against the cached total so writes within the
|
|
412
|
+
// TTL accumulate rather than each re-admitting against the same snapshot.
|
|
413
|
+
cached.usageBytes += incomingBytes;
|
|
414
|
+
return headroom;
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* A pass-through `Transform` that counts the bytes flowing through it and
|
|
418
|
+
* aborts the pipeline with `QuotaExceededError` (507) once the cumulative total
|
|
419
|
+
* would exceed `headroomBytes`. Hard-caps a streamed blob write whose size is
|
|
420
|
+
* not known up front (so the pre-flight check alone cannot catch it), e.g. a
|
|
421
|
+
* multipart upload or a body without `Content-Length`.
|
|
422
|
+
* @param options {object}
|
|
423
|
+
* @param options.spaceId {string}
|
|
424
|
+
* @param options.capacityBytes {number} the configured per-Space limit
|
|
425
|
+
* @param options.headroomBytes {number} max bytes this write may add
|
|
426
|
+
* @returns {Transform}
|
|
427
|
+
*/
|
|
428
|
+
_quotaGuard({ spaceId, capacityBytes, headroomBytes }) {
|
|
429
|
+
let written = 0;
|
|
430
|
+
return new Transform({
|
|
431
|
+
transform(chunk, _encoding, callback) {
|
|
432
|
+
written += chunk.length;
|
|
433
|
+
if (written > headroomBytes) {
|
|
434
|
+
callback(new QuotaExceededError({ spaceId, capacityBytes }));
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
callback(null, chunk);
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
/**
|
|
442
|
+
* A pass-through `Transform` that counts the bytes flowing through it and
|
|
443
|
+
* aborts the pipeline with `PayloadTooLargeError` (413) once the cumulative
|
|
444
|
+
* total exceeds `maxUploadBytes`. Caps a single streamed upload whose size is
|
|
445
|
+
* not known up front (e.g. a multipart part or a body without
|
|
446
|
+
* `Content-Length`), independently of the cumulative Space quota.
|
|
447
|
+
* @param options {object}
|
|
448
|
+
* @param options.maxUploadBytes {number} the per-upload cap in bytes
|
|
449
|
+
* @returns {Transform}
|
|
450
|
+
*/
|
|
451
|
+
_uploadCapGuard({ maxUploadBytes }) {
|
|
452
|
+
const backendId = this.describe().id;
|
|
453
|
+
let written = 0;
|
|
454
|
+
return new Transform({
|
|
455
|
+
transform(chunk, _encoding, callback) {
|
|
456
|
+
written += chunk.length;
|
|
457
|
+
if (written > maxUploadBytes) {
|
|
458
|
+
callback(new PayloadTooLargeError({ maxUploadBytes, backendId }));
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
callback(null, chunk);
|
|
462
|
+
}
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Defense in depth: asserts that a built path stays within the given storage
|
|
467
|
+
* root (`spacesDir` by default; `keystoresDir` for the keystore tree), so a
|
|
468
|
+
* malformed id that somehow slips past request-layer validation can never
|
|
469
|
+
* escape it (path traversal). The request and tar-import layers reject such
|
|
470
|
+
* ids first; this is the last line of defense.
|
|
471
|
+
* @param targetPath {string}
|
|
472
|
+
* @param [rootDir] {string} the containing root; defaults to `spacesDir`
|
|
473
|
+
* @returns {void}
|
|
474
|
+
*/
|
|
475
|
+
_assertContained(targetPath, rootDir = this.spacesDir) {
|
|
476
|
+
const root = path.resolve(rootDir);
|
|
477
|
+
const resolved = path.resolve(targetPath);
|
|
478
|
+
if (resolved !== root && !resolved.startsWith(root + path.sep)) {
|
|
479
|
+
throw new StorageError({
|
|
480
|
+
cause: new Error(`Resolved path "${resolved}" escapes the storage root.`)
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
_spaceDir(spaceId) {
|
|
485
|
+
const spaceDir = path.join(this.spacesDir, spaceId);
|
|
486
|
+
this._assertContained(spaceDir);
|
|
487
|
+
return spaceDir;
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* The directory holding one Space's zcap revocation records, under the
|
|
491
|
+
* sibling `spaceRevocationsDir` root (see that property's doc for why it is
|
|
492
|
+
* not inside the Space dir), guarded against escaping it.
|
|
493
|
+
* @param spaceId {string}
|
|
494
|
+
* @returns {string}
|
|
495
|
+
*/
|
|
496
|
+
_spaceRevocationDir(spaceId) {
|
|
497
|
+
const dir = path.join(this.spaceRevocationsDir, spaceId);
|
|
498
|
+
this._assertContained(dir, this.spaceRevocationsDir);
|
|
499
|
+
return dir;
|
|
500
|
+
}
|
|
501
|
+
_collectionDir({ spaceId, collectionId }) {
|
|
502
|
+
const collectionDir = path.join(this._spaceDir(spaceId), collectionId);
|
|
503
|
+
this._assertContained(collectionDir);
|
|
504
|
+
return collectionDir;
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* @param options {object}
|
|
508
|
+
* @param options.spaceId {string}
|
|
509
|
+
* @returns {Promise<string>} Created space storage directory path.
|
|
510
|
+
*/
|
|
511
|
+
async _ensureSpaceDir({ spaceId }) {
|
|
512
|
+
const spaceDir = this._spaceDir(spaceId);
|
|
513
|
+
// Ensure the parent spaces/ directory exists (the dataDir may be brand new,
|
|
514
|
+
// e.g. a per-suite temp dir); the space dir itself is created non-recursively
|
|
515
|
+
// below so its EEXIST case can be detected.
|
|
516
|
+
await mkdir(this.spacesDir, { recursive: true });
|
|
517
|
+
try {
|
|
518
|
+
await mkdir(spaceDir);
|
|
519
|
+
}
|
|
520
|
+
catch (err) {
|
|
521
|
+
if (err.code === 'EEXIST') {
|
|
522
|
+
this.logger.info(`Space "${spaceId}" already exists, overwriting.`);
|
|
523
|
+
}
|
|
524
|
+
else {
|
|
525
|
+
throw new StorageError({ cause: err });
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
return spaceDir;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* @param options {object}
|
|
532
|
+
* @param options.spaceId {string}
|
|
533
|
+
* @param options.collectionId {string}
|
|
534
|
+
* @returns {Promise<string>} Created collection storage directory path.
|
|
535
|
+
*/
|
|
536
|
+
async _ensureCollectionDir({ spaceId, collectionId }) {
|
|
537
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
538
|
+
try {
|
|
539
|
+
await mkdir(collectionDir);
|
|
540
|
+
}
|
|
541
|
+
catch (err) {
|
|
542
|
+
if (err.code === 'EEXIST') {
|
|
543
|
+
this.logger.info(`Collection "${collectionId}" already exists, overwriting.`);
|
|
544
|
+
}
|
|
545
|
+
else {
|
|
546
|
+
this.logger.error({ err }, 'Error creating directory');
|
|
547
|
+
throw err; // http 500
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
return collectionDir;
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Lists every on-disk file belonging to a single Resource: the
|
|
554
|
+
* representation(s) whose name starts with `r.<encodedResourceId>.` in the
|
|
555
|
+
* Collection dir. The trailing `.` anchors to the filename's segment boundary
|
|
556
|
+
* (`r.<encodedResourceId>.<encodedType>.<ext>`) so a resourceId that is a
|
|
557
|
+
* prefix of another (e.g. `note` vs `notebook`) does not match the longer one;
|
|
558
|
+
* the id is dot-escaped to match the stored name (see `fileNameFor`). A Resource
|
|
559
|
+
* normally has a single current representation, so this usually returns one
|
|
560
|
+
* path; it returns more only transiently while a prior representation under a
|
|
561
|
+
* different content-type is being pruned. An absent Collection dir resolves an
|
|
562
|
+
* empty list (it holds no such files).
|
|
563
|
+
* @param options {object}
|
|
564
|
+
* @param options.collectionDir {string}
|
|
565
|
+
* @param options.resourceId {string}
|
|
566
|
+
* @returns {Promise<string[]>} full paths, in directory order
|
|
567
|
+
*/
|
|
568
|
+
async _resourceFilesFor({ collectionDir, resourceId }) {
|
|
569
|
+
const prefix = `r.${encodeFilenameSegment(resourceId)}.`;
|
|
570
|
+
let entries;
|
|
571
|
+
try {
|
|
572
|
+
entries = await fs.promises.readdir(collectionDir, {
|
|
573
|
+
withFileTypes: true
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
catch (err) {
|
|
577
|
+
if (err.code === 'ENOENT') {
|
|
578
|
+
return [];
|
|
579
|
+
}
|
|
580
|
+
throw err;
|
|
581
|
+
}
|
|
582
|
+
return entries
|
|
583
|
+
.filter(entry => entry.isFile() && entry.name.startsWith(prefix))
|
|
584
|
+
.map(entry => path.join(collectionDir, entry.name));
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* @param options {object}
|
|
588
|
+
* @param options.collectionDir {string}
|
|
589
|
+
* @param options.resourceId {string}
|
|
590
|
+
* @returns {Promise<string|undefined>} First matching resource file path.
|
|
591
|
+
*/
|
|
592
|
+
async _findFile({ collectionDir, resourceId }) {
|
|
593
|
+
const [filePath] = await this._resourceFilesFor({
|
|
594
|
+
collectionDir,
|
|
595
|
+
resourceId
|
|
596
|
+
});
|
|
597
|
+
return filePath;
|
|
598
|
+
}
|
|
599
|
+
// Spaces
|
|
600
|
+
/**
|
|
601
|
+
* @param options {object}
|
|
602
|
+
* @param options.spaceId {string}
|
|
603
|
+
* @param options.spaceDescription {SpaceDescription}
|
|
604
|
+
* @param [options.createdBy] {string} DID of the invoker, recorded as the
|
|
605
|
+
* Space's `createdBy` on first write only
|
|
606
|
+
* @returns {Promise<void>} Resolved value is implementation-defined and ignored.
|
|
607
|
+
*/
|
|
608
|
+
async writeSpace({ spaceId, spaceDescription, createdBy }) {
|
|
609
|
+
// Prior description, read once and reused below: for the create-path quota
|
|
610
|
+
// check (a brand-new Space has none yet) and to resolve `createdBy`.
|
|
611
|
+
const prior = await this.getSpaceDescription({ spaceId });
|
|
612
|
+
// Count quota (create path only): a brand-new Space (no description yet)
|
|
613
|
+
// must not push its controller past `maxSpacesPerController`. Overwriting an
|
|
614
|
+
// existing Space's description never trips it. Space creation is rare, so
|
|
615
|
+
// the O(all Spaces) enumeration is acceptable; soft under concurrency, like
|
|
616
|
+
// the byte quota.
|
|
617
|
+
if (this.maxSpacesPerController !== undefined && !prior) {
|
|
618
|
+
const { controller } = spaceDescription;
|
|
619
|
+
const spaces = await this.listSpaces();
|
|
620
|
+
const owned = spaces.filter(space => space.controller === controller).length;
|
|
621
|
+
if (owned >= this.maxSpacesPerController) {
|
|
622
|
+
throw new CountQuotaExceededError({
|
|
623
|
+
scope: 'Spaces per controller',
|
|
624
|
+
limit: this.maxSpacesPerController
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
// `createdBy` names the Space's creator, not its last writer: taken from
|
|
629
|
+
// this write's invoker only when this write CREATES the description, and
|
|
630
|
+
// preserved verbatim afterward -- including preserved-as-absent, so a Space
|
|
631
|
+
// created with no invoker (a token-provisioned create) never has a later
|
|
632
|
+
// writer backfilled into it as its creator. The client-supplied
|
|
633
|
+
// `spaceDescription` is wire input and may carry its own `createdBy` --
|
|
634
|
+
// discard it, since the server alone is authoritative for this field.
|
|
635
|
+
const { createdBy: _suppliedCreatedBy, ...rest } = spaceDescription;
|
|
636
|
+
const creator = prior ? prior.createdBy : createdBy;
|
|
637
|
+
const spaceDir = await this._ensureSpaceDir({ spaceId });
|
|
638
|
+
const filename = `.space.${spaceId}.json`;
|
|
639
|
+
// Durable full replacement: `MetadataJsonStore.read` parses plain JSON, so
|
|
640
|
+
// an atomically-written JSON string round-trips through the same read path.
|
|
641
|
+
await atomicWriteFile({
|
|
642
|
+
filePath: path.join(spaceDir, filename),
|
|
643
|
+
data: JSON.stringify({
|
|
644
|
+
...rest,
|
|
645
|
+
...(creator !== undefined && { createdBy: creator })
|
|
646
|
+
})
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* @param options {object}
|
|
651
|
+
* @param options.spaceId {string}
|
|
652
|
+
* @returns {Promise<SpaceDescription|undefined>}
|
|
653
|
+
* Resolves falsy when the Space does not exist (must not throw).
|
|
654
|
+
*/
|
|
655
|
+
async getSpaceDescription({ spaceId }) {
|
|
656
|
+
const spaceDir = this._spaceDir(spaceId);
|
|
657
|
+
const filename = `.space.${spaceId}.json`;
|
|
658
|
+
const metaStore = new MetadataJsonStore({
|
|
659
|
+
file: path.join(spaceDir, filename)
|
|
660
|
+
});
|
|
661
|
+
return await metaStore.read();
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* Removes a Space and everything scoped to it. The Space's zcap revocations
|
|
665
|
+
* live in a sibling root (`spaceRevocationsDir`), not under the Space dir, so
|
|
666
|
+
* they are removed explicitly here rather than falling out of the Space dir
|
|
667
|
+
* rm.
|
|
668
|
+
* @param options {object}
|
|
669
|
+
* @param options.spaceId {string}
|
|
670
|
+
* @returns {Promise<void>}
|
|
671
|
+
*/
|
|
672
|
+
async deleteSpace({ spaceId }) {
|
|
673
|
+
// Freed bytes: drop the cached quota usage so the next write re-measures.
|
|
674
|
+
this._usageCache.delete(spaceId);
|
|
675
|
+
// Remove this Space's revocations, which sit outside the Space dir.
|
|
676
|
+
await rm(this._spaceRevocationDir(spaceId), {
|
|
677
|
+
recursive: true,
|
|
678
|
+
force: true
|
|
679
|
+
});
|
|
680
|
+
// `force: true` keeps delete idempotent (the `StorageBackend` contract):
|
|
681
|
+
// removing an absent Space resolves rather than rejecting with `ENOENT`.
|
|
682
|
+
return await rm(this._spaceDir(spaceId), { recursive: true, force: true });
|
|
683
|
+
}
|
|
684
|
+
/**
|
|
685
|
+
* Enumerates every Space stored on this backend (each immediate subdirectory
|
|
686
|
+
* of the spaces root), sorted by Space id. An absent spaces root (nothing
|
|
687
|
+
* stored yet) resolves an empty list, not an error; a directory without a
|
|
688
|
+
* readable description file (e.g. a partially deleted Space) is skipped.
|
|
689
|
+
* @returns {Promise<SpaceDescription[]>}
|
|
690
|
+
*/
|
|
691
|
+
async listSpaces() {
|
|
692
|
+
let rootEntries;
|
|
693
|
+
try {
|
|
694
|
+
rootEntries = await fs.promises.readdir(this.spacesDir, {
|
|
695
|
+
withFileTypes: true
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
catch (err) {
|
|
699
|
+
if (err.code === 'ENOENT') {
|
|
700
|
+
return [];
|
|
701
|
+
}
|
|
702
|
+
throw new StorageError({ cause: err });
|
|
703
|
+
}
|
|
704
|
+
const spaceEntries = rootEntries
|
|
705
|
+
.filter(entry => entry.isDirectory())
|
|
706
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
707
|
+
const spaces = [];
|
|
708
|
+
for (const entry of spaceEntries) {
|
|
709
|
+
const spaceDescription = await this.getSpaceDescription({
|
|
710
|
+
spaceId: entry.name
|
|
711
|
+
});
|
|
712
|
+
if (spaceDescription) {
|
|
713
|
+
spaces.push(spaceDescription);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
return spaces;
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* Counts the live Resources across every Collection of a Space, for the
|
|
720
|
+
* Resource count quota (`maxResourcesPerSpace`). Enumerates each Collection
|
|
721
|
+
* dir and counts distinct Resource ids that have a live representation file
|
|
722
|
+
* (`r.<id>...`); a tombstone (a `.meta.` sidecar with no `r.` file) does not
|
|
723
|
+
* count. An absent Space dir counts zero (not yet provisioned). Soft under
|
|
724
|
+
* concurrency, like the byte quota -- measured at check time by enumeration.
|
|
725
|
+
* @param options {object}
|
|
726
|
+
* @param options.spaceId {string}
|
|
727
|
+
* @returns {Promise<number>} the number of live Resources
|
|
728
|
+
*/
|
|
729
|
+
async _countLiveResources({ spaceId }) {
|
|
730
|
+
const spaceDir = this._spaceDir(spaceId);
|
|
731
|
+
let spaceEntries;
|
|
732
|
+
try {
|
|
733
|
+
spaceEntries = await fs.promises.readdir(spaceDir, {
|
|
734
|
+
withFileTypes: true
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
catch (err) {
|
|
738
|
+
if (err.code === 'ENOENT') {
|
|
739
|
+
return 0;
|
|
740
|
+
}
|
|
741
|
+
throw new StorageError({ cause: err });
|
|
742
|
+
}
|
|
743
|
+
let count = 0;
|
|
744
|
+
for (const entry of spaceEntries) {
|
|
745
|
+
if (!entry.isDirectory()) {
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
const collectionDir = path.join(spaceDir, entry.name);
|
|
749
|
+
const files = await fs.promises.readdir(collectionDir);
|
|
750
|
+
// A live Resource has one representation file; count distinct ids so a
|
|
751
|
+
// transient second representation (mid content-type swap) is not
|
|
752
|
+
// double-counted.
|
|
753
|
+
const liveIds = new Set();
|
|
754
|
+
for (const fileName of files) {
|
|
755
|
+
if (!fileName.startsWith('r.')) {
|
|
756
|
+
continue;
|
|
757
|
+
}
|
|
758
|
+
liveIds.add(parseResourceFileName(fileName).resourceId);
|
|
759
|
+
}
|
|
760
|
+
count += liveIds.size;
|
|
761
|
+
}
|
|
762
|
+
return count;
|
|
763
|
+
}
|
|
764
|
+
/**
|
|
765
|
+
* @param options {object}
|
|
766
|
+
* @param options.spaceId {string}
|
|
767
|
+
* @returns {Promise<CollectionSummary[]>}
|
|
768
|
+
*/
|
|
769
|
+
async listCollections({ spaceId }) {
|
|
770
|
+
const spaceDir = this._spaceDir(spaceId);
|
|
771
|
+
const spaceEntries = await fs.promises.readdir(spaceDir, {
|
|
772
|
+
withFileTypes: true
|
|
773
|
+
});
|
|
774
|
+
const collectionEntries = spaceEntries
|
|
775
|
+
.filter(entry => entry.isDirectory())
|
|
776
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
777
|
+
const collections = [];
|
|
778
|
+
for (const entry of collectionEntries) {
|
|
779
|
+
const collectionDescription = await this.getCollectionDescription({
|
|
780
|
+
spaceId,
|
|
781
|
+
collectionId: entry.name
|
|
782
|
+
});
|
|
783
|
+
collections.push({
|
|
784
|
+
id: entry.name,
|
|
785
|
+
url: collectionPath({ spaceId, collectionId: entry.name }),
|
|
786
|
+
// `name` is optional on the wire type; a stored Collection normally has
|
|
787
|
+
// one (create defaults it to the id). Fall back to the dir name for a
|
|
788
|
+
// description-less directory too (e.g. one left by a policy write to a
|
|
789
|
+
// never-created Collection) -- reading `.name` off `undefined` here would
|
|
790
|
+
// 500 the entire Space listing.
|
|
791
|
+
name: collectionDescription?.name ?? entry.name
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
return collections;
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* @param options {object}
|
|
798
|
+
* @param options.spaceId {string}
|
|
799
|
+
* @returns {Promise<Readable>} tar-stream pack
|
|
800
|
+
*/
|
|
801
|
+
async exportSpace({ spaceId }) {
|
|
802
|
+
const spaceDescription = await this.getSpaceDescription({ spaceId });
|
|
803
|
+
if (!spaceDescription) {
|
|
804
|
+
throw new SpaceNotFoundError({ requestName: 'Export Space' });
|
|
805
|
+
}
|
|
806
|
+
const sourceSpaceDir = this._spaceDir(spaceId);
|
|
807
|
+
const spaceEntries = (await fs.promises.readdir(sourceSpaceDir, { withFileTypes: true })).filter(
|
|
808
|
+
// Backend registration records (.backend.<id>.json) hold plaintext
|
|
809
|
+
// connection material and do NOT travel in a Space export; after import
|
|
810
|
+
// the user re-registers (re-runs consent + POST /backends). importSpace
|
|
811
|
+
// ignores unrecognized space-level files, so this is symmetric.
|
|
812
|
+
entry => !(entry.isFile() && entry.name.startsWith('.backend.')));
|
|
813
|
+
spaceEntries.sort((a, b) => a.name.localeCompare(b.name));
|
|
814
|
+
const collectionEntriesByDir = {};
|
|
815
|
+
for (const entry of spaceEntries) {
|
|
816
|
+
if (!entry.isDirectory()) {
|
|
817
|
+
continue;
|
|
818
|
+
}
|
|
819
|
+
const entries = await fs.promises.readdir(path.join(sourceSpaceDir, entry.name), { withFileTypes: true });
|
|
820
|
+
collectionEntriesByDir[entry.name] = entries
|
|
821
|
+
.filter(e => e.isFile())
|
|
822
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
823
|
+
}
|
|
824
|
+
// Space-scoped zcap revocations travel with the export. They live in the
|
|
825
|
+
// sibling `spaceRevocationsDir` root (see that property's doc), so the
|
|
826
|
+
// Space-dir walk above never sees them; they pack under a top-level
|
|
827
|
+
// `revocations/` dir -- outside `space/<spaceId>/`, where a subdirectory
|
|
828
|
+
// would read as a Collection on import.
|
|
829
|
+
const revocationsDir = this._spaceRevocationDir(spaceId);
|
|
830
|
+
let revocationFiles = [];
|
|
831
|
+
try {
|
|
832
|
+
revocationFiles = (await fs.promises.readdir(revocationsDir, { withFileTypes: true }))
|
|
833
|
+
.filter(entry => entry.isFile() && entry.name.endsWith('.json'))
|
|
834
|
+
.map(entry => entry.name)
|
|
835
|
+
.sort((a, b) => a.localeCompare(b));
|
|
836
|
+
}
|
|
837
|
+
catch (err) {
|
|
838
|
+
if (err.code !== 'ENOENT') {
|
|
839
|
+
throw new StorageError({ cause: err });
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
const manifest = buildExportManifest({
|
|
843
|
+
spaceId,
|
|
844
|
+
entries: spaceEntries.map(entry => entry.isDirectory()
|
|
845
|
+
? {
|
|
846
|
+
name: entry.name,
|
|
847
|
+
files: (collectionEntriesByDir[entry.name] ?? []).map(file => file.name)
|
|
848
|
+
}
|
|
849
|
+
: // top-level files in space (e.g. .space.<spaceId>.json)
|
|
850
|
+
{ name: entry.name }),
|
|
851
|
+
revocationFiles
|
|
852
|
+
});
|
|
853
|
+
const pack = tar.pack();
|
|
854
|
+
pack.entry({ name: 'manifest.yml' }, YAML.stringify(manifest));
|
|
855
|
+
pack.entry({ name: 'space/', type: 'directory' });
|
|
856
|
+
pack.entry({ name: `space/${spaceId}/`, type: 'directory' });
|
|
857
|
+
for (const entry of spaceEntries) {
|
|
858
|
+
const entryTarget = `space/${spaceId}/${entry.name}`;
|
|
859
|
+
if (entry.isDirectory()) {
|
|
860
|
+
pack.entry({ name: `${entryTarget}/`, type: 'directory' });
|
|
861
|
+
for (const file of collectionEntriesByDir[entry.name] ?? []) {
|
|
862
|
+
const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name, file.name));
|
|
863
|
+
pack.entry({ name: `${entryTarget}/${file.name}` }, bytes);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
else if (entry.isFile()) {
|
|
867
|
+
const bytes = await fs.promises.readFile(path.join(sourceSpaceDir, entry.name));
|
|
868
|
+
pack.entry({ name: entryTarget }, bytes);
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
if (revocationFiles.length > 0) {
|
|
872
|
+
pack.entry({ name: 'revocations/', type: 'directory' });
|
|
873
|
+
for (const name of revocationFiles) {
|
|
874
|
+
const bytes = await fs.promises.readFile(path.join(revocationsDir, name));
|
|
875
|
+
pack.entry({ name: `revocations/${name}` }, bytes);
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
pack.finalize();
|
|
879
|
+
return pack;
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* Merges a WAS space-export tarball into an existing Space (collections and
|
|
883
|
+
* resources that already exist are skipped, not overwritten).
|
|
884
|
+
* @param options {object}
|
|
885
|
+
* @param options.spaceId {string}
|
|
886
|
+
* @param options.tarStream {Readable}
|
|
887
|
+
* @returns {Promise<ImportStats>}
|
|
888
|
+
*/
|
|
889
|
+
async importSpace({ spaceId, tarStream }) {
|
|
890
|
+
const entries = await extractTarEntries(tarStream);
|
|
891
|
+
const { spacePolicy, collections, revocations } = buildImportPlan(entries);
|
|
892
|
+
// Pre-flight pass over every staged resource, before writing anything, so a
|
|
893
|
+
// rejected import leaves the Space untouched. Three checks, each of which the
|
|
894
|
+
// PUT/POST write routes already enforce and which import MUST inherit too:
|
|
895
|
+
// - per-upload size cap (413): reject any single body over `maxUploadBytes`;
|
|
896
|
+
// - fail-closed encryption (422): when the target Collection declares a
|
|
897
|
+
// recognized `encryption` scheme, every incoming resource body MUST be a
|
|
898
|
+
// conforming envelope of it (a plaintext body under an encrypted
|
|
899
|
+
// Collection would otherwise store server-visible plaintext);
|
|
900
|
+
// - cumulative quota (507): sum bodies and check remaining Space headroom.
|
|
901
|
+
// The effective encryption marker is the merged-into Collection's existing
|
|
902
|
+
// one, else the import's own Collection description (a new Collection). Skips
|
|
903
|
+
// (existing ids) are counted conservatively, as for the quota estimate.
|
|
904
|
+
const { capacityBytes, maxUploadBytes, maxCollectionsPerSpace, maxResourcesPerSpace } = this;
|
|
905
|
+
let incomingBytes = 0;
|
|
906
|
+
for (const { collectionId, collectionDescription, resources } of collections) {
|
|
907
|
+
const existing = await this.getCollectionDescription({
|
|
908
|
+
spaceId,
|
|
909
|
+
collectionId
|
|
910
|
+
});
|
|
911
|
+
const effectiveEncryption = existing
|
|
912
|
+
? existing.encryption
|
|
913
|
+
: collectionDescription.encryption;
|
|
914
|
+
for (const { fileName, body } of resources) {
|
|
915
|
+
if (maxUploadBytes !== undefined && body.length > maxUploadBytes) {
|
|
916
|
+
throw new PayloadTooLargeError({
|
|
917
|
+
maxUploadBytes,
|
|
918
|
+
backendId: this.describe().id,
|
|
919
|
+
uploadBytes: body.length
|
|
920
|
+
});
|
|
921
|
+
}
|
|
922
|
+
if (effectiveEncryption?.scheme !== undefined) {
|
|
923
|
+
const { contentType } = parseResourceFileName(fileName);
|
|
924
|
+
let parsedBody;
|
|
925
|
+
try {
|
|
926
|
+
parsedBody = JSON.parse(body.toString('utf8'));
|
|
927
|
+
}
|
|
928
|
+
catch {
|
|
929
|
+
parsedBody = undefined;
|
|
930
|
+
}
|
|
931
|
+
assertEncryptedWriteConforms({
|
|
932
|
+
collectionDescription: { encryption: effectiveEncryption },
|
|
933
|
+
contentType,
|
|
934
|
+
body: parsedBody
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
incomingBytes += body.length;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
if (capacityBytes !== undefined) {
|
|
941
|
+
await this._assertSpaceHeadroom({
|
|
942
|
+
spaceId,
|
|
943
|
+
capacityBytes,
|
|
944
|
+
incomingBytes
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
const stats = {
|
|
948
|
+
collectionsCreated: 0,
|
|
949
|
+
collectionsSkipped: 0,
|
|
950
|
+
resourcesCreated: 0,
|
|
951
|
+
resourcesSkipped: 0,
|
|
952
|
+
policiesCreated: 0,
|
|
953
|
+
policiesSkipped: 0
|
|
954
|
+
};
|
|
955
|
+
// Space-level policy: restore it when the destination has none (the import
|
|
956
|
+
// target Space pre-exists, so this fills in a missing policy without
|
|
957
|
+
// clobbering one the destination already carries).
|
|
958
|
+
if (spacePolicy) {
|
|
959
|
+
if (await this.getPolicy({ spaceId })) {
|
|
960
|
+
stats.policiesSkipped++;
|
|
961
|
+
}
|
|
962
|
+
else {
|
|
963
|
+
await this.writePolicy({ spaceId, policy: spacePolicy });
|
|
964
|
+
stats.policiesCreated++;
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
// Count quotas: measure the Space's existing live Collections/Resources
|
|
968
|
+
// ONCE here, then track running totals as the apply loop creates items, so
|
|
969
|
+
// an import cannot push the Space past `maxCollectionsPerSpace` /
|
|
970
|
+
// `maxResourcesPerSpace`. Only brand-new items count -- a re-imported
|
|
971
|
+
// existing id is skipped and does not -- mirroring the per-create
|
|
972
|
+
// write-path guards without re-enumerating the Space per item.
|
|
973
|
+
const collectionIds = new Set((await this.listCollections({ spaceId })).map(entry => entry.id));
|
|
974
|
+
let liveResourceCount = maxResourcesPerSpace !== undefined
|
|
975
|
+
? await this._countLiveResources({ spaceId })
|
|
976
|
+
: 0;
|
|
977
|
+
for (const { collectionId, collectionDescription, collectionPolicy, resources, resourcePolicies, resourceMetadata } of collections) {
|
|
978
|
+
// check if collection already exists
|
|
979
|
+
const collectionExisted = Boolean(await this.getCollectionDescription({ spaceId, collectionId }));
|
|
980
|
+
if (collectionExisted) {
|
|
981
|
+
stats.collectionsSkipped++;
|
|
982
|
+
}
|
|
983
|
+
else {
|
|
984
|
+
// A brand-new Collection (one whose id the Space did not already hold,
|
|
985
|
+
// even as a description-less directory) counts against the cap; filling
|
|
986
|
+
// in the description of an existing directory does not.
|
|
987
|
+
if (maxCollectionsPerSpace !== undefined &&
|
|
988
|
+
!collectionIds.has(collectionId) &&
|
|
989
|
+
collectionIds.size >= maxCollectionsPerSpace) {
|
|
990
|
+
throw new CountQuotaExceededError({
|
|
991
|
+
scope: 'Collections per Space',
|
|
992
|
+
limit: maxCollectionsPerSpace
|
|
993
|
+
});
|
|
994
|
+
}
|
|
995
|
+
collectionIds.add(collectionId);
|
|
996
|
+
await this._persistCollection({
|
|
997
|
+
spaceId,
|
|
998
|
+
collectionId,
|
|
999
|
+
collectionDescription
|
|
1000
|
+
});
|
|
1001
|
+
stats.collectionsCreated++;
|
|
1002
|
+
}
|
|
1003
|
+
// A collection-level policy travels with a newly-created collection; for
|
|
1004
|
+
// an existing (skipped) collection, leave its access policy untouched.
|
|
1005
|
+
if (collectionPolicy) {
|
|
1006
|
+
if (collectionExisted) {
|
|
1007
|
+
stats.policiesSkipped++;
|
|
1008
|
+
}
|
|
1009
|
+
else {
|
|
1010
|
+
await this.writePolicy({
|
|
1011
|
+
spaceId,
|
|
1012
|
+
collectionId,
|
|
1013
|
+
policy: collectionPolicy
|
|
1014
|
+
});
|
|
1015
|
+
stats.policiesCreated++;
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1019
|
+
for (const { fileName, resourceId, body } of resources) {
|
|
1020
|
+
// Skip anything the destination already has for this id: a live
|
|
1021
|
+
// representation (`_findFile`) OR a sidecar (`_readMetaSidecar`, which
|
|
1022
|
+
// includes a `deleted:true` tombstone). Checking only `_findFile` would
|
|
1023
|
+
// let an import write content back over a soft-deleted (tombstoned)
|
|
1024
|
+
// resource -- resurrecting it while its `deleted:true` sidecar remains,
|
|
1025
|
+
// yielding a served-but-tombstoned resource and an inconsistent feed.
|
|
1026
|
+
const resourceExists = Boolean(await this._findFile({ collectionDir, resourceId })) ||
|
|
1027
|
+
Boolean(await this._readMetaSidecar({ collectionDir, resourceId }));
|
|
1028
|
+
if (resourceExists) {
|
|
1029
|
+
stats.resourcesSkipped++;
|
|
1030
|
+
// A resource-level policy travels with a newly-created resource only.
|
|
1031
|
+
if (resourcePolicies.has(resourceId)) {
|
|
1032
|
+
stats.policiesSkipped++;
|
|
1033
|
+
}
|
|
1034
|
+
continue;
|
|
1035
|
+
}
|
|
1036
|
+
// A new live Resource counts against the per-Space cap.
|
|
1037
|
+
if (maxResourcesPerSpace !== undefined) {
|
|
1038
|
+
if (liveResourceCount >= maxResourcesPerSpace) {
|
|
1039
|
+
throw new CountQuotaExceededError({
|
|
1040
|
+
scope: 'Resources per Space',
|
|
1041
|
+
limit: maxResourcesPerSpace
|
|
1042
|
+
});
|
|
1043
|
+
}
|
|
1044
|
+
liveResourceCount++;
|
|
1045
|
+
}
|
|
1046
|
+
await atomicWriteFile({
|
|
1047
|
+
filePath: path.join(collectionDir, fileName),
|
|
1048
|
+
data: body
|
|
1049
|
+
});
|
|
1050
|
+
stats.resourcesCreated++;
|
|
1051
|
+
// A metadata sidecar travels with a newly-created resource (preserving
|
|
1052
|
+
// its timestamps and user-writable `custom`); an absent one leaves
|
|
1053
|
+
// `getResourceMetadata` to fall back to the file's stat times.
|
|
1054
|
+
const metadataBytes = resourceMetadata.get(resourceId);
|
|
1055
|
+
if (metadataBytes) {
|
|
1056
|
+
await atomicWriteFile({
|
|
1057
|
+
filePath: this._metaSidecarPath({ collectionDir, resourceId }),
|
|
1058
|
+
data: metadataBytes
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
const resourcePolicy = resourcePolicies.get(resourceId);
|
|
1062
|
+
if (resourcePolicy) {
|
|
1063
|
+
await this.writePolicy({
|
|
1064
|
+
spaceId,
|
|
1065
|
+
collectionId,
|
|
1066
|
+
resourceId,
|
|
1067
|
+
policy: resourcePolicy
|
|
1068
|
+
});
|
|
1069
|
+
stats.policiesCreated++;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
// Carry tombstones: a soft-deleted Resource (see `deleteResource`) exports
|
|
1073
|
+
// as a `.meta.` sidecar with no paired `r.` content file, so it never
|
|
1074
|
+
// appears in `resources` above. Restore each such ORPHAN sidecar that is a
|
|
1075
|
+
// tombstone (`deleted: true`) -- writing only the sidecar re-creates the
|
|
1076
|
+
// tombstone. A non-tombstone orphan sidecar is anomalous (a Resource with
|
|
1077
|
+
// no representation) and is skipped. Merge semantics match resources:
|
|
1078
|
+
// anything the destination already has for that id (a live Resource or an
|
|
1079
|
+
// existing tombstone) is left untouched.
|
|
1080
|
+
const importedResourceIds = new Set(resources.map(r => r.resourceId));
|
|
1081
|
+
for (const [resourceId, metadataBytes] of resourceMetadata) {
|
|
1082
|
+
if (importedResourceIds.has(resourceId)) {
|
|
1083
|
+
continue;
|
|
1084
|
+
}
|
|
1085
|
+
let sidecar;
|
|
1086
|
+
try {
|
|
1087
|
+
sidecar = JSON.parse(metadataBytes.toString('utf8'));
|
|
1088
|
+
}
|
|
1089
|
+
catch {
|
|
1090
|
+
continue;
|
|
1091
|
+
}
|
|
1092
|
+
if (sidecar?.deleted !== true) {
|
|
1093
|
+
continue;
|
|
1094
|
+
}
|
|
1095
|
+
const exists = Boolean(await this._findFile({ collectionDir, resourceId })) ||
|
|
1096
|
+
Boolean(await this._readMetaSidecar({ collectionDir, resourceId }));
|
|
1097
|
+
if (exists) {
|
|
1098
|
+
stats.resourcesSkipped++;
|
|
1099
|
+
continue;
|
|
1100
|
+
}
|
|
1101
|
+
await atomicWriteFile({
|
|
1102
|
+
filePath: this._metaSidecarPath({ collectionDir, resourceId }),
|
|
1103
|
+
data: metadataBytes
|
|
1104
|
+
});
|
|
1105
|
+
stats.resourcesCreated++;
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
// Restore the archive's Space-scoped zcap revocations under this Space's
|
|
1109
|
+
// scope: a capability revoked before the export must stay revoked after
|
|
1110
|
+
// an import (a backup/restore round-trip must not resurrect revoked
|
|
1111
|
+
// access). Merge semantics match the rest of the import -- an
|
|
1112
|
+
// already-stored record is skipped -- and a record past its GC horizon is
|
|
1113
|
+
// dropped (the capability itself has expired; `isRevoked` would prune it).
|
|
1114
|
+
const now = Date.now();
|
|
1115
|
+
for (const record of revocations) {
|
|
1116
|
+
if (record.meta.expires && Date.parse(record.meta.expires) <= now) {
|
|
1117
|
+
continue;
|
|
1118
|
+
}
|
|
1119
|
+
try {
|
|
1120
|
+
await this.insertRevocation({ scope: { spaceId }, record });
|
|
1121
|
+
}
|
|
1122
|
+
catch (err) {
|
|
1123
|
+
if (!(err instanceof DuplicateRevocationError)) {
|
|
1124
|
+
throw err;
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
return stats;
|
|
1129
|
+
}
|
|
1130
|
+
// Collections
|
|
1131
|
+
/**
|
|
1132
|
+
* @param options {object}
|
|
1133
|
+
* @param options.spaceId {string}
|
|
1134
|
+
* @param options.collectionId {string}
|
|
1135
|
+
* @param options.collectionDescription {CollectionDescription}
|
|
1136
|
+
* @param [options.createdBy] {string} DID of the invoker, recorded as the
|
|
1137
|
+
* Collection's `createdBy` on first write only
|
|
1138
|
+
* @param [options.ifMatch] {string} an `If-Match` compare-and-swap on the
|
|
1139
|
+
* current description version (the `key-epochs` feature); a stale validator
|
|
1140
|
+
* throws `PreconditionFailedError` (412)
|
|
1141
|
+
* @returns {Promise<{ version: number }>} the Collection's new description
|
|
1142
|
+
* version (the `ETag` validator)
|
|
1143
|
+
*/
|
|
1144
|
+
async writeCollection({ spaceId, collectionId, collectionDescription, createdBy, ifMatch, assertTransition }) {
|
|
1145
|
+
// Serialize the read-check-write under a per-Collection-description lock so
|
|
1146
|
+
// the `If-Match` compare-and-swap and the monotonic version bump are atomic
|
|
1147
|
+
// with the write (two concurrent recipient edits cannot clobber one
|
|
1148
|
+
// another). A distinct lock namespace from the per-Resource / unique-scan
|
|
1149
|
+
// locks: a description write and a Resource write touch different files.
|
|
1150
|
+
return this._writeMutex.run(this._collectionDescLockKey({ spaceId, collectionId }), async () => {
|
|
1151
|
+
// Prior description, read once and reused below: for the create-path
|
|
1152
|
+
// quota check, `createdBy` resolution, and the CAS version.
|
|
1153
|
+
const prior = await this.getCollectionDescription({
|
|
1154
|
+
spaceId,
|
|
1155
|
+
collectionId
|
|
1156
|
+
});
|
|
1157
|
+
// Compare-and-swap on the current description version (opt-in): a stale
|
|
1158
|
+
// `If-Match` throws 412. An unconditional write skips this.
|
|
1159
|
+
assertCollectionWritePrecondition({
|
|
1160
|
+
collectionId,
|
|
1161
|
+
currentVersion: prior?.descriptionVersion ?? 0,
|
|
1162
|
+
ifMatch
|
|
1163
|
+
});
|
|
1164
|
+
// The request layer's state-transition rails (e.g. epoch append-only),
|
|
1165
|
+
// re-evaluated here against the description just read under the lock.
|
|
1166
|
+
assertTransition?.(prior);
|
|
1167
|
+
// Count quota (create path only): a new Collection must not push its
|
|
1168
|
+
// Space past `maxCollectionsPerSpace`; overwriting an existing
|
|
1169
|
+
// Collection's description never trips it.
|
|
1170
|
+
if (this.maxCollectionsPerSpace !== undefined && !prior) {
|
|
1171
|
+
const collections = await this.listCollections({ spaceId });
|
|
1172
|
+
if (collections.length >= this.maxCollectionsPerSpace) {
|
|
1173
|
+
throw new CountQuotaExceededError({
|
|
1174
|
+
scope: 'Collections per Space',
|
|
1175
|
+
limit: this.maxCollectionsPerSpace
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
// `createdBy` names the Collection's creator, not its last writer: taken
|
|
1180
|
+
// from this write's invoker only when this write CREATES the
|
|
1181
|
+
// description, and preserved verbatim afterward -- including
|
|
1182
|
+
// preserved-as-absent. The client-supplied `collectionDescription` is
|
|
1183
|
+
// wire input and may carry its own `createdBy` (and the out-of-band
|
|
1184
|
+
// `descriptionVersion` when a caller spread a read result back in) --
|
|
1185
|
+
// discard both, since the server alone is authoritative for them.
|
|
1186
|
+
const { createdBy: _suppliedCreatedBy, descriptionVersion: _suppliedVersion, ...rest } = collectionDescription;
|
|
1187
|
+
const creator = prior ? prior.createdBy : createdBy;
|
|
1188
|
+
const version = (prior?.descriptionVersion ?? 0) + 1;
|
|
1189
|
+
await this._persistCollection({
|
|
1190
|
+
spaceId,
|
|
1191
|
+
collectionId,
|
|
1192
|
+
collectionDescription: {
|
|
1193
|
+
...rest,
|
|
1194
|
+
...(creator !== undefined && { createdBy: creator })
|
|
1195
|
+
},
|
|
1196
|
+
descriptionVersion: version
|
|
1197
|
+
});
|
|
1198
|
+
return { version };
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
/**
|
|
1202
|
+
* Builds the per-Collection-description serialization key for `_writeMutex`
|
|
1203
|
+
* (`desc:<spaceId>/<collectionId>`), so a Collection Description compare-and-
|
|
1204
|
+
* swap serializes with itself while staying disjoint from the per-Resource and
|
|
1205
|
+
* unique-scan lock namespaces.
|
|
1206
|
+
* @param options {object}
|
|
1207
|
+
* @param options.spaceId {string}
|
|
1208
|
+
* @param options.collectionId {string}
|
|
1209
|
+
* @returns {string}
|
|
1210
|
+
*/
|
|
1211
|
+
_collectionDescLockKey({ spaceId, collectionId }) {
|
|
1212
|
+
return `desc:${spaceId}/${collectionId}`;
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* Writes a Collection's description file (creating the Collection dir if
|
|
1216
|
+
* needed), with NO count-quota check. The count guard lives in the public
|
|
1217
|
+
* `writeCollection`; `importSpace` tracks the Space's Collection count itself
|
|
1218
|
+
* (measured once up front) and calls this directly, so it does not
|
|
1219
|
+
* re-enumerate the Space per created Collection.
|
|
1220
|
+
* @param options {object}
|
|
1221
|
+
* @param options.spaceId {string}
|
|
1222
|
+
* @param options.collectionId {string}
|
|
1223
|
+
* @param options.collectionDescription {CollectionDescription}
|
|
1224
|
+
* @param [options.descriptionVersion] {number} the monotonic description
|
|
1225
|
+
* version to persist (the `ETag` validator; the `key-epochs` feature). Kept
|
|
1226
|
+
* OUT of the wire body -- stored under a reserved `_version` member that
|
|
1227
|
+
* `getCollectionDescription` strips and re-surfaces as `descriptionVersion`.
|
|
1228
|
+
* When omitted (the import path), a `_version` already on the incoming
|
|
1229
|
+
* description is preserved, else it defaults to `1`.
|
|
1230
|
+
* @returns {Promise<void>}
|
|
1231
|
+
*/
|
|
1232
|
+
async _persistCollection({ spaceId, collectionId, collectionDescription, descriptionVersion }) {
|
|
1233
|
+
const collectionDir = await this._ensureCollectionDir({
|
|
1234
|
+
spaceId,
|
|
1235
|
+
collectionId
|
|
1236
|
+
});
|
|
1237
|
+
const filename = `.collection.${collectionId}.json`;
|
|
1238
|
+
// Shared normalization (lib/collectionDescription.ts): strip the
|
|
1239
|
+
// version-bearing members from the incoming body and re-stamp -- an
|
|
1240
|
+
// explicit `descriptionVersion` wins, else a `_version` already on the
|
|
1241
|
+
// incoming (imported) description is kept, else the first write starts
|
|
1242
|
+
// at 1.
|
|
1243
|
+
const { body, version } = normalizeDescriptionWrite({
|
|
1244
|
+
collectionDescription,
|
|
1245
|
+
descriptionVersion
|
|
1246
|
+
});
|
|
1247
|
+
await atomicWriteFile({
|
|
1248
|
+
filePath: path.join(collectionDir, filename),
|
|
1249
|
+
data: JSON.stringify({ ...body, _version: version })
|
|
1250
|
+
});
|
|
1251
|
+
}
|
|
1252
|
+
/**
|
|
1253
|
+
* @param options {object}
|
|
1254
|
+
* @param options.spaceId {string}
|
|
1255
|
+
* @param options.collectionId {string}
|
|
1256
|
+
* @returns {Promise<CollectionDescription|undefined>}
|
|
1257
|
+
* Resolves falsy when the Collection does not exist (must not throw).
|
|
1258
|
+
*/
|
|
1259
|
+
async getCollectionDescription({ spaceId, collectionId }) {
|
|
1260
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1261
|
+
const filename = `.collection.${collectionId}.json`;
|
|
1262
|
+
const metaStore = new MetadataJsonStore({
|
|
1263
|
+
file: path.join(collectionDir, filename)
|
|
1264
|
+
});
|
|
1265
|
+
const raw = await metaStore.read();
|
|
1266
|
+
if (!raw) {
|
|
1267
|
+
return undefined;
|
|
1268
|
+
}
|
|
1269
|
+
// `_version` is the internal ETag validator, kept out of the wire body:
|
|
1270
|
+
// strip it and re-surface it out-of-band as `descriptionVersion` (the
|
|
1271
|
+
// handler sets the `ETag` header from it). Absent for a legacy Collection.
|
|
1272
|
+
const { _version, ...description } = raw;
|
|
1273
|
+
return {
|
|
1274
|
+
...description,
|
|
1275
|
+
...(_version !== undefined && { descriptionVersion: _version })
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
/**
|
|
1279
|
+
* @param options {object}
|
|
1280
|
+
* @param options.spaceId {string}
|
|
1281
|
+
* @param options.collectionId {string}
|
|
1282
|
+
* @returns {Promise<void>}
|
|
1283
|
+
*/
|
|
1284
|
+
async deleteCollection({ spaceId, collectionId }) {
|
|
1285
|
+
// Freed bytes: drop the cached quota usage so the next write re-measures.
|
|
1286
|
+
this._usageCache.delete(spaceId);
|
|
1287
|
+
// `force: true` keeps delete idempotent (spec / `StorageBackend` contract):
|
|
1288
|
+
// removing an absent (or already-deleted) Collection resolves rather than
|
|
1289
|
+
// rejecting with `ENOENT` (which the request layer would wrap as a 500).
|
|
1290
|
+
return await rm(this._collectionDir({ spaceId, collectionId }), {
|
|
1291
|
+
recursive: true,
|
|
1292
|
+
force: true
|
|
1293
|
+
});
|
|
1294
|
+
}
|
|
1295
|
+
/**
|
|
1296
|
+
* Lists a Collection's Resources, OPTIONALLY cursor-paginated (spec
|
|
1297
|
+
* "Pagination"). Items are returned in a stable total order -- ascending by
|
|
1298
|
+
* `resourceId`, read straight from the `r.<resourceId>.<type>.<ext>` filename
|
|
1299
|
+
* (the keyset). A `cursor` resumes the scan at the first id strictly greater
|
|
1300
|
+
* than the cursor's anchor, so paging stays correct even if the anchor id was
|
|
1301
|
+
* deleted between pages; `limit` bounds the page (clamped to
|
|
1302
|
+
* `[1, MAX_PAGE_SIZE]`, default `DEFAULT_PAGE_SIZE`). `next` is built (and
|
|
1303
|
+
* present) only when a further page may follow.
|
|
1304
|
+
* @param options {object}
|
|
1305
|
+
* @param options.spaceId {string}
|
|
1306
|
+
* @param options.collectionId {string}
|
|
1307
|
+
* @param [options.limit] {number} requested page size
|
|
1308
|
+
* @param [options.cursor] {string} opaque cursor from a prior page's `next`
|
|
1309
|
+
* @returns {Promise<CollectionResourcesList>}
|
|
1310
|
+
*/
|
|
1311
|
+
async listCollectionItems({ spaceId, collectionId, limit, cursor, collectionDescription: providedDescription }) {
|
|
1312
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1313
|
+
// Prefer the caller's control-plane description. When this backend serves a
|
|
1314
|
+
// Collection's data plane (an external backend), it does NOT hold the
|
|
1315
|
+
// description locally, so its own `getCollectionDescription` would resolve
|
|
1316
|
+
// `undefined` and reading `.name`/`.type` off it would 500.
|
|
1317
|
+
const collectionDescription = providedDescription ??
|
|
1318
|
+
(await this.getCollectionDescription({ spaceId, collectionId }));
|
|
1319
|
+
// Enumerate the Collection dir directly rather than globbing: glob v13 does
|
|
1320
|
+
// not sort, so its order is nondeterministic -- pagination needs a stable
|
|
1321
|
+
// keyset. Keep only resource representations (`r.<id>.<type>.<ext>`), which
|
|
1322
|
+
// drops the `.meta.` / `.collection.` / `.policy.` dot-files.
|
|
1323
|
+
let entries = [];
|
|
1324
|
+
try {
|
|
1325
|
+
entries = await fs.promises.readdir(collectionDir, {
|
|
1326
|
+
withFileTypes: true
|
|
1327
|
+
});
|
|
1328
|
+
}
|
|
1329
|
+
catch (err) {
|
|
1330
|
+
this.logger.error({ err }, 'Error reading collection directory');
|
|
1331
|
+
}
|
|
1332
|
+
const resources = entries
|
|
1333
|
+
.filter(entry => entry.isFile() && entry.name.startsWith('r.'))
|
|
1334
|
+
.map(entry => parseResourceFileName(entry.name))
|
|
1335
|
+
// Sort by `resourceId` ascending in code-unit order -- the SAME ordering
|
|
1336
|
+
// the cursor seek (`resourceId > after`) uses, so the keyset is consistent
|
|
1337
|
+
// (localeCompare could disagree with the `>` operator and break paging).
|
|
1338
|
+
.sort((left, right) => compareCodeUnits(left.resourceId, right.resourceId));
|
|
1339
|
+
// The full count is free here (we enumerated the whole dir), so keep
|
|
1340
|
+
// returning `totalItems` -- the count of the entire Collection, not the page.
|
|
1341
|
+
const totalItems = resources.length;
|
|
1342
|
+
// Seek to the first entry strictly after the cursor's anchor id. Keyset
|
|
1343
|
+
// stability: a missing anchor (deleted between pages) does not break the
|
|
1344
|
+
// scan, since we resume at the first id greater than it.
|
|
1345
|
+
let startIndex = 0;
|
|
1346
|
+
if (cursor !== undefined) {
|
|
1347
|
+
const { after } = decodeCursor(cursor);
|
|
1348
|
+
const found = resources.findIndex(({ resourceId }) => resourceId > after);
|
|
1349
|
+
startIndex = found === -1 ? resources.length : found;
|
|
1350
|
+
}
|
|
1351
|
+
// Clamp `limit` to `[1, MAX_PAGE_SIZE]`, defaulting when absent.
|
|
1352
|
+
const pageSize = limit === undefined ? DEFAULT_PAGE_SIZE : clampPageSize(limit);
|
|
1353
|
+
// Take `pageSize + 1` from the seek point to detect a further page without a
|
|
1354
|
+
// second pass; the page is the first `pageSize`, and `hasMore` is whether we
|
|
1355
|
+
// got the extra one (so a page that exactly fills the Collection has no
|
|
1356
|
+
// spurious empty trailing page).
|
|
1357
|
+
const window = resources.slice(startIndex, startIndex + pageSize + 1);
|
|
1358
|
+
const hasMore = window.length > pageSize;
|
|
1359
|
+
const pageEntries = hasMore ? window.slice(0, pageSize) : window;
|
|
1360
|
+
// Read `.meta` sidecars ONLY for the items on this page (the previous
|
|
1361
|
+
// implementation read a sidecar for every resource on every list). Surface
|
|
1362
|
+
// the user-writable `custom.name` (spec: updating it updates the name shown
|
|
1363
|
+
// in Collection listings) -- but only for a plaintext Collection. On an
|
|
1364
|
+
// encrypted Collection `custom` is the opaque encryption envelope, so the
|
|
1365
|
+
// server cannot project a `name`; the listing omits it (spec "List
|
|
1366
|
+
// Collection", encrypted-Collection note). (A bare `custom?.name` on a JWE
|
|
1367
|
+
// envelope already yields `undefined`; the guard makes that explicit.)
|
|
1368
|
+
const encrypted = collectionDescription?.encryption !== undefined;
|
|
1369
|
+
const items = await Promise.all(pageEntries.map(async ({ resourceId, contentType }) => {
|
|
1370
|
+
const sidecar = await this._readMetaSidecar({
|
|
1371
|
+
collectionDir,
|
|
1372
|
+
resourceId
|
|
1373
|
+
});
|
|
1374
|
+
const name = encrypted
|
|
1375
|
+
? undefined
|
|
1376
|
+
: sidecar?.custom?.name;
|
|
1377
|
+
return {
|
|
1378
|
+
id: resourceId,
|
|
1379
|
+
url: resourcePath({ spaceId, collectionId, resourceId }),
|
|
1380
|
+
contentType,
|
|
1381
|
+
...(name !== undefined && { name }),
|
|
1382
|
+
// The client-declared key epoch (the `key-epochs` feature) rides each
|
|
1383
|
+
// listing item so a reader can pick the right epoch key without a
|
|
1384
|
+
// `/meta` fetch per Resource.
|
|
1385
|
+
...(sidecar?.epoch !== undefined && { epoch: sidecar.epoch })
|
|
1386
|
+
};
|
|
1387
|
+
}));
|
|
1388
|
+
// `next` is present iff a further page may follow; its absence marks the last
|
|
1389
|
+
// page (the authoritative end-of-list signal). The cursor (the last id on
|
|
1390
|
+
// this page) and the page size are baked into the URL so the client follows
|
|
1391
|
+
// it verbatim without constructing query parameters.
|
|
1392
|
+
let next;
|
|
1393
|
+
if (hasMore) {
|
|
1394
|
+
const lastId = pageEntries[pageEntries.length - 1].resourceId;
|
|
1395
|
+
const base = collectionPath({
|
|
1396
|
+
spaceId,
|
|
1397
|
+
collectionId,
|
|
1398
|
+
trailingSlash: true
|
|
1399
|
+
});
|
|
1400
|
+
next = `${base}?limit=${pageSize}&cursor=${encodeCursor(lastId)}`;
|
|
1401
|
+
}
|
|
1402
|
+
return {
|
|
1403
|
+
id: collectionId,
|
|
1404
|
+
url: collectionPath({ spaceId, collectionId }),
|
|
1405
|
+
name: collectionDescription?.name ?? collectionId,
|
|
1406
|
+
type: collectionDescription?.type || ['Collection'],
|
|
1407
|
+
totalItems,
|
|
1408
|
+
items,
|
|
1409
|
+
...(next !== undefined && { next })
|
|
1410
|
+
};
|
|
1411
|
+
}
|
|
1412
|
+
// Resources
|
|
1413
|
+
/**
|
|
1414
|
+
* Writes a resource representation (JSON value or byte stream) to disk, under
|
|
1415
|
+
* the per-Resource write lock (the `conditional-writes` feature), and bumps
|
|
1416
|
+
* the Resource's monotonic `version`. The new version is returned so the
|
|
1417
|
+
* request layer can surface it as the response `ETag`.
|
|
1418
|
+
*
|
|
1419
|
+
* When a conditional-write precondition is supplied it is evaluated against
|
|
1420
|
+
* the Resource's current state atomically with the write (under the lock),
|
|
1421
|
+
* throwing `PreconditionFailedError` (412) on a mismatch:
|
|
1422
|
+
* - `ifNoneMatch` (a create-if-absent `If-None-Match: *`) fails if the
|
|
1423
|
+
* Resource already exists.
|
|
1424
|
+
* - `ifMatch` (an update-if-unchanged `If-Match: "<etag>"`) fails if the
|
|
1425
|
+
* Resource is absent or its current `ETag` does not equal `ifMatch`.
|
|
1426
|
+
* `ifNoneMatch` takes precedence when both are supplied (RFC9110).
|
|
1427
|
+
*
|
|
1428
|
+
* @param options {object}
|
|
1429
|
+
* @param options.spaceId {string}
|
|
1430
|
+
* @param options.collectionId {string}
|
|
1431
|
+
* @param options.resourceId {string}
|
|
1432
|
+
* @param options.input {ResourceInput}
|
|
1433
|
+
* @param [options.createdBy] {string} DID of the invoker, recorded as the
|
|
1434
|
+
* Resource's `createdBy` on first write only
|
|
1435
|
+
* @param [options.ifMatch] {string} `If-Match` precondition (a quoted ETag)
|
|
1436
|
+
* @param [options.ifNoneMatch] {boolean} `If-None-Match: *` (create-if-absent)
|
|
1437
|
+
* @returns {Promise<{ version: number }>} the Resource's new version
|
|
1438
|
+
*/
|
|
1439
|
+
async writeResource({ spaceId, collectionId, resourceId, input, createdBy, epoch, ifMatch, ifNoneMatch }) {
|
|
1440
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1441
|
+
const lockKey = this._resourceLockKey({
|
|
1442
|
+
spaceId,
|
|
1443
|
+
collectionId,
|
|
1444
|
+
resourceId
|
|
1445
|
+
});
|
|
1446
|
+
const write = () => this._writeMutex.run(lockKey, () => this._writeResourceLocked({
|
|
1447
|
+
spaceId,
|
|
1448
|
+
collectionDir,
|
|
1449
|
+
resourceId,
|
|
1450
|
+
input,
|
|
1451
|
+
createdBy,
|
|
1452
|
+
epoch,
|
|
1453
|
+
ifMatch,
|
|
1454
|
+
ifNoneMatch
|
|
1455
|
+
}));
|
|
1456
|
+
// The EDV unique-attribute invariant (`unique: true` blinded attributes;
|
|
1457
|
+
// the `blinded-index-query` feature): a write claiming a unique blinded
|
|
1458
|
+
// triple must not collide with another live document's claim. Only a
|
|
1459
|
+
// unique-carrying JSON write can create a new claim, so only such writes
|
|
1460
|
+
// pay for it: they serialize per Collection (the outer lock, so two racing
|
|
1461
|
+
// claimants cannot both pass the scan), evaluate the conflict against the
|
|
1462
|
+
// Collection's other live JSON documents, then take the ordinary
|
|
1463
|
+
// per-Resource lock nested inside. Distinct-key nesting cannot deadlock:
|
|
1464
|
+
// plain writes never hold a Resource key while waiting on a Collection key.
|
|
1465
|
+
if (input.kind === 'json' &&
|
|
1466
|
+
collectUniqueBlindedTerms({ document: input.data }).length > 0) {
|
|
1467
|
+
return this._writeMutex.run(this._collectionLockKey({ spaceId, collectionId }), async () => {
|
|
1468
|
+
assertNoUniqueBlindedConflict({
|
|
1469
|
+
document: input.data,
|
|
1470
|
+
candidates: await this._readJsonCandidates({
|
|
1471
|
+
spaceId,
|
|
1472
|
+
collectionId,
|
|
1473
|
+
excludeResourceId: resourceId
|
|
1474
|
+
})
|
|
1475
|
+
});
|
|
1476
|
+
return write();
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1479
|
+
return write();
|
|
1480
|
+
}
|
|
1481
|
+
/**
|
|
1482
|
+
* The critical section of `writeResource`, run under the per-Resource lock:
|
|
1483
|
+
* evaluates any precondition, writes the representation, prunes a stale
|
|
1484
|
+
* representation under a different content-type, and persists the bumped
|
|
1485
|
+
* `version` in the sidecar. See `writeResource` for the parameters.
|
|
1486
|
+
* @returns {Promise<{ version: number }>}
|
|
1487
|
+
*/
|
|
1488
|
+
async _writeResourceLocked({ spaceId, collectionDir, resourceId, input, createdBy, epoch, ifMatch, ifNoneMatch }) {
|
|
1489
|
+
const filename = fileNameFor({ resourceId, contentType: input.contentType });
|
|
1490
|
+
const filePath = path.join(collectionDir, filename);
|
|
1491
|
+
this._assertContained(filePath);
|
|
1492
|
+
// Evaluate any conditional-write precondition against the current state
|
|
1493
|
+
// before writing (still inside the lock, so the check and write are atomic).
|
|
1494
|
+
if (ifMatch !== undefined || ifNoneMatch) {
|
|
1495
|
+
await this._assertWritePrecondition({
|
|
1496
|
+
collectionDir,
|
|
1497
|
+
resourceId,
|
|
1498
|
+
ifMatch,
|
|
1499
|
+
ifNoneMatch
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
// Count quota (create path only): a new live Resource must not push its
|
|
1503
|
+
// Space past `maxResourcesPerSpace`. A write over an existing live
|
|
1504
|
+
// representation is an update (never trips it); a write over a tombstone
|
|
1505
|
+
// (no `r.` file) is a create and does count. Soft under concurrency.
|
|
1506
|
+
if (this.maxResourcesPerSpace !== undefined) {
|
|
1507
|
+
const isLive = (await this._findFile({ collectionDir, resourceId })) !== undefined;
|
|
1508
|
+
if (!isLive) {
|
|
1509
|
+
const liveCount = await this._countLiveResources({ spaceId });
|
|
1510
|
+
if (liveCount >= this.maxResourcesPerSpace) {
|
|
1511
|
+
throw new CountQuotaExceededError({
|
|
1512
|
+
scope: 'Resources per Space',
|
|
1513
|
+
limit: this.maxResourcesPerSpace
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
const { capacityBytes, maxUploadBytes } = this;
|
|
1519
|
+
if (input.kind === 'json') {
|
|
1520
|
+
// JSON bodies are fully in memory, so their serialized size is known up
|
|
1521
|
+
// front and the pre-flight checks alone suffice (no streaming guard). The
|
|
1522
|
+
// per-upload cap (413) is checked before the cumulative quota (507).
|
|
1523
|
+
// Serialize once and reuse for both the size pre-flight and the write.
|
|
1524
|
+
const serialized = JSON.stringify(input.data);
|
|
1525
|
+
const incomingBytes = Buffer.byteLength(serialized);
|
|
1526
|
+
if (maxUploadBytes !== undefined && incomingBytes > maxUploadBytes) {
|
|
1527
|
+
throw new PayloadTooLargeError({
|
|
1528
|
+
maxUploadBytes,
|
|
1529
|
+
backendId: this.describe().id,
|
|
1530
|
+
uploadBytes: incomingBytes
|
|
1531
|
+
});
|
|
1532
|
+
}
|
|
1533
|
+
if (capacityBytes !== undefined) {
|
|
1534
|
+
await this._assertSpaceHeadroom({
|
|
1535
|
+
spaceId,
|
|
1536
|
+
capacityBytes,
|
|
1537
|
+
incomingBytes
|
|
1538
|
+
});
|
|
1539
|
+
}
|
|
1540
|
+
// Write the serialized JSON directly rather than through fs-json-store,
|
|
1541
|
+
// whose `write` verifies the result via `readExisting` and treats a falsy
|
|
1542
|
+
// round-tripped value (`null`, `false`, `0`, `""`) as "file does not
|
|
1543
|
+
// exist" -- which would 500 a legitimate top-level primitive Resource. The
|
|
1544
|
+
// read path (`getResource`) streams the bytes back verbatim, so any
|
|
1545
|
+
// top-level JSON value -- object, array, or bare primitive -- round-trips.
|
|
1546
|
+
// Ensure the Collection dir exists first (fs-json-store used to create it
|
|
1547
|
+
// on the fly; a data-plane backend may not have seen this Collection yet).
|
|
1548
|
+
this.logger.info('Creating JSON resource');
|
|
1549
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
1550
|
+
// Durable, atomic replacement (write-temp + fsync + rename + dir fsync):
|
|
1551
|
+
// the final path never observes a torn write, even across a crash.
|
|
1552
|
+
await atomicWriteFile({ filePath, data: serialized });
|
|
1553
|
+
}
|
|
1554
|
+
else {
|
|
1555
|
+
this.logger.info('Writing blob');
|
|
1556
|
+
// Pre-flight the declared size (when present) against the per-upload cap,
|
|
1557
|
+
// then stream through guards that hard-cap a body whose size is omitted or
|
|
1558
|
+
// understated: the upload cap (413) and, when a quota is configured, the
|
|
1559
|
+
// Space headroom (507). On overflow either guard removes the partial file
|
|
1560
|
+
// before surfacing the error.
|
|
1561
|
+
if (maxUploadBytes !== undefined &&
|
|
1562
|
+
input.declaredBytes !== undefined &&
|
|
1563
|
+
input.declaredBytes > maxUploadBytes) {
|
|
1564
|
+
throw new PayloadTooLargeError({
|
|
1565
|
+
maxUploadBytes,
|
|
1566
|
+
backendId: this.describe().id,
|
|
1567
|
+
uploadBytes: input.declaredBytes
|
|
1568
|
+
});
|
|
1569
|
+
}
|
|
1570
|
+
const guards = [];
|
|
1571
|
+
if (maxUploadBytes !== undefined) {
|
|
1572
|
+
guards.push(this._uploadCapGuard({ maxUploadBytes }));
|
|
1573
|
+
}
|
|
1574
|
+
if (capacityBytes !== undefined) {
|
|
1575
|
+
const headroomBytes = await this._assertSpaceHeadroom({
|
|
1576
|
+
spaceId,
|
|
1577
|
+
capacityBytes,
|
|
1578
|
+
incomingBytes: input.declaredBytes ?? 0
|
|
1579
|
+
});
|
|
1580
|
+
guards.push(this._quotaGuard({ spaceId, capacityBytes, headroomBytes }));
|
|
1581
|
+
}
|
|
1582
|
+
// Stream into a temp file in the same directory, then durably commit it
|
|
1583
|
+
// (fsync + rename + dir fsync) once the whole body has been written and
|
|
1584
|
+
// verified. Streaming to the final path directly would expose a truncated
|
|
1585
|
+
// representation under the resource's name if the process crashed
|
|
1586
|
+
// mid-stream; staging + rename makes the resource appear only whole.
|
|
1587
|
+
const tempPath = tempPathFor(filePath);
|
|
1588
|
+
try {
|
|
1589
|
+
await pipeline([
|
|
1590
|
+
input.stream,
|
|
1591
|
+
...guards,
|
|
1592
|
+
fs.createWriteStream(tempPath)
|
|
1593
|
+
]);
|
|
1594
|
+
await commitTempFile({ tempPath, filePath });
|
|
1595
|
+
}
|
|
1596
|
+
catch (err) {
|
|
1597
|
+
// Remove the partial file on ANY failure: a guard rejection (413/507),
|
|
1598
|
+
// an aborted upload, or a streamed `Digest` mismatch (the request layer
|
|
1599
|
+
// verifies the body's digest as it flows). A failed write must not leave
|
|
1600
|
+
// a truncated or unverified representation behind. Only the temp path is
|
|
1601
|
+
// ever staged into, so a failed write never touches the final path.
|
|
1602
|
+
await rm(tempPath, { force: true });
|
|
1603
|
+
throw err;
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
// A Resource has a single current representation: remove any prior
|
|
1607
|
+
// representation stored under a different content-type (its filename
|
|
1608
|
+
// differs). Write-new-then-prune (not delete-then-write) so the resource is
|
|
1609
|
+
// never momentarily absent.
|
|
1610
|
+
const existing = await this._resourceFilesFor({ collectionDir, resourceId });
|
|
1611
|
+
await Promise.all(existing
|
|
1612
|
+
.filter(name => path.resolve(name) !== path.resolve(filePath))
|
|
1613
|
+
.map(name => rm(name)));
|
|
1614
|
+
// Maintain the server-managed timestamps and the monotonic `version`: a
|
|
1615
|
+
// content write sets `createdAt` on first write, bumps `updatedAt`, and
|
|
1616
|
+
// increments `version` (the ETag validator) from its prior value, preserving
|
|
1617
|
+
// any user-writable `custom` and the independent `metaVersion` already stored
|
|
1618
|
+
// in the sidecar (a content write does not touch the metadata sub-resource).
|
|
1619
|
+
//
|
|
1620
|
+
// `createdBy` pairs with `createdAt`: taken from this write's invoker only
|
|
1621
|
+
// when this write creates the sidecar, so it names the creator rather than
|
|
1622
|
+
// the last writer, and is preserved verbatim afterward -- including
|
|
1623
|
+
// preserved-as-absent, so a Resource created with no invoker never has a
|
|
1624
|
+
// later writer backfilled into it. A tombstone keeps both, so re-creating a
|
|
1625
|
+
// deleted id under a different invoker preserves the original creator, as
|
|
1626
|
+
// it does the original `createdAt`.
|
|
1627
|
+
const now = new Date().toISOString();
|
|
1628
|
+
const prior = await this._readMetaSidecar({ collectionDir, resourceId });
|
|
1629
|
+
const version = (prior?.version ?? 0) + 1;
|
|
1630
|
+
const creator = prior ? prior.createdBy : createdBy;
|
|
1631
|
+
await this._writeMetaSidecar({
|
|
1632
|
+
collectionDir,
|
|
1633
|
+
resourceId,
|
|
1634
|
+
sidecar: {
|
|
1635
|
+
createdAt: prior?.createdAt ?? now,
|
|
1636
|
+
updatedAt: now,
|
|
1637
|
+
...(creator !== undefined && { createdBy: creator }),
|
|
1638
|
+
version,
|
|
1639
|
+
...(prior?.metaVersion !== undefined && {
|
|
1640
|
+
metaVersion: prior.metaVersion
|
|
1641
|
+
}),
|
|
1642
|
+
...(prior?.custom && { custom: prior.custom }),
|
|
1643
|
+
// The key-epoch stamp is set from this write's declaration and CLEARED
|
|
1644
|
+
// when absent (the new ciphertext's epoch is unknown -- a stale stamp is
|
|
1645
|
+
// worse than none), so it is NOT preserved from `prior` like `custom`.
|
|
1646
|
+
...(epoch !== undefined && { epoch })
|
|
1647
|
+
}
|
|
1648
|
+
});
|
|
1649
|
+
return { version };
|
|
1650
|
+
}
|
|
1651
|
+
/**
|
|
1652
|
+
* Builds the per-Resource serialization key for `_writeMutex`
|
|
1653
|
+
* (`<spaceId>/<collectionId>/<resourceId>`), so conditional writes to distinct
|
|
1654
|
+
* Resources run concurrently while writes to the same Resource are ordered.
|
|
1655
|
+
* @param options {object}
|
|
1656
|
+
* @param options.spaceId {string}
|
|
1657
|
+
* @param options.collectionId {string}
|
|
1658
|
+
* @param options.resourceId {string}
|
|
1659
|
+
* @returns {string}
|
|
1660
|
+
*/
|
|
1661
|
+
_resourceLockKey({ spaceId, collectionId, resourceId }) {
|
|
1662
|
+
return `${spaceId}/${collectionId}/${resourceId}`;
|
|
1663
|
+
}
|
|
1664
|
+
/**
|
|
1665
|
+
* The Collection-level mutex key, held by unique-blinded-attribute writes to
|
|
1666
|
+
* serialize their conflict scans. Namespaced (`unique:` prefix) so it can
|
|
1667
|
+
* never collide with a per-Resource key.
|
|
1668
|
+
* @param options {object}
|
|
1669
|
+
* @param options.spaceId {string}
|
|
1670
|
+
* @param options.collectionId {string}
|
|
1671
|
+
* @returns {string}
|
|
1672
|
+
*/
|
|
1673
|
+
_collectionLockKey({ spaceId, collectionId }) {
|
|
1674
|
+
return `unique:${spaceId}/${collectionId}`;
|
|
1675
|
+
}
|
|
1676
|
+
/**
|
|
1677
|
+
* Evaluates a conditional-write precondition against a Resource's current
|
|
1678
|
+
* on-disk state. MUST be called inside the per-Resource write lock so the
|
|
1679
|
+
* check is atomic with the write that follows. Throws
|
|
1680
|
+
* `PreconditionFailedError` (412) when the precondition is not met.
|
|
1681
|
+
* @param options {object}
|
|
1682
|
+
* @param options.collectionDir {string}
|
|
1683
|
+
* @param options.resourceId {string}
|
|
1684
|
+
* @param [options.ifMatch] {string} a quoted ETag (`If-Match`)
|
|
1685
|
+
* @param [options.ifNoneMatch] {boolean} `If-None-Match: *` (create-if-absent)
|
|
1686
|
+
* @returns {Promise<void>}
|
|
1687
|
+
*/
|
|
1688
|
+
async _assertWritePrecondition({ collectionDir, resourceId, ifMatch, ifNoneMatch }) {
|
|
1689
|
+
const exists = (await this._findFile({ collectionDir, resourceId })) !== undefined;
|
|
1690
|
+
const prior = exists
|
|
1691
|
+
? await this._readMetaSidecar({ collectionDir, resourceId })
|
|
1692
|
+
: undefined;
|
|
1693
|
+
assertWritePrecondition({
|
|
1694
|
+
resourceId,
|
|
1695
|
+
exists,
|
|
1696
|
+
currentVersion: prior?.version ?? 0,
|
|
1697
|
+
ifMatch,
|
|
1698
|
+
ifNoneMatch
|
|
1699
|
+
});
|
|
1700
|
+
}
|
|
1701
|
+
/**
|
|
1702
|
+
* @param options {object}
|
|
1703
|
+
* @param options.spaceId {string}
|
|
1704
|
+
* @param options.collectionId {string}
|
|
1705
|
+
* @param options.resourceId {string}
|
|
1706
|
+
* @param [options.contentType] {string}
|
|
1707
|
+
* @returns {Promise<ResourceResult>} includes the Resource's current
|
|
1708
|
+
* `version` (the ETag validator) when one is recorded in its sidecar.
|
|
1709
|
+
*/
|
|
1710
|
+
async getResource({ spaceId, collectionId, resourceId }) {
|
|
1711
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1712
|
+
const filePath = await this._findFile({ collectionDir, resourceId });
|
|
1713
|
+
if (!filePath) {
|
|
1714
|
+
throw new ResourceNotFoundError({ requestName: 'Get Resource' });
|
|
1715
|
+
}
|
|
1716
|
+
try {
|
|
1717
|
+
await fsStat(filePath);
|
|
1718
|
+
}
|
|
1719
|
+
catch (err) {
|
|
1720
|
+
if (err.code === 'ENOENT') {
|
|
1721
|
+
throw new ResourceNotFoundError({ requestName: 'Get Resource' });
|
|
1722
|
+
}
|
|
1723
|
+
throw err;
|
|
1724
|
+
}
|
|
1725
|
+
// Derive the stored content-type from the filename segment (the exact type
|
|
1726
|
+
// it was written under), not from `mime.lookup` on the extension.
|
|
1727
|
+
const { contentType: storedResourceType } = parseResourceFileName(path.basename(filePath));
|
|
1728
|
+
// Surface the ETag validator: the per-Resource `version` from the sidecar
|
|
1729
|
+
// (absent only for a legacy Resource written before versioning).
|
|
1730
|
+
const sidecar = await this._readMetaSidecar({ collectionDir, resourceId });
|
|
1731
|
+
return {
|
|
1732
|
+
resourceStream: await openFileStream(filePath, this.logger),
|
|
1733
|
+
storedResourceType,
|
|
1734
|
+
...(sidecar?.version !== undefined && { version: sidecar.version })
|
|
1735
|
+
};
|
|
1736
|
+
}
|
|
1737
|
+
/**
|
|
1738
|
+
* Builds the on-disk path for a Resource's metadata sidecar
|
|
1739
|
+
* (`.meta.<resourceId>.json`) in its Collection dir.
|
|
1740
|
+
* @param options {object}
|
|
1741
|
+
* @param options.collectionDir {string}
|
|
1742
|
+
* @param options.resourceId {string}
|
|
1743
|
+
* @returns {string}
|
|
1744
|
+
*/
|
|
1745
|
+
_metaSidecarPath({ collectionDir, resourceId }) {
|
|
1746
|
+
const filePath = path.join(collectionDir, metaSidecarFileName(resourceId));
|
|
1747
|
+
this._assertContained(filePath);
|
|
1748
|
+
return filePath;
|
|
1749
|
+
}
|
|
1750
|
+
/**
|
|
1751
|
+
* Reads a Resource's metadata sidecar. Resolves `undefined` when none has been
|
|
1752
|
+
* written yet (e.g. a Resource created before sidecars existed).
|
|
1753
|
+
* @param options {object}
|
|
1754
|
+
* @param options.collectionDir {string}
|
|
1755
|
+
* @param options.resourceId {string}
|
|
1756
|
+
* @returns {Promise<MetaSidecar|undefined>}
|
|
1757
|
+
*/
|
|
1758
|
+
async _readMetaSidecar({ collectionDir, resourceId }) {
|
|
1759
|
+
const metaStore = new MetadataJsonStore({
|
|
1760
|
+
file: this._metaSidecarPath({ collectionDir, resourceId })
|
|
1761
|
+
});
|
|
1762
|
+
return await metaStore.read();
|
|
1763
|
+
}
|
|
1764
|
+
/**
|
|
1765
|
+
* Writes a Resource's metadata sidecar (full replacement).
|
|
1766
|
+
* @param options {object}
|
|
1767
|
+
* @param options.collectionDir {string}
|
|
1768
|
+
* @param options.resourceId {string}
|
|
1769
|
+
* @param options.sidecar {MetaSidecar}
|
|
1770
|
+
* @returns {Promise<void>}
|
|
1771
|
+
*/
|
|
1772
|
+
async _writeMetaSidecar({ collectionDir, resourceId, sidecar }) {
|
|
1773
|
+
await atomicWriteFile({
|
|
1774
|
+
filePath: this._metaSidecarPath({ collectionDir, resourceId }),
|
|
1775
|
+
data: JSON.stringify(sidecar)
|
|
1776
|
+
});
|
|
1777
|
+
}
|
|
1778
|
+
/**
|
|
1779
|
+
* Reads the metadata of a Resource's current representation: the REQUIRED
|
|
1780
|
+
* server-managed fields (`contentType`, `size`, both derived from the stored
|
|
1781
|
+
* file), plus the OPTIONAL `createdAt` / `updatedAt` timestamps and the
|
|
1782
|
+
* user-writable `custom` object read from the sidecar. For a Resource written
|
|
1783
|
+
* before sidecars existed, the timestamps fall back to the file's birth/modify
|
|
1784
|
+
* times and `custom` is omitted. Resolves `undefined` when the Resource is
|
|
1785
|
+
* absent (including a delete race on `stat`).
|
|
1786
|
+
*
|
|
1787
|
+
* Also surfaces the Resource's content `version` and its `metaVersion` (the
|
|
1788
|
+
* two ETag validators) when recorded in the sidecar, so the request layer can
|
|
1789
|
+
* set the `ETag` header: HEAD / the resource itself use the content `version`,
|
|
1790
|
+
* while `GET /meta` uses `metaVersion`. Both are out-of-band fields the request
|
|
1791
|
+
* layer reads for the header; neither is part of the Resource Metadata wire
|
|
1792
|
+
* body.
|
|
1793
|
+
* @param options {object}
|
|
1794
|
+
* @param options.spaceId {string}
|
|
1795
|
+
* @param options.collectionId {string}
|
|
1796
|
+
* @param options.resourceId {string}
|
|
1797
|
+
* @returns {Promise<(ResourceMetadata & { version?: number; metaVersion?:
|
|
1798
|
+
* number }) | undefined>}
|
|
1799
|
+
*/
|
|
1800
|
+
async getResourceMetadata({ spaceId, collectionId, resourceId }) {
|
|
1801
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1802
|
+
const filePath = await this._findFile({ collectionDir, resourceId });
|
|
1803
|
+
if (!filePath) {
|
|
1804
|
+
return undefined;
|
|
1805
|
+
}
|
|
1806
|
+
let stats;
|
|
1807
|
+
try {
|
|
1808
|
+
stats = await fsStat(filePath);
|
|
1809
|
+
}
|
|
1810
|
+
catch (err) {
|
|
1811
|
+
if (err.code === 'ENOENT') {
|
|
1812
|
+
return undefined;
|
|
1813
|
+
}
|
|
1814
|
+
throw err;
|
|
1815
|
+
}
|
|
1816
|
+
// Derive the stored content-type from the filename segment (the exact type
|
|
1817
|
+
// it was written under), as `getResource` does.
|
|
1818
|
+
const { contentType } = parseResourceFileName(path.basename(filePath));
|
|
1819
|
+
const sidecar = await this._readMetaSidecar({ collectionDir, resourceId });
|
|
1820
|
+
const createdAt = sidecar?.createdAt ?? stats.birthtime.toISOString();
|
|
1821
|
+
const updatedAt = sidecar?.updatedAt ?? stats.mtime.toISOString();
|
|
1822
|
+
const hasCustom = sidecar?.custom && Object.keys(sidecar.custom).length > 0;
|
|
1823
|
+
return {
|
|
1824
|
+
contentType,
|
|
1825
|
+
size: stats.size,
|
|
1826
|
+
createdAt,
|
|
1827
|
+
updatedAt,
|
|
1828
|
+
// Absent for a Resource created before `createdBy` was recorded; there is
|
|
1829
|
+
// no stat-based fallback for it, as there is for the timestamps.
|
|
1830
|
+
...(sidecar?.createdBy !== undefined && { createdBy: sidecar.createdBy }),
|
|
1831
|
+
// `custom` is returned verbatim -- `{ name, tags }` on a plaintext
|
|
1832
|
+
// Collection, the opaque encryption envelope on an encrypted one.
|
|
1833
|
+
...(hasCustom && { custom: sidecar.custom }),
|
|
1834
|
+
// The client-declared key epoch (the `key-epochs` feature), when stamped.
|
|
1835
|
+
...(sidecar?.epoch !== undefined && { epoch: sidecar.epoch }),
|
|
1836
|
+
...(sidecar?.version !== undefined && { version: sidecar.version }),
|
|
1837
|
+
...(sidecar?.metaVersion !== undefined && {
|
|
1838
|
+
metaVersion: sidecar.metaVersion
|
|
1839
|
+
})
|
|
1840
|
+
};
|
|
1841
|
+
}
|
|
1842
|
+
/**
|
|
1843
|
+
* Replaces the user-writable `custom` object of a Resource's metadata sidecar
|
|
1844
|
+
* (full replacement; `{}` clears it), bumping `updatedAt` and the independent
|
|
1845
|
+
* `metaVersion` (the `/meta` ETag). Does not create a Resource: resolves
|
|
1846
|
+
* `undefined` when the Resource is absent so the handler can 404. The content
|
|
1847
|
+
* `version` and the two REQUIRED server-managed fields are untouched (a
|
|
1848
|
+
* metadata write does not change the stored representation, preserving the
|
|
1849
|
+
* content ETag contract). On an encrypted Collection `custom` is the opaque
|
|
1850
|
+
* encryption envelope, stored verbatim.
|
|
1851
|
+
*
|
|
1852
|
+
* Runs under the per-Resource write lock -- the same lock content writes take
|
|
1853
|
+
* -- so an `If-Match` / `If-None-Match` precondition (evaluated on
|
|
1854
|
+
* `metaVersion`) is atomic with the write and serializes with concurrent
|
|
1855
|
+
* content/metadata writes to the same Resource. A precondition mismatch throws
|
|
1856
|
+
* `PreconditionFailedError` (412).
|
|
1857
|
+
* @param options {object}
|
|
1858
|
+
* @param options.spaceId {string}
|
|
1859
|
+
* @param options.collectionId {string}
|
|
1860
|
+
* @param options.resourceId {string}
|
|
1861
|
+
* @param options.custom {ResourceMetadataCustom | Record<string, unknown>}
|
|
1862
|
+
* @param [options.ifMatch] {string} `If-Match` on the current `metaVersion`
|
|
1863
|
+
* @param [options.ifNoneMatch] {boolean} `If-None-Match: *` -- write only if
|
|
1864
|
+
* no metadata has been written yet (`metaVersion` unset)
|
|
1865
|
+
* @returns {Promise<{ metaVersion: number } | undefined>} the new
|
|
1866
|
+
* `metaVersion`, or `undefined` when the Resource does not exist
|
|
1867
|
+
*/
|
|
1868
|
+
async writeResourceMetadata({ spaceId, collectionId, resourceId, custom, epoch, ifMatch, ifNoneMatch }) {
|
|
1869
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1870
|
+
const writeMeta = async () => {
|
|
1871
|
+
const filePath = await this._findFile({ collectionDir, resourceId });
|
|
1872
|
+
if (!filePath) {
|
|
1873
|
+
return undefined;
|
|
1874
|
+
}
|
|
1875
|
+
const prior = await this._readMetaSidecar({ collectionDir, resourceId });
|
|
1876
|
+
// Evaluate the `/meta` precondition against the current `metaVersion`
|
|
1877
|
+
// atomically under the lock, before writing. `If-None-Match: *` means
|
|
1878
|
+
// "only if no metadata has been written yet"; `If-Match` pins the current
|
|
1879
|
+
// `metaVersion` ETag.
|
|
1880
|
+
assertMetaWritePrecondition({
|
|
1881
|
+
resourceId,
|
|
1882
|
+
metaVersion: prior?.metaVersion,
|
|
1883
|
+
ifMatch,
|
|
1884
|
+
ifNoneMatch
|
|
1885
|
+
});
|
|
1886
|
+
const now = new Date().toISOString();
|
|
1887
|
+
// Fall back to the file's birth time for `createdAt` if the Resource
|
|
1888
|
+
// predates sidecars (so a meta write does not lose its creation time).
|
|
1889
|
+
let createdAt = prior?.createdAt;
|
|
1890
|
+
if (!createdAt) {
|
|
1891
|
+
try {
|
|
1892
|
+
createdAt = (await fsStat(filePath)).birthtime.toISOString();
|
|
1893
|
+
}
|
|
1894
|
+
catch {
|
|
1895
|
+
createdAt = now;
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
const metaVersion = (prior?.metaVersion ?? 0) + 1;
|
|
1899
|
+
const hasCustom = Object.keys(custom).length > 0;
|
|
1900
|
+
// The key-epoch stamp describes the CONTENT write, not this metadata
|
|
1901
|
+
// write, so a supplied `epoch` replaces it but an omitted one PRESERVES
|
|
1902
|
+
// the stored value (unlike `custom`, which is full-replace).
|
|
1903
|
+
const resolvedEpoch = epoch ?? prior?.epoch;
|
|
1904
|
+
await this._writeMetaSidecar({
|
|
1905
|
+
collectionDir,
|
|
1906
|
+
resourceId,
|
|
1907
|
+
sidecar: {
|
|
1908
|
+
createdAt,
|
|
1909
|
+
updatedAt: now,
|
|
1910
|
+
// Preserve the server-managed creator: a metadata write replaces only
|
|
1911
|
+
// `custom`, and `createdBy` is not user-writable.
|
|
1912
|
+
...(prior?.createdBy !== undefined && {
|
|
1913
|
+
createdBy: prior.createdBy
|
|
1914
|
+
}),
|
|
1915
|
+
// Preserve the content `version` (ETag) -- a metadata write does not
|
|
1916
|
+
// change the stored representation.
|
|
1917
|
+
...(prior?.version !== undefined && { version: prior.version }),
|
|
1918
|
+
metaVersion,
|
|
1919
|
+
...(hasCustom && { custom }),
|
|
1920
|
+
...(resolvedEpoch !== undefined && { epoch: resolvedEpoch })
|
|
1921
|
+
}
|
|
1922
|
+
});
|
|
1923
|
+
return { metaVersion };
|
|
1924
|
+
};
|
|
1925
|
+
return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), writeMeta);
|
|
1926
|
+
}
|
|
1927
|
+
/**
|
|
1928
|
+
* Soft-deletes a Resource: drops its content representation but keeps the
|
|
1929
|
+
* sidecar as a **tombstone** (`deleted: true`, a bumped `version` and
|
|
1930
|
+
* `updatedAt`, the last-known `contentType` retained) so the change feed
|
|
1931
|
+
* (replication) still surfaces it until clients catch up (GC of tombstones is
|
|
1932
|
+
* future work). With no content file left, the tombstone is invisible to
|
|
1933
|
+
* every normal read path
|
|
1934
|
+
* (`getResource` / `getResourceMetadata` / `listCollectionItems` all gate on
|
|
1935
|
+
* the content file via `_findFile`, so they 404 / skip it), making soft delete
|
|
1936
|
+
* transparent to the existing API.
|
|
1937
|
+
*
|
|
1938
|
+
* When `ifMatch` is supplied (the `conditional-writes` feature), the delete
|
|
1939
|
+
* proceeds only if the Resource exists and its current `ETag` matches, else
|
|
1940
|
+
* `PreconditionFailedError` (412). The whole read-modify-write runs under the
|
|
1941
|
+
* per-Resource write lock so it serializes with concurrent writes. The delete
|
|
1942
|
+
* is idempotent: an already-absent Resource (never created, or an existing
|
|
1943
|
+
* tombstone) is a no-op, leaving any tombstone's change-feed entry stable.
|
|
1944
|
+
* @param options {object}
|
|
1945
|
+
* @param options.spaceId {string}
|
|
1946
|
+
* @param options.collectionId {string}
|
|
1947
|
+
* @param options.resourceId {string}
|
|
1948
|
+
* @param [options.ifMatch] {string} `If-Match` precondition (a quoted ETag)
|
|
1949
|
+
* @returns {Promise<void>}
|
|
1950
|
+
*/
|
|
1951
|
+
async deleteResource({ spaceId, collectionId, resourceId, ifMatch }) {
|
|
1952
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
1953
|
+
// Freed bytes: drop the cached quota usage so the next write re-measures.
|
|
1954
|
+
this._usageCache.delete(spaceId);
|
|
1955
|
+
const softDelete = async () => {
|
|
1956
|
+
if (ifMatch !== undefined) {
|
|
1957
|
+
await this._assertWritePrecondition({
|
|
1958
|
+
collectionDir,
|
|
1959
|
+
resourceId,
|
|
1960
|
+
ifMatch
|
|
1961
|
+
});
|
|
1962
|
+
}
|
|
1963
|
+
// A Resource has a single current representation, so this normally matches
|
|
1964
|
+
// one file. The segment-anchored match in `_resourceFilesFor` keeps a
|
|
1965
|
+
// prefix id (e.g. `note` vs `notebook`) from being swept up too.
|
|
1966
|
+
const filesForResource = await this._resourceFilesFor({
|
|
1967
|
+
collectionDir,
|
|
1968
|
+
resourceId
|
|
1969
|
+
});
|
|
1970
|
+
if (filesForResource.length === 0) {
|
|
1971
|
+
// Already absent (never existed, or already a tombstone): idempotent
|
|
1972
|
+
// no-op. Leaving an existing tombstone untouched keeps its change-feed
|
|
1973
|
+
// entry (its `updatedAt` / `version`) stable.
|
|
1974
|
+
return;
|
|
1975
|
+
}
|
|
1976
|
+
// Capture the representation's last-known content-type from its filename
|
|
1977
|
+
// before removing it: once the content file is gone the tombstone sidecar
|
|
1978
|
+
// is the only record of it, and the change feed reports it.
|
|
1979
|
+
const { contentType } = parseResourceFileName(path.basename(filesForResource[0]));
|
|
1980
|
+
// Drop the content representation(s) but KEEP the sidecar as the tombstone.
|
|
1981
|
+
await Promise.all(filesForResource.map(filename => rm(filename)));
|
|
1982
|
+
// Bump `version` / `updatedAt` so the tombstone sorts after the Resource's
|
|
1983
|
+
// prior state in the change feed, and continues the monotonic version (a
|
|
1984
|
+
// later re-create reads this sidecar and keeps counting up). `custom` is
|
|
1985
|
+
// dropped: the user Metadata goes with the deleted Resource. `createdAt` /
|
|
1986
|
+
// `createdBy` are kept: they are the server's record of the Resource's
|
|
1987
|
+
// origin, which a re-create under the same id continues.
|
|
1988
|
+
const now = new Date().toISOString();
|
|
1989
|
+
const prior = await this._readMetaSidecar({ collectionDir, resourceId });
|
|
1990
|
+
await this._writeMetaSidecar({
|
|
1991
|
+
collectionDir,
|
|
1992
|
+
resourceId,
|
|
1993
|
+
sidecar: {
|
|
1994
|
+
createdAt: prior?.createdAt ?? now,
|
|
1995
|
+
updatedAt: now,
|
|
1996
|
+
...(prior?.createdBy !== undefined && {
|
|
1997
|
+
createdBy: prior.createdBy
|
|
1998
|
+
}),
|
|
1999
|
+
version: (prior?.version ?? 0) + 1,
|
|
2000
|
+
deleted: true,
|
|
2001
|
+
contentType
|
|
2002
|
+
}
|
|
2003
|
+
});
|
|
2004
|
+
};
|
|
2005
|
+
// The soft delete is a read-modify-write on the sidecar, so it always
|
|
2006
|
+
// serializes with concurrent writes under the per-Resource lock (not only
|
|
2007
|
+
// for a conditional delete, as the old unconditional removal did).
|
|
2008
|
+
return this._writeMutex.run(this._resourceLockKey({ spaceId, collectionId, resourceId }), softDelete);
|
|
2009
|
+
}
|
|
2010
|
+
/**
|
|
2011
|
+
* Replication change feed (the `changes` query profile; see the
|
|
2012
|
+
* `StorageBackend.changesSince` contract.
|
|
2013
|
+
* Enumerates the Collection once, builds a lightweight descriptor for every
|
|
2014
|
+
* JSON-document Resource (live) and JSON tombstone, orders them by
|
|
2015
|
+
* `(updatedAt, resourceId)`, seeks strictly past `checkpoint`, takes a page of
|
|
2016
|
+
* `limit`, and reads JSON bodies ONLY for that page. O(n) over the Collection
|
|
2017
|
+
* per call (it must read every sidecar to order by `updatedAt`) -- acceptable
|
|
2018
|
+
* for this teaching backend; an indexed backend would answer it from an
|
|
2019
|
+
* `updatedAt` index.
|
|
2020
|
+
* @param options {object}
|
|
2021
|
+
* @param options.spaceId {string}
|
|
2022
|
+
* @param options.collectionId {string}
|
|
2023
|
+
* @param [options.checkpoint] {{ id: string, updatedAt: string }} resume position
|
|
2024
|
+
* @param options.limit {number} page cap (clamped to the backend maximum)
|
|
2025
|
+
* @returns {Promise<{ documents: Array<object>, checkpoint: object | null }>}
|
|
2026
|
+
*/
|
|
2027
|
+
async changesSince({ spaceId, collectionId, checkpoint, limit }) {
|
|
2028
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
2029
|
+
let entries = [];
|
|
2030
|
+
try {
|
|
2031
|
+
entries = await fs.promises.readdir(collectionDir, {
|
|
2032
|
+
withFileTypes: true
|
|
2033
|
+
});
|
|
2034
|
+
}
|
|
2035
|
+
catch (err) {
|
|
2036
|
+
if (err.code !== 'ENOENT') {
|
|
2037
|
+
throw err;
|
|
2038
|
+
}
|
|
2039
|
+
}
|
|
2040
|
+
// Index the dir: live content files by id, and the set of ids that have a
|
|
2041
|
+
// `.meta.` sidecar (a sidecar with no live file is a tombstone candidate).
|
|
2042
|
+
const liveFileById = new Map();
|
|
2043
|
+
const sidecarIds = new Set();
|
|
2044
|
+
for (const entry of entries) {
|
|
2045
|
+
if (!entry.isFile()) {
|
|
2046
|
+
continue;
|
|
2047
|
+
}
|
|
2048
|
+
if (entry.name.startsWith('r.')) {
|
|
2049
|
+
const { resourceId, contentType } = parseResourceFileName(entry.name);
|
|
2050
|
+
liveFileById.set(resourceId, { fileName: entry.name, contentType });
|
|
2051
|
+
}
|
|
2052
|
+
else {
|
|
2053
|
+
const sidecarId = metaSidecarFileId(entry.name);
|
|
2054
|
+
if (sidecarId !== undefined) {
|
|
2055
|
+
sidecarIds.add(sidecarId);
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
const liveDescriptors = [...liveFileById].map(async ([resourceId, live]) => {
|
|
2060
|
+
if (!isJson({ contentType: live.contentType })) {
|
|
2061
|
+
return undefined;
|
|
2062
|
+
}
|
|
2063
|
+
const sidecar = await this._readMetaSidecar({
|
|
2064
|
+
collectionDir,
|
|
2065
|
+
resourceId
|
|
2066
|
+
});
|
|
2067
|
+
// `updatedAt` / `version` come from the sidecar; fall back to the file's
|
|
2068
|
+
// mtime for a legacy Resource written before sidecars existed.
|
|
2069
|
+
let updatedAt = sidecar?.updatedAt;
|
|
2070
|
+
if (!updatedAt) {
|
|
2071
|
+
try {
|
|
2072
|
+
updatedAt = (await fsStat(path.join(collectionDir, live.fileName))).mtime.toISOString();
|
|
2073
|
+
}
|
|
2074
|
+
catch {
|
|
2075
|
+
return undefined;
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
return {
|
|
2079
|
+
resourceId,
|
|
2080
|
+
version: sidecar?.version ?? 0,
|
|
2081
|
+
...(sidecar?.metaVersion !== undefined && {
|
|
2082
|
+
metaVersion: sidecar.metaVersion
|
|
2083
|
+
}),
|
|
2084
|
+
// The creator's DID rides the feed so provenance replicates with the
|
|
2085
|
+
// document, rather than needing a `/meta` fetch per Resource.
|
|
2086
|
+
...(sidecar?.createdBy !== undefined && {
|
|
2087
|
+
createdBy: sidecar.createdBy
|
|
2088
|
+
}),
|
|
2089
|
+
updatedAt,
|
|
2090
|
+
deleted: false,
|
|
2091
|
+
fileName: live.fileName,
|
|
2092
|
+
// The user-writable `custom` (the opaque encryption envelope on an
|
|
2093
|
+
// encrypted Collection) rides the feed so metadata replicates
|
|
2094
|
+
// alongside content; read from the sidecar already loaded here.
|
|
2095
|
+
...(sidecar?.custom !== undefined && { custom: sidecar.custom }),
|
|
2096
|
+
// The client-declared key epoch (the `key-epochs` feature) rides the
|
|
2097
|
+
// feed so a replicating reader picks the right epoch key.
|
|
2098
|
+
...(sidecar?.epoch !== undefined && { epoch: sidecar.epoch })
|
|
2099
|
+
};
|
|
2100
|
+
});
|
|
2101
|
+
// A sidecar with no live file is a tombstone candidate; keep only the ones
|
|
2102
|
+
// that are actually tombstones (`deleted: true`) and JSON.
|
|
2103
|
+
const tombstoneDescriptors = [...sidecarIds]
|
|
2104
|
+
.filter(resourceId => !liveFileById.has(resourceId))
|
|
2105
|
+
.map(async (resourceId) => {
|
|
2106
|
+
const sidecar = await this._readMetaSidecar({
|
|
2107
|
+
collectionDir,
|
|
2108
|
+
resourceId
|
|
2109
|
+
});
|
|
2110
|
+
if (sidecar?.deleted !== true ||
|
|
2111
|
+
!isJson({ contentType: sidecar.contentType })) {
|
|
2112
|
+
return undefined;
|
|
2113
|
+
}
|
|
2114
|
+
return {
|
|
2115
|
+
resourceId,
|
|
2116
|
+
version: sidecar.version ?? 0,
|
|
2117
|
+
...(sidecar.metaVersion !== undefined && {
|
|
2118
|
+
metaVersion: sidecar.metaVersion
|
|
2119
|
+
}),
|
|
2120
|
+
// A tombstone keeps its creator, as it keeps its `createdAt`.
|
|
2121
|
+
...(sidecar.createdBy !== undefined && {
|
|
2122
|
+
createdBy: sidecar.createdBy
|
|
2123
|
+
}),
|
|
2124
|
+
updatedAt: sidecar.updatedAt,
|
|
2125
|
+
deleted: true
|
|
2126
|
+
};
|
|
2127
|
+
});
|
|
2128
|
+
const descriptors = (await Promise.all([...liveDescriptors, ...tombstoneDescriptors])).filter((desc) => desc !== undefined);
|
|
2129
|
+
// Order by `(updatedAt, resourceId)` ascending -- the SAME total order the
|
|
2130
|
+
// checkpoint seek uses (ISO-8601 `updatedAt` sorts chronologically as a
|
|
2131
|
+
// string; `resourceId` breaks same-instant ties), so the keyset is stable.
|
|
2132
|
+
descriptors.sort((left, right) => compareCodeUnits(left.updatedAt, right.updatedAt) ||
|
|
2133
|
+
compareCodeUnits(left.resourceId, right.resourceId));
|
|
2134
|
+
// Seek to the first descriptor strictly after the checkpoint's position.
|
|
2135
|
+
let startIndex = 0;
|
|
2136
|
+
if (checkpoint !== undefined) {
|
|
2137
|
+
const found = descriptors.findIndex(desc => desc.updatedAt > checkpoint.updatedAt ||
|
|
2138
|
+
(desc.updatedAt === checkpoint.updatedAt &&
|
|
2139
|
+
desc.resourceId > checkpoint.id));
|
|
2140
|
+
startIndex = found === -1 ? descriptors.length : found;
|
|
2141
|
+
}
|
|
2142
|
+
const pageSize = clampPageSize(limit);
|
|
2143
|
+
const pageDescriptors = descriptors.slice(startIndex, startIndex + pageSize);
|
|
2144
|
+
// Read JSON bodies only for this page. A tombstone carries no `data` (the
|
|
2145
|
+
// delete replicates on `deleted: true` alone).
|
|
2146
|
+
const documents = await Promise.all(pageDescriptors.map(async (desc) => {
|
|
2147
|
+
if (desc.deleted) {
|
|
2148
|
+
return {
|
|
2149
|
+
resourceId: desc.resourceId,
|
|
2150
|
+
version: desc.version,
|
|
2151
|
+
...(desc.metaVersion !== undefined && {
|
|
2152
|
+
metaVersion: desc.metaVersion
|
|
2153
|
+
}),
|
|
2154
|
+
...(desc.createdBy !== undefined && { createdBy: desc.createdBy }),
|
|
2155
|
+
updatedAt: desc.updatedAt,
|
|
2156
|
+
deleted: true
|
|
2157
|
+
};
|
|
2158
|
+
}
|
|
2159
|
+
let data;
|
|
2160
|
+
try {
|
|
2161
|
+
data = JSON.parse(await fs.promises.readFile(path.join(collectionDir, desc.fileName), 'utf8'));
|
|
2162
|
+
}
|
|
2163
|
+
catch {
|
|
2164
|
+
data = undefined;
|
|
2165
|
+
}
|
|
2166
|
+
return {
|
|
2167
|
+
resourceId: desc.resourceId,
|
|
2168
|
+
version: desc.version,
|
|
2169
|
+
...(desc.metaVersion !== undefined && {
|
|
2170
|
+
metaVersion: desc.metaVersion
|
|
2171
|
+
}),
|
|
2172
|
+
...(desc.createdBy !== undefined && { createdBy: desc.createdBy }),
|
|
2173
|
+
updatedAt: desc.updatedAt,
|
|
2174
|
+
deleted: false,
|
|
2175
|
+
data,
|
|
2176
|
+
// Surface the user-writable `custom` (opaque envelope on an encrypted
|
|
2177
|
+
// Collection) so a metadata-only edit replicates.
|
|
2178
|
+
...(desc.custom !== undefined && { custom: desc.custom }),
|
|
2179
|
+
// Surface the client-declared key epoch (the `key-epochs` feature).
|
|
2180
|
+
...(desc.epoch !== undefined && { epoch: desc.epoch })
|
|
2181
|
+
};
|
|
2182
|
+
}));
|
|
2183
|
+
const last = documents[documents.length - 1];
|
|
2184
|
+
return {
|
|
2185
|
+
documents,
|
|
2186
|
+
checkpoint: last
|
|
2187
|
+
? { id: last.resourceId, updatedAt: last.updatedAt }
|
|
2188
|
+
: null
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
/**
|
|
2192
|
+
* Blinded-index query (the `blinded-index` query profile; see the
|
|
2193
|
+
* `StorageBackend.queryByBlindedIndex` contract). Enumerates the Collection
|
|
2194
|
+
* dir, reads and parses every live JSON Resource, and hands the candidates
|
|
2195
|
+
* to the shared evaluator (`lib/blindedIndex.ts`) for matching, ordering,
|
|
2196
|
+
* and cursor pagination. O(n) over the Collection per call, with every JSON
|
|
2197
|
+
* body read -- acceptable for this teaching backend; an indexed backend
|
|
2198
|
+
* would answer from flattened attribute tokens. Tombstones are excluded
|
|
2199
|
+
* naturally (no live content file); binary Resources and unparsable JSON
|
|
2200
|
+
* are skipped.
|
|
2201
|
+
* @param options {object}
|
|
2202
|
+
* @param options.spaceId {string}
|
|
2203
|
+
* @param options.collectionId {string}
|
|
2204
|
+
* @param options.query {BlindedIndexQuery}
|
|
2205
|
+
* @param [options.count] {boolean} return only the match count
|
|
2206
|
+
* @param [options.limit] {number} requested page size
|
|
2207
|
+
* @param [options.cursor] {string} opaque cursor from a prior page
|
|
2208
|
+
* @returns {Promise<{ count: number } | BlindedIndexQueryPage>}
|
|
2209
|
+
*/
|
|
2210
|
+
async queryByBlindedIndex({ spaceId, collectionId, query, count, limit, cursor }) {
|
|
2211
|
+
const candidates = await this._readJsonCandidates({
|
|
2212
|
+
spaceId,
|
|
2213
|
+
collectionId
|
|
2214
|
+
});
|
|
2215
|
+
return runBlindedIndexQuery({ candidates, query, count, limit, cursor });
|
|
2216
|
+
}
|
|
2217
|
+
/**
|
|
2218
|
+
* Reads and parses every live JSON Resource of a Collection -- the candidate
|
|
2219
|
+
* set for both the blinded-index query and the unique-attribute conflict
|
|
2220
|
+
* scan. Tombstones are excluded naturally (no live content file); binary
|
|
2221
|
+
* Resources, unparsable JSON, and (optionally) one excluded Resource are
|
|
2222
|
+
* skipped. An absent Collection dir resolves empty.
|
|
2223
|
+
* @param options {object}
|
|
2224
|
+
* @param options.spaceId {string}
|
|
2225
|
+
* @param options.collectionId {string}
|
|
2226
|
+
* @param [options.excludeResourceId] {string} omit this Resource (the
|
|
2227
|
+
* conflict scan excludes the document being written)
|
|
2228
|
+
* @returns {Promise<Array<{ resourceId: string, document: unknown }>>}
|
|
2229
|
+
*/
|
|
2230
|
+
async _readJsonCandidates({ spaceId, collectionId, excludeResourceId }) {
|
|
2231
|
+
const collectionDir = this._collectionDir({ spaceId, collectionId });
|
|
2232
|
+
let entries = [];
|
|
2233
|
+
try {
|
|
2234
|
+
entries = await fs.promises.readdir(collectionDir, {
|
|
2235
|
+
withFileTypes: true
|
|
2236
|
+
});
|
|
2237
|
+
}
|
|
2238
|
+
catch (err) {
|
|
2239
|
+
if (err.code !== 'ENOENT') {
|
|
2240
|
+
throw err;
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
return (await Promise.all(entries
|
|
2244
|
+
.filter(entry => entry.isFile() && entry.name.startsWith('r.'))
|
|
2245
|
+
.map(entry => ({
|
|
2246
|
+
fileName: entry.name,
|
|
2247
|
+
...parseResourceFileName(entry.name)
|
|
2248
|
+
}))
|
|
2249
|
+
.filter(({ resourceId, contentType }) => resourceId !== excludeResourceId && isJson({ contentType }))
|
|
2250
|
+
.map(async ({ resourceId, fileName }) => {
|
|
2251
|
+
try {
|
|
2252
|
+
return {
|
|
2253
|
+
resourceId,
|
|
2254
|
+
document: JSON.parse(await fs.promises.readFile(path.join(collectionDir, fileName), 'utf8'))
|
|
2255
|
+
};
|
|
2256
|
+
}
|
|
2257
|
+
catch {
|
|
2258
|
+
return undefined;
|
|
2259
|
+
}
|
|
2260
|
+
}))).filter(candidate => candidate !== undefined);
|
|
2261
|
+
}
|
|
2262
|
+
// Policies
|
|
2263
|
+
/**
|
|
2264
|
+
* Builds the on-disk path for a policy document. Stored as a dot-file keyed by
|
|
2265
|
+
* the entity id, alongside the matching `.space.` / `.collection.` description:
|
|
2266
|
+
* Space policy in the space dir, Collection and Resource policy in the
|
|
2267
|
+
* collection dir (the keying id differs, so they never collide).
|
|
2268
|
+
* @param options {object}
|
|
2269
|
+
* @param options.spaceId {string}
|
|
2270
|
+
* @param [options.collectionId] {string}
|
|
2271
|
+
* @param [options.resourceId] {string}
|
|
2272
|
+
* @returns {string}
|
|
2273
|
+
*/
|
|
2274
|
+
_policyFile({ spaceId, collectionId, resourceId }) {
|
|
2275
|
+
const dir = collectionId !== undefined
|
|
2276
|
+
? this._collectionDir({ spaceId, collectionId })
|
|
2277
|
+
: this._spaceDir(spaceId);
|
|
2278
|
+
const filename = `.policy.${resourceId ?? collectionId ?? spaceId}.json`;
|
|
2279
|
+
const filePath = path.join(dir, filename);
|
|
2280
|
+
this._assertContained(filePath);
|
|
2281
|
+
return filePath;
|
|
2282
|
+
}
|
|
2283
|
+
/**
|
|
2284
|
+
* @param options {object}
|
|
2285
|
+
* @param options.spaceId {string}
|
|
2286
|
+
* @param [options.collectionId] {string}
|
|
2287
|
+
* @param [options.resourceId] {string}
|
|
2288
|
+
* @returns {Promise<PolicyDocument|undefined>}
|
|
2289
|
+
* Resolves falsy when no policy is set at that level (must not throw).
|
|
2290
|
+
*/
|
|
2291
|
+
async getPolicy({ spaceId, collectionId, resourceId }) {
|
|
2292
|
+
const metaStore = new MetadataJsonStore({
|
|
2293
|
+
file: this._policyFile({ spaceId, collectionId, resourceId })
|
|
2294
|
+
});
|
|
2295
|
+
return await metaStore.read();
|
|
2296
|
+
}
|
|
2297
|
+
/**
|
|
2298
|
+
* @param options {object}
|
|
2299
|
+
* @param options.spaceId {string}
|
|
2300
|
+
* @param [options.collectionId] {string}
|
|
2301
|
+
* @param [options.resourceId] {string}
|
|
2302
|
+
* @param options.policy {PolicyDocument}
|
|
2303
|
+
* @returns {Promise<void>}
|
|
2304
|
+
*/
|
|
2305
|
+
async writePolicy({ spaceId, collectionId, resourceId, policy }) {
|
|
2306
|
+
// Ensure the containing directory exists (Space or Collection dir).
|
|
2307
|
+
if (collectionId !== undefined) {
|
|
2308
|
+
await this._ensureCollectionDir({ spaceId, collectionId });
|
|
2309
|
+
}
|
|
2310
|
+
else {
|
|
2311
|
+
await this._ensureSpaceDir({ spaceId });
|
|
2312
|
+
}
|
|
2313
|
+
await atomicWriteFile({
|
|
2314
|
+
filePath: this._policyFile({ spaceId, collectionId, resourceId }),
|
|
2315
|
+
data: JSON.stringify(policy)
|
|
2316
|
+
});
|
|
2317
|
+
}
|
|
2318
|
+
/**
|
|
2319
|
+
* @param options {object}
|
|
2320
|
+
* @param options.spaceId {string}
|
|
2321
|
+
* @param [options.collectionId] {string}
|
|
2322
|
+
* @param [options.resourceId] {string}
|
|
2323
|
+
* @returns {Promise<void>} idempotent (no error if absent)
|
|
2324
|
+
*/
|
|
2325
|
+
async deletePolicy({ spaceId, collectionId, resourceId }) {
|
|
2326
|
+
await rm(this._policyFile({ spaceId, collectionId, resourceId }), {
|
|
2327
|
+
force: true
|
|
2328
|
+
});
|
|
2329
|
+
}
|
|
2330
|
+
// Registered external backends (spec "Backends")
|
|
2331
|
+
/**
|
|
2332
|
+
* Builds the on-disk path for a registered backend record: a
|
|
2333
|
+
* `.backend.<backendId>.json` dot-file in the Space dir (the same per-file
|
|
2334
|
+
* convention as `.policy.` / `.space.`). One file per backend id.
|
|
2335
|
+
* @param options {object}
|
|
2336
|
+
* @param options.spaceId {string}
|
|
2337
|
+
* @param options.backendId {string}
|
|
2338
|
+
* @returns {string}
|
|
2339
|
+
*/
|
|
2340
|
+
_backendFile({ spaceId, backendId }) {
|
|
2341
|
+
const filePath = path.join(this._spaceDir(spaceId), `.backend.${backendId}.json`);
|
|
2342
|
+
this._assertContained(filePath);
|
|
2343
|
+
return filePath;
|
|
2344
|
+
}
|
|
2345
|
+
/**
|
|
2346
|
+
* Persists a full (secret-bearing) backend-registration record. Upsert.
|
|
2347
|
+
* @param options {object}
|
|
2348
|
+
* @param options.spaceId {string}
|
|
2349
|
+
* @param options.backendId {string}
|
|
2350
|
+
* @param options.record {StoredBackendRecord}
|
|
2351
|
+
* @returns {Promise<void>}
|
|
2352
|
+
*/
|
|
2353
|
+
async writeBackend({ spaceId, backendId, record }) {
|
|
2354
|
+
await this._ensureSpaceDir({ spaceId });
|
|
2355
|
+
await atomicWriteFile({
|
|
2356
|
+
filePath: this._backendFile({ spaceId, backendId }),
|
|
2357
|
+
data: JSON.stringify(record)
|
|
2358
|
+
});
|
|
2359
|
+
}
|
|
2360
|
+
/**
|
|
2361
|
+
* Reads the full (secret-bearing) record for one backend, for internal use
|
|
2362
|
+
* (existence checks, the future provider adapter). Resolves `undefined` when
|
|
2363
|
+
* absent. The only method that exposes secret connection material.
|
|
2364
|
+
* @param options {object}
|
|
2365
|
+
* @param options.spaceId {string}
|
|
2366
|
+
* @param options.backendId {string}
|
|
2367
|
+
* @returns {Promise<StoredBackendRecord|undefined>}
|
|
2368
|
+
*/
|
|
2369
|
+
async getBackend({ spaceId, backendId }) {
|
|
2370
|
+
const metaStore = new MetadataJsonStore({
|
|
2371
|
+
file: this._backendFile({ spaceId, backendId })
|
|
2372
|
+
});
|
|
2373
|
+
return await metaStore.read();
|
|
2374
|
+
}
|
|
2375
|
+
/**
|
|
2376
|
+
* Enumerates the Space's registered backends and returns them **sanitized**
|
|
2377
|
+
* (each mapped through `sanitizeBackendRecord`, so the secret connection
|
|
2378
|
+
* material never reaches the listing), sorted by id. An absent Space dir
|
|
2379
|
+
* reports no registered backends rather than throwing.
|
|
2380
|
+
* @param options {object}
|
|
2381
|
+
* @param options.spaceId {string}
|
|
2382
|
+
* @returns {Promise<BackendDescriptor[]>}
|
|
2383
|
+
*/
|
|
2384
|
+
async listBackends({ spaceId }) {
|
|
2385
|
+
const spaceDir = this._spaceDir(spaceId);
|
|
2386
|
+
let entries;
|
|
2387
|
+
try {
|
|
2388
|
+
entries = await fs.promises.readdir(spaceDir, { withFileTypes: true });
|
|
2389
|
+
}
|
|
2390
|
+
catch (err) {
|
|
2391
|
+
if (err.code === 'ENOENT') {
|
|
2392
|
+
return [];
|
|
2393
|
+
}
|
|
2394
|
+
throw err;
|
|
2395
|
+
}
|
|
2396
|
+
const backendFile = /^\.backend\.(.+)\.json$/;
|
|
2397
|
+
const reads = entries
|
|
2398
|
+
.filter(entry => entry.isFile() && backendFile.test(entry.name))
|
|
2399
|
+
.map(entry => new MetadataJsonStore({
|
|
2400
|
+
file: path.join(spaceDir, entry.name)
|
|
2401
|
+
}).read());
|
|
2402
|
+
const records = (await Promise.all(reads)).filter((record) => Boolean(record));
|
|
2403
|
+
records.sort((a, b) => a.id.localeCompare(b.id));
|
|
2404
|
+
return records.map(sanitizeBackendRecord);
|
|
2405
|
+
}
|
|
2406
|
+
/**
|
|
2407
|
+
* Removes a registered backend record. Idempotent (no error if absent).
|
|
2408
|
+
* @param options {object}
|
|
2409
|
+
* @param options.spaceId {string}
|
|
2410
|
+
* @param options.backendId {string}
|
|
2411
|
+
* @returns {Promise<void>}
|
|
2412
|
+
*/
|
|
2413
|
+
async deleteBackend({ spaceId, backendId }) {
|
|
2414
|
+
await rm(this._backendFile({ spaceId, backendId }), { force: true });
|
|
2415
|
+
}
|
|
2416
|
+
/**
|
|
2417
|
+
* WebKMS keystores (the `/kms` facet). A sibling tree to Spaces:
|
|
2418
|
+
* `keystores/<localId>/` holds a keystore's `config.json` now, and its key
|
|
2419
|
+
* records / revocations in later tracks -- hence a directory per keystore
|
|
2420
|
+
* rather than a flat file.
|
|
2421
|
+
*
|
|
2422
|
+
* The directory holding one keystore's records, contained in `keystoresDir`.
|
|
2423
|
+
* @param keystoreId {string} the keystore's server-generated local id
|
|
2424
|
+
* @returns {string}
|
|
2425
|
+
*/
|
|
2426
|
+
_keystoreDir(keystoreId) {
|
|
2427
|
+
const keystoreDir = path.join(this.keystoresDir, keystoreId);
|
|
2428
|
+
this._assertContained(keystoreDir, this.keystoresDir);
|
|
2429
|
+
return keystoreDir;
|
|
2430
|
+
}
|
|
2431
|
+
_keystoreConfigFile(keystoreId) {
|
|
2432
|
+
return path.join(this._keystoreDir(keystoreId), 'config.json');
|
|
2433
|
+
}
|
|
2434
|
+
/**
|
|
2435
|
+
* Persists a keystore config unconditionally (the create path -- local ids
|
|
2436
|
+
* are server-generated random values, so create never collides). The
|
|
2437
|
+
* sequence-gated update path is `updateKeystore`.
|
|
2438
|
+
* @param options {object}
|
|
2439
|
+
* @param options.keystoreId {string} the keystore's local id
|
|
2440
|
+
* @param options.config {KeystoreConfig}
|
|
2441
|
+
* @returns {Promise<void>}
|
|
2442
|
+
*/
|
|
2443
|
+
async writeKeystore({ keystoreId, config }) {
|
|
2444
|
+
await mkdir(this._keystoreDir(keystoreId), { recursive: true });
|
|
2445
|
+
await atomicWriteFile({
|
|
2446
|
+
filePath: this._keystoreConfigFile(keystoreId),
|
|
2447
|
+
data: JSON.stringify(config)
|
|
2448
|
+
});
|
|
2449
|
+
}
|
|
2450
|
+
/**
|
|
2451
|
+
* @param options {object}
|
|
2452
|
+
* @param options.keystoreId {string} the keystore's local id
|
|
2453
|
+
* @returns {Promise<KeystoreConfig|undefined>}
|
|
2454
|
+
* Resolves falsy when the keystore does not exist (must not throw).
|
|
2455
|
+
*/
|
|
2456
|
+
async getKeystore({ keystoreId }) {
|
|
2457
|
+
const metaStore = new MetadataJsonStore({
|
|
2458
|
+
file: this._keystoreConfigFile(keystoreId)
|
|
2459
|
+
});
|
|
2460
|
+
return await metaStore.read();
|
|
2461
|
+
}
|
|
2462
|
+
/**
|
|
2463
|
+
* Replaces a keystore config, gated atomically (under the per-keystore write
|
|
2464
|
+
* mutex) on: the keystore existing, `config.sequence` being exactly the
|
|
2465
|
+
* stored sequence + 1, and `config.kmsModule` matching the stored one (the
|
|
2466
|
+
* module is immutable). Any other state rejects with the protocol's single
|
|
2467
|
+
* merged 409 conflict.
|
|
2468
|
+
* @param options {object}
|
|
2469
|
+
* @param options.keystoreId {string} the keystore's local id
|
|
2470
|
+
* @param options.config {KeystoreConfig}
|
|
2471
|
+
* @returns {Promise<void>}
|
|
2472
|
+
*/
|
|
2473
|
+
async updateKeystore({ keystoreId, config }) {
|
|
2474
|
+
await this._writeMutex.run(`keystore:${keystoreId}`, async () => {
|
|
2475
|
+
const existing = await this.getKeystore({ keystoreId });
|
|
2476
|
+
if (!existing ||
|
|
2477
|
+
config.sequence !== existing.sequence + 1 ||
|
|
2478
|
+
config.kmsModule !== existing.kmsModule) {
|
|
2479
|
+
throw new KeystoreStateConflictError();
|
|
2480
|
+
}
|
|
2481
|
+
await atomicWriteFile({
|
|
2482
|
+
filePath: this._keystoreConfigFile(keystoreId),
|
|
2483
|
+
data: JSON.stringify(config)
|
|
2484
|
+
});
|
|
2485
|
+
});
|
|
2486
|
+
}
|
|
2487
|
+
/**
|
|
2488
|
+
* Every stored keystore config whose `controller` matches, sorted by local
|
|
2489
|
+
* id (the request layer caps the wire result). An absent keystores root
|
|
2490
|
+
* (nothing stored yet) resolves an empty list; a directory without a
|
|
2491
|
+
* readable config file is skipped.
|
|
2492
|
+
* @param options {object}
|
|
2493
|
+
* @param options.controller {IDID}
|
|
2494
|
+
* @returns {Promise<KeystoreConfig[]>}
|
|
2495
|
+
*/
|
|
2496
|
+
async listKeystoresByController({ controller }) {
|
|
2497
|
+
let rootEntries;
|
|
2498
|
+
try {
|
|
2499
|
+
rootEntries = await fs.promises.readdir(this.keystoresDir, {
|
|
2500
|
+
withFileTypes: true
|
|
2501
|
+
});
|
|
2502
|
+
}
|
|
2503
|
+
catch (err) {
|
|
2504
|
+
if (err.code === 'ENOENT') {
|
|
2505
|
+
return [];
|
|
2506
|
+
}
|
|
2507
|
+
throw new StorageError({ cause: err });
|
|
2508
|
+
}
|
|
2509
|
+
const keystoreEntries = rootEntries
|
|
2510
|
+
.filter(entry => entry.isDirectory())
|
|
2511
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
2512
|
+
const configs = [];
|
|
2513
|
+
for (const entry of keystoreEntries) {
|
|
2514
|
+
const config = await this.getKeystore({ keystoreId: entry.name });
|
|
2515
|
+
if (config && config.controller === controller) {
|
|
2516
|
+
configs.push(config);
|
|
2517
|
+
}
|
|
2518
|
+
}
|
|
2519
|
+
return configs;
|
|
2520
|
+
}
|
|
2521
|
+
/**
|
|
2522
|
+
* The file holding one key record, contained in its keystore's `keys/`
|
|
2523
|
+
* subdirectory. The record is a plain JSON file (not a metadata store):
|
|
2524
|
+
* records are immutable once inserted, so there is no read-modify-write to
|
|
2525
|
+
* protect.
|
|
2526
|
+
* @param options {object}
|
|
2527
|
+
* @param options.keystoreId {string} the owning keystore's local id
|
|
2528
|
+
* @param options.localId {string} the key's local id
|
|
2529
|
+
* @returns {string}
|
|
2530
|
+
*/
|
|
2531
|
+
_keyFile({ keystoreId, localId }) {
|
|
2532
|
+
const keysDir = path.join(this._keystoreDir(keystoreId), 'keys');
|
|
2533
|
+
const keyFile = path.join(keysDir, `${localId}.json`);
|
|
2534
|
+
this._assertContained(keyFile, keysDir);
|
|
2535
|
+
return keyFile;
|
|
2536
|
+
}
|
|
2537
|
+
/**
|
|
2538
|
+
* Inserts a key record, create-only: the exclusive-create write (`wx`)
|
|
2539
|
+
* enforces the `(keystoreId, localId)` uniqueness atomically, rejecting a
|
|
2540
|
+
* duplicate with the protocol's 409 (`KeyIdConflictError`).
|
|
2541
|
+
* @param options {object}
|
|
2542
|
+
* @param options.keystoreId {string} the owning keystore's local id
|
|
2543
|
+
* @param options.localId {string} the key's local id
|
|
2544
|
+
* @param options.record {KmsKeyRecord} the full (secret-bearing) record
|
|
2545
|
+
* @returns {Promise<void>}
|
|
2546
|
+
*/
|
|
2547
|
+
async insertKey({ keystoreId, localId, record }) {
|
|
2548
|
+
const keyFile = this._keyFile({ keystoreId, localId });
|
|
2549
|
+
await mkdir(path.dirname(keyFile), { recursive: true });
|
|
2550
|
+
try {
|
|
2551
|
+
// Durable, atomic create-only write: `atomicCreateFile` preserves the
|
|
2552
|
+
// `wx` uniqueness (an existing key file rejects with EEXIST) while also
|
|
2553
|
+
// fsyncing the record and its directory.
|
|
2554
|
+
await atomicCreateFile({
|
|
2555
|
+
filePath: keyFile,
|
|
2556
|
+
data: JSON.stringify(record, null, 2)
|
|
2557
|
+
});
|
|
2558
|
+
}
|
|
2559
|
+
catch (err) {
|
|
2560
|
+
if (err.code === 'EEXIST') {
|
|
2561
|
+
throw new KeyIdConflictError();
|
|
2562
|
+
}
|
|
2563
|
+
throw new StorageError({ cause: err });
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
/**
|
|
2567
|
+
* @param options {object}
|
|
2568
|
+
* @param options.keystoreId {string} the owning keystore's local id
|
|
2569
|
+
* @param options.localId {string} the key's local id
|
|
2570
|
+
* @returns {Promise<KmsKeyRecord|undefined>}
|
|
2571
|
+
* Resolves falsy when the key does not exist (must not throw).
|
|
2572
|
+
*/
|
|
2573
|
+
async getKey({ keystoreId, localId }) {
|
|
2574
|
+
try {
|
|
2575
|
+
const raw = await fs.promises.readFile(this._keyFile({ keystoreId, localId }), 'utf8');
|
|
2576
|
+
return JSON.parse(raw);
|
|
2577
|
+
}
|
|
2578
|
+
catch (err) {
|
|
2579
|
+
if (err.code === 'ENOENT') {
|
|
2580
|
+
return undefined;
|
|
2581
|
+
}
|
|
2582
|
+
throw new StorageError({ cause: err });
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
/**
|
|
2586
|
+
* Every stored key record under the keystore (`keys/*.json`), sorted by local
|
|
2587
|
+
* id (the file name's stem). An absent keystore or `keys/` directory (no keys
|
|
2588
|
+
* yet) resolves an empty list; a non-`.json` entry is skipped. The record is
|
|
2589
|
+
* returned verbatim -- the at-rest cipher applies above the backend.
|
|
2590
|
+
* @param options {object}
|
|
2591
|
+
* @param options.keystoreId {string} the owning keystore's local id
|
|
2592
|
+
* @returns {Promise<Array<{ localId: string, record: KmsKeyRecord }>>}
|
|
2593
|
+
*/
|
|
2594
|
+
async listKeys({ keystoreId }) {
|
|
2595
|
+
const keysDir = path.join(this._keystoreDir(keystoreId), 'keys');
|
|
2596
|
+
let entries;
|
|
2597
|
+
try {
|
|
2598
|
+
entries = await fs.promises.readdir(keysDir, { withFileTypes: true });
|
|
2599
|
+
}
|
|
2600
|
+
catch (err) {
|
|
2601
|
+
if (err.code === 'ENOENT') {
|
|
2602
|
+
return [];
|
|
2603
|
+
}
|
|
2604
|
+
throw new StorageError({ cause: err });
|
|
2605
|
+
}
|
|
2606
|
+
const localIds = entries
|
|
2607
|
+
.filter(entry => entry.isFile() && entry.name.endsWith('.json'))
|
|
2608
|
+
.map(entry => entry.name.slice(0, -'.json'.length))
|
|
2609
|
+
.sort((a, b) => a.localeCompare(b));
|
|
2610
|
+
const keys = [];
|
|
2611
|
+
for (const localId of localIds) {
|
|
2612
|
+
const record = await this.getKey({ keystoreId, localId });
|
|
2613
|
+
// A record readable at readdir time but gone by getKey (a concurrent
|
|
2614
|
+
// prune) is simply skipped; the listing is a snapshot, not a lock.
|
|
2615
|
+
if (record) {
|
|
2616
|
+
keys.push({ localId, record });
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
return keys;
|
|
2620
|
+
}
|
|
2621
|
+
/**
|
|
2622
|
+
* The file holding one zcap revocation record. For a keystore scope it lives
|
|
2623
|
+
* in that keystore's `revocations/` subdirectory; for a Space scope it lives
|
|
2624
|
+
* under the sibling `spaceRevocationsDir` root (NOT inside the Space dir --
|
|
2625
|
+
* see the `spaceRevocationsDir` property doc). The file name folds the
|
|
2626
|
+
* `(delegator, capabilityId)` unique key into a SHA-256 digest (the shared
|
|
2627
|
+
* `revocationFileName` codec).
|
|
2628
|
+
* @param options {object}
|
|
2629
|
+
* @param options.scope {RevocationScope} the owning keystore or Space
|
|
2630
|
+
* @param options.delegator {string} the revoked capability's delegator
|
|
2631
|
+
* @param options.capabilityId {string} the revoked capability's id
|
|
2632
|
+
* @returns {string}
|
|
2633
|
+
*/
|
|
2634
|
+
_revocationFile({ scope, delegator, capabilityId }) {
|
|
2635
|
+
const fileName = revocationFileName({ delegator, capabilityId });
|
|
2636
|
+
if ('keystoreId' in scope) {
|
|
2637
|
+
return path.join(this._keystoreDir(scope.keystoreId), 'revocations', fileName);
|
|
2638
|
+
}
|
|
2639
|
+
return path.join(this._spaceRevocationDir(scope.spaceId), fileName);
|
|
2640
|
+
}
|
|
2641
|
+
/**
|
|
2642
|
+
* Inserts a revocation record, create-only: the exclusive-create write
|
|
2643
|
+
* (`wx`) enforces the `(delegator, capability.id)` uniqueness atomically,
|
|
2644
|
+
* rejecting a duplicate with the protocol's 409
|
|
2645
|
+
* (`DuplicateRevocationError`).
|
|
2646
|
+
* @param options {object}
|
|
2647
|
+
* @param options.scope {RevocationScope} the owning keystore or Space
|
|
2648
|
+
* @param options.record {RevocationRecord} the revocation to store
|
|
2649
|
+
* @returns {Promise<void>}
|
|
2650
|
+
*/
|
|
2651
|
+
async insertRevocation({ scope, record }) {
|
|
2652
|
+
// The scope must already exist -- the postgres backend enforces this via
|
|
2653
|
+
// its foreign keys, so an absent-parent insert rejects identically on
|
|
2654
|
+
// both backends instead of mkdir-ing an orphan record dir here. (The HTTP
|
|
2655
|
+
// route 404-masks unknown scopes before reaching the store; this guards
|
|
2656
|
+
// direct backend use.)
|
|
2657
|
+
const scopeExists = 'keystoreId' in scope
|
|
2658
|
+
? Boolean(await this.getKeystore({ keystoreId: scope.keystoreId }))
|
|
2659
|
+
: Boolean(await this.getSpaceDescription({ spaceId: scope.spaceId }));
|
|
2660
|
+
if (!scopeExists) {
|
|
2661
|
+
throw new StorageError({
|
|
2662
|
+
cause: new Error('Cannot insert a revocation under an absent keystore or Space.')
|
|
2663
|
+
});
|
|
2664
|
+
}
|
|
2665
|
+
const revocationFile = this._revocationFile({
|
|
2666
|
+
scope,
|
|
2667
|
+
delegator: record.meta.delegator,
|
|
2668
|
+
capabilityId: record.capability.id
|
|
2669
|
+
});
|
|
2670
|
+
await mkdir(path.dirname(revocationFile), { recursive: true });
|
|
2671
|
+
try {
|
|
2672
|
+
// Durable, atomic create-only write: `atomicCreateFile` preserves the
|
|
2673
|
+
// `wx` uniqueness (an existing revocation rejects with EEXIST) while also
|
|
2674
|
+
// fsyncing the record and its directory.
|
|
2675
|
+
await atomicCreateFile({
|
|
2676
|
+
filePath: revocationFile,
|
|
2677
|
+
data: JSON.stringify(record, null, 2)
|
|
2678
|
+
});
|
|
2679
|
+
}
|
|
2680
|
+
catch (err) {
|
|
2681
|
+
if (err.code === 'EEXIST') {
|
|
2682
|
+
throw new DuplicateRevocationError();
|
|
2683
|
+
}
|
|
2684
|
+
throw new StorageError({ cause: err });
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
2687
|
+
/**
|
|
2688
|
+
* True when any of the given capabilities has a stored, unexpired
|
|
2689
|
+
* revocation under the scope (keystore or Space). A record past its
|
|
2690
|
+
* `meta.expires` GC horizon is pruned on the way through and counts as not
|
|
2691
|
+
* revoked -- the capability itself has expired, so verification already
|
|
2692
|
+
* rejects it on expiry (this is the filesystem analogue of a TTL index).
|
|
2693
|
+
* @param options {object}
|
|
2694
|
+
* @param options.scope {RevocationScope} the owning keystore or Space
|
|
2695
|
+
* @param options.capabilities {CapabilitySummary[]} the
|
|
2696
|
+
* `(capabilityId, delegator)` pairs to check
|
|
2697
|
+
* @returns {Promise<boolean>}
|
|
2698
|
+
*/
|
|
2699
|
+
async isRevoked({ scope, capabilities }) {
|
|
2700
|
+
const now = Date.now();
|
|
2701
|
+
for (const { capabilityId, delegator } of capabilities) {
|
|
2702
|
+
const revocationFile = this._revocationFile({
|
|
2703
|
+
scope,
|
|
2704
|
+
delegator,
|
|
2705
|
+
capabilityId
|
|
2706
|
+
});
|
|
2707
|
+
let raw;
|
|
2708
|
+
try {
|
|
2709
|
+
raw = await fs.promises.readFile(revocationFile, 'utf8');
|
|
2710
|
+
}
|
|
2711
|
+
catch (err) {
|
|
2712
|
+
if (err.code === 'ENOENT') {
|
|
2713
|
+
continue;
|
|
2714
|
+
}
|
|
2715
|
+
throw new StorageError({ cause: err });
|
|
2716
|
+
}
|
|
2717
|
+
const record = JSON.parse(raw);
|
|
2718
|
+
if (record.meta.expires && Date.parse(record.meta.expires) <= now) {
|
|
2719
|
+
await rm(revocationFile, { force: true });
|
|
2720
|
+
continue;
|
|
2721
|
+
}
|
|
2722
|
+
return true;
|
|
2723
|
+
}
|
|
2724
|
+
return false;
|
|
2725
|
+
}
|
|
2726
|
+
}
|
|
2727
|
+
//# sourceMappingURL=filesystem.js.map
|