swarm-deploy 0.1.0 → 0.3.0

package/dist/storage/tar-session-store.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TarSessionStore = void 0;
+exports.TarSessionStore = exports.TREE_SESSION_VERSION = void 0;
 const b4a_1 = __importDefault(require("b4a"));
 const error_code_js_1 = require("../error-code.js");
 const _fs_1 = __importDefault(require("#fs"));
@@ -12,12 +12,19 @@ const sodium_native_1 = __importDefault(require("sodium-native"));
 const errors_js_1 = require("../errors.js");
 const controls_js_1 = require("../tar-protocol/controls.js");
 const extract_js_1 = require("../tar-protocol/extract.js");
+const tree_extract_js_1 = require("../tar-protocol/tree-extract.js");
 const manifest_js_1 = require("../tar-protocol/manifest.js");
+const tree_manifest_js_1 = require("../tar-protocol/tree-manifest.js");
 const atomic_file_js_1 = require("./atomic-file.js");
 const commit_journal_js_1 = require("./commit-journal.js");
 const layout_js_1 = require("./layout.js");
+const tree_staging_js_1 = require("./tree-staging.js");
+const tree_fs_js_1 = require("./tree-fs.js");
 const validation_js_1 = require("../validation.js");
-const VERSION = 2;
+const VERSION = 3;
+const LEGACY_VERSION = 2;
+/** Directory sessions are persisted at a version no file-only server understands. */
+exports.TREE_SESSION_VERSION = 4;
 const RECEIVING = 'receiving';
 const VERIFIED = 'verified';
 const DELETING = 'deleting';
@@ -43,12 +50,22 @@ function key(value, name) {
         throw problem(`Invalid ${name}`);
 }
 function sameMetadata(session, metadata, owner) {
-    return (sodium_native_1.default.sodium_memcmp(session.ownerKey, owner) &&
-        session.name === metadata.name &&
+    if (!sodium_native_1.default.sodium_memcmp(session.ownerKey, owner) ||
+        session.name !== metadata.name ||
+        session.sourceParent !== metadata.sourceParent ||
+        session.tarSize !== metadata.tarSize ||
+        !sodium_native_1.default.sodium_memcmp(session.tarDigest, b4a_1.default.from(metadata.tarSha256, 'hex'))) {
+        return false;
+    }
+    if ((0, controls_js_1.isTreeMetadata)(metadata)) {
+        return (session.kind === 'directory' &&
+            session.entryCount === metadata.entryCount &&
+            session.size === metadata.payloadBytes &&
+            sodium_native_1.default.sodium_memcmp(session.digest, b4a_1.default.from(metadata.treeSha256, 'hex')));
+    }
+    return (session.kind === 'file' &&
         session.size === metadata.fileSize &&
-        session.tarSize === metadata.tarSize &&
-        sodium_native_1.default.sodium_memcmp(session.digest, b4a_1.default.from(metadata.fileSha256, 'hex')) &&
-        sodium_native_1.default.sodium_memcmp(session.tarDigest, b4a_1.default.from(metadata.tarSha256, 'hex')));
+        sodium_native_1.default.sodium_memcmp(session.digest, b4a_1.default.from(metadata.fileSha256, 'hex')));
 }
 async function writeAll(handle, value, position) {
     let offset = 0;
@@ -140,6 +157,15 @@ class TarSessionStore {
     filePath(id) {
         return _path_1.default.join(this.layout.staging, `${id}.part`);
     }
+    treeStagingPath(id) {
+        return _path_1.default.join(this.layout.staging, `${id}.tree`);
+    }
+    /** Removes a staged tree through the contained primitives; never `storage.rm`. */
+    removeStagedTree(session) {
+        return (0, tree_fs_js_1.removeTree)(this.treeStagingPath(session.id), this.layout.staging, this.storage, {
+            layout: this.layout
+        });
+    }
     journalPath(id) {
         return _path_1.default.join(this.layout.journals, `${id}.json`);
     }
@@ -151,12 +177,76 @@ class TarSessionStore {
             throw problem('Session store is not available');
         }
     }
-    metadata(metadata) {
-        return (0, controls_js_1.decodeMetadataRecord)((0, controls_js_1.encodeMetadataRecord)(metadata));
+    metadata(input) {
+        return (0, controls_js_1.decodeAnyMetadataRecord)((0, controls_js_1.encodeAnyMetadataRecord)(input));
+    }
+    fileMetadata(input) {
+        const metadata = this.metadata(input);
+        if ((0, controls_js_1.isTreeMetadata)(metadata))
+            throw problem('Invalid file session metadata');
+        return metadata;
+    }
+    treeFromDisk(id, record) {
+        const persisted = record;
+        if (persisted.kind !== 'directory' || persisted.transferId !== id) {
+            throw problem('Invalid tree session');
+        }
+        const ownerKey = bytes(persisted.ownerKey, 'owner key');
+        uint(persisted.entryCount, 'tree entry count');
+        uint(persisted.payloadBytes, 'tree payload size');
+        uint(persisted.tarSize, 'TAR size');
+        uint(persisted.createdAt, 'creation time');
+        uint(persisted.updatedAt, 'update time');
+        if (!persisted.partialTar ||
+            persisted.partialTar.path !== `${id}.tar.part` ||
+            ![RECEIVING, VERIFIED, DELETING].includes(persisted.state)) {
+            throw problem('Invalid tree session metadata');
+        }
+        uint(persisted.partialTar.size, 'partial TAR size');
+        if (persisted.partialTar.size > persisted.tarSize)
+            throw problem('Oversized partial TAR');
+        const metadata = (0, controls_js_1.decodeTreeMetadataRecord)((0, controls_js_1.encodeTreeMetadataRecord)({
+            v: 1,
+            kind: 'directory',
+            name: persisted.name,
+            ...(persisted.sourceParent === undefined ? {} : { sourceParent: persisted.sourceParent }),
+            entryCount: persisted.entryCount,
+            payloadBytes: persisted.payloadBytes,
+            treeSha256: hex(bytes(persisted.treeSha256, 'tree digest')),
+            tarSize: persisted.tarSize,
+            tarSha256: hex(bytes(persisted.tarSha256, 'TAR digest')),
+            transferId: id,
+            reset: false
+        }));
+        (0, tree_manifest_js_1.assertTreeMetadataTransferId)(ownerKey, metadata);
+        return {
+            id,
+            transferId: b4a_1.default.from(id, 'hex'),
+            ownerKey,
+            kind: 'directory',
+            name: metadata.name,
+            sourceParent: metadata.sourceParent,
+            size: metadata.payloadBytes,
+            digest: b4a_1.default.from(metadata.treeSha256, 'hex'),
+            entryCount: metadata.entryCount,
+            tarSize: metadata.tarSize,
+            tarDigest: b4a_1.default.from(metadata.tarSha256, 'hex'),
+            tarPath: this.tarPath(id),
+            treePath: this.treeStagingPath(id),
+            partialTarSize: persisted.partialTar.size,
+            createdAt: persisted.createdAt,
+            updatedAt: persisted.updatedAt,
+            state: persisted.state
+        };
     }
     fromDisk(id, record) {
+        if (record.version === exports.TREE_SESSION_VERSION)
+            return this.treeFromDisk(id, record);
         const persisted = record;
-        if (persisted.version !== VERSION || persisted.transferId !== id) {
+        if ((persisted.version !== LEGACY_VERSION && persisted.version !== VERSION) ||
+            persisted.transferId !== id ||
+            (persisted.version === LEGACY_VERSION &&
+                Object.prototype.hasOwnProperty.call(record, 'sourceParent'))) {
             throw problem('Invalid TAR session');
         }
         const ownerKey = bytes(persisted.ownerKey, 'owner key');
@@ -174,9 +264,10 @@ class TarSessionStore {
         if (persisted.partialTar.size > persisted.tarSize) {
             throw problem('Oversized partial TAR');
         }
-        const metadata = this.metadata({
+        const metadata = this.fileMetadata({
             v: 1,
             name: persisted.name,
+            ...(persisted.sourceParent === undefined ? {} : { sourceParent: persisted.sourceParent }),
             fileSize: persisted.fileSize,
             fileSha256: hex(bytes(persisted.fileSha256, 'file digest')),
             tarSize: persisted.tarSize,
@@ -189,7 +280,9 @@ class TarSessionStore {
             id,
             transferId,
             ownerKey,
+            kind: 'file',
             name: metadata.name,
+            sourceParent: metadata.sourceParent,
             size: metadata.fileSize,
             digest: b4a_1.default.from(metadata.fileSha256, 'hex'),
             tarSize: metadata.tarSize,
@@ -201,12 +294,38 @@ class TarSessionStore {
             state: persisted.state
         };
     }
+    /**
+     * Version 3 only adds `sourceParent`, so a session without one is written as
+     * version 2. An older server can then still read, and resume, every session
+     * whose record it fully understands; only parent-bearing ones are lost to a
+     * downgrade.
+     */
     serialize(session) {
+        if (session.kind === 'directory') {
+            return {
+                version: exports.TREE_SESSION_VERSION,
+                kind: 'directory',
+                transferId: session.id,
+                ownerKey: hex(session.ownerKey),
+                name: session.name,
+                sourceParent: session.sourceParent,
+                entryCount: session.entryCount,
+                payloadBytes: session.size,
+                treeSha256: hex(session.digest),
+                tarSize: session.tarSize,
+                tarSha256: hex(session.tarDigest),
+                partialTar: { path: `${session.id}.tar.part`, size: session.partialTarSize },
+                createdAt: session.createdAt,
+                updatedAt: session.updatedAt,
+                state: session.state
+            };
+        }
         return {
-            version: VERSION,
+            version: session.sourceParent === undefined ? LEGACY_VERSION : VERSION,
             transferId: session.id,
             ownerKey: hex(session.ownerKey),
             name: session.name,
+            sourceParent: session.sourceParent,
             fileSize: session.size,
             fileSha256: hex(session.digest),
             tarSize: session.tarSize,
@@ -237,6 +356,23 @@ class TarSessionStore {
         });
         await (0, layout_js_1.withSafeDirectoryIdentity)(directory, this.storage, () => syncDirectory(directory, this.storage));
     }
+    /** A verified directory session must still own its staging tree. */
+    async assertStagedTree(session) {
+        const state = await (0, tree_fs_js_1.inspectTreePath)(this.treeStagingPath(session.id), this.layout.staging, this.storage, { layout: this.layout });
+        if (state !== 'DIRECTORY')
+            throw problem('Session is not verified');
+    }
+    /** Verified directory sessions mid-commit may only have a durable journal tree. */
+    async assertVerifiedDirectorySession(session) {
+        const treeState = await (0, tree_fs_js_1.inspectTreePath)(this.treeStagingPath(session.id), this.layout.staging, this.storage, { layout: this.layout });
+        if (treeState === 'DIRECTORY') {
+            await this.assertStagedTree(session);
+            return;
+        }
+        const journal = await (0, commit_journal_js_1.readCommitJournal)(session.id, this.layout, this.storage);
+        if (!journal || !(0, commit_journal_js_1.isDirectoryJournal)(journal))
+            throw problem('Session is not verified');
+    }
     async purgeCorruptSession(id) {
         await this.remove(this.tarPath(id), this.layout.staging);
         await this.remove(this.sessionPath(id), this.layout.sessions);
@@ -361,7 +497,10 @@ class TarSessionStore {
                 }
                 if (session.state === DELETING) {
                     await this.remove(session.tarPath, this.layout.staging);
-                    await this.remove(this.filePath(id), this.layout.staging);
+                    if (session.kind === 'directory')
+                        await this.removeStagedTree(session);
+                    else
+                        await this.remove(this.filePath(id), this.layout.staging);
                     await this.remove(this.sessionPath(id), this.layout.sessions);
                     continue;
                 }
@@ -372,6 +511,24 @@ class TarSessionStore {
                         await this.writeSession(session);
                     }
                 }
+                else if (session.kind === 'directory') {
+                    const treeState = await (0, tree_fs_js_1.inspectTreePath)(this.treeStagingPath(id), this.layout.staging, this.storage, { layout: this.layout });
+                    if (treeState !== 'DIRECTORY') {
+                        const journal = await (0, commit_journal_js_1.readCommitJournal)(id, this.layout, this.storage);
+                        if (!journal || !(0, commit_journal_js_1.isDirectoryJournal)(journal)) {
+                            await this.purgeCorruptSession(id);
+                            if (treeState !== 'MISSING') {
+                                await (0, tree_fs_js_1.removeTree)(this.treeStagingPath(id), this.layout.staging, this.storage, {
+                                    layout: this.layout
+                                }).catch(() => { });
+                            }
+                            continue;
+                        }
+                    }
+                    else {
+                        await this.assertVerifiedDirectorySession(session);
+                    }
+                }
                 else {
                     await (0, layout_js_1.assertSafeFile)(this.filePath(id), this.storage);
                 }
@@ -384,12 +541,24 @@ class TarSessionStore {
             const expected = new Set();
             for (const session of this.sessions.values()) {
                 expected.add(`${session.id}.tar.part`);
-                if (session.state === VERIFIED)
-                    expected.add(`${session.id}.part`);
+                if (session.state === VERIFIED) {
+                    expected.add(session.kind === 'directory' ? `${session.id}.tree` : `${session.id}.part`);
+                }
             }
             for (const name of await this.storage.readdir(this.layout.staging)) {
                 if (expected.has(name))
                     continue;
+                const tree = /^([0-9a-f]{64})\.tree$/.exec(name);
+                if (tree) {
+                    // A journal-owned tree belongs to a commit in flight; anything else is residue.
+                    if (await (0, commit_journal_js_1.readCommitJournal)(tree[1], this.layout, this.storage))
+                        continue;
+                    await (0, tree_fs_js_1.removeTree)(_path_1.default.join(this.layout.staging, name), this.layout.staging, this.storage, {
+                        layout: this.layout
+                    });
+                    this.purgedSessions++;
+                    continue;
+                }
                 const staged = /^([0-9a-f]{64})\.(?:part|tar\.part)$/.exec(name);
                 if (staged) {
                     const journal = await (0, commit_journal_js_1.readCommitJournal)(staged[1], this.layout, this.storage);
@@ -408,7 +577,10 @@ class TarSessionStore {
             this.assertReady();
             key(ownerKey, 'owner key');
             const metadata = this.metadata(input);
-            (0, manifest_js_1.assertMetadataTransferId)(ownerKey, metadata);
+            if ((0, controls_js_1.isTreeMetadata)(metadata))
+                (0, tree_manifest_js_1.assertTreeMetadataTransferId)(ownerKey, metadata);
+            else
+                (0, manifest_js_1.assertMetadataTransferId)(ownerKey, metadata);
             const id = metadata.transferId;
             if (this.resumeTtl !== undefined)
                 await this.expireUnlocked(this.resumeTtl);
@@ -446,27 +618,49 @@ class TarSessionStore {
                     throw new errors_js_1.SwarmDeployError(errors_js_1.ERRORS.FILE_BUSY, 'Destination has an active session');
                 }
             }
-            const reservation = metadata.tarSize + metadata.fileSize;
+            const reservation = metadata.tarSize + ((0, controls_js_1.isTreeMetadata)(metadata) ? metadata.payloadBytes : metadata.fileSize);
             if (this.reservedBytes > this.maxStagingBytes - reservation) {
                 throw new errors_js_1.SwarmDeployError(errors_js_1.ERRORS.STAGING_LIMIT, 'Staging capacity exceeded');
             }
             await this.diskReserve(reservation);
             const now = this.clock.now();
-            session = {
-                id,
-                transferId: b4a_1.default.from(id, 'hex'),
-                ownerKey: b4a_1.default.from(ownerKey),
-                name: metadata.name,
-                size: metadata.fileSize,
-                digest: b4a_1.default.from(metadata.fileSha256, 'hex'),
-                tarSize: metadata.tarSize,
-                tarDigest: b4a_1.default.from(metadata.tarSha256, 'hex'),
-                tarPath: this.tarPath(id),
-                partialTarSize: 0,
-                createdAt: now,
-                updatedAt: now,
-                state: RECEIVING
-            };
+            session = (0, controls_js_1.isTreeMetadata)(metadata)
+                ? {
+                    id,
+                    transferId: b4a_1.default.from(id, 'hex'),
+                    ownerKey: b4a_1.default.from(ownerKey),
+                    kind: 'directory',
+                    name: metadata.name,
+                    sourceParent: metadata.sourceParent,
+                    size: metadata.payloadBytes,
+                    digest: b4a_1.default.from(metadata.treeSha256, 'hex'),
+                    entryCount: metadata.entryCount,
+                    tarSize: metadata.tarSize,
+                    tarDigest: b4a_1.default.from(metadata.tarSha256, 'hex'),
+                    tarPath: this.tarPath(id),
+                    treePath: this.treeStagingPath(id),
+                    partialTarSize: 0,
+                    createdAt: now,
+                    updatedAt: now,
+                    state: RECEIVING
+                }
+                : {
+                    id,
+                    transferId: b4a_1.default.from(id, 'hex'),
+                    ownerKey: b4a_1.default.from(ownerKey),
+                    kind: 'file',
+                    name: metadata.name,
+                    sourceParent: metadata.sourceParent,
+                    size: metadata.fileSize,
+                    digest: b4a_1.default.from(metadata.fileSha256, 'hex'),
+                    tarSize: metadata.tarSize,
+                    tarDigest: b4a_1.default.from(metadata.tarSha256, 'hex'),
+                    tarPath: this.tarPath(id),
+                    partialTarSize: 0,
+                    createdAt: now,
+                    updatedAt: now,
+                    state: RECEIVING
+                };
             await (0, layout_js_1.withSafeDirectoryIdentity)(this.layout.staging, this.storage, async () => {
                 const handle = await (0, layout_js_1.openSafeRegularFile)(session.tarPath, 'create', this.storage);
                 await handle.sync();
@@ -621,26 +815,31 @@ class TarSessionStore {
             let file = null;
             let fileOffset = 0;
             try {
-                file = await (0, layout_js_1.openSafeRegularFile)(this.filePath(session.id), 'create', this.storage);
-                const source = this.readTar(session);
-                await (0, extract_js_1.validateAndExtractTar)(source, metadata, {
-                    writeTar: () => { },
-                    writeFile: async (chunk) => {
-                        await writeAll(file, chunk, fileOffset);
-                        fileOffset += chunk.byteLength;
-                    },
-                    complete: async () => {
-                        await file.sync();
-                    },
-                    abort: async () => {
-                        await file?.close().catch(() => { });
-                        file = null;
-                        await this.remove(this.filePath(session.id), this.layout.staging).catch(() => { });
-                    }
-                });
-                await file.close();
-                file = null;
-                await (0, layout_js_1.withSafeDirectoryIdentity)(this.layout.staging, this.storage, () => syncDirectory(this.layout.staging, this.storage));
+                if (session.kind === 'directory') {
+                    await this.extractTree(session, metadata);
+                }
+                else {
+                    file = await (0, layout_js_1.openSafeRegularFile)(this.filePath(session.id), 'create', this.storage);
+                    const source = this.readTar(session);
+                    await (0, extract_js_1.validateAndExtractTar)(source, metadata, {
+                        writeTar: () => { },
+                        writeFile: async (chunk) => {
+                            await writeAll(file, chunk, fileOffset);
+                            fileOffset += chunk.byteLength;
+                        },
+                        complete: async () => {
+                            await file.sync();
+                        },
+                        abort: async () => {
+                            await file?.close().catch(() => { });
+                            file = null;
+                            await this.remove(this.filePath(session.id), this.layout.staging).catch(() => { });
+                        }
+                    });
+                    await file.close();
+                    file = null;
+                    await (0, layout_js_1.withSafeDirectoryIdentity)(this.layout.staging, this.storage, () => syncDirectory(this.layout.staging, this.storage));
+                }
                 await this.run(async () => {
                     if (this.sessions.get(session.id) !== session ||
                         session.state !== RECEIVING ||
@@ -668,7 +867,10 @@ class TarSessionStore {
                     await file.close().catch(() => { });
                 if (session.state !== VERIFIED) {
                     try {
-                        await this.remove(this.filePath(session.id), this.layout.staging);
+                        if (session.kind === 'directory')
+                            await this.removeStagedTree(session);
+                        else
+                            await this.remove(this.filePath(session.id), this.layout.staging);
                     }
                     catch (cleanupCause) {
                         await this.quarantineVerificationFailure(session, error, cleanupCause);
@@ -689,6 +891,26 @@ class TarSessionStore {
         task.then(settled, settled);
         return task;
     }
+    /**
+     * Extracts the complete directory archive into its staging tree, then proves
+     * the tree from the bytes on disk alone: the recomputed canonical digest,
+     * entry count, and payload size must equal the offered values.
+     */
+    async extractTree(session, metadata) {
+        const treePath = this.treeStagingPath(session.id);
+        const options = { layout: this.layout };
+        // Residue of an interrupted attempt never survives into a new extraction.
+        await (0, tree_fs_js_1.removeTree)(treePath, this.layout.staging, this.storage, options);
+        const target = (0, tree_staging_js_1.createTreeStagingTarget)(treePath, this.layout.staging, this.storage, this.layout);
+        await (0, tree_extract_js_1.validateAndExtractTreeTar)(this.readTar(session), metadata, target);
+        const staged = await (0, tree_fs_js_1.digestTree)(treePath, this.storage);
+        if (staged.entryCount !== metadata.entryCount ||
+            staged.payloadBytes !== metadata.payloadBytes ||
+            !sodium_native_1.default.sodium_memcmp(staged.treeSha256, b4a_1.default.from(metadata.treeSha256, 'hex'))) {
+            throw new errors_js_1.SwarmDeployError(errors_js_1.ERRORS.CHECKSUM_MISMATCH, 'Staged tree digest mismatch');
+        }
+        await (0, layout_js_1.withSafeDirectoryIdentity)(this.layout.staging, this.storage, () => syncDirectory(this.layout.staging, this.storage));
+    }
     async *readTar(session) {
         const handle = await (0, layout_js_1.openSafeRegularFile)(session.tarPath, 'read', this.storage);
         try {
@@ -709,7 +931,10 @@ class TarSessionStore {
             const session = this.sessions.get(hex(transferId));
             if (!session || session.state !== VERIFIED)
                 throw problem('Session is not verified');
-            await (0, layout_js_1.assertSafeFile)(this.filePath(session.id), this.storage);
+            if (session.kind === 'directory')
+                await this.assertVerifiedDirectorySession(session);
+            else
+                await (0, layout_js_1.assertSafeFile)(this.filePath(session.id), this.storage);
             return session;
         });
     }
@@ -728,7 +953,10 @@ class TarSessionStore {
         try {
             await this.writeSession(session);
             await this.remove(session.tarPath, this.layout.staging);
-            await this.remove(this.filePath(session.id), this.layout.staging);
+            if (session.kind === 'directory')
+                await this.removeStagedTree(session);
+            else
+                await this.remove(this.filePath(session.id), this.layout.staging);
             await this.remove(this.sessionPath(session.id), this.layout.sessions);
         }
         finally {