sliftutils 1.7.130 → 1.7.131

package/storage/remoteStorage/serverConfig.ts

@@ -1,19 +1,13 @@
 import path from "path";
-import { getFileStorageNested2 } from "../FileFolderAPI";
-import { TransactionStorage } from "../TransactionStorage";
-import { JSONStorage } from "../JSONStorage";
-import type { IStorage } from "../IStorage";
-import type { AccessRequest, TrustRecord } from "./storageController";
 import { getArg } from "./cliArgs";
 
-// The storage server's process-level identity and system state: which server we are (config, extra ports), whether writes are allowed, and the trust/request stores.
+// The storage server's process-level identity and system state: which server we are (config, extra
+// ports), and whether writes are allowed.
 
 export type StorageServerConfig = {
     domain: string;
     port: number;
     rootDomain: string;
-    sshTarget: string;
-    serverCommand: string;
     folder: string;
 };
 
@@ -52,46 +46,10 @@ export function getStorageFolder(): string {
     return path.resolve(folder);
 }
 
-const systemStorages = new Map<string, Promise<IStorage<unknown>>>();
-function getSystemStorage<T>(name: string): Promise<IStorage<T>> {
-    let storage = systemStorages.get(name);
-    if (!storage) {
-        storage = (async () => {
-            let root = await getFileStorageNested2(getStorageFolder());
-            let system = await root.folder.getStorage("system2");
-            let transactionName = "storage" + name[0].toUpperCase() + name.slice(1);
-            return new JSONStorage<unknown>(new TransactionStorage(await system.folder.getStorage(name), transactionName));
-        })();
-        systemStorages.set(name, storage);
-    }
-    return storage as Promise<IStorage<T>>;
-}
-export function getTrust(): Promise<IStorage<TrustRecord>> {
-    return getSystemStorage<TrustRecord>("trust");
-}
-export function getRequests(): Promise<IStorage<AccessRequest[]>> {
-    return getSystemStorage<AccessRequest[]>("requests");
-}
-
-export async function setTrustedMachines(config: { account: string; machineIds: string[] }): Promise<void> {
-    let trust = await getTrust();
-    let prefix = `${config.account}|`;
-    let desired = new Set(config.machineIds);
-    for (let key of await trust.getKeys()) {
-        if (!key.startsWith(prefix)) continue;
-        let machineId = key.slice(prefix.length);
-        if (desired.has(machineId)) {
-            desired.delete(machineId);
-            continue;
-        }
-        console.log(`Removing trust for machine ${machineId} on account ${config.account}`);
-        await trust.remove(key);
-    }
-    for (let machineId of desired) {
-        console.log(`Adding trust for machine ${machineId} on account ${config.account}`);
-        await trust.set(`${prefix}${machineId}`, { account: config.account, machineId, ip: "", time: Date.now() });
-    }
-}
+// Which machines this server trusts is not kept here any more. It lives in the signed
+// authorized_keys repo, read through security/machines, so every machine shares one list, with
+// revocation and address checking that a per-server store never had. setTrustedMachines, the trust
+// store and the access request store all went with it.
 
 const extraListenPorts = new Set<number>();
 export function addExtraListenPort(port: number): void {

package/storage/remoteStorage/storageController.d.ts

@@ -2,6 +2,7 @@
 /// <reference types="node" />
 import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, FindConfig, SourceConfig } from "../IArchives";
 import { ActiveBucketInfo, ServerBucketInfo } from "./storageServerState";
+import { MachineState } from "../../security/machines/machines";
 import { AccessTotals, AccessSummaryState } from "./accessStats";
 import { LogFileInfo } from "../StreamingLogs";
 import type { SummaryEntry } from "../../treeSummary";
@@ -19,25 +20,13 @@ export type AuthToken = {
     signature: string;
     data: AuthTokenData;
 };
-export type AccessRequest = {
-    requestId: string;
-    account: string;
-    machineId: string;
-    ip: string;
-    time: number;
-};
-export type TrustRecord = {
-    account: string;
-    machineId: string;
-    ip: string;
-    time: number;
-};
 export type AccessState = {
     machineId: string;
     ip: string;
     hasAccess: boolean;
-    grantAccessCommand?: string;
-    trustedMachines?: TrustRecord[];
+    reason?: string;
+    addMachineCommand?: string;
+    trustedMachines?: MachineState[];
 };
 export declare function broadcastRoutingChanged(): void;
 export declare const RemoteStorageController: import("socket-function/SocketFunctionTypes").SocketRegistered<{
@@ -46,34 +35,13 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
         machineId: string;
         ip: string;
     }>;
-    requestAccess: (config: {
-        account: string;
-    }) => Promise<{
-        machineId: string;
-        ip: string;
-        requestId: string;
-        grantAccessCommand: string;
-    }>;
     getAccessState: (config: {
         account: string;
     }) => Promise<AccessState>;
-    listRequestsForIP: (config: {
-        account: string;
-        ip: string;
-    }) => Promise<AccessRequest[]>;
-    grantAccess: (config: {
-        requestId: string;
-    }) => Promise<TrustRecord>;
     adminListActiveBuckets: () => Promise<{
         account: string;
         bucketName: string;
     }[]>;
-    adminListRequests: (config: {
-        ip: string;
-    }) => Promise<AccessRequest[]>;
-    adminGrantAccess: (config: {
-        requestId: string;
-    }) => Promise<TrustRecord>;
     get2: (config: {
         account: string;
         bucketName: string;

package/storage/remoteStorage/storageController.ts

@@ -15,7 +15,8 @@ import {
     debugGetActiveBucket, activateBucket, getActiveBucketKeys,
 } from "./storageServerState";
 import { debugClearAccountWriteStats } from "./accessStats";
-import { getStorageServerConfig, getTrust, getRequests, assertWritesAllowed } from "./serverConfig";
+import { getStorageServerConfig, assertWritesAllowed } from "./serverConfig";
+import { getTrustedMachines, isMachineAccepted, MachineState } from "../../security/machines/machines";
 import { BlobStore } from "./blobStore";
 import { getRoutingFileResult } from "./bucketDisk";
 import { StorageClientController } from "./storageClientController";
@@ -45,25 +46,15 @@ export type AuthToken = {
     signature: string;
     data: AuthTokenData;
 };
-export type AccessRequest = {
-    requestId: string;
-    account: string;
-    machineId: string;
-    ip: string;
-    time: number;
-};
-export type TrustRecord = {
-    account: string;
-    machineId: string;
-    ip: string;
-    time: number;
-};
 export type AccessState = {
     machineId: string;
     ip: string;
     hasAccess: boolean;
-    grantAccessCommand?: string;
-    trustedMachines?: TrustRecord[];
+    // Why not, and what to run about it, when there is no access.
+    reason?: string;
+    addMachineCommand?: string;
+    // Everything the repo trusts, when there is.
+    trustedMachines?: MachineState[];
 };
 
 const sessions = new Map<string, string>();
@@ -114,24 +105,29 @@ function requireAdmin(): string {
     }
     return machineId;
 }
+/** Who may talk to this server, decided by the signed authorized_keys repo rather than by anything
+    this server keeps for itself.
+
+    The identity is the same machine id it always was - a machine that was trusted here before is
+    trusted the moment its id is in the repo - but the record of who is trusted now lives in one
+    signed place shared by every machine, with revocation and everything else that comes with it.
+    The address is checked too, which the trust store here never did. */
 async function requireAccess(account: string): Promise<string> {
     assertValidName(account, "account");
     let machineId = getCallerMachineId();
     if (isAdmin(machineId)) return machineId;
-    let trust = await getTrust();
-    let trusted = await trust.get(`${account}|${machineId}`);
-    if (!trusted) {
-        let { domain, port } = getStorageServerConfig();
-        throw new Error(`${STORAGE_ACCESS_DENIED} Machine has no access to this account. Machine ${machineId}, account ${JSON.stringify(account)}. Visit https://${domain}:${port}/${account} for access instructions.`);
+    let { rootDomain } = getStorageServerConfig();
+    let ip = getCallerIP();
+    let verdict = await isMachineAccepted({ machineId, ip, domain: rootDomain });
+    if (!verdict.accepted) {
+        throw new Error(
+            `${STORAGE_ACCESS_DENIED} ${verdict.reason}`
+            + ` Machine ${machineId}, address ${ip}, account ${JSON.stringify(account)}.`
+        );
     }
     return machineId;
 }
 
-function getGrantAccessCommand(requestId: string): string {
-    let { sshTarget, serverCommand } = getStorageServerConfig();
-    return `ssh ${sshTarget} '${serverCommand} --requestId ${requestId}'`;
-}
-
 class RemoteStorageControllerBase {
     async ping(): Promise<{}> {
         trackCaller();
@@ -166,114 +162,37 @@ class RemoteStorageControllerBase {
         return { machineId, ip: getCallerIP() };
     }
 
-    @assertValidArgs
-    async requestAccess(config: { account: string }): Promise<{ machineId: string; ip: string; requestId: string; grantAccessCommand: string }> {
-        let account = config.account;
-        let machineId = getCallerMachineId();
-        let ip = getCallerIP();
-        let requestsStorage = await getRequests();
-        let requests = await requestsStorage.get(ip) || [];
-        let existing = requests.find(x => x.account === account && x.machineId === machineId);
-        if (existing) {
-            existing.time = Date.now();
-        } else {
-            existing = {
-                requestId: Math.random().toString(36).slice(2, 10),
-                account,
-                machineId,
-                ip,
-                time: Date.now(),
-            };
-            requests.push(existing);
-        }
-        while (requests.length > MAX_REQUESTS_PER_IP) requests.shift();
-        await requestsStorage.set(ip, requests);
-        return { machineId, ip, requestId: existing.requestId, grantAccessCommand: getGrantAccessCommand(existing.requestId) };
-    }
+    /** Whether the caller may use this account, and if not, exactly what to run to change that.
 
+        There is nothing to request and nothing to grant here any more: trust lives in the signed
+        authorized_keys repo, so the answer is "run addmachine in that repo", which only somebody
+        holding the hardware key can do. */
     @assertValidArgs
     async getAccessState(config: { account: string }): Promise<AccessState> {
-        let account = config.account;
+        assertValidName(config.account, "account");
         let machineId = getCallerMachineId();
         let ip = getCallerIP();
-        let trust = await getTrust();
-        let hasAccess = isAdmin(machineId) || !!await trust.get(`${account}|${machineId}`);
-        let result: AccessState = { machineId, ip, hasAccess };
-        if (!hasAccess) {
-            let requests = await getRequests();
-            let ownRequest = (await requests.get(ip) || []).find(x => x.account === account && x.machineId === machineId);
-            if (ownRequest) {
-                result.grantAccessCommand = getGrantAccessCommand(ownRequest.requestId);
-            }
-            return result;
+        let { rootDomain } = getStorageServerConfig();
+        if (isAdmin(machineId)) {
+            return { machineId, ip, hasAccess: true, trustedMachines: await getTrustedMachines() };
         }
-
-        let trustedMachines: TrustRecord[] = [];
-        for (let key of await trust.getKeys()) {
-            if (!key.startsWith(`${account}|`)) continue;
-            let record = await trust.get(key);
-            if (record) trustedMachines.push(record);
-        }
-        result.trustedMachines = trustedMachines;
-        return result;
-    }
-
-    async listRequestsForIP(config: { account: string; ip: string }): Promise<AccessRequest[]> {
-        let requests = await getRequests();
-        return (await requests.get(config.ip) || []).filter(x => x.account === config.account);
-    }
-
-    async grantAccess(config: { requestId: string }): Promise<TrustRecord> {
-        let requestId = config.requestId;
-        let callerMachineId = getCallerMachineId();
-        let trust = await getTrust();
-        let requests = await getRequests();
-        for (let ip of await requests.getKeys()) {
-            for (let request of await requests.get(ip) || []) {
-                if (request.requestId !== requestId) continue;
-                if (!isAdmin(callerMachineId) && !await trust.get(`${request.account}|${callerMachineId}`)) {
-                    throw new Error(`${STORAGE_ACCESS_DENIED} Machine has no access to this account. Machine ${callerMachineId}, account ${JSON.stringify(request.account)}`);
-                }
-                let record: TrustRecord = {
-                    account: request.account,
-                    machineId: request.machineId,
-                    ip: request.ip,
-                    time: Date.now(),
-                };
-                await trust.set(`${request.account}|${request.machineId}`, record);
-                return record;
-            }
+        let verdict = await isMachineAccepted({ machineId, ip, domain: rootDomain });
+        if (!verdict.accepted) {
+            return {
+                machineId,
+                ip,
+                hasAccess: false,
+                reason: verdict.reason,
+                addMachineCommand: `yarn addmachine ${rootDomain} ${machineId} ${ip} git`,
+            };
         }
-        throw new Error(`No access request found with id ${JSON.stringify(requestId)}. It may have already been granted or expired.`);
+        return { machineId, ip, hasAccess: true, trustedMachines: await getTrustedMachines() };
     }
 
     /** Admin (so only this machine's own processes can call it): the buckets this process has loaded. A deploy successor asks its predecessor for this, so it activates exactly the buckets that are in use instead of every bucket on disk. */
     async adminListActiveBuckets(): Promise<{ account: string; bucketName: string }[]> {
         return getActiveBucketKeys();
     }
-    async adminListRequests(config: { ip: string }): Promise<AccessRequest[]> {
-        let requests = await getRequests();
-        return await requests.get(config.ip) || [];
-    }
-    async adminGrantAccess(config: { requestId: string }): Promise<TrustRecord> {
-        let requestId = config.requestId;
-        let trust = await getTrust();
-        let requests = await getRequests();
-        for (let ip of await requests.getKeys()) {
-            for (let request of await requests.get(ip) || []) {
-                if (request.requestId !== requestId) continue;
-                let record: TrustRecord = {
-                    account: request.account,
-                    machineId: request.machineId,
-                    ip: request.ip,
-                    time: Date.now(),
-                };
-                await trust.set(`${request.account}|${request.machineId}`, record);
-                return record;
-            }
-        }
-        throw new Error(`No access request found with id ${JSON.stringify(requestId)}. It may have already been granted or expired.`);
-    }
 
     @assertValidArgs
     @trackAccessCall("get")
@@ -586,13 +505,8 @@ export const RemoteStorageController = SocketFunction.register(
     () => ({
         ping: {},
         authenticate: {},
-        requestAccess: {},
         getAccessState: {},
-        listRequestsForIP: { hooks: [accountAccess] },
-        grantAccess: {},
-        adminListRequests: { hooks: [adminAccess] },
         adminListActiveBuckets: { hooks: [adminAccess] },
-        adminGrantAccess: { hooks: [adminAccess] },
         get2: { hooks: [accountAccess] },
         set: { hooks: [accountAccess] },
         del: { hooks: [accountAccess] },

package/storage/remoteStorage/storageServer.ts

@@ -36,7 +36,7 @@ function getInternalIP(): string {
     return candidates.find(ip => /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(ip)) || candidates[0];
 }
 
-// The remote storage server, as a library function: consumers call hostStorageServer() from their own process to start hosting (or use the storageserver bin, see storageServerCli.ts). The grantAccess.js bootstrap (next to this file) is what the access page's shown SSH command points at.
+// The remote storage server, as a library function: consumers call hostStorageServer() from their own process to start hosting (or use the storageserver bin, see storageServerCli.ts). Who may talk to it comes from the signed authorized_keys repo, read through security/machines - there is nothing to grant here.
 
 export type HostStorageServerConfig = {
     // Full URL of this storage server, e.g. "https://1-2-3-4.example.com:4444". The subdomain must be an ip domain - this machine's external IP with dashes, or 127-0-0-1 for local testing (see the validation in hostStorageServer). The domain and port are extracted from it (bucket routing URLs clients use look like
@@ -82,11 +82,6 @@ async function checkDiskSpace(config: { folder: string; threshold: number }): Pr
     }
 }
 
-// Full path to the grantAccess CLI bootstrap that lives next to this file. The SSH command shown on the access page invokes it via `node <path> ...` (found through __dirname so consumers don't have to know where our source lives).
-function getGrantAccessCliPath(): string {
-    return path.join(__dirname, "grantAccess.js");
-}
-
 export async function hostStorageServer(config: HostStorageServerConfig): Promise<void> {
     let { url, folder } = config;
     let { address: domain, port } = parseStorageUrl(url);
@@ -104,8 +99,6 @@ export async function hostStorageServer(config: HostStorageServerConfig): Promis
         domain,
         port,
         rootDomain,
-        sshTarget: `${os.userInfo().username}@${ip}`,
-        serverCommand: `node ${getGrantAccessCliPath()} --url ${url}`,
         folder: path.resolve(folder),
     });
 

package/storage/remoteStorage/storageServerCli.ts

@@ -1,5 +1,6 @@
 import { hostStorageServer } from "./storageServer";
 import { getArg, getFlag } from "./cliArgs";
+import { keysRepo } from "../../security/machines/machines";
 
 // Hosts a storage server from the command line (via the storageserver bin, see package.json).
 
@@ -11,6 +12,19 @@ process.on("uncaughtException", (error) => {
 });
 
 async function main() {
+    // Checked before anything is hosted. Who may talk to this server comes from the authorized_keys
+    // repo, so a machine without one would start up and refuse every caller with an error about a
+    // repo, which is a confusing way to learn the machine was never set up.
+    try {
+        let { repoPath, sourceURL } = await keysRepo();
+        console.log(`Trusting the machines in ${sourceURL} (${repoPath})`);
+    } catch (e) {
+        throw new Error(
+            `This machine has no authorized_keys repo, so it has no way to know which machines to`
+            + ` trust, and every caller would be refused.\n${e}`
+        );
+    }
+
     let url = getArg("url");
     if (!url) throw new Error(`--url is required (ex: --url https://storage.example.com:4444)`);
     let folder = getArg("folder");

package/storage/remoteStorage/grantAccess.js

@@ -1,4 +0,0 @@
-#!/usr/bin/env node
-"use strict";
-require("typenode");
-require("./grantAccessCli");

package/storage/remoteStorage/grantAccessCli.d.ts

@@ -1 +0,0 @@
-export {};

package/storage/remoteStorage/grantAccessCli.ts

@@ -1,25 +0,0 @@
-import { SocketFunction } from "socket-function/SocketFunction";
-import { RemoteStorageController } from "./storageController";
-import { authenticateStorage, parseStorageUrl } from "./ArchivesRemote";
-import { getArg } from "./cliArgs";
-
-// The grantAccess CLI. Invoked via the sibling grantAccess.js bootstrap (which loads typenode and then requires this file). Grants a specific access request by its requestId; must be run on the storage machine itself (its certs.ts identity is what the server trusts as admin).
-
-async function main() {
-    let url = getArg("url");
-    if (!url) throw new Error(`--url is required (ex: --url https://storage.example.com:4444)`);
-    let requestId = getArg("requestId");
-    if (!requestId) throw new Error(`--requestId is required (the request id shown on the access page)`);
-
-    let { address, port } = parseStorageUrl(url);
-    let nodeId = SocketFunction.connect({ address, port });
-    await authenticateStorage({ address, port, nodeId });
-    let record = await RemoteStorageController.nodes[nodeId].adminGrantAccess({ requestId });
-    console.log(`Granted machine ${record.machineId} access to account ${JSON.stringify(record.account)}`);
-    process.exit(0);
-}
-
-main().catch(e => {
-    console.error(e);
-    process.exit(1);
-});