querysub 0.591.0 → 0.593.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +5 -1
- package/bin/stop-machine.js +4 -0
- package/package.json +6 -3
- package/spec.txt +12 -0
- package/src/-a-archives/archiveCache.ts +28 -7
- package/src/-a-archives/archiveCache2.ts +4 -1
- package/src/-a-archives/archives.ts +2 -63
- package/src/-a-archives/archives2.ts +21 -17
- package/src/-a-archives/archivesDisk.ts +2 -13
- package/src/-a-archives/archivesMemoryCache.ts +27 -17
- package/src/-a-archives/archivesMemoryCache2.ts +23 -0
- package/src/-d-trust/NetworkTrust2.ts +3 -3
- package/src/-e-certs/certAuthority.ts +2 -2
- package/src/-f-node-discovery/NodeDiscovery.ts +3 -3
- package/src/0-path-value-core/AuthorityLookup.ts +0 -4
- package/src/0-path-value-core/PathRouter.ts +17 -16
- package/src/0-path-value-core/ShardPrefixes.ts +2 -4
- package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +133 -38
- package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +0 -1
- package/src/0-path-value-core/pathValueArchives.ts +9 -5
- package/src/2-proxy/archiveMoveHarness.ts +1 -1
- package/src/4-deploy/edgeBootstrap.ts +50 -11
- package/src/4-deploy/edgeNodes.ts +7 -13
- package/src/4-querysub/Querysub.ts +2 -2
- package/src/archiveapps/archiveJoinEntry.ts +1 -1
- package/src/config2.ts +1 -7
- package/src/deployManager/components/ServicesListPage.tsx +27 -8
- package/src/deployManager/machineDaemonShared.ts +2 -0
- package/src/deployManager/machineSchema.ts +5 -6
- package/src/deployManager/setupMachineMain.ts +1 -2
- package/src/deployManager/stopMachineMain.ts +45 -0
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +34 -21
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +3 -3
- package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +11 -2
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +3 -4
- package/src/diagnostics/logs/errorTickets/tickets.ts +3 -4
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +2 -4
- package/src/storageSetup.ts +1 -1
- package/src/-a-archives/archivesBackBlaze.ts +0 -156
- package/src/-a-archives/archivesCborT.ts +0 -52
- package/src/-a-archives/archivesLimitedCache.ts +0 -307
- package/src/-a-archives/archivesPrivateFileSystem.ts +0 -326
- package/src/-a-archives/copyLocalToBackblaze.ts +0 -24
- package/src/-b-authorities/cdnAuthority.ts +0 -53
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { isNodeTrue, list, timeInDay, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
2
|
-
import {
|
|
3
|
-
import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
|
|
2
|
+
import { getArchives2 } from "../-a-archives/archives2";
|
|
4
3
|
import { getDomain } from "../config";
|
|
5
4
|
import { archiveJSONT } from "../-a-archives/archivesJSONT";
|
|
6
5
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
@@ -187,9 +186,9 @@ export type MachineConfig = {
|
|
|
187
186
|
machineId: string;
|
|
188
187
|
disabled: boolean;
|
|
189
188
|
};
|
|
190
|
-
export const machineInfos = archiveJSONT<MachineInfo>(() =>
|
|
191
|
-
export const serviceConfigs = archiveJSONT<ServiceConfig>(() =>
|
|
192
|
-
export const machineConfigs = archiveJSONT<MachineConfig>(() =>
|
|
189
|
+
export const machineInfos = archiveJSONT<MachineInfo>(() => getArchives2("machines/machine-heartbeats/"));
|
|
190
|
+
export const serviceConfigs = archiveJSONT<ServiceConfig>(() => getArchives2("machines/service-configs/"));
|
|
191
|
+
export const machineConfigs = archiveJSONT<MachineConfig>(() => getArchives2("machines/machine-configs/"));
|
|
193
192
|
|
|
194
193
|
export type LaunchRecord = {
|
|
195
194
|
serviceId: string;
|
|
@@ -207,7 +206,7 @@ export type LaunchSummary = {
|
|
|
207
206
|
key: string;
|
|
208
207
|
};
|
|
209
208
|
|
|
210
|
-
const launches = lazy(() =>
|
|
209
|
+
const launches = lazy(() => getArchives2("machines/launches/"));
|
|
211
210
|
|
|
212
211
|
function formatLaunchDay(time: number): string {
|
|
213
212
|
let date = new Date(time);
|
|
@@ -8,13 +8,12 @@ import readline from "readline";
|
|
|
8
8
|
import open from "open";
|
|
9
9
|
import { fsExistsAsync } from "../fs";
|
|
10
10
|
import { delay } from "socket-function/src/batching";
|
|
11
|
+
import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
|
|
11
12
|
// Import querysub, to fix missing dependencies
|
|
12
13
|
Querysub;
|
|
13
14
|
|
|
14
15
|
const pinnedNodeVersion = 22;
|
|
15
16
|
|
|
16
|
-
const SERVICE_NAME = "machine-alwaysup";
|
|
17
|
-
const SERVICE_UNIT_NAME = `${SERVICE_NAME}.service`;
|
|
18
17
|
const DAEMON_SCRIPT_NAME = "machine-daemon.sh";
|
|
19
18
|
// How long after the process dies before it is started again. Short, because until it is back nothing on the machine is being deployed or repaired.
|
|
20
19
|
const SERVICE_RESTART_DELAY_SECONDS = 5;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { runPromise } from "../functional/runCommand";
|
|
2
|
+
import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
|
|
3
|
+
|
|
4
|
+
const DPLY_SCREEN_SUFFIX = "-dply";
|
|
5
|
+
|
|
6
|
+
async function main() {
|
|
7
|
+
let sshRemote = process.argv.slice(2).join(" ");
|
|
8
|
+
if (!sshRemote) {
|
|
9
|
+
console.error("Incorrect usage. Examples:\nyarn stop-machine 153.34.64.2\nyarn stop-machine devops@153.34.64.2");
|
|
10
|
+
process.exit(1);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
await runPromise(`ssh ${sshRemote} whoami`);
|
|
14
|
+
|
|
15
|
+
let daemonState = (await runPromise(`ssh ${sshRemote} "systemctl is-active ${SERVICE_UNIT_NAME} || true"`, { nothrow: true })).trim();
|
|
16
|
+
if (daemonState === "active") {
|
|
17
|
+
console.log(`Stopping the ${SERVICE_UNIT_NAME} daemon...`);
|
|
18
|
+
await runPromise(`ssh ${sshRemote} "sudo systemctl stop ${SERVICE_UNIT_NAME}"`);
|
|
19
|
+
console.log(`✅ Daemon stopped`);
|
|
20
|
+
} else {
|
|
21
|
+
console.log(`✅ The ${SERVICE_UNIT_NAME} daemon is not running (state: ${daemonState || "unknown"})`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
let sessions = (await runPromise(`ssh ${sshRemote} "tmux list-sessions -F '#{session_name}' 2>/dev/null || true"`, { nothrow: true }))
|
|
25
|
+
.split("\n").map(x => x.trim()).filter(x => x);
|
|
26
|
+
// The machine's own screen first, so it cannot restart a service screen while the rest are being killed
|
|
27
|
+
let toKill = [
|
|
28
|
+
...sessions.filter(name => name === SERVICE_NAME),
|
|
29
|
+
...sessions.filter(name => name.endsWith(DPLY_SCREEN_SUFFIX)),
|
|
30
|
+
];
|
|
31
|
+
if (toKill.length === 0) {
|
|
32
|
+
console.log(`✅ No ${SERVICE_NAME} or *${DPLY_SCREEN_SUFFIX} tmux sessions to kill (sessions: ${sessions.join(", ") || "none"})`);
|
|
33
|
+
}
|
|
34
|
+
for (let name of toKill) {
|
|
35
|
+
console.log(`Killing tmux session ${name}...`);
|
|
36
|
+
await runPromise(`ssh ${sshRemote} "tmux kill-session -t ${name}"`);
|
|
37
|
+
}
|
|
38
|
+
if (toKill.length > 0) {
|
|
39
|
+
console.log(`✅ Killed ${toKill.length} tmux session(s): ${toKill.join(", ")}`);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
console.log(`\n🛑 Machine stopped. Start it again with: ssh ${sshRemote} "~/machine-startup.sh"`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
main().catch(console.error).finally(() => process.exit(0));
|
|
@@ -5,15 +5,17 @@ import { BufferIndex } from "./BufferIndex";
|
|
|
5
5
|
import { delay, runInParallel, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
6
6
|
import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION, mergeIndexedLogResults } from "./BufferIndexHelpers";
|
|
7
7
|
import { getDomain, isPublic } from "../../../config";
|
|
8
|
-
import { getArchivesHome
|
|
9
|
-
import {
|
|
8
|
+
import { getArchivesHome } from "../../../-a-archives/archivesDisk";
|
|
9
|
+
import { getArchives2 } from "../../../-a-archives/archives2";
|
|
10
10
|
import { getMachineId, getOwnThreadId } from "sliftutils/misc/https/certs";
|
|
11
11
|
import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
|
|
12
|
+
import { createArchivesMemoryCache2 } from "../../../-a-archives/archivesMemoryCache2";
|
|
13
|
+
import { wrapArchivesWithCache2 } from "../../../-a-archives/archiveCache2";
|
|
12
14
|
import { registerShutdownHandler } from "../../periodic";
|
|
13
15
|
import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
|
|
14
16
|
import { isNode } from "typesafecss";
|
|
15
17
|
import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
|
|
16
|
-
import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
|
|
18
|
+
import { LogArchives, TimeFilePath, TimeFileTree } from "./TimeFileTree";
|
|
17
19
|
import { LogStreamer } from "./LogStreamer";
|
|
18
20
|
import { moveLogsToPublic } from "./moveIndexLogsToPublic";
|
|
19
21
|
import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT } from "./BufferIndexLogsOptimizationConstants";
|
|
@@ -83,7 +85,7 @@ export class IndexedLogs<T> {
|
|
|
83
85
|
|
|
84
86
|
private getLocalLogs = lazy((): Archives => {
|
|
85
87
|
let baseDisk = getArchivesHome(getDomain());
|
|
86
|
-
let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk);
|
|
88
|
+
let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk) as Archives;
|
|
87
89
|
archives = createArchivesMemoryCache(archives, {
|
|
88
90
|
maxSize: 1024 * 1024 * 512,
|
|
89
91
|
maxCount: 1000 * 100,
|
|
@@ -94,32 +96,43 @@ export class IndexedLogs<T> {
|
|
|
94
96
|
});
|
|
95
97
|
return archives;
|
|
96
98
|
});
|
|
97
|
-
private getPublicLogsBase(loadPublic: boolean):
|
|
98
|
-
let basePublic: Archives = getArchivesHome(getDomain());
|
|
99
|
-
// NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever.
|
|
100
|
-
let extraReadSize = 1024 * 1024 * 10;
|
|
99
|
+
private getPublicLogsBase(loadPublic: boolean): LogArchives {
|
|
101
100
|
if (loadPublic) {
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
101
|
+
return createArchivesMemoryCache2(getArchives2("final-indexed-logs/" + this.config.name), {
|
|
102
|
+
maxSize: 1024 * 1024 * 1024 * 2,
|
|
103
|
+
maxCount: 1000 * 500,
|
|
104
|
+
fullyImmutable: true,
|
|
105
|
+
// NOTE: While the latency to the remote is high, now we're reading in parallel, so it shouldn't be as big of an issue.
|
|
106
|
+
extraReadSize: 1024 * 1024 * 1,
|
|
107
|
+
stats: this.backblazeLogsStats,
|
|
108
|
+
sizeCache: this.fileSizeCache,
|
|
109
|
+
});
|
|
105
110
|
}
|
|
106
|
-
let archives = nestArchives("final-indexed-logs/" + this.config.name,
|
|
107
|
-
|
|
111
|
+
let archives = nestArchives("final-indexed-logs/" + this.config.name, getArchivesHome(getDomain()));
|
|
112
|
+
return createArchivesMemoryCache(archives, {
|
|
108
113
|
maxSize: 1024 * 1024 * 1024 * 2,
|
|
109
114
|
maxCount: 1000 * 500,
|
|
110
115
|
fullyImmutable: true,
|
|
111
|
-
|
|
116
|
+
// NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever.
|
|
117
|
+
extraReadSize: 1024 * 1024 * 10,
|
|
112
118
|
stats: this.backblazeLogsStats,
|
|
113
119
|
sizeCache: this.fileSizeCache,
|
|
114
120
|
});
|
|
115
|
-
return archives;
|
|
116
121
|
};
|
|
117
122
|
public debugGetCachedLogs = cache((config: {
|
|
118
123
|
type: "local" | "public";
|
|
119
|
-
}) => {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
124
|
+
}): LogArchives => {
|
|
125
|
+
if (config.type === "public") {
|
|
126
|
+
let archives2 = wrapArchivesWithCache2(getArchives2("final-indexed-logs/" + this.config.name), {
|
|
127
|
+
immutable: true,
|
|
128
|
+
});
|
|
129
|
+
return createArchivesMemoryCache2(archives2, {
|
|
130
|
+
maxSize: 1024 * 1024 * 1024 * 12,
|
|
131
|
+
maxCount: 1000 * 500,
|
|
132
|
+
fullyImmutable: true,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
let archives = nestArchives("final-indexed-logs/" + this.config.name, getArchivesHome(getDomain())) as Archives;
|
|
123
136
|
archives = wrapArchivesWithCache(archives, {
|
|
124
137
|
immutable: true,
|
|
125
138
|
});
|
|
@@ -133,10 +146,10 @@ export class IndexedLogs<T> {
|
|
|
133
146
|
public debugIsPublic() {
|
|
134
147
|
return isPublic();
|
|
135
148
|
}
|
|
136
|
-
private getPublicLogs = lazy(():
|
|
149
|
+
private getPublicLogs = lazy((): LogArchives => {
|
|
137
150
|
return this.getPublicLogsBase(isPublic());
|
|
138
151
|
});
|
|
139
|
-
private getRealPublicLogs = lazy(():
|
|
152
|
+
private getRealPublicLogs = lazy((): LogArchives => {
|
|
140
153
|
return this.getPublicLogsBase(true);
|
|
141
154
|
});
|
|
142
155
|
|
|
@@ -5,7 +5,7 @@ import { getDomain } from "../../../config";
|
|
|
5
5
|
import { getAllNodeIds, getOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
|
|
6
6
|
import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
|
|
7
7
|
import { getLoggers2Async, LogDatum } from "../diskLogger";
|
|
8
|
-
import {
|
|
8
|
+
import { LogArchives } from "./TimeFileTree";
|
|
9
9
|
import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
|
|
10
10
|
import { BufferReader, INDEX_EXTENSION, createEmptyIndexedLogResults } from "./BufferIndexHelpers";
|
|
11
11
|
import { BufferIndex } from "./BufferIndex";
|
|
@@ -104,7 +104,7 @@ type Direction = "fromStart" | "fromEnd";
|
|
|
104
104
|
type FileEntry = {
|
|
105
105
|
path: TimeFilePath;
|
|
106
106
|
loggerName: LoggerName;
|
|
107
|
-
archives:
|
|
107
|
+
archives: LogArchives;
|
|
108
108
|
};
|
|
109
109
|
|
|
110
110
|
// Mutable accumulators the file/block walk appends into, instead of returning
|
|
@@ -570,7 +570,7 @@ export class MCPIndexedLogs {
|
|
|
570
570
|
// Caches TimeFileTree.findAllPaths by hour-aligned bucket so repeated
|
|
571
571
|
// searches over similar windows reuse the slow folder walk.
|
|
572
572
|
private async getCachedPaths(config: {
|
|
573
|
-
archives:
|
|
573
|
+
archives: LogArchives;
|
|
574
574
|
type: "local" | "public";
|
|
575
575
|
loggerName: LoggerName;
|
|
576
576
|
startTime: number;
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { timeInDay } from "socket-function/src/misc";
|
|
2
|
-
import { Archives } from "../../../-a-archives/archives";
|
|
3
2
|
import { getOwnThreadId, getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
|
|
4
3
|
|
|
4
|
+
/** The subset of archive operations the log system uses, so both the old disk Archives and archives2 (IArchives) work interchangeably. */
|
|
5
|
+
export type LogArchives = {
|
|
6
|
+
getDebugName(): string;
|
|
7
|
+
get(path: string, config?: { range?: { start: number; end: number; } }): Promise<Buffer | undefined>;
|
|
8
|
+
set(path: string, data: Buffer): Promise<unknown>;
|
|
9
|
+
del(path: string): Promise<void>;
|
|
10
|
+
getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
|
|
11
|
+
find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
|
|
12
|
+
};
|
|
13
|
+
|
|
5
14
|
export type TimeFilePath = {
|
|
6
15
|
fullPath: string;
|
|
7
16
|
|
|
@@ -113,7 +122,7 @@ export function decodeLogFilePath(path: string): TimeFilePath | undefined {
|
|
|
113
122
|
|
|
114
123
|
// NOTE: This class will add its own extensions to the files, and it'll use it to detect these files. You can append your own extension and then we will ignore those files.
|
|
115
124
|
export class TimeFileTree {
|
|
116
|
-
public constructor(public archives:
|
|
125
|
+
public constructor(public archives: LogArchives) { }
|
|
117
126
|
|
|
118
127
|
public async findAllPaths(config: {
|
|
119
128
|
startTime: number;
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { delay } from "socket-function/src/batching";
|
|
2
2
|
import { sort, keyByArray } from "socket-function/src/misc";
|
|
3
|
-
import { Archives } from "../../../-a-archives/archives";
|
|
4
3
|
import { getOwnThreadId } from "../../../-f-node-discovery/NodeDiscovery";
|
|
5
4
|
import { BufferIndex } from "./BufferIndex";
|
|
6
5
|
import { LogStreamer } from "./LogStreamer";
|
|
7
|
-
import { TimeFileTree } from "./TimeFileTree";
|
|
6
|
+
import { LogArchives, TimeFileTree } from "./TimeFileTree";
|
|
8
7
|
import { blue, green, magenta } from "socket-function/src/formatting/logColors";
|
|
9
8
|
import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
10
9
|
|
|
@@ -17,8 +16,8 @@ export async function moveLogsToPublic(config: {
|
|
|
17
16
|
movingTimeout: number;
|
|
18
17
|
// Unsafe, but useful for testing
|
|
19
18
|
forceAll: boolean;
|
|
20
|
-
localLogs:
|
|
21
|
-
publicLogs:
|
|
19
|
+
localLogs: LogArchives;
|
|
20
|
+
publicLogs: LogArchives;
|
|
22
21
|
indexExtension: string;
|
|
23
22
|
}) {
|
|
24
23
|
let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, indexExtension, movingTimeout } = config;
|
|
@@ -2,10 +2,9 @@ import { lazy } from "socket-function/src/caching";
|
|
|
2
2
|
import { batchFunction, runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
3
3
|
import { timeInMinute } from "socket-function/src/misc";
|
|
4
4
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
5
|
-
import {
|
|
6
|
-
import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
|
|
5
|
+
import { getArchives2 } from "../../../-a-archives/archives2";
|
|
7
6
|
import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
|
|
8
|
-
import {
|
|
7
|
+
import { isPublic } from "../../../config";
|
|
9
8
|
import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
|
|
10
9
|
import { assertIsManagementUser } from "../../managementPages";
|
|
11
10
|
import { getSyncedController } from "../../../library-components/SyncedController";
|
|
@@ -14,7 +13,7 @@ import { Ticket, TicketComment, TicketPatchStatus, TicketState } from "./ticketT
|
|
|
14
13
|
|
|
15
14
|
const TICKET_CACHE_POLL_INTERVAL = timeInMinute * 5;
|
|
16
15
|
|
|
17
|
-
export const ticketsArchive = archiveJSONT<Ticket>(() =>
|
|
16
|
+
export const ticketsArchive = archiveJSONT<Ticket>(() => getArchives2("logs/error-tickets/"));
|
|
18
17
|
|
|
19
18
|
let ticketCache = new Map<string, Ticket>();
|
|
20
19
|
let ensureWatching = lazy(async () => {
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { lazy } from "socket-function/src/caching";
|
|
2
|
-
import {
|
|
3
|
-
import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
|
|
2
|
+
import { getArchives2 } from "../../../-a-archives/archives2";
|
|
4
3
|
import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
|
|
5
|
-
import { getDomain } from "../../../config";
|
|
6
4
|
import { runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
7
5
|
import { timeInMinute } from "socket-function/src/misc";
|
|
8
6
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
@@ -56,7 +54,7 @@ export function getVariables(entry: LifeCycleEntry): { key: string, title?: stri
|
|
|
56
54
|
return variables;
|
|
57
55
|
}
|
|
58
56
|
|
|
59
|
-
const lifeCycles = archiveJSONT<LifeCycle>(() =>
|
|
57
|
+
const lifeCycles = archiveJSONT<LifeCycle>(() => getArchives2("logs/life-cycles/"));
|
|
60
58
|
let lifeCyclesCache: LifeCycle[] = [];
|
|
61
59
|
|
|
62
60
|
let ensureWatching = lazy(async () => {
|
package/src/storageSetup.ts
CHANGED
|
@@ -15,7 +15,7 @@ async function main() {
|
|
|
15
15
|
|
|
16
16
|
let trustedMachineIds = await getTrustedMachineIds();
|
|
17
17
|
|
|
18
|
-
let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/
|
|
18
|
+
let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/serverConfig");
|
|
19
19
|
await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
|
|
20
20
|
console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);
|
|
21
21
|
|
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
import { cache } from "socket-function/src/caching";
|
|
2
|
-
import { Archives } from "./archives";
|
|
3
|
-
import fs from "fs";
|
|
4
|
-
import { isNode, timeInMinute } from "socket-function/src/misc";
|
|
5
|
-
import { isLogBackblaze } from "../config";
|
|
6
|
-
import { getBackblazePath } from "../misc/appPaths";
|
|
7
|
-
import { ArchivesBackblaze as ArchivesBackblazeBase } from "sliftutils/storage/backblaze";
|
|
8
|
-
import type { IArchives, ArchivesConfig } from "sliftutils/storage/IArchives";
|
|
9
|
-
|
|
10
|
-
export function hasBackblazePermissions() {
|
|
11
|
-
return isNode() && fs.existsSync(getBackblazePath());
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
// The implementation lives in sliftutils (it reads credentials via getSecret, which our appSecrets.ts serves from getBackblazePath). We forward to an inner instance instead of subclassing, because our Archives interface's move/copy take a querysub Archives target (with getBaseArchives unwrapping) which is type-incompatible with the base class's IArchives signatures.
|
|
15
|
-
export class ArchivesBackblaze {
|
|
16
|
-
private archives: ArchivesBackblazeBase;
|
|
17
|
-
|
|
18
|
-
public constructor(config: {
|
|
19
|
-
bucketName: string;
|
|
20
|
-
public?: boolean;
|
|
21
|
-
immutable?: boolean;
|
|
22
|
-
cacheTime?: number;
|
|
23
|
-
allowedOrigins?: string[];
|
|
24
|
-
}) {
|
|
25
|
-
this.archives = new ArchivesBackblazeBase(config);
|
|
26
|
-
if (isLogBackblaze()) {
|
|
27
|
-
this.archives.enableLogging();
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
public enableLogging(): void {
|
|
32
|
-
this.archives.enableLogging();
|
|
33
|
-
}
|
|
34
|
-
public getDebugName(): string {
|
|
35
|
-
return this.archives.getDebugName();
|
|
36
|
-
}
|
|
37
|
-
public async hasWriteAccess(): Promise<boolean> {
|
|
38
|
-
return this.archives.hasWriteAccess();
|
|
39
|
-
}
|
|
40
|
-
public async getConfig(): Promise<ArchivesConfig> {
|
|
41
|
-
return this.archives.getConfig();
|
|
42
|
-
}
|
|
43
|
-
public async get(fileName: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
|
|
44
|
-
return this.archives.get(fileName, config);
|
|
45
|
-
}
|
|
46
|
-
public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
|
|
47
|
-
return this.archives.get2(fileName, config);
|
|
48
|
-
}
|
|
49
|
-
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
|
|
50
|
-
return this.archives.set(fileName, data, config);
|
|
51
|
-
}
|
|
52
|
-
public async append(fileName: string, data: Buffer): Promise<void> {
|
|
53
|
-
throw new Error(`ArchivesBackblaze does not support append. Use set instead.`);
|
|
54
|
-
}
|
|
55
|
-
public async del(fileName: string): Promise<void> {
|
|
56
|
-
return this.archives.del(fileName);
|
|
57
|
-
}
|
|
58
|
-
public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
|
|
59
|
-
return this.archives.setLargeFile(config);
|
|
60
|
-
}
|
|
61
|
-
public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
|
|
62
|
-
return this.archives.getInfo(fileName);
|
|
63
|
-
}
|
|
64
|
-
public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
|
|
65
|
-
return this.archives.find(prefix, config);
|
|
66
|
-
}
|
|
67
|
-
public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
|
|
68
|
-
return this.archives.findInfo(prefix, config);
|
|
69
|
-
}
|
|
70
|
-
public async assertPathValid(path: string): Promise<void> {
|
|
71
|
-
return this.archives.assertPathValid(path);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
public async move(config: {
|
|
75
|
-
path: string;
|
|
76
|
-
target: Archives;
|
|
77
|
-
targetPath: string;
|
|
78
|
-
copyInstead?: boolean;
|
|
79
|
-
}): Promise<void> {
|
|
80
|
-
let { target, targetPath } = config;
|
|
81
|
-
let base = target.getBaseArchives?.();
|
|
82
|
-
if (base) {
|
|
83
|
-
target = base.archives;
|
|
84
|
-
targetPath = base.parentPath + targetPath;
|
|
85
|
-
}
|
|
86
|
-
// Hand the inner instance across so the base implementation's bucket-to-bucket copyFile fast path (an instanceof check on its own class) still triggers.
|
|
87
|
-
let innerTarget: IArchives = target instanceof ArchivesBackblaze
|
|
88
|
-
? target.archives
|
|
89
|
-
: target as unknown as IArchives;
|
|
90
|
-
return this.archives.move({
|
|
91
|
-
path: config.path,
|
|
92
|
-
target: innerTarget as any,
|
|
93
|
-
targetPath,
|
|
94
|
-
copyInstead: config.copyInstead,
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
public async copy(config: {
|
|
99
|
-
path: string;
|
|
100
|
-
target: Archives;
|
|
101
|
-
targetPath: string;
|
|
102
|
-
}): Promise<void> {
|
|
103
|
-
return this.move({ ...config, copyInstead: true });
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
public async getURL(path: string): Promise<string> {
|
|
107
|
-
return this.archives.getURL(path);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
public async getDownloadAuthorization(config: {
|
|
111
|
-
fileNamePrefix?: string;
|
|
112
|
-
validDurationInSeconds: number;
|
|
113
|
-
b2ContentDisposition?: string;
|
|
114
|
-
b2ContentLanguage?: string;
|
|
115
|
-
b2Expires?: string;
|
|
116
|
-
b2CacheControl?: string;
|
|
117
|
-
b2ContentEncoding?: string;
|
|
118
|
-
b2ContentType?: string;
|
|
119
|
-
}): Promise<{
|
|
120
|
-
bucketId: string;
|
|
121
|
-
fileNamePrefix: string;
|
|
122
|
-
authorizationToken: string;
|
|
123
|
-
}> {
|
|
124
|
-
return this.archives.getDownloadAuthorization(config);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
export const getArchivesBackblaze = cache((domain: string) => {
|
|
129
|
-
return new ArchivesBackblaze({ bucketName: domain });
|
|
130
|
-
});
|
|
131
|
-
export const getArchivesBackblazePrivateImmutable = cache((domain: string) => {
|
|
132
|
-
return new ArchivesBackblaze({
|
|
133
|
-
bucketName: domain + "-private-immutable",
|
|
134
|
-
immutable: true
|
|
135
|
-
});
|
|
136
|
-
});
|
|
137
|
-
export const getArchivesBackblazePublicImmutable = cache((domain: string) => {
|
|
138
|
-
return new ArchivesBackblaze({
|
|
139
|
-
bucketName: domain + "-public-immutable",
|
|
140
|
-
public: true,
|
|
141
|
-
immutable: true,
|
|
142
|
-
allowedOrigins: [`https://${domain}`, `https://127-0-0-1.${domain}:7007`],
|
|
143
|
-
});
|
|
144
|
-
});
|
|
145
|
-
|
|
146
|
-
// NOTE: Cache by a minute. This might be a bad idea, but... usually whole reason for public is
|
|
147
|
-
// for cloudflare caching (as otherwise we can just access it through a server), or for large files
|
|
148
|
-
// (which should be cached anyways, and probably even use immutable caching).
|
|
149
|
-
export const getArchivesBackblazePublic = cache((domain: string) => {
|
|
150
|
-
return new ArchivesBackblaze({
|
|
151
|
-
bucketName: domain + "-public",
|
|
152
|
-
public: true,
|
|
153
|
-
cacheTime: timeInMinute,
|
|
154
|
-
allowedOrigins: [`https://${domain}`, `https://127-0-0-1.${domain}:7007`],
|
|
155
|
-
});
|
|
156
|
-
});
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { lazy } from "socket-function/src/caching";
|
|
2
|
-
import { Archives } from "./archives";
|
|
3
|
-
import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
|
|
4
|
-
import { ArchiveT } from "./archivesJSONT";
|
|
5
|
-
|
|
6
|
-
export function archiveCborT<T>(archives: () => Archives): ArchiveT<T> {
|
|
7
|
-
archives = lazy(archives);
|
|
8
|
-
async function get(key: string) {
|
|
9
|
-
let buffer = await archives().get(key);
|
|
10
|
-
if (!buffer) return undefined;
|
|
11
|
-
return decodeCborx(buffer) as T;
|
|
12
|
-
}
|
|
13
|
-
async function set(key: string, value: T) {
|
|
14
|
-
await archives().set(key, encodeCborx(value));
|
|
15
|
-
}
|
|
16
|
-
async function deleteFnc(key: string) {
|
|
17
|
-
await archives().del(key);
|
|
18
|
-
}
|
|
19
|
-
async function keys() {
|
|
20
|
-
return (await archives().find("")).map(value => value.toString());
|
|
21
|
-
}
|
|
22
|
-
async function values() {
|
|
23
|
-
let keysArray = await keys();
|
|
24
|
-
let results: T[] = [];
|
|
25
|
-
await Promise.all(keysArray.map(async key => {
|
|
26
|
-
let value = await get(key);
|
|
27
|
-
if (value) {
|
|
28
|
-
results.push(value);
|
|
29
|
-
}
|
|
30
|
-
}));
|
|
31
|
-
return results;
|
|
32
|
-
}
|
|
33
|
-
async function entries(): Promise<[string, T][]> {
|
|
34
|
-
let keysArray = await keys();
|
|
35
|
-
let results: [string, T][] = [];
|
|
36
|
-
await Promise.all(keysArray.map(async key => {
|
|
37
|
-
let value = await get(key);
|
|
38
|
-
if (value) {
|
|
39
|
-
results.push([key, value]);
|
|
40
|
-
}
|
|
41
|
-
}));
|
|
42
|
-
return results;
|
|
43
|
-
}
|
|
44
|
-
async function* asyncIterator(): AsyncIterator<[string, T]> {
|
|
45
|
-
for (let [key, value] of await entries()) {
|
|
46
|
-
yield [key, value];
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
return {
|
|
50
|
-
get, set, delete: deleteFnc, keys, values, entries, [Symbol.asyncIterator]: asyncIterator
|
|
51
|
-
};
|
|
52
|
-
}
|