querysub 0.591.0 → 0.593.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.claude/settings.local.json +5 -1
  2. package/bin/stop-machine.js +4 -0
  3. package/package.json +6 -3
  4. package/spec.txt +12 -0
  5. package/src/-a-archives/archiveCache.ts +28 -7
  6. package/src/-a-archives/archiveCache2.ts +4 -1
  7. package/src/-a-archives/archives.ts +2 -63
  8. package/src/-a-archives/archives2.ts +21 -17
  9. package/src/-a-archives/archivesDisk.ts +2 -13
  10. package/src/-a-archives/archivesMemoryCache.ts +27 -17
  11. package/src/-a-archives/archivesMemoryCache2.ts +23 -0
  12. package/src/-d-trust/NetworkTrust2.ts +3 -3
  13. package/src/-e-certs/certAuthority.ts +2 -2
  14. package/src/-f-node-discovery/NodeDiscovery.ts +3 -3
  15. package/src/0-path-value-core/AuthorityLookup.ts +0 -4
  16. package/src/0-path-value-core/PathRouter.ts +17 -16
  17. package/src/0-path-value-core/ShardPrefixes.ts +2 -4
  18. package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +133 -38
  19. package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +0 -1
  20. package/src/0-path-value-core/pathValueArchives.ts +9 -5
  21. package/src/2-proxy/archiveMoveHarness.ts +1 -1
  22. package/src/4-deploy/edgeBootstrap.ts +50 -11
  23. package/src/4-deploy/edgeNodes.ts +7 -13
  24. package/src/4-querysub/Querysub.ts +2 -2
  25. package/src/archiveapps/archiveJoinEntry.ts +1 -1
  26. package/src/config2.ts +1 -7
  27. package/src/deployManager/components/ServicesListPage.tsx +27 -8
  28. package/src/deployManager/machineDaemonShared.ts +2 -0
  29. package/src/deployManager/machineSchema.ts +5 -6
  30. package/src/deployManager/setupMachineMain.ts +1 -2
  31. package/src/deployManager/stopMachineMain.ts +45 -0
  32. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +34 -21
  33. package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +3 -3
  34. package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +11 -2
  35. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +3 -4
  36. package/src/diagnostics/logs/errorTickets/tickets.ts +3 -4
  37. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +2 -4
  38. package/src/storageSetup.ts +1 -1
  39. package/src/-a-archives/archivesBackBlaze.ts +0 -156
  40. package/src/-a-archives/archivesCborT.ts +0 -52
  41. package/src/-a-archives/archivesLimitedCache.ts +0 -307
  42. package/src/-a-archives/archivesPrivateFileSystem.ts +0 -326
  43. package/src/-a-archives/copyLocalToBackblaze.ts +0 -24
  44. package/src/-b-authorities/cdnAuthority.ts +0 -53
@@ -1,6 +1,5 @@
1
1
  import { isNodeTrue, list, timeInDay, timeInMinute, timeInSecond } from "socket-function/src/misc";
2
- import { nestArchives } from "../-a-archives/archives";
3
- import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
2
+ import { getArchives2 } from "../-a-archives/archives2";
4
3
  import { getDomain } from "../config";
5
4
  import { archiveJSONT } from "../-a-archives/archivesJSONT";
6
5
  import { SocketFunction } from "socket-function/SocketFunction";
@@ -187,9 +186,9 @@ export type MachineConfig = {
187
186
  machineId: string;
188
187
  disabled: boolean;
189
188
  };
190
- export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machines/machine-heartbeats/", getArchivesBackblaze(getDomain())));
191
- export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
192
- export const machineConfigs = archiveJSONT<MachineConfig>(() => nestArchives("machines/machine-configs/", getArchivesBackblaze(getDomain())));
189
+ export const machineInfos = archiveJSONT<MachineInfo>(() => getArchives2("machines/machine-heartbeats/"));
190
+ export const serviceConfigs = archiveJSONT<ServiceConfig>(() => getArchives2("machines/service-configs/"));
191
+ export const machineConfigs = archiveJSONT<MachineConfig>(() => getArchives2("machines/machine-configs/"));
193
192
 
194
193
  export type LaunchRecord = {
195
194
  serviceId: string;
@@ -207,7 +206,7 @@ export type LaunchSummary = {
207
206
  key: string;
208
207
  };
209
208
 
210
- const launches = lazy(() => nestArchives("machines/launches/", getArchivesBackblaze(getDomain())));
209
+ const launches = lazy(() => getArchives2("machines/launches/"));
211
210
 
212
211
  function formatLaunchDay(time: number): string {
213
212
  let date = new Date(time);
@@ -8,13 +8,12 @@ import readline from "readline";
8
8
  import open from "open";
9
9
  import { fsExistsAsync } from "../fs";
10
10
  import { delay } from "socket-function/src/batching";
11
+ import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
11
12
  // Import querysub, to fix missing dependencies
12
13
  Querysub;
13
14
 
14
15
  const pinnedNodeVersion = 22;
15
16
 
16
- const SERVICE_NAME = "machine-alwaysup";
17
- const SERVICE_UNIT_NAME = `${SERVICE_NAME}.service`;
18
17
  const DAEMON_SCRIPT_NAME = "machine-daemon.sh";
19
18
  // How long after the process dies before it is started again. Short, because until it is back nothing on the machine is being deployed or repaired.
20
19
  const SERVICE_RESTART_DELAY_SECONDS = 5;
@@ -0,0 +1,45 @@
1
+ import { runPromise } from "../functional/runCommand";
2
+ import { SERVICE_NAME, SERVICE_UNIT_NAME } from "./machineDaemonShared";
3
+
4
+ const DPLY_SCREEN_SUFFIX = "-dply";
5
+
6
+ async function main() {
7
+ let sshRemote = process.argv.slice(2).join(" ");
8
+ if (!sshRemote) {
9
+ console.error("Incorrect usage. Examples:\nyarn stop-machine 153.34.64.2\nyarn stop-machine devops@153.34.64.2");
10
+ process.exit(1);
11
+ }
12
+
13
+ await runPromise(`ssh ${sshRemote} whoami`);
14
+
15
+ let daemonState = (await runPromise(`ssh ${sshRemote} "systemctl is-active ${SERVICE_UNIT_NAME} || true"`, { nothrow: true })).trim();
16
+ if (daemonState === "active") {
17
+ console.log(`Stopping the ${SERVICE_UNIT_NAME} daemon...`);
18
+ await runPromise(`ssh ${sshRemote} "sudo systemctl stop ${SERVICE_UNIT_NAME}"`);
19
+ console.log(`✅ Daemon stopped`);
20
+ } else {
21
+ console.log(`✅ The ${SERVICE_UNIT_NAME} daemon is not running (state: ${daemonState || "unknown"})`);
22
+ }
23
+
24
+ let sessions = (await runPromise(`ssh ${sshRemote} "tmux list-sessions -F '#{session_name}' 2>/dev/null || true"`, { nothrow: true }))
25
+ .split("\n").map(x => x.trim()).filter(x => x);
26
+ // The machine's own screen first, so it cannot restart a service screen while the rest are being killed
27
+ let toKill = [
28
+ ...sessions.filter(name => name === SERVICE_NAME),
29
+ ...sessions.filter(name => name.endsWith(DPLY_SCREEN_SUFFIX)),
30
+ ];
31
+ if (toKill.length === 0) {
32
+ console.log(`✅ No ${SERVICE_NAME} or *${DPLY_SCREEN_SUFFIX} tmux sessions to kill (sessions: ${sessions.join(", ") || "none"})`);
33
+ }
34
+ for (let name of toKill) {
35
+ console.log(`Killing tmux session ${name}...`);
36
+ await runPromise(`ssh ${sshRemote} "tmux kill-session -t ${name}"`);
37
+ }
38
+ if (toKill.length > 0) {
39
+ console.log(`✅ Killed ${toKill.length} tmux session(s): ${toKill.join(", ")}`);
40
+ }
41
+
42
+ console.log(`\n🛑 Machine stopped. Start it again with: ssh ${sshRemote} "~/machine-startup.sh"`);
43
+ }
44
+
45
+ main().catch(console.error).finally(() => process.exit(0));
@@ -5,15 +5,17 @@ import { BufferIndex } from "./BufferIndex";
5
5
  import { delay, runInParallel, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
6
6
  import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION, mergeIndexedLogResults } from "./BufferIndexHelpers";
7
7
  import { getDomain, isPublic } from "../../../config";
8
- import { getArchivesHome, getArchivesLocal } from "../../../-a-archives/archivesDisk";
9
- import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../../../-a-archives/archivesBackBlaze";
8
+ import { getArchivesHome } from "../../../-a-archives/archivesDisk";
9
+ import { getArchives2 } from "../../../-a-archives/archives2";
10
10
  import { getMachineId, getOwnThreadId } from "sliftutils/misc/https/certs";
11
11
  import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
12
+ import { createArchivesMemoryCache2 } from "../../../-a-archives/archivesMemoryCache2";
13
+ import { wrapArchivesWithCache2 } from "../../../-a-archives/archiveCache2";
12
14
  import { registerShutdownHandler } from "../../periodic";
13
15
  import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
14
16
  import { isNode } from "typesafecss";
15
17
  import { getOwnMachineId, isNodeIdOnOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
16
- import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
18
+ import { LogArchives, TimeFilePath, TimeFileTree } from "./TimeFileTree";
17
19
  import { LogStreamer } from "./LogStreamer";
18
20
  import { moveLogsToPublic } from "./moveIndexLogsToPublic";
19
21
  import { MAX_SINGLE_FILE_DATA, MAX_COUNT_PER_FILE, DISK_FLUSH_INTERVAL, PUBLIC_MOVE_THRESHOLD, MOVING_TIMEOUT } from "./BufferIndexLogsOptimizationConstants";
@@ -83,7 +85,7 @@ export class IndexedLogs<T> {
83
85
 
84
86
  private getLocalLogs = lazy((): Archives => {
85
87
  let baseDisk = getArchivesHome(getDomain());
86
- let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk);
88
+ let archives = nestArchives("indexed-logs/" + this.config.name, baseDisk) as Archives;
87
89
  archives = createArchivesMemoryCache(archives, {
88
90
  maxSize: 1024 * 1024 * 512,
89
91
  maxCount: 1000 * 100,
@@ -94,32 +96,43 @@ export class IndexedLogs<T> {
94
96
  });
95
97
  return archives;
96
98
  });
97
- private getPublicLogsBase(loadPublic: boolean): Archives {
98
- let basePublic: Archives = getArchivesHome(getDomain());
99
- // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever.
100
- let extraReadSize = 1024 * 1024 * 10;
99
+ private getPublicLogsBase(loadPublic: boolean): LogArchives {
101
100
  if (loadPublic) {
102
- basePublic = getArchivesBackblaze(getDomain());
103
- // NOTE: While the latency to back plays is high, now we're reading in parallel, so it shouldn't be as big of an issue.
104
- extraReadSize = 1024 * 1024 * 1;
101
+ return createArchivesMemoryCache2(getArchives2("final-indexed-logs/" + this.config.name), {
102
+ maxSize: 1024 * 1024 * 1024 * 2,
103
+ maxCount: 1000 * 500,
104
+ fullyImmutable: true,
105
+ // NOTE: While the latency to the remote is high, now we're reading in parallel, so it shouldn't be as big of an issue.
106
+ extraReadSize: 1024 * 1024 * 1,
107
+ stats: this.backblazeLogsStats,
108
+ sizeCache: this.fileSizeCache,
109
+ });
105
110
  }
106
- let archives = nestArchives("final-indexed-logs/" + this.config.name, basePublic);
107
- archives = createArchivesMemoryCache(archives, {
111
+ let archives = nestArchives("final-indexed-logs/" + this.config.name, getArchivesHome(getDomain()));
112
+ return createArchivesMemoryCache(archives, {
108
113
  maxSize: 1024 * 1024 * 1024 * 2,
109
114
  maxCount: 1000 * 500,
110
115
  fullyImmutable: true,
111
- extraReadSize,
116
+ // NOTE: The local disk is so fast that reading in 10 megabytes is nothing, And if we read in too small of a value, the overhead per read ends up making this take forever.
117
+ extraReadSize: 1024 * 1024 * 10,
112
118
  stats: this.backblazeLogsStats,
113
119
  sizeCache: this.fileSizeCache,
114
120
  });
115
- return archives;
116
121
  };
117
122
  public debugGetCachedLogs = cache((config: {
118
123
  type: "local" | "public";
119
- }) => {
120
- let usePublic = config.type === "public";
121
- let archives = usePublic ? getArchivesBackblaze(getDomain()) : getArchivesHome(getDomain());
122
- archives = nestArchives("final-indexed-logs/" + this.config.name, archives);
124
+ }): LogArchives => {
125
+ if (config.type === "public") {
126
+ let archives2 = wrapArchivesWithCache2(getArchives2("final-indexed-logs/" + this.config.name), {
127
+ immutable: true,
128
+ });
129
+ return createArchivesMemoryCache2(archives2, {
130
+ maxSize: 1024 * 1024 * 1024 * 12,
131
+ maxCount: 1000 * 500,
132
+ fullyImmutable: true,
133
+ });
134
+ }
135
+ let archives = nestArchives("final-indexed-logs/" + this.config.name, getArchivesHome(getDomain())) as Archives;
123
136
  archives = wrapArchivesWithCache(archives, {
124
137
  immutable: true,
125
138
  });
@@ -133,10 +146,10 @@ export class IndexedLogs<T> {
133
146
  public debugIsPublic() {
134
147
  return isPublic();
135
148
  }
136
- private getPublicLogs = lazy((): Archives => {
149
+ private getPublicLogs = lazy((): LogArchives => {
137
150
  return this.getPublicLogsBase(isPublic());
138
151
  });
139
- private getRealPublicLogs = lazy((): Archives => {
152
+ private getRealPublicLogs = lazy((): LogArchives => {
140
153
  return this.getPublicLogsBase(true);
141
154
  });
142
155
 
@@ -5,7 +5,7 @@ import { getDomain } from "../../../config";
5
5
  import { getAllNodeIds, getOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
6
6
  import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
7
7
  import { getLoggers2Async, LogDatum } from "../diskLogger";
8
- import { Archives } from "../../../-a-archives/archives";
8
+ import { LogArchives } from "./TimeFileTree";
9
9
  import { TimeFilePath, TimeFileTree } from "./TimeFileTree";
10
10
  import { BufferReader, INDEX_EXTENSION, createEmptyIndexedLogResults } from "./BufferIndexHelpers";
11
11
  import { BufferIndex } from "./BufferIndex";
@@ -104,7 +104,7 @@ type Direction = "fromStart" | "fromEnd";
104
104
  type FileEntry = {
105
105
  path: TimeFilePath;
106
106
  loggerName: LoggerName;
107
- archives: Archives;
107
+ archives: LogArchives;
108
108
  };
109
109
 
110
110
  // Mutable accumulators the file/block walk appends into, instead of returning
@@ -570,7 +570,7 @@ export class MCPIndexedLogs {
570
570
  // Caches TimeFileTree.findAllPaths by hour-aligned bucket so repeated
571
571
  // searches over similar windows reuse the slow folder walk.
572
572
  private async getCachedPaths(config: {
573
- archives: Archives;
573
+ archives: LogArchives;
574
574
  type: "local" | "public";
575
575
  loggerName: LoggerName;
576
576
  startTime: number;
@@ -1,7 +1,16 @@
1
1
  import { timeInDay } from "socket-function/src/misc";
2
- import { Archives } from "../../../-a-archives/archives";
3
2
  import { getOwnThreadId, getOwnMachineId } from "../../../-f-node-discovery/NodeDiscovery";
4
3
 
4
+ /** The subset of archive operations the log system uses, so both the old disk Archives and archives2 (IArchives) work interchangeably. */
5
+ export type LogArchives = {
6
+ getDebugName(): string;
7
+ get(path: string, config?: { range?: { start: number; end: number; } }): Promise<Buffer | undefined>;
8
+ set(path: string, data: Buffer): Promise<unknown>;
9
+ del(path: string): Promise<void>;
10
+ getInfo(path: string): Promise<{ writeTime: number; size: number; } | undefined>;
11
+ find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
12
+ };
13
+
5
14
  export type TimeFilePath = {
6
15
  fullPath: string;
7
16
 
@@ -113,7 +122,7 @@ export function decodeLogFilePath(path: string): TimeFilePath | undefined {
113
122
 
114
123
  // NOTE: This class will add its own extensions to the files, and it'll use it to detect these files. You can append your own extension and then we will ignore those files.
115
124
  export class TimeFileTree {
116
- public constructor(public archives: Archives) { }
125
+ public constructor(public archives: LogArchives) { }
117
126
 
118
127
  public async findAllPaths(config: {
119
128
  startTime: number;
@@ -1,10 +1,9 @@
1
1
  import { delay } from "socket-function/src/batching";
2
2
  import { sort, keyByArray } from "socket-function/src/misc";
3
- import { Archives } from "../../../-a-archives/archives";
4
3
  import { getOwnThreadId } from "../../../-f-node-discovery/NodeDiscovery";
5
4
  import { BufferIndex } from "./BufferIndex";
6
5
  import { LogStreamer } from "./LogStreamer";
7
- import { TimeFileTree } from "./TimeFileTree";
6
+ import { LogArchives, TimeFileTree } from "./TimeFileTree";
8
7
  import { blue, green, magenta } from "socket-function/src/formatting/logColors";
9
8
  import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
10
9
 
@@ -17,8 +16,8 @@ export async function moveLogsToPublic(config: {
17
16
  movingTimeout: number;
18
17
  // Unsafe, but useful for testing
19
18
  forceAll: boolean;
20
- localLogs: Archives;
21
- publicLogs: Archives;
19
+ localLogs: LogArchives;
20
+ publicLogs: LogArchives;
22
21
  indexExtension: string;
23
22
  }) {
24
23
  let { forceAll, localLogs, publicLogs, publicMoveThreshold, maxSingleFileData, indexExtension, movingTimeout } = config;
@@ -2,10 +2,9 @@ import { lazy } from "socket-function/src/caching";
2
2
  import { batchFunction, runInfinitePollCallAtStart } from "socket-function/src/batching";
3
3
  import { timeInMinute } from "socket-function/src/misc";
4
4
  import { SocketFunction } from "socket-function/SocketFunction";
5
- import { nestArchives } from "../../../-a-archives/archives";
6
- import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
5
+ import { getArchives2 } from "../../../-a-archives/archives2";
7
6
  import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
8
- import { getDomain, isPublic } from "../../../config";
7
+ import { isPublic } from "../../../config";
9
8
  import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
10
9
  import { assertIsManagementUser } from "../../managementPages";
11
10
  import { getSyncedController } from "../../../library-components/SyncedController";
@@ -14,7 +13,7 @@ import { Ticket, TicketComment, TicketPatchStatus, TicketState } from "./ticketT
14
13
 
15
14
  const TICKET_CACHE_POLL_INTERVAL = timeInMinute * 5;
16
15
 
17
- export const ticketsArchive = archiveJSONT<Ticket>(() => nestArchives("logs/error-tickets/", getArchivesBackblaze(getDomain())));
16
+ export const ticketsArchive = archiveJSONT<Ticket>(() => getArchives2("logs/error-tickets/"));
18
17
 
19
18
  let ticketCache = new Map<string, Ticket>();
20
19
  let ensureWatching = lazy(async () => {
@@ -1,8 +1,6 @@
1
1
  import { lazy } from "socket-function/src/caching";
2
- import { nestArchives } from "../../../-a-archives/archives";
3
- import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
2
+ import { getArchives2 } from "../../../-a-archives/archives2";
4
3
  import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
5
- import { getDomain } from "../../../config";
6
4
  import { runInfinitePollCallAtStart } from "socket-function/src/batching";
7
5
  import { timeInMinute } from "socket-function/src/misc";
8
6
  import { SocketFunction } from "socket-function/SocketFunction";
@@ -56,7 +54,7 @@ export function getVariables(entry: LifeCycleEntry): { key: string, title?: stri
56
54
  return variables;
57
55
  }
58
56
 
59
- const lifeCycles = archiveJSONT<LifeCycle>(() => nestArchives("logs/life-cycles/", getArchivesBackblaze(getDomain())));
57
+ const lifeCycles = archiveJSONT<LifeCycle>(() => getArchives2("logs/life-cycles/"));
60
58
  let lifeCyclesCache: LifeCycle[] = [];
61
59
 
62
60
  let ensureWatching = lazy(async () => {
@@ -15,7 +15,7 @@ async function main() {
15
15
 
16
16
  let trustedMachineIds = await getTrustedMachineIds();
17
17
 
18
- let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/storageServerState");
18
+ let { setTrustedMachines } = await import("sliftutils/storage/remoteStorage/serverConfig");
19
19
  await setTrustedMachines({ account: "root", machineIds: trustedMachineIds });
20
20
  console.log(`Synchronized ${trustedMachineIds.length} trusted machines`);
21
21
 
@@ -1,156 +0,0 @@
1
- import { cache } from "socket-function/src/caching";
2
- import { Archives } from "./archives";
3
- import fs from "fs";
4
- import { isNode, timeInMinute } from "socket-function/src/misc";
5
- import { isLogBackblaze } from "../config";
6
- import { getBackblazePath } from "../misc/appPaths";
7
- import { ArchivesBackblaze as ArchivesBackblazeBase } from "sliftutils/storage/backblaze";
8
- import type { IArchives, ArchivesConfig } from "sliftutils/storage/IArchives";
9
-
10
- export function hasBackblazePermissions() {
11
- return isNode() && fs.existsSync(getBackblazePath());
12
- }
13
-
14
- // The implementation lives in sliftutils (it reads credentials via getSecret, which our appSecrets.ts serves from getBackblazePath). We forward to an inner instance instead of subclassing, because our Archives interface's move/copy take a querysub Archives target (with getBaseArchives unwrapping) which is type-incompatible with the base class's IArchives signatures.
15
- export class ArchivesBackblaze {
16
- private archives: ArchivesBackblazeBase;
17
-
18
- public constructor(config: {
19
- bucketName: string;
20
- public?: boolean;
21
- immutable?: boolean;
22
- cacheTime?: number;
23
- allowedOrigins?: string[];
24
- }) {
25
- this.archives = new ArchivesBackblazeBase(config);
26
- if (isLogBackblaze()) {
27
- this.archives.enableLogging();
28
- }
29
- }
30
-
31
- public enableLogging(): void {
32
- this.archives.enableLogging();
33
- }
34
- public getDebugName(): string {
35
- return this.archives.getDebugName();
36
- }
37
- public async hasWriteAccess(): Promise<boolean> {
38
- return this.archives.hasWriteAccess();
39
- }
40
- public async getConfig(): Promise<ArchivesConfig> {
41
- return this.archives.getConfig();
42
- }
43
- public async get(fileName: string, config?: { range?: { start: number; end: number; }; retryCount?: number }): Promise<Buffer | undefined> {
44
- return this.archives.get(fileName, config);
45
- }
46
- public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
47
- return this.archives.get2(fileName, config);
48
- }
49
- public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<string> {
50
- return this.archives.set(fileName, data, config);
51
- }
52
- public async append(fileName: string, data: Buffer): Promise<void> {
53
- throw new Error(`ArchivesBackblaze does not support append. Use set instead.`);
54
- }
55
- public async del(fileName: string): Promise<void> {
56
- return this.archives.del(fileName);
57
- }
58
- public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined>; }): Promise<void> {
59
- return this.archives.setLargeFile(config);
60
- }
61
- public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
62
- return this.archives.getInfo(fileName);
63
- }
64
- public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
65
- return this.archives.find(prefix, config);
66
- }
67
- public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<{ path: string; createTime: number; size: number; }[]> {
68
- return this.archives.findInfo(prefix, config);
69
- }
70
- public async assertPathValid(path: string): Promise<void> {
71
- return this.archives.assertPathValid(path);
72
- }
73
-
74
- public async move(config: {
75
- path: string;
76
- target: Archives;
77
- targetPath: string;
78
- copyInstead?: boolean;
79
- }): Promise<void> {
80
- let { target, targetPath } = config;
81
- let base = target.getBaseArchives?.();
82
- if (base) {
83
- target = base.archives;
84
- targetPath = base.parentPath + targetPath;
85
- }
86
- // Hand the inner instance across so the base implementation's bucket-to-bucket copyFile fast path (an instanceof check on its own class) still triggers.
87
- let innerTarget: IArchives = target instanceof ArchivesBackblaze
88
- ? target.archives
89
- : target as unknown as IArchives;
90
- return this.archives.move({
91
- path: config.path,
92
- target: innerTarget as any,
93
- targetPath,
94
- copyInstead: config.copyInstead,
95
- });
96
- }
97
-
98
- public async copy(config: {
99
- path: string;
100
- target: Archives;
101
- targetPath: string;
102
- }): Promise<void> {
103
- return this.move({ ...config, copyInstead: true });
104
- }
105
-
106
- public async getURL(path: string): Promise<string> {
107
- return this.archives.getURL(path);
108
- }
109
-
110
- public async getDownloadAuthorization(config: {
111
- fileNamePrefix?: string;
112
- validDurationInSeconds: number;
113
- b2ContentDisposition?: string;
114
- b2ContentLanguage?: string;
115
- b2Expires?: string;
116
- b2CacheControl?: string;
117
- b2ContentEncoding?: string;
118
- b2ContentType?: string;
119
- }): Promise<{
120
- bucketId: string;
121
- fileNamePrefix: string;
122
- authorizationToken: string;
123
- }> {
124
- return this.archives.getDownloadAuthorization(config);
125
- }
126
- }
127
-
128
- export const getArchivesBackblaze = cache((domain: string) => {
129
- return new ArchivesBackblaze({ bucketName: domain });
130
- });
131
- export const getArchivesBackblazePrivateImmutable = cache((domain: string) => {
132
- return new ArchivesBackblaze({
133
- bucketName: domain + "-private-immutable",
134
- immutable: true
135
- });
136
- });
137
- export const getArchivesBackblazePublicImmutable = cache((domain: string) => {
138
- return new ArchivesBackblaze({
139
- bucketName: domain + "-public-immutable",
140
- public: true,
141
- immutable: true,
142
- allowedOrigins: [`https://${domain}`, `https://127-0-0-1.${domain}:7007`],
143
- });
144
- });
145
-
146
- // NOTE: Cache by a minute. This might be a bad idea, but... usually whole reason for public is
147
- // for cloudflare caching (as otherwise we can just access it through a server), or for large files
148
- // (which should be cached anyways, and probably even use immutable caching).
149
- export const getArchivesBackblazePublic = cache((domain: string) => {
150
- return new ArchivesBackblaze({
151
- bucketName: domain + "-public",
152
- public: true,
153
- cacheTime: timeInMinute,
154
- allowedOrigins: [`https://${domain}`, `https://127-0-0-1.${domain}:7007`],
155
- });
156
- });
@@ -1,52 +0,0 @@
1
- import { lazy } from "socket-function/src/caching";
2
- import { Archives } from "./archives";
3
- import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
4
- import { ArchiveT } from "./archivesJSONT";
5
-
6
- export function archiveCborT<T>(archives: () => Archives): ArchiveT<T> {
7
- archives = lazy(archives);
8
- async function get(key: string) {
9
- let buffer = await archives().get(key);
10
- if (!buffer) return undefined;
11
- return decodeCborx(buffer) as T;
12
- }
13
- async function set(key: string, value: T) {
14
- await archives().set(key, encodeCborx(value));
15
- }
16
- async function deleteFnc(key: string) {
17
- await archives().del(key);
18
- }
19
- async function keys() {
20
- return (await archives().find("")).map(value => value.toString());
21
- }
22
- async function values() {
23
- let keysArray = await keys();
24
- let results: T[] = [];
25
- await Promise.all(keysArray.map(async key => {
26
- let value = await get(key);
27
- if (value) {
28
- results.push(value);
29
- }
30
- }));
31
- return results;
32
- }
33
- async function entries(): Promise<[string, T][]> {
34
- let keysArray = await keys();
35
- let results: [string, T][] = [];
36
- await Promise.all(keysArray.map(async key => {
37
- let value = await get(key);
38
- if (value) {
39
- results.push([key, value]);
40
- }
41
- }));
42
- return results;
43
- }
44
- async function* asyncIterator(): AsyncIterator<[string, T]> {
45
- for (let [key, value] of await entries()) {
46
- yield [key, value];
47
- }
48
- }
49
- return {
50
- get, set, delete: deleteFnc, keys, values, entries, [Symbol.asyncIterator]: asyncIterator
51
- };
52
- }