sliftutils 1.7.65 → 1.7.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1 -0
- package/dist/test.ts.cache +187 -36
- package/dist/treeSummary.ts.cache +359 -0
- package/index.d.ts +56 -18
- package/package.json +1 -1
- package/storage/ArchivesDisk.d.ts +2 -2
- package/storage/ArchivesDisk.ts +3 -2
- package/storage/IArchives.d.ts +19 -2
- package/storage/IArchives.ts +23 -5
- package/storage/backblaze.d.ts +2 -2
- package/storage/backblaze.ts +5 -4
- package/storage/dist/ArchivesDisk.ts.cache +6 -4
- package/storage/dist/IArchives.ts.cache +6 -6
- package/storage/dist/backblaze.ts.cache +7 -6
- package/storage/remoteStorage/ArchivesRemote.d.ts +2 -2
- package/storage/remoteStorage/ArchivesRemote.ts +5 -5
- package/storage/remoteStorage/ArchivesUrl.d.ts +2 -2
- package/storage/remoteStorage/ArchivesUrl.ts +8 -6
- package/storage/remoteStorage/blobStore.d.ts +21 -3
- package/storage/remoteStorage/blobStore.ts +43 -12
- package/storage/remoteStorage/createArchives.d.ts +2 -2
- package/storage/remoteStorage/createArchives.ts +5 -6
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +7 -7
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +12 -8
- package/storage/remoteStorage/dist/blobStore.ts.cache +43 -13
- package/storage/remoteStorage/dist/createArchives.ts.cache +6 -7
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +3 -4
- package/storage/remoteStorage/dist/storageController.ts.cache +11 -8
- package/storage/remoteStorage/dist/storageServerState.ts.cache +38 -9
- package/storage/remoteStorage/sourceWrapper.ts +2 -3
- package/storage/remoteStorage/storageController.d.ts +3 -3
- package/storage/remoteStorage/storageController.ts +8 -5
- package/storage/remoteStorage/storageServerState.d.ts +3 -0
- package/storage/remoteStorage/storageServerState.ts +34 -7
- package/test-files-cache.json +1 -0
- package/test.ts +127 -10
- package/treeSummary.ts +331 -54
package/index.d.ts
CHANGED
|
@@ -858,7 +858,7 @@ declare module "sliftutils/render-utils/observer" {
|
|
|
858
858
|
declare module "sliftutils/storage/ArchivesDisk" {
|
|
859
859
|
/// <reference types="node" />
|
|
860
860
|
/// <reference types="node" />
|
|
861
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig } from "./IArchives";
|
|
861
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
|
|
862
862
|
export declare class ArchivesDisk implements IArchives {
|
|
863
863
|
private folder;
|
|
864
864
|
constructor(folder: string);
|
|
@@ -885,7 +885,7 @@ declare module "sliftutils/storage/ArchivesDisk" {
|
|
|
885
885
|
writeTime: number;
|
|
886
886
|
size: number;
|
|
887
887
|
} | undefined>;
|
|
888
|
-
getInfo(key: string): Promise<{
|
|
888
|
+
getInfo(key: string, config?: GetInfoConfig): Promise<{
|
|
889
889
|
writeTime: number;
|
|
890
890
|
size: number;
|
|
891
891
|
} | undefined>;
|
|
@@ -2146,6 +2146,12 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2146
2146
|
};
|
|
2147
2147
|
/** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
|
|
2148
2148
|
noFallbacks?: boolean;
|
|
2149
|
+
/** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
|
|
2150
|
+
internal?: boolean;
|
|
2151
|
+
};
|
|
2152
|
+
export type GetInfoConfig = {
|
|
2153
|
+
/** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
|
|
2154
|
+
includeTombstones?: boolean;
|
|
2149
2155
|
};
|
|
2150
2156
|
export type ChangesAfterConfig = {
|
|
2151
2157
|
time: number;
|
|
@@ -2158,6 +2164,8 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2158
2164
|
forceSetImmutable?: boolean;
|
|
2159
2165
|
/** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
|
|
2160
2166
|
noChecks?: boolean;
|
|
2167
|
+
/** Store-to-store push: the receiving node writes purely to its own disk and index, with NO downstream fan-out (the pushing store owns propagation - fanning its pushes back out is how write loops between stores form). Window and route ARE still checked: the stamp must fall inside one of the receiver's configured windows and routes, so a confused peer cannot stuff data onto a node that was never meant to hold it. Requires lastModified. */
|
|
2168
|
+
internal?: boolean;
|
|
2161
2169
|
};
|
|
2162
2170
|
export type ArchiveFileInfo = {
|
|
2163
2171
|
path: string;
|
|
@@ -2212,6 +2220,7 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2212
2220
|
writeTime?: number;
|
|
2213
2221
|
forceSetImmutable?: boolean;
|
|
2214
2222
|
noChecks?: boolean;
|
|
2223
|
+
internal?: boolean;
|
|
2215
2224
|
}): Promise<{
|
|
2216
2225
|
writeTime: number;
|
|
2217
2226
|
size: number;
|
|
@@ -2271,16 +2280,24 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2271
2280
|
lastModified?: number;
|
|
2272
2281
|
getNextData(): Promise<Buffer | undefined>;
|
|
2273
2282
|
}): Promise<void>;
|
|
2274
|
-
/** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. */
|
|
2275
|
-
getInfo(fileName: string): Promise<{
|
|
2283
|
+
/** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. Size-0 entries (tombstones) report undefined unless config.includeTombstones. */
|
|
2284
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
2276
2285
|
writeTime: number;
|
|
2277
2286
|
size: number;
|
|
2278
2287
|
url?: string;
|
|
2279
2288
|
} | undefined>;
|
|
2289
|
+
/**
|
|
2290
|
+
* Empty (size-0) files are NEVER returned by index-backed stores (BlobStore, and therefore the
|
|
2291
|
+
* chain): an empty file IS a missing file - the tombstone of a deletion. If you want a marker
|
|
2292
|
+
* file that shows up in listings, add some content to it. Raw sources (disk, backblaze) DO list
|
|
2293
|
+
* their empty files - that is how scans learn of deletions - but nothing built on the index
|
|
2294
|
+
* ever surfaces them.
|
|
2295
|
+
*/
|
|
2280
2296
|
find(prefix: string, config?: {
|
|
2281
2297
|
shallow?: boolean;
|
|
2282
2298
|
type: "files" | "folders";
|
|
2283
2299
|
}): Promise<string[]>;
|
|
2300
|
+
/** See find for the empty-file (tombstone) rule. */
|
|
2284
2301
|
findInfo(prefix: string, config?: {
|
|
2285
2302
|
shallow?: boolean;
|
|
2286
2303
|
type: "files" | "folders";
|
|
@@ -2559,7 +2576,7 @@ declare module "sliftutils/storage/TransactionStorage" {
|
|
|
2559
2576
|
declare module "sliftutils/storage/backblaze" {
|
|
2560
2577
|
/// <reference types="node" />
|
|
2561
2578
|
/// <reference types="node" />
|
|
2562
|
-
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig } from "./IArchives";
|
|
2579
|
+
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
|
|
2563
2580
|
export declare class ArchivesBackblaze implements IArchives {
|
|
2564
2581
|
private config;
|
|
2565
2582
|
constructor(config: {
|
|
@@ -2594,7 +2611,7 @@ declare module "sliftutils/storage/backblaze" {
|
|
|
2594
2611
|
lastModified?: number;
|
|
2595
2612
|
getNextData(): Promise<Buffer | undefined>;
|
|
2596
2613
|
}): Promise<void>;
|
|
2597
|
-
getInfo(fileName: string): Promise<{
|
|
2614
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
2598
2615
|
writeTime: number;
|
|
2599
2616
|
size: number;
|
|
2600
2617
|
} | undefined>;
|
|
@@ -2863,7 +2880,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
|
|
|
2863
2880
|
declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
2864
2881
|
/// <reference types="node" />
|
|
2865
2882
|
/// <reference types="node" />
|
|
2866
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SetConfig } from "../IArchives";
|
|
2883
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "../IArchives";
|
|
2867
2884
|
export type ArchivesRemoteConfig = {
|
|
2868
2885
|
url: string;
|
|
2869
2886
|
waitForAccess?: boolean;
|
|
@@ -2910,7 +2927,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
|
2910
2927
|
} | undefined>;
|
|
2911
2928
|
set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
|
|
2912
2929
|
del(fileName: string): Promise<void>;
|
|
2913
|
-
getInfo(fileName: string): Promise<{
|
|
2930
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
2914
2931
|
writeTime: number;
|
|
2915
2932
|
size: number;
|
|
2916
2933
|
} | undefined>;
|
|
@@ -2938,7 +2955,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
|
2938
2955
|
declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
|
|
2939
2956
|
/// <reference types="node" />
|
|
2940
2957
|
/// <reference types="node" />
|
|
2941
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig } from "../IArchives";
|
|
2958
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig } from "../IArchives";
|
|
2942
2959
|
export declare class ArchivesUrl implements IArchives {
|
|
2943
2960
|
private base;
|
|
2944
2961
|
constructor(base: string);
|
|
@@ -2950,7 +2967,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
|
|
|
2950
2967
|
writeTime: number;
|
|
2951
2968
|
size: number;
|
|
2952
2969
|
} | undefined>;
|
|
2953
|
-
getInfo(fileName: string): Promise<{
|
|
2970
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
2954
2971
|
writeTime: number;
|
|
2955
2972
|
size: number;
|
|
2956
2973
|
} | undefined>;
|
|
@@ -2986,7 +3003,7 @@ declare module "sliftutils/storage/remoteStorage/accessPage" {
|
|
|
2986
3003
|
declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
2987
3004
|
/// <reference types="node" />
|
|
2988
3005
|
/// <reference types="node" />
|
|
2989
|
-
import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SyncActivity } from "../IArchives";
|
|
3006
|
+
import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SyncActivity } from "../IArchives";
|
|
2990
3007
|
export declare const DEFAULT_FAST_WRITE_DELAY: number;
|
|
2991
3008
|
export declare const WINDOW_END_FLUSH_MARGIN: number;
|
|
2992
3009
|
export type WriteConfig = {
|
|
@@ -3001,9 +3018,17 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3001
3018
|
writeTime: number;
|
|
3002
3019
|
size: number;
|
|
3003
3020
|
} | undefined>;
|
|
3021
|
+
getInternal2?(fileName: string, config?: GetConfig): Promise<{
|
|
3022
|
+
data: Buffer;
|
|
3023
|
+
writeTime: number;
|
|
3024
|
+
size: number;
|
|
3025
|
+
} | undefined>;
|
|
3026
|
+
setInternal?(fileName: string, data: Buffer, config: {
|
|
3027
|
+
lastModified: number;
|
|
3028
|
+
}): Promise<void>;
|
|
3004
3029
|
set(fileName: string, data: Buffer, config?: WriteConfig): Promise<string>;
|
|
3005
3030
|
del(fileName: string, config?: WriteConfig): Promise<void>;
|
|
3006
|
-
getInfo(fileName: string): Promise<{
|
|
3031
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
3007
3032
|
writeTime: number;
|
|
3008
3033
|
size: number;
|
|
3009
3034
|
} | undefined>;
|
|
@@ -3140,12 +3165,22 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3140
3165
|
writeTime: number;
|
|
3141
3166
|
size: number;
|
|
3142
3167
|
} | undefined>;
|
|
3168
|
+
/** Internal (store-to-store) read: purely the local disk, completely short-circuiting the index and holder resolution - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form. No window or route checks: if the bytes are on our disk, the caller may have them. Note fast writes still sitting in the overlay are invisible here; the caller re-finds them after our flush. */
|
|
3169
|
+
getInternal2(key: string, config?: GetConfig): Promise<{
|
|
3170
|
+
data: Buffer;
|
|
3171
|
+
writeTime: number;
|
|
3172
|
+
size: number;
|
|
3173
|
+
} | undefined>;
|
|
3174
|
+
/** Internal (store-to-store) write: the local disk plus our index, with NO downstream fan-out - the pushing store owns propagation, and fanning its pushes back out is how write loops between stores form. Window/route acceptance is the caller's (writeBucketFile's) job; only-take-latest still applies here. */
|
|
3175
|
+
setInternal(key: string, data: Buffer, config: {
|
|
3176
|
+
lastModified: number;
|
|
3177
|
+
}): Promise<void>;
|
|
3143
3178
|
private cacheRead;
|
|
3144
3179
|
set(key: string, data: Buffer, config?: WriteConfig): Promise<string>;
|
|
3145
3180
|
del(key: string, config?: WriteConfig): Promise<void>;
|
|
3146
3181
|
private getWritableSources;
|
|
3147
3182
|
private writeToSources;
|
|
3148
|
-
getInfo(key: string): Promise<{
|
|
3183
|
+
getInfo(key: string, config?: GetInfoConfig): Promise<{
|
|
3149
3184
|
writeTime: number;
|
|
3150
3185
|
size: number;
|
|
3151
3186
|
} | undefined>;
|
|
@@ -3176,7 +3211,7 @@ declare module "sliftutils/storage/remoteStorage/cliArgs" {
|
|
|
3176
3211
|
declare module "sliftutils/storage/remoteStorage/createArchives" {
|
|
3177
3212
|
/// <reference types="node" />
|
|
3178
3213
|
/// <reference types="node" />
|
|
3179
|
-
import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SetConfig } from "../IArchives";
|
|
3214
|
+
import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "../IArchives";
|
|
3180
3215
|
import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
|
|
3181
3216
|
/** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
|
|
3182
3217
|
export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
|
|
@@ -3251,7 +3286,7 @@ declare module "sliftutils/storage/remoteStorage/createArchives" {
|
|
|
3251
3286
|
size?: undefined;
|
|
3252
3287
|
url: string;
|
|
3253
3288
|
}>;
|
|
3254
|
-
getInfo(fileName: string): Promise<{
|
|
3289
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
3255
3290
|
writeTime: number;
|
|
3256
3291
|
size: number;
|
|
3257
3292
|
url: string;
|
|
@@ -3572,14 +3607,14 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
|
|
|
3572
3607
|
get2: (account: string, bucketName: string, path: string, range?: {
|
|
3573
3608
|
start: number;
|
|
3574
3609
|
end: number;
|
|
3575
|
-
}) => Promise<{
|
|
3610
|
+
}, internal?: boolean) => Promise<{
|
|
3576
3611
|
data: Buffer;
|
|
3577
3612
|
writeTime: number;
|
|
3578
3613
|
size: number;
|
|
3579
3614
|
} | undefined>;
|
|
3580
|
-
set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean) => Promise<void>;
|
|
3615
|
+
set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean, internal?: boolean) => Promise<void>;
|
|
3581
3616
|
del: (account: string, bucketName: string, path: string) => Promise<void>;
|
|
3582
|
-
getInfo: (account: string, bucketName: string, path: string) => Promise<{
|
|
3617
|
+
getInfo: (account: string, bucketName: string, path: string, includeTombstones?: boolean) => Promise<{
|
|
3583
3618
|
writeTime: number;
|
|
3584
3619
|
size: number;
|
|
3585
3620
|
} | undefined>;
|
|
@@ -3682,6 +3717,8 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
|
|
|
3682
3717
|
};
|
|
3683
3718
|
export declare function addExtraListenPort(port: number): void;
|
|
3684
3719
|
export declare function removeExtraListenPort(port: number): void;
|
|
3720
|
+
/** Whether address:port is this server process. The ONE self test - findSelfIndexes, createApiArchives, and SourceWrapper all consult it, so "is this me" cannot disagree between the routing plan and connection building: a URL that is us on an extra listen port must never become a network client to ourselves, which is how infinite self-request loops form. */
|
|
3721
|
+
export declare function isOwnAddress(address: string, port: number): boolean;
|
|
3685
3722
|
/** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
|
|
3686
3723
|
export declare function resolveSourceArchives(url: string): IArchives;
|
|
3687
3724
|
/** Our role in a bucket's routing config, summarized across ALL currently-valid self entries. Stored instead of a single representative HostedConfig, so nothing can accidentally use one entry's route or flags where the union is required - the standard config has the same URL twice: a routed write-shard entry plus an unrouted read-everything entry. */
|
|
@@ -3699,6 +3736,7 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
|
|
|
3699
3736
|
export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
|
|
3700
3737
|
lastModified?: number;
|
|
3701
3738
|
forceSetImmutable?: boolean;
|
|
3739
|
+
internal?: boolean;
|
|
3702
3740
|
}): Promise<void>;
|
|
3703
3741
|
export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
|
|
3704
3742
|
/** Which buckets this process currently has loaded - what a deploy successor asks its predecessor for, so it activates exactly the buckets that are actually in use. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig } from "./IArchives";
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
|
|
4
4
|
export declare class ArchivesDisk implements IArchives {
|
|
5
5
|
private folder;
|
|
6
6
|
constructor(folder: string);
|
|
@@ -27,7 +27,7 @@ export declare class ArchivesDisk implements IArchives {
|
|
|
27
27
|
writeTime: number;
|
|
28
28
|
size: number;
|
|
29
29
|
} | undefined>;
|
|
30
|
-
getInfo(key: string): Promise<{
|
|
30
|
+
getInfo(key: string, config?: GetInfoConfig): Promise<{
|
|
31
31
|
writeTime: number;
|
|
32
32
|
size: number;
|
|
33
33
|
} | undefined>;
|
package/storage/ArchivesDisk.ts
CHANGED
|
@@ -3,7 +3,7 @@ import path from "path";
|
|
|
3
3
|
import { lazy } from "socket-function/src/caching";
|
|
4
4
|
import { runInfinitePoll } from "socket-function/src/batching";
|
|
5
5
|
import { sort, binarySearchBasic } from "socket-function/src/misc";
|
|
6
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig, assertValidLastModified } from "./IArchives";
|
|
6
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig, assertValidLastModified } from "./IArchives";
|
|
7
7
|
import { filterChanges } from "./remoteStorage/remoteConfig";
|
|
8
8
|
|
|
9
9
|
// The base file-system IArchives: storage is one-to-one with the file system, every key is exactly one real file under <folder>/files, so the file system itself is the index. File handles are cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in serial, so they can't collide with each other or with handle closing. Used as the disk synchronization source of BlobStore (see remoteStorage/blobStore.ts).
|
|
@@ -218,12 +218,13 @@ export class ArchivesDisk implements IArchives {
|
|
|
218
218
|
});
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
-
public async getInfo(key: string): Promise<{ writeTime: number; size: number } | undefined> {
|
|
221
|
+
public async getInfo(key: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number } | undefined> {
|
|
222
222
|
await this.init();
|
|
223
223
|
let filePath = this.filePath(key);
|
|
224
224
|
return await this.handles.run(filePath, async () => {
|
|
225
225
|
let stats = await statOrUndefined(filePath);
|
|
226
226
|
if (!stats || !stats.isFile()) return undefined;
|
|
227
|
+
if (!stats.size && !config?.includeTombstones) return undefined;
|
|
227
228
|
return { writeTime: stats.mtimeMs, size: stats.size };
|
|
228
229
|
});
|
|
229
230
|
}
|
package/storage/IArchives.d.ts
CHANGED
|
@@ -55,6 +55,12 @@ export type GetConfig = {
|
|
|
55
55
|
};
|
|
56
56
|
/** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
|
|
57
57
|
noFallbacks?: boolean;
|
|
58
|
+
/** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
|
|
59
|
+
internal?: boolean;
|
|
60
|
+
};
|
|
61
|
+
export type GetInfoConfig = {
|
|
62
|
+
/** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
|
|
63
|
+
includeTombstones?: boolean;
|
|
58
64
|
};
|
|
59
65
|
export type ChangesAfterConfig = {
|
|
60
66
|
time: number;
|
|
@@ -67,6 +73,8 @@ export type SetConfig = {
|
|
|
67
73
|
forceSetImmutable?: boolean;
|
|
68
74
|
/** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
|
|
69
75
|
noChecks?: boolean;
|
|
76
|
+
/** Store-to-store push: the receiving node writes purely to its own disk and index, with NO downstream fan-out (the pushing store owns propagation - fanning its pushes back out is how write loops between stores form). Window and route ARE still checked: the stamp must fall inside one of the receiver's configured windows and routes, so a confused peer cannot stuff data onto a node that was never meant to hold it. Requires lastModified. */
|
|
77
|
+
internal?: boolean;
|
|
70
78
|
};
|
|
71
79
|
export type ArchiveFileInfo = {
|
|
72
80
|
path: string;
|
|
@@ -121,6 +129,7 @@ export declare function copyArchiveFile(config: {
|
|
|
121
129
|
writeTime?: number;
|
|
122
130
|
forceSetImmutable?: boolean;
|
|
123
131
|
noChecks?: boolean;
|
|
132
|
+
internal?: boolean;
|
|
124
133
|
}): Promise<{
|
|
125
134
|
writeTime: number;
|
|
126
135
|
size: number;
|
|
@@ -180,16 +189,24 @@ export interface IArchives {
|
|
|
180
189
|
lastModified?: number;
|
|
181
190
|
getNextData(): Promise<Buffer | undefined>;
|
|
182
191
|
}): Promise<void>;
|
|
183
|
-
/** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. */
|
|
184
|
-
getInfo(fileName: string): Promise<{
|
|
192
|
+
/** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. Size-0 entries (tombstones) report undefined unless config.includeTombstones. */
|
|
193
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
185
194
|
writeTime: number;
|
|
186
195
|
size: number;
|
|
187
196
|
url?: string;
|
|
188
197
|
} | undefined>;
|
|
198
|
+
/**
|
|
199
|
+
* Empty (size-0) files are NEVER returned by index-backed stores (BlobStore, and therefore the
|
|
200
|
+
* chain): an empty file IS a missing file - the tombstone of a deletion. If you want a marker
|
|
201
|
+
* file that shows up in listings, add some content to it. Raw sources (disk, backblaze) DO list
|
|
202
|
+
* their empty files - that is how scans learn of deletions - but nothing built on the index
|
|
203
|
+
* ever surfaces them.
|
|
204
|
+
*/
|
|
189
205
|
find(prefix: string, config?: {
|
|
190
206
|
shallow?: boolean;
|
|
191
207
|
type: "files" | "folders";
|
|
192
208
|
}): Promise<string[]>;
|
|
209
|
+
/** See find for the empty-file (tombstone) rule. */
|
|
193
210
|
findInfo(prefix: string, config?: {
|
|
194
211
|
shallow?: boolean;
|
|
195
212
|
type: "files" | "folders";
|
package/storage/IArchives.ts
CHANGED
|
@@ -90,6 +90,13 @@ export type GetConfig = {
|
|
|
90
90
|
range?: { start: number; end: number };
|
|
91
91
|
/** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
|
|
92
92
|
noFallbacks?: boolean;
|
|
93
|
+
/** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
|
|
94
|
+
internal?: boolean;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
export type GetInfoConfig = {
|
|
98
|
+
/** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
|
|
99
|
+
includeTombstones?: boolean;
|
|
93
100
|
};
|
|
94
101
|
|
|
95
102
|
export type ChangesAfterConfig = {
|
|
@@ -104,6 +111,8 @@ export type SetConfig = {
|
|
|
104
111
|
forceSetImmutable?: boolean;
|
|
105
112
|
/** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
|
|
106
113
|
noChecks?: boolean;
|
|
114
|
+
/** Store-to-store push: the receiving node writes purely to its own disk and index, with NO downstream fan-out (the pushing store owns propagation - fanning its pushes back out is how write loops between stores form). Window and route ARE still checked: the stamp must fall inside one of the receiver's configured windows and routes, so a confused peer cannot stuff data onto a node that was never meant to hold it. Requires lastModified. */
|
|
115
|
+
internal?: boolean;
|
|
107
116
|
};
|
|
108
117
|
|
|
109
118
|
// createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written".
|
|
@@ -182,6 +191,7 @@ export async function copyArchiveFile(config: {
|
|
|
182
191
|
writeTime?: number;
|
|
183
192
|
forceSetImmutable?: boolean;
|
|
184
193
|
noChecks?: boolean;
|
|
194
|
+
internal?: boolean;
|
|
185
195
|
}): Promise<{ writeTime: number; size: number } | undefined> {
|
|
186
196
|
let { from, to, path } = config;
|
|
187
197
|
let size = config.size;
|
|
@@ -193,9 +203,9 @@ export async function copyArchiveFile(config: {
|
|
|
193
203
|
writeTime = info.writeTime;
|
|
194
204
|
}
|
|
195
205
|
if (size <= LARGE_COPY_THRESHOLD) {
|
|
196
|
-
let result = await from.get2(path);
|
|
206
|
+
let result = await from.get2(path, { internal: config.internal });
|
|
197
207
|
if (!result || !result.data) return undefined;
|
|
198
|
-
await to.set(path, result.data, { lastModified: result.writeTime, forceSetImmutable: config.forceSetImmutable, noChecks: config.noChecks });
|
|
208
|
+
await to.set(path, result.data, { lastModified: result.writeTime, forceSetImmutable: config.forceSetImmutable, noChecks: config.noChecks, internal: config.internal });
|
|
199
209
|
return { writeTime: result.writeTime, size: result.data.length };
|
|
200
210
|
}
|
|
201
211
|
// Consts so the closure keeps the narrowed types
|
|
@@ -208,7 +218,7 @@ export async function copyArchiveFile(config: {
|
|
|
208
218
|
getNextData: async () => {
|
|
209
219
|
if (offset >= totalSize) return undefined;
|
|
210
220
|
let end = Math.min(offset + LARGE_COPY_CHUNK, totalSize);
|
|
211
|
-
let data = await from.get(path, { range: { start: offset, end } });
|
|
221
|
+
let data = await from.get(path, { range: { start: offset, end }, internal: config.internal });
|
|
212
222
|
if (!data || !data.length) {
|
|
213
223
|
throw new Error(`Ranged read of ${JSON.stringify(path)} from ${from.getDebugName()} returned ${data && data.length || "nothing"} at ${offset}-${end} (expected ${end - offset} bytes of a ${totalSize} byte file - it changed or vanished mid-copy)`);
|
|
214
224
|
}
|
|
@@ -263,9 +273,17 @@ export interface IArchives {
|
|
|
263
273
|
del(fileName: string): Promise<void>;
|
|
264
274
|
/** Streams a file too large to hold in memory. getNextData returns undefined when done. lastModified stamps the finished file like set's (synchronized copies need it to keep write ordering); backends that stamp their own times (backblaze) accept and ignore it. */
|
|
265
275
|
setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined> }): Promise<void>;
|
|
266
|
-
/** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. */
|
|
267
|
-
getInfo(fileName: string): Promise<{ writeTime: number; size: number; url?: string } | undefined>;
|
|
276
|
+
/** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. Size-0 entries (tombstones) report undefined unless config.includeTombstones. */
|
|
277
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number; url?: string } | undefined>;
|
|
278
|
+
/**
|
|
279
|
+
* Empty (size-0) files are NEVER returned by index-backed stores (BlobStore, and therefore the
|
|
280
|
+
* chain): an empty file IS a missing file - the tombstone of a deletion. If you want a marker
|
|
281
|
+
* file that shows up in listings, add some content to it. Raw sources (disk, backblaze) DO list
|
|
282
|
+
* their empty files - that is how scans learn of deletions - but nothing built on the index
|
|
283
|
+
* ever surfaces them.
|
|
284
|
+
*/
|
|
268
285
|
find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
|
|
286
|
+
/** See find for the empty-file (tombstone) rule. */
|
|
269
287
|
findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]>;
|
|
270
288
|
/** Only works for public buckets (private buckets are API-access only). */
|
|
271
289
|
getURL(path: string): Promise<string>;
|
package/storage/backblaze.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig } from "./IArchives";
|
|
3
|
+
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
|
|
4
4
|
export declare class ArchivesBackblaze implements IArchives {
|
|
5
5
|
private config;
|
|
6
6
|
constructor(config: {
|
|
@@ -35,7 +35,7 @@ export declare class ArchivesBackblaze implements IArchives {
|
|
|
35
35
|
lastModified?: number;
|
|
36
36
|
getNextData(): Promise<Buffer | undefined>;
|
|
37
37
|
}): Promise<void>;
|
|
38
|
-
getInfo(fileName: string): Promise<{
|
|
38
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
39
39
|
writeTime: number;
|
|
40
40
|
size: number;
|
|
41
41
|
} | undefined>;
|
package/storage/backblaze.ts
CHANGED
|
@@ -9,7 +9,7 @@ import debugbreak from "debugbreak";
|
|
|
9
9
|
import dns from "dns";
|
|
10
10
|
import { getSecret } from "../misc/getSecret";
|
|
11
11
|
import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
|
|
12
|
-
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
|
|
12
|
+
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, GetInfoConfig, SetConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
|
|
13
13
|
import { filterChanges } from "./remoteStorage/remoteConfig";
|
|
14
14
|
|
|
15
15
|
type BackblazeCreds = {
|
|
@@ -709,7 +709,8 @@ export class ArchivesBackblaze implements IArchives {
|
|
|
709
709
|
if (config?.lastModified) {
|
|
710
710
|
assertValidLastModified(config.lastModified);
|
|
711
711
|
if (!config.noChecks) {
|
|
712
|
-
|
|
712
|
+
// includeTombstones: a deletion pushed to b2 is a real size-0 file, and it must still win the older-write comparison - or an older-stamped write would resurrect the deleted file
|
|
713
|
+
let existing = await this.getInfo(fileName, { includeTombstones: true });
|
|
713
714
|
// An older write never overwrites a newer one (see IArchives.set). B2 stamps its own upload time, so the exact lastModified is not preserved on the stored file.
|
|
714
715
|
if (existing && config.lastModified < existing.writeTime) return fileName;
|
|
715
716
|
// Immutability wins: a synchronization push never overwrites an existing path on an immutable bucket (see SetConfig.forceSetImmutable)
|
|
@@ -882,11 +883,11 @@ export class ArchivesBackblaze implements IArchives {
|
|
|
882
883
|
}
|
|
883
884
|
}
|
|
884
885
|
|
|
885
|
-
public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
|
|
886
|
+
public async getInfo(fileName: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number; } | undefined> {
|
|
886
887
|
return await this.apiRetryLogic(`getInfo ${fileName}`, async (api) => {
|
|
887
888
|
try {
|
|
888
889
|
let file = await api.headFileByName({ bucketName: this.bucketName, fileName });
|
|
889
|
-
if (!file) {
|
|
890
|
+
if (!file || !file.size && !config?.includeTombstones) {
|
|
890
891
|
this.log(`Backblaze file not exists ${fileName}`);
|
|
891
892
|
return undefined;
|
|
892
893
|
}
|