sliftutils 1.7.65 → 1.7.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CLAUDE.md +1 -0
  2. package/dist/test.ts.cache +187 -36
  3. package/dist/treeSummary.ts.cache +359 -0
  4. package/index.d.ts +56 -18
  5. package/package.json +1 -1
  6. package/storage/ArchivesDisk.d.ts +2 -2
  7. package/storage/ArchivesDisk.ts +3 -2
  8. package/storage/IArchives.d.ts +19 -2
  9. package/storage/IArchives.ts +23 -5
  10. package/storage/backblaze.d.ts +2 -2
  11. package/storage/backblaze.ts +5 -4
  12. package/storage/dist/ArchivesDisk.ts.cache +6 -4
  13. package/storage/dist/IArchives.ts.cache +6 -6
  14. package/storage/dist/backblaze.ts.cache +7 -6
  15. package/storage/remoteStorage/ArchivesRemote.d.ts +2 -2
  16. package/storage/remoteStorage/ArchivesRemote.ts +5 -5
  17. package/storage/remoteStorage/ArchivesUrl.d.ts +2 -2
  18. package/storage/remoteStorage/ArchivesUrl.ts +8 -6
  19. package/storage/remoteStorage/blobStore.d.ts +21 -3
  20. package/storage/remoteStorage/blobStore.ts +43 -12
  21. package/storage/remoteStorage/createArchives.d.ts +2 -2
  22. package/storage/remoteStorage/createArchives.ts +5 -6
  23. package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +7 -7
  24. package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +12 -8
  25. package/storage/remoteStorage/dist/blobStore.ts.cache +43 -13
  26. package/storage/remoteStorage/dist/createArchives.ts.cache +6 -7
  27. package/storage/remoteStorage/dist/sourceWrapper.ts.cache +3 -4
  28. package/storage/remoteStorage/dist/storageController.ts.cache +11 -8
  29. package/storage/remoteStorage/dist/storageServerState.ts.cache +38 -9
  30. package/storage/remoteStorage/sourceWrapper.ts +2 -3
  31. package/storage/remoteStorage/storageController.d.ts +3 -3
  32. package/storage/remoteStorage/storageController.ts +8 -5
  33. package/storage/remoteStorage/storageServerState.d.ts +3 -0
  34. package/storage/remoteStorage/storageServerState.ts +34 -7
  35. package/test-files-cache.json +1 -0
  36. package/test.ts +127 -10
  37. package/treeSummary.ts +331 -54
package/index.d.ts CHANGED
@@ -858,7 +858,7 @@ declare module "sliftutils/render-utils/observer" {
858
858
  declare module "sliftutils/storage/ArchivesDisk" {
859
859
  /// <reference types="node" />
860
860
  /// <reference types="node" />
861
- import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig } from "./IArchives";
861
+ import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
862
862
  export declare class ArchivesDisk implements IArchives {
863
863
  private folder;
864
864
  constructor(folder: string);
@@ -885,7 +885,7 @@ declare module "sliftutils/storage/ArchivesDisk" {
885
885
  writeTime: number;
886
886
  size: number;
887
887
  } | undefined>;
888
- getInfo(key: string): Promise<{
888
+ getInfo(key: string, config?: GetInfoConfig): Promise<{
889
889
  writeTime: number;
890
890
  size: number;
891
891
  } | undefined>;
@@ -2146,6 +2146,12 @@ declare module "sliftutils/storage/IArchives" {
2146
2146
  };
2147
2147
  /** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
2148
2148
  noFallbacks?: boolean;
2149
+ /** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
2150
+ internal?: boolean;
2151
+ };
2152
+ export type GetInfoConfig = {
2153
+ /** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
2154
+ includeTombstones?: boolean;
2149
2155
  };
2150
2156
  export type ChangesAfterConfig = {
2151
2157
  time: number;
@@ -2158,6 +2164,8 @@ declare module "sliftutils/storage/IArchives" {
2158
2164
  forceSetImmutable?: boolean;
2159
2165
  /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
2160
2166
  noChecks?: boolean;
2167
+ /** Store-to-store push: the receiving node writes purely to its own disk and index, with NO downstream fan-out (the pushing store owns propagation - fanning its pushes back out is how write loops between stores form). Window and route ARE still checked: the stamp must fall inside one of the receiver's configured windows and routes, so a confused peer cannot stuff data onto a node that was never meant to hold it. Requires lastModified. */
2168
+ internal?: boolean;
2161
2169
  };
2162
2170
  export type ArchiveFileInfo = {
2163
2171
  path: string;
@@ -2212,6 +2220,7 @@ declare module "sliftutils/storage/IArchives" {
2212
2220
  writeTime?: number;
2213
2221
  forceSetImmutable?: boolean;
2214
2222
  noChecks?: boolean;
2223
+ internal?: boolean;
2215
2224
  }): Promise<{
2216
2225
  writeTime: number;
2217
2226
  size: number;
@@ -2271,16 +2280,24 @@ declare module "sliftutils/storage/IArchives" {
2271
2280
  lastModified?: number;
2272
2281
  getNextData(): Promise<Buffer | undefined>;
2273
2282
  }): Promise<void>;
2274
- /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. */
2275
- getInfo(fileName: string): Promise<{
2283
+ /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. Size-0 entries (tombstones) report undefined unless config.includeTombstones. */
2284
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
2276
2285
  writeTime: number;
2277
2286
  size: number;
2278
2287
  url?: string;
2279
2288
  } | undefined>;
2289
+ /**
2290
+ * Empty (size-0) files are NEVER returned by index-backed stores (BlobStore, and therefore the
2291
+ * chain): an empty file IS a missing file - the tombstone of a deletion. If you want a marker
2292
+ * file that shows up in listings, add some content to it. Raw sources (disk, backblaze) DO list
2293
+ * their empty files - that is how scans learn of deletions - but nothing built on the index
2294
+ * ever surfaces them.
2295
+ */
2280
2296
  find(prefix: string, config?: {
2281
2297
  shallow?: boolean;
2282
2298
  type: "files" | "folders";
2283
2299
  }): Promise<string[]>;
2300
+ /** See find for the empty-file (tombstone) rule. */
2284
2301
  findInfo(prefix: string, config?: {
2285
2302
  shallow?: boolean;
2286
2303
  type: "files" | "folders";
@@ -2559,7 +2576,7 @@ declare module "sliftutils/storage/TransactionStorage" {
2559
2576
  declare module "sliftutils/storage/backblaze" {
2560
2577
  /// <reference types="node" />
2561
2578
  /// <reference types="node" />
2562
- import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig } from "./IArchives";
2579
+ import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
2563
2580
  export declare class ArchivesBackblaze implements IArchives {
2564
2581
  private config;
2565
2582
  constructor(config: {
@@ -2594,7 +2611,7 @@ declare module "sliftutils/storage/backblaze" {
2594
2611
  lastModified?: number;
2595
2612
  getNextData(): Promise<Buffer | undefined>;
2596
2613
  }): Promise<void>;
2597
- getInfo(fileName: string): Promise<{
2614
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
2598
2615
  writeTime: number;
2599
2616
  size: number;
2600
2617
  } | undefined>;
@@ -2863,7 +2880,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
2863
2880
  declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
2864
2881
  /// <reference types="node" />
2865
2882
  /// <reference types="node" />
2866
- import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SetConfig } from "../IArchives";
2883
+ import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "../IArchives";
2867
2884
  export type ArchivesRemoteConfig = {
2868
2885
  url: string;
2869
2886
  waitForAccess?: boolean;
@@ -2910,7 +2927,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
2910
2927
  } | undefined>;
2911
2928
  set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
2912
2929
  del(fileName: string): Promise<void>;
2913
- getInfo(fileName: string): Promise<{
2930
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
2914
2931
  writeTime: number;
2915
2932
  size: number;
2916
2933
  } | undefined>;
@@ -2938,7 +2955,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
2938
2955
  declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
2939
2956
  /// <reference types="node" />
2940
2957
  /// <reference types="node" />
2941
- import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig } from "../IArchives";
2958
+ import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig } from "../IArchives";
2942
2959
  export declare class ArchivesUrl implements IArchives {
2943
2960
  private base;
2944
2961
  constructor(base: string);
@@ -2950,7 +2967,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
2950
2967
  writeTime: number;
2951
2968
  size: number;
2952
2969
  } | undefined>;
2953
- getInfo(fileName: string): Promise<{
2970
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
2954
2971
  writeTime: number;
2955
2972
  size: number;
2956
2973
  } | undefined>;
@@ -2986,7 +3003,7 @@ declare module "sliftutils/storage/remoteStorage/accessPage" {
2986
3003
  declare module "sliftutils/storage/remoteStorage/blobStore" {
2987
3004
  /// <reference types="node" />
2988
3005
  /// <reference types="node" />
2989
- import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SyncActivity } from "../IArchives";
3006
+ import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SyncActivity } from "../IArchives";
2990
3007
  export declare const DEFAULT_FAST_WRITE_DELAY: number;
2991
3008
  export declare const WINDOW_END_FLUSH_MARGIN: number;
2992
3009
  export type WriteConfig = {
@@ -3001,9 +3018,17 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
3001
3018
  writeTime: number;
3002
3019
  size: number;
3003
3020
  } | undefined>;
3021
+ getInternal2?(fileName: string, config?: GetConfig): Promise<{
3022
+ data: Buffer;
3023
+ writeTime: number;
3024
+ size: number;
3025
+ } | undefined>;
3026
+ setInternal?(fileName: string, data: Buffer, config: {
3027
+ lastModified: number;
3028
+ }): Promise<void>;
3004
3029
  set(fileName: string, data: Buffer, config?: WriteConfig): Promise<string>;
3005
3030
  del(fileName: string, config?: WriteConfig): Promise<void>;
3006
- getInfo(fileName: string): Promise<{
3031
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
3007
3032
  writeTime: number;
3008
3033
  size: number;
3009
3034
  } | undefined>;
@@ -3140,12 +3165,22 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
3140
3165
  writeTime: number;
3141
3166
  size: number;
3142
3167
  } | undefined>;
3168
+ /** Internal (store-to-store) read: purely the local disk, completely short-circuiting the index and holder resolution - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form. No window or route checks: if the bytes are on our disk, the caller may have them. Note fast writes still sitting in the overlay are invisible here; the caller re-finds them after our flush. */
3169
+ getInternal2(key: string, config?: GetConfig): Promise<{
3170
+ data: Buffer;
3171
+ writeTime: number;
3172
+ size: number;
3173
+ } | undefined>;
3174
+ /** Internal (store-to-store) write: the local disk plus our index, with NO downstream fan-out - the pushing store owns propagation, and fanning its pushes back out is how write loops between stores form. Window/route acceptance is the caller's (writeBucketFile's) job; only-take-latest still applies here. */
3175
+ setInternal(key: string, data: Buffer, config: {
3176
+ lastModified: number;
3177
+ }): Promise<void>;
3143
3178
  private cacheRead;
3144
3179
  set(key: string, data: Buffer, config?: WriteConfig): Promise<string>;
3145
3180
  del(key: string, config?: WriteConfig): Promise<void>;
3146
3181
  private getWritableSources;
3147
3182
  private writeToSources;
3148
- getInfo(key: string): Promise<{
3183
+ getInfo(key: string, config?: GetInfoConfig): Promise<{
3149
3184
  writeTime: number;
3150
3185
  size: number;
3151
3186
  } | undefined>;
@@ -3176,7 +3211,7 @@ declare module "sliftutils/storage/remoteStorage/cliArgs" {
3176
3211
  declare module "sliftutils/storage/remoteStorage/createArchives" {
3177
3212
  /// <reference types="node" />
3178
3213
  /// <reference types="node" />
3179
- import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, SetConfig } from "../IArchives";
3214
+ import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "../IArchives";
3180
3215
  import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
3181
3216
  /** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
3182
3217
  export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
@@ -3251,7 +3286,7 @@ declare module "sliftutils/storage/remoteStorage/createArchives" {
3251
3286
  size?: undefined;
3252
3287
  url: string;
3253
3288
  }>;
3254
- getInfo(fileName: string): Promise<{
3289
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
3255
3290
  writeTime: number;
3256
3291
  size: number;
3257
3292
  url: string;
@@ -3572,14 +3607,14 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
3572
3607
  get2: (account: string, bucketName: string, path: string, range?: {
3573
3608
  start: number;
3574
3609
  end: number;
3575
- }) => Promise<{
3610
+ }, internal?: boolean) => Promise<{
3576
3611
  data: Buffer;
3577
3612
  writeTime: number;
3578
3613
  size: number;
3579
3614
  } | undefined>;
3580
- set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean) => Promise<void>;
3615
+ set: (account: string, bucketName: string, path: string, data: Buffer, lastModified?: number, forceSetImmutable?: boolean, internal?: boolean) => Promise<void>;
3581
3616
  del: (account: string, bucketName: string, path: string) => Promise<void>;
3582
- getInfo: (account: string, bucketName: string, path: string) => Promise<{
3617
+ getInfo: (account: string, bucketName: string, path: string, includeTombstones?: boolean) => Promise<{
3583
3618
  writeTime: number;
3584
3619
  size: number;
3585
3620
  } | undefined>;
@@ -3682,6 +3717,8 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
3682
3717
  };
3683
3718
  export declare function addExtraListenPort(port: number): void;
3684
3719
  export declare function removeExtraListenPort(port: number): void;
3720
+ /** Whether address:port is this server process. The ONE self test - findSelfIndexes, createApiArchives, and SourceWrapper all consult it, so "is this me" cannot disagree between the routing plan and connection building: a URL that is us on an extra listen port must never become a network client to ourselves, which is how infinite self-request loops form. */
3721
+ export declare function isOwnAddress(address: string, port: number): boolean;
3685
3722
  /** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
3686
3723
  export declare function resolveSourceArchives(url: string): IArchives;
3687
3724
  /** Our role in a bucket's routing config, summarized across ALL currently-valid self entries. Stored instead of a single representative HostedConfig, so nothing can accidentally use one entry's route or flags where the union is required - the standard config has the same URL twice: a routed write-shard entry plus an unrouted read-everything entry. */
@@ -3699,6 +3736,7 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
3699
3736
  export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
3700
3737
  lastModified?: number;
3701
3738
  forceSetImmutable?: boolean;
3739
+ internal?: boolean;
3702
3740
  }): Promise<void>;
3703
3741
  export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
3704
3742
  /** Which buckets this process currently has loaded - what a deploy successor asks its predecessor for, so it activates exactly the buckets that are actually in use. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sliftutils",
3
- "version": "1.7.65",
3
+ "version": "1.7.67",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "files": [
@@ -1,6 +1,6 @@
1
1
  /// <reference types="node" />
2
2
  /// <reference types="node" />
3
- import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig } from "./IArchives";
3
+ import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
4
4
  export declare class ArchivesDisk implements IArchives {
5
5
  private folder;
6
6
  constructor(folder: string);
@@ -27,7 +27,7 @@ export declare class ArchivesDisk implements IArchives {
27
27
  writeTime: number;
28
28
  size: number;
29
29
  } | undefined>;
30
- getInfo(key: string): Promise<{
30
+ getInfo(key: string, config?: GetInfoConfig): Promise<{
31
31
  writeTime: number;
32
32
  size: number;
33
33
  } | undefined>;
@@ -3,7 +3,7 @@ import path from "path";
3
3
  import { lazy } from "socket-function/src/caching";
4
4
  import { runInfinitePoll } from "socket-function/src/batching";
5
5
  import { sort, binarySearchBasic } from "socket-function/src/misc";
6
- import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, SetConfig, assertValidLastModified } from "./IArchives";
6
+ import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, GetConfig, GetInfoConfig, SetConfig, assertValidLastModified } from "./IArchives";
7
7
  import { filterChanges } from "./remoteStorage/remoteConfig";
8
8
 
9
9
  // The base file-system IArchives: storage is one-to-one with the file system, every key is exactly one real file under <folder>/files, so the file system itself is the index. File handles are cached and reused, and closed once idle (see FileHandleCache). All operations on a file run in serial, so they can't collide with each other or with handle closing. Used as the disk synchronization source of BlobStore (see remoteStorage/blobStore.ts).
@@ -218,12 +218,13 @@ export class ArchivesDisk implements IArchives {
218
218
  });
219
219
  }
220
220
 
221
- public async getInfo(key: string): Promise<{ writeTime: number; size: number } | undefined> {
221
+ public async getInfo(key: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number } | undefined> {
222
222
  await this.init();
223
223
  let filePath = this.filePath(key);
224
224
  return await this.handles.run(filePath, async () => {
225
225
  let stats = await statOrUndefined(filePath);
226
226
  if (!stats || !stats.isFile()) return undefined;
227
+ if (!stats.size && !config?.includeTombstones) return undefined;
227
228
  return { writeTime: stats.mtimeMs, size: stats.size };
228
229
  });
229
230
  }
@@ -55,6 +55,12 @@ export type GetConfig = {
55
55
  };
56
56
  /** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
57
57
  noFallbacks?: boolean;
58
+ /** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
59
+ internal?: boolean;
60
+ };
61
+ export type GetInfoConfig = {
62
+ /** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
63
+ includeTombstones?: boolean;
58
64
  };
59
65
  export type ChangesAfterConfig = {
60
66
  time: number;
@@ -67,6 +73,8 @@ export type SetConfig = {
67
73
  forceSetImmutable?: boolean;
68
74
  /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
69
75
  noChecks?: boolean;
76
+ /** Store-to-store push: the receiving node writes purely to its own disk and index, with NO downstream fan-out (the pushing store owns propagation - fanning its pushes back out is how write loops between stores form). Window and route ARE still checked: the stamp must fall inside one of the receiver's configured windows and routes, so a confused peer cannot stuff data onto a node that was never meant to hold it. Requires lastModified. */
77
+ internal?: boolean;
70
78
  };
71
79
  export type ArchiveFileInfo = {
72
80
  path: string;
@@ -121,6 +129,7 @@ export declare function copyArchiveFile(config: {
121
129
  writeTime?: number;
122
130
  forceSetImmutable?: boolean;
123
131
  noChecks?: boolean;
132
+ internal?: boolean;
124
133
  }): Promise<{
125
134
  writeTime: number;
126
135
  size: number;
@@ -180,16 +189,24 @@ export interface IArchives {
180
189
  lastModified?: number;
181
190
  getNextData(): Promise<Buffer | undefined>;
182
191
  }): Promise<void>;
183
- /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. */
184
- getInfo(fileName: string): Promise<{
192
+ /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. Size-0 entries (tombstones) report undefined unless config.includeTombstones. */
193
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
185
194
  writeTime: number;
186
195
  size: number;
187
196
  url?: string;
188
197
  } | undefined>;
198
+ /**
199
+ * Empty (size-0) files are NEVER returned by index-backed stores (BlobStore, and therefore the
200
+ * chain): an empty file IS a missing file - the tombstone of a deletion. If you want a marker
201
+ * file that shows up in listings, add some content to it. Raw sources (disk, backblaze) DO list
202
+ * their empty files - that is how scans learn of deletions - but nothing built on the index
203
+ * ever surfaces them.
204
+ */
189
205
  find(prefix: string, config?: {
190
206
  shallow?: boolean;
191
207
  type: "files" | "folders";
192
208
  }): Promise<string[]>;
209
+ /** See find for the empty-file (tombstone) rule. */
193
210
  findInfo(prefix: string, config?: {
194
211
  shallow?: boolean;
195
212
  type: "files" | "folders";
@@ -90,6 +90,13 @@ export type GetConfig = {
90
90
  range?: { start: number; end: number };
91
91
  /** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
92
92
  noFallbacks?: boolean;
93
+ /** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
94
+ internal?: boolean;
95
+ };
96
+
97
+ export type GetInfoConfig = {
98
+ /** Also report size-0 entries (tombstones - an empty file IS a missing file). Off by default, so a deleted key reports undefined, matching get. Synchronization-style callers pass this when they need a deletion's write time (e.g. to compare it against a write they are about to make). */
99
+ includeTombstones?: boolean;
93
100
  };
94
101
 
95
102
  export type ChangesAfterConfig = {
@@ -104,6 +111,8 @@ export type SetConfig = {
104
111
  forceSetImmutable?: boolean;
105
112
  /** Skips the target-side safety reads around the write (backblaze: the pre-write getInfo comparison and the post-upload existence poll). For writers whose own bookkeeping already decides what to write and orders it by write time (BlobStore's index-driven writes and synchronization), those reads are pure extra API calls - but the default stays checked, because other users of the raw backends rely on the checks. */
106
113
  noChecks?: boolean;
114
+ /** Store-to-store push: the receiving node writes purely to its own disk and index, with NO downstream fan-out (the pushing store owns propagation - fanning its pushes back out is how write loops between stores form). Window and route ARE still checked: the stamp must fall inside one of the receiver's configured windows and routes, so a confused peer cannot stuff data onto a node that was never meant to hold it. Requires lastModified. */
115
+ internal?: boolean;
107
116
  };
108
117
 
109
118
  // createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written".
@@ -182,6 +191,7 @@ export async function copyArchiveFile(config: {
182
191
  writeTime?: number;
183
192
  forceSetImmutable?: boolean;
184
193
  noChecks?: boolean;
194
+ internal?: boolean;
185
195
  }): Promise<{ writeTime: number; size: number } | undefined> {
186
196
  let { from, to, path } = config;
187
197
  let size = config.size;
@@ -193,9 +203,9 @@ export async function copyArchiveFile(config: {
193
203
  writeTime = info.writeTime;
194
204
  }
195
205
  if (size <= LARGE_COPY_THRESHOLD) {
196
- let result = await from.get2(path);
206
+ let result = await from.get2(path, { internal: config.internal });
197
207
  if (!result || !result.data) return undefined;
198
- await to.set(path, result.data, { lastModified: result.writeTime, forceSetImmutable: config.forceSetImmutable, noChecks: config.noChecks });
208
+ await to.set(path, result.data, { lastModified: result.writeTime, forceSetImmutable: config.forceSetImmutable, noChecks: config.noChecks, internal: config.internal });
199
209
  return { writeTime: result.writeTime, size: result.data.length };
200
210
  }
201
211
  // Consts so the closure keeps the narrowed types
@@ -208,7 +218,7 @@ export async function copyArchiveFile(config: {
208
218
  getNextData: async () => {
209
219
  if (offset >= totalSize) return undefined;
210
220
  let end = Math.min(offset + LARGE_COPY_CHUNK, totalSize);
211
- let data = await from.get(path, { range: { start: offset, end } });
221
+ let data = await from.get(path, { range: { start: offset, end }, internal: config.internal });
212
222
  if (!data || !data.length) {
213
223
  throw new Error(`Ranged read of ${JSON.stringify(path)} from ${from.getDebugName()} returned ${data && data.length || "nothing"} at ${offset}-${end} (expected ${end - offset} bytes of a ${totalSize} byte file - it changed or vanished mid-copy)`);
214
224
  }
@@ -263,9 +273,17 @@ export interface IArchives {
263
273
  del(fileName: string): Promise<void>;
264
274
  /** Streams a file too large to hold in memory. getNextData returns undefined when done. lastModified stamps the finished file like set's (synchronized copies need it to keep write ordering); backends that stamp their own times (backblaze) accept and ignore it. */
265
275
  setLargeFile(config: { path: string; lastModified?: number; getNextData(): Promise<Buffer | undefined> }): Promise<void>;
266
- /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. */
267
- getInfo(fileName: string): Promise<{ writeTime: number; size: number; url?: string } | undefined>;
276
+ /** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. Size-0 entries (tombstones) report undefined unless config.includeTombstones. */
277
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number; url?: string } | undefined>;
278
+ /**
279
+ * Empty (size-0) files are NEVER returned by index-backed stores (BlobStore, and therefore the
280
+ * chain): an empty file IS a missing file - the tombstone of a deletion. If you want a marker
281
+ * file that shows up in listings, add some content to it. Raw sources (disk, backblaze) DO list
282
+ * their empty files - that is how scans learn of deletions - but nothing built on the index
283
+ * ever surfaces them.
284
+ */
268
285
  find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
286
+ /** See find for the empty-file (tombstone) rule. */
269
287
  findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]>;
270
288
  /** Only works for public buckets (private buckets are API-access only). */
271
289
  getURL(path: string): Promise<string>;
@@ -1,6 +1,6 @@
1
1
  /// <reference types="node" />
2
2
  /// <reference types="node" />
3
- import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig } from "./IArchives";
3
+ import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
4
4
  export declare class ArchivesBackblaze implements IArchives {
5
5
  private config;
6
6
  constructor(config: {
@@ -35,7 +35,7 @@ export declare class ArchivesBackblaze implements IArchives {
35
35
  lastModified?: number;
36
36
  getNextData(): Promise<Buffer | undefined>;
37
37
  }): Promise<void>;
38
- getInfo(fileName: string): Promise<{
38
+ getInfo(fileName: string, config?: GetInfoConfig): Promise<{
39
39
  writeTime: number;
40
40
  size: number;
41
41
  } | undefined>;
@@ -9,7 +9,7 @@ import debugbreak from "debugbreak";
9
9
  import dns from "dns";
10
10
  import { getSecret } from "../misc/getSecret";
11
11
  import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
12
- import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, SetConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
12
+ import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, GetConfig, GetInfoConfig, SetConfig, assertValidLastModified, IMMUTABLE_CACHE_TIME } from "./IArchives";
13
13
  import { filterChanges } from "./remoteStorage/remoteConfig";
14
14
 
15
15
  type BackblazeCreds = {
@@ -709,7 +709,8 @@ export class ArchivesBackblaze implements IArchives {
709
709
  if (config?.lastModified) {
710
710
  assertValidLastModified(config.lastModified);
711
711
  if (!config.noChecks) {
712
- let existing = await this.getInfo(fileName);
712
+ // includeTombstones: a deletion pushed to b2 is a real size-0 file, and it must still win the older-write comparison - or an older-stamped write would resurrect the deleted file
713
+ let existing = await this.getInfo(fileName, { includeTombstones: true });
713
714
  // An older write never overwrites a newer one (see IArchives.set). B2 stamps its own upload time, so the exact lastModified is not preserved on the stored file.
714
715
  if (existing && config.lastModified < existing.writeTime) return fileName;
715
716
  // Immutability wins: a synchronization push never overwrites an existing path on an immutable bucket (see SetConfig.forceSetImmutable)
@@ -882,11 +883,11 @@ export class ArchivesBackblaze implements IArchives {
882
883
  }
883
884
  }
884
885
 
885
- public async getInfo(fileName: string): Promise<{ writeTime: number; size: number; } | undefined> {
886
+ public async getInfo(fileName: string, config?: GetInfoConfig): Promise<{ writeTime: number; size: number; } | undefined> {
886
887
  return await this.apiRetryLogic(`getInfo ${fileName}`, async (api) => {
887
888
  try {
888
889
  let file = await api.headFileByName({ bucketName: this.bucketName, fileName });
889
- if (!file) {
890
+ if (!file || !file.size && !config?.includeTombstones) {
890
891
  this.log(`Backblaze file not exists ${fileName}`);
891
892
  return undefined;
892
893
  }