sliftutils 1.7.115 → 1.7.117

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/index.d.ts +12 -5
  2. package/package.json +2 -2
  3. package/spec.txt +4 -5
  4. package/storage/ArchivesDisk.ts +8 -8
  5. package/storage/IArchives.d.ts +6 -2
  6. package/storage/IArchives.ts +7 -3
  7. package/storage/TransactionFile.d.ts +1 -1
  8. package/storage/TransactionFile.ts +11 -11
  9. package/storage/archiveHelpers.d.ts +3 -1
  10. package/storage/archiveHelpers.ts +19 -14
  11. package/storage/dist/ArchivesDisk.ts.cache +10 -10
  12. package/storage/dist/IArchives.ts.cache +2 -2
  13. package/storage/dist/TransactionFile.ts.cache +13 -13
  14. package/storage/dist/archiveHelpers.ts.cache +19 -16
  15. package/storage/remoteStorage/ArchivesDelayed.ts +1 -1
  16. package/storage/remoteStorage/ArchivesRemote.ts +2 -2
  17. package/storage/remoteStorage/blobStore.d.ts +1 -1
  18. package/storage/remoteStorage/blobStore.ts +15 -9
  19. package/storage/remoteStorage/bucketDisk.ts +2 -2
  20. package/storage/remoteStorage/chainStartup.ts +1 -1
  21. package/storage/remoteStorage/createArchives.ts +1 -1
  22. package/storage/remoteStorage/dist/ArchivesDelayed.ts.cache +3 -3
  23. package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +4 -4
  24. package/storage/remoteStorage/dist/blobStore.ts.cache +20 -11
  25. package/storage/remoteStorage/dist/bucketDisk.ts.cache +4 -4
  26. package/storage/remoteStorage/dist/chainStartup.ts.cache +3 -3
  27. package/storage/remoteStorage/dist/createArchives.ts.cache +3 -3
  28. package/storage/remoteStorage/dist/storageController.ts.cache +2 -2
  29. package/storage/remoteStorage/dist/storeSync.ts.cache +8 -8
  30. package/storage/remoteStorage/storageController.d.ts +1 -0
  31. package/storage/remoteStorage/storageController.ts +1 -1
  32. package/storage/remoteStorage/storeSync.ts +6 -6
  33. package/yarn.lock +4 -4
package/index.d.ts CHANGED
@@ -2166,7 +2166,7 @@ declare module "sliftutils/storage/IArchives" {
2166
2166
  };
2167
2167
  /** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
2168
2168
  noFallbacks?: boolean;
2169
- /** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
2169
+ /** Store-to-store call: the serving node never consults its OTHER sources - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). That is the flag's ENTIRE meaning: no fallbacks, nothing else. The read is otherwise fully correct - the node's index still gates it (a key its index says is deleted answers as deleted, never as the history bytes still sitting on its disk). No window or route checks on reads. */
2170
2170
  internal?: boolean;
2171
2171
  /** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
2172
2172
  includeTombstones?: boolean;
@@ -2184,6 +2184,8 @@ declare module "sliftutils/storage/IArchives" {
2184
2184
  includeMarked?: boolean;
2185
2185
  /** Listings normally come ONLY from the authoritative sources (the same nodes writes go to - read-your-writes). With fallbacks, a failing shard's routes are covered by the next source holding them (e.g. a wide read replica) instead of the call failing - high availability at the cost of possibly missing just-written data. Single-source archives ignore the flag. */
2186
2186
  fallbacks?: boolean;
2187
+ /** Store-to-store listing: only entries whose bytes the node ITSELF holds - never entries its index redirects to its own other sources. A peer reads with GetConfig.internal (which never chases those redirects), so listing a redirect would just make the peer flag the file missing, purge it, re-list it, and loop forever; the peer hears about such files from the source actually holding them instead. */
2188
+ internal?: boolean;
2187
2189
  };
2188
2190
  export type DelConfig = {
2189
2191
  /** Stamps the deletion (its tombstone) with this write time instead of now. Synchronization passes the ORIGINAL deletion time, so deletion ordering survives propagation exactly like any other write's ordering. */
@@ -2211,9 +2213,11 @@ declare module "sliftutils/storage/IArchives" {
2211
2213
  time: number;
2212
2214
  /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
2213
2215
  routes?: [number, number][];
2216
+ /** See FindConfig.internal - the change feed is a listing too, and redirect entries fail a peer's internal reads the same way. Deletions are always reported (they are index-only, there are no bytes to hold). */
2217
+ internal?: boolean;
2214
2218
  };
2215
2219
  export type SetConfig = {
2216
- /** The write time to stamp (see IArchives.set). FLOORED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. */
2220
+ /** The write time to stamp (see IArchives.set). ROUNDED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. Rounded rather than floored because utimes goes through a seconds double and can read back a hair below the stamped millisecond (see ArchivesDisk.get2). */
2217
2221
  lastModified?: number;
2218
2222
  /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
2219
2223
  forceSetImmutable?: boolean;
@@ -2773,7 +2777,7 @@ declare module "sliftutils/storage/TransactionFile" {
2773
2777
  entries(): IterableIterator<[string, LogEntry<T>]>;
2774
2778
  /** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
2775
2779
  deletedEntries(): IterableIterator<[string, LogTombstone<T>]>;
2776
- /** Stores a value as of `time` (floored to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
2780
+ /** Stores a value as of `time` (rounded to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
2777
2781
  set(key: string, value: T, time: number): boolean;
2778
2782
  /** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
2779
2783
  delete(key: string, time: number): boolean;
@@ -2856,13 +2860,15 @@ declare module "sliftutils/storage/TransactionStorage" {
2856
2860
 
2857
2861
  declare module "sliftutils/storage/archiveHelpers" {
2858
2862
  import type { IArchives } from "./IArchives";
2859
- /** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, the destination already had a NEWER file (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
2863
+ /** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, and with preserveWriteTime the two guarded cases - the destination already held something NEWER (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
2860
2864
  export declare function copyArchiveFile(config: {
2861
2865
  from: IArchives;
2862
2866
  to: IArchives;
2863
2867
  path: string;
2864
2868
  /** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
2865
2869
  toPath?: string;
2870
+ /** Stamps the destination with the SOURCE's write time instead of now, and turns on the ordering guards around it (the newer-destination refusal up front, and the getInfo confirm after). ONLY for synchronization between replicas of the same key, where the higher write time must win and ordering must survive propagation - never for a user-triggered copy: a plain copy is a NEW write, and the source's old stamp would make it LOSE to any newer write or tombstone at the destination, silently (move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely). */
2871
+ preserveWriteTime?: boolean;
2866
2872
  forceSetImmutable?: boolean;
2867
2873
  noChecks?: boolean;
2868
2874
  internal?: boolean;
@@ -3713,7 +3719,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
3713
3719
  private assertFreshWriteTarget;
3714
3720
  private assertMutable;
3715
3721
  private assertInternalWriteAccepted;
3716
- /** Internal (store-to-store) read: purely the local disk, completely short-circuiting the index and holder resolution - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form. No window or route checks: if the bytes are on our disk, the caller may have them. Note this reads the disk past any write delay, so a fast write still buffered in memory is invisible here; the caller re-finds it once it flushes. */
3722
+ /** Internal (store-to-store) read: never goes to OTHER sources - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form - but the INDEX still gates, because it is the source of truth: a marked deletion keeps its bytes on disk as history (see writeToSources), so the disk alone would happily serve a DELETED file as live. Index says live -> the disk provides the bytes (past any write delay, so a fast write still buffered in memory is invisible here; the caller re-finds it once it flushes). Index says deleted -> the tombstone is the answer, never the disk. No window or route checks. */
3717
3723
  private getInternal2;
3718
3724
  /** Internal (store-to-store) write: the local disk plus our index, with NO downstream fan-out - the pushing store owns propagation, and fanning its pushes back out is how write loops between stores form. Only-take-latest still applies here. */
3719
3725
  private setInternal;
@@ -4482,6 +4488,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
4482
4488
  sourceConfig: SourceConfig;
4483
4489
  time: number;
4484
4490
  routes?: [number, number][];
4491
+ internal?: boolean;
4485
4492
  }) => Promise<ArchiveFileInfo[]>;
4486
4493
  getArchivesConfig: (config: {
4487
4494
  account: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sliftutils",
3
- "version": "1.7.115",
3
+ "version": "1.7.117",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "files": [
@@ -57,7 +57,7 @@
57
57
  "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
58
58
  "preact-old-types": "^10.28.1",
59
59
  "shell-quote": "^1.8.3",
60
- "socket-function": "^1.2.32",
60
+ "socket-function": "^1.2.33",
61
61
  "typenode": "^6.6.1",
62
62
  "typesafecss": "*",
63
63
  "ws": "^8.18.3",
package/spec.txt CHANGED
@@ -1,11 +1,10 @@
1
- Refactor
2
1
 
3
- createArchives => storageController => storageServerState => BlobStore
4
-
5
- So create Archives has to decide who we're going to talk to, and then Storage Controller, it just acts like adds security and stuff. and then it gets into load it bucket, and then load it bucket. It then decides what back-end store we're talking to, and the back-end store, it might talk to multiple sources.
6
- So there's which server we're going to talk to, which bucket on the server it maps to once we get there, and then there's which back-end stores that we talk to. So I guess it's fine.
7
2
 
3
+ test
4
+ --internal
5
+ --selcert
8
6
 
7
+ with a test.ts script
9
8
 
10
9
 
11
10
  TODO: Use wildcard certificates, and then expose urls which match up to the shard fraction, to 2 decimal places:
@@ -164,8 +164,8 @@ export class ArchivesDisk implements IArchives {
164
164
  await this.handles.run(filePath, async () => {
165
165
  if (lastModified) {
166
166
  let existing = await statOrUndefined(filePath);
167
- // An older write never overwrites a newer one (see IArchives.set). Both sides floored: sub-millisecond mtime digits are storage artifacts, not ordering (see get2)
168
- if (existing && Math.floor(lastModified) < Math.floor(existing.mtimeMs)) return;
167
+ // An older write never overwrites a newer one (see IArchives.set). Both sides rounded: sub-millisecond mtime digits are storage artifacts, not ordering (see get2)
168
+ if (existing && Math.round(lastModified) < Math.round(existing.mtimeMs)) return;
169
169
  }
170
170
  await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
171
171
  let handle = await this.handles.getHandle(filePath, fs.constants.O_RDWR | fs.constants.O_CREAT);
@@ -238,14 +238,14 @@ export class ArchivesDisk implements IArchives {
238
238
  if (!size && !config?.includeTombstones) return undefined;
239
239
  let start = range && Math.min(range.start, size) || 0;
240
240
  let end = range && Math.min(range.end, size) || size;
241
- // mtimeMs is floored everywhere this disk reports a time: it carries fractional milliseconds (kernel-stamped files have nanosecond mtimes), but utimes round-trips only whole ones - so a fractional time handed to the rest of the system could never be reproduced by propagation, and every copy of the file would compare "older" than the original forever
242
- if (end <= start) return { data: Buffer.alloc(0), writeTime: Math.floor(stats.mtimeMs), size };
241
+ // mtimeMs is ROUNDED everywhere this disk reports a time: it carries fractional milliseconds (kernel-stamped files have nanosecond mtimes), but utimes round-trips only whole ones - so a fractional time handed to the rest of the system could never be reproduced by propagation, and every copy of the file would compare "older" than the original forever. Round rather than floor, because utimes goes through a seconds DOUBLE and can land a hair BELOW the stamped millisecond - flooring turned that representation error into a full lost millisecond, un-reproducing our own stamps.
242
+ if (end <= start) return { data: Buffer.alloc(0), writeTime: Math.round(stats.mtimeMs), size };
243
243
  let buffer = Buffer.alloc(end - start);
244
244
  let { bytesRead } = await handle.read(buffer, 0, buffer.length, start);
245
245
  if (bytesRead !== buffer.length) {
246
246
  throw new Error(`Expected ${buffer.length} bytes at ${filePath}:${start}, read ${bytesRead}`);
247
247
  }
248
- return { data: buffer, writeTime: Math.floor(stats.mtimeMs), size };
248
+ return { data: buffer, writeTime: Math.round(stats.mtimeMs), size };
249
249
  });
250
250
  }
251
251
 
@@ -256,7 +256,7 @@ export class ArchivesDisk implements IArchives {
256
256
  let stats = await statOrUndefined(filePath);
257
257
  if (!stats || !stats.isFile()) return undefined;
258
258
  if (!stats.size && !config?.includeTombstones) return undefined;
259
- return { writeTime: Math.floor(stats.mtimeMs), size: stats.size };
259
+ return { writeTime: Math.round(stats.mtimeMs), size: stats.size };
260
260
  });
261
261
  }
262
262
 
@@ -297,7 +297,7 @@ export class ArchivesDisk implements IArchives {
297
297
  let stats = await statOrUndefined(path.join(this.filesDir, relPath));
298
298
  // Deleted while we were walking
299
299
  if (!stats) continue;
300
- infos.set(relPath, { path: relPath, createTime: Math.floor(stats.mtimeMs), size: stats.size });
300
+ infos.set(relPath, { path: relPath, createTime: Math.round(stats.mtimeMs), size: stats.size });
301
301
  }
302
302
  }
303
303
 
@@ -355,7 +355,7 @@ export class ArchivesDisk implements IArchives {
355
355
  if (lastModified) {
356
356
  // An older write never overwrites a newer one (see IArchives.set) - re-checked HERE, not just when the upload started, because a large upload streams for minutes and that window is exactly when a newer write lands
357
357
  let existing = await statOrUndefined(filePath);
358
- if (existing && Math.floor(lastModified) < Math.floor(existing.mtimeMs)) {
358
+ if (existing && Math.round(lastModified) < Math.round(existing.mtimeMs)) {
359
359
  await fs.promises.rm(tmpPath, { force: true });
360
360
  return;
361
361
  }
@@ -71,7 +71,7 @@ export type GetConfig = {
71
71
  };
72
72
  /** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
73
73
  noFallbacks?: boolean;
74
- /** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
74
+ /** Store-to-store call: the serving node never consults its OTHER sources - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). That is the flag's ENTIRE meaning: no fallbacks, nothing else. The read is otherwise fully correct - the node's index still gates it (a key its index says is deleted answers as deleted, never as the history bytes still sitting on its disk). No window or route checks on reads. */
75
75
  internal?: boolean;
76
76
  /** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
77
77
  includeTombstones?: boolean;
@@ -89,6 +89,8 @@ export type FindConfig = {
89
89
  includeMarked?: boolean;
90
90
  /** Listings normally come ONLY from the authoritative sources (the same nodes writes go to - read-your-writes). With fallbacks, a failing shard's routes are covered by the next source holding them (e.g. a wide read replica) instead of the call failing - high availability at the cost of possibly missing just-written data. Single-source archives ignore the flag. */
91
91
  fallbacks?: boolean;
92
+ /** Store-to-store listing: only entries whose bytes the node ITSELF holds - never entries its index redirects to its own other sources. A peer reads with GetConfig.internal (which never chases those redirects), so listing a redirect would just make the peer flag the file missing, purge it, re-list it, and loop forever; the peer hears about such files from the source actually holding them instead. */
93
+ internal?: boolean;
92
94
  };
93
95
  export type DelConfig = {
94
96
  /** Stamps the deletion (its tombstone) with this write time instead of now. Synchronization passes the ORIGINAL deletion time, so deletion ordering survives propagation exactly like any other write's ordering. */
@@ -116,9 +118,11 @@ export type ChangesAfterConfig = {
116
118
  time: number;
117
119
  /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
118
120
  routes?: [number, number][];
121
+ /** See FindConfig.internal - the change feed is a listing too, and redirect entries fail a peer's internal reads the same way. Deletions are always reported (they are index-only, there are no bytes to hold). */
122
+ internal?: boolean;
119
123
  };
120
124
  export type SetConfig = {
121
- /** The write time to stamp (see IArchives.set). FLOORED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. */
125
+ /** The write time to stamp (see IArchives.set). ROUNDED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. Rounded rather than floored because utimes goes through a seconds double and can read back a hair below the stamped millisecond (see ArchivesDisk.get2). */
122
126
  lastModified?: number;
123
127
  /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
124
128
  forceSetImmutable?: boolean;
@@ -106,7 +106,7 @@ export type GetConfig = {
106
106
  range?: { start: number; end: number };
107
107
  /** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
108
108
  noFallbacks?: boolean;
109
- /** Store-to-store call: the serving node answers purely from its own disk, completely short-circuiting its index holders - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). No window or route checks on reads: if the bytes are on its disk, the caller may have them. */
109
+ /** Store-to-store call: the serving node never consults its OTHER sources - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). That is the flag's ENTIRE meaning: no fallbacks, nothing else. The read is otherwise fully correct - the node's index still gates it (a key its index says is deleted answers as deleted, never as the history bytes still sitting on its disk). No window or route checks on reads. */
110
110
  internal?: boolean;
111
111
  /** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
112
112
  includeTombstones?: boolean;
@@ -125,6 +125,8 @@ export type FindConfig = {
125
125
  includeMarked?: boolean;
126
126
  /** Listings normally come ONLY from the authoritative sources (the same nodes writes go to - read-your-writes). With fallbacks, a failing shard's routes are covered by the next source holding them (e.g. a wide read replica) instead of the call failing - high availability at the cost of possibly missing just-written data. Single-source archives ignore the flag. */
127
127
  fallbacks?: boolean;
128
+ /** Store-to-store listing: only entries whose bytes the node ITSELF holds - never entries its index redirects to its own other sources. A peer reads with GetConfig.internal (which never chases those redirects), so listing a redirect would just make the peer flag the file missing, purge it, re-list it, and loop forever; the peer hears about such files from the source actually holding them instead. */
129
+ internal?: boolean;
128
130
  };
129
131
 
130
132
  export type DelConfig = {
@@ -155,10 +157,12 @@ export type ChangesAfterConfig = {
155
157
  time: number;
156
158
  /** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
157
159
  routes?: [number, number][];
160
+ /** See FindConfig.internal - the change feed is a listing too, and redirect entries fail a peer's internal reads the same way. Deletions are always reported (they are index-only, there are no bytes to hold). */
161
+ internal?: boolean;
158
162
  };
159
163
 
160
164
  export type SetConfig = {
161
- /** The write time to stamp (see IArchives.set). FLOORED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. */
165
+ /** The write time to stamp (see IArchives.set). ROUNDED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. Rounded rather than floored because utimes goes through a seconds double and can read back a hair below the stamped millisecond (see ArchivesDisk.get2). */
162
166
  lastModified?: number;
163
167
  /** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
164
168
  forceSetImmutable?: boolean;
@@ -182,7 +186,7 @@ export type SetLargeFileConfig = SetConfig & {
182
186
  restartStream?(): Promise<void> | void;
183
187
  };
184
188
 
185
- // createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written". Always WHOLE milliseconds: write times are floored at every producer (see SetConfig.lastModified).
189
+ // createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written". Always WHOLE milliseconds: write times are rounded at every producer (see SetConfig.lastModified).
186
190
  export type ArchiveFileInfo = { path: string; createTime: number; size: number };
187
191
 
188
192
  // An in-progress background synchronization task (see ArchivesConfig.syncing)
@@ -39,7 +39,7 @@ export declare class TransactionFile<T> {
39
39
  entries(): IterableIterator<[string, LogEntry<T>]>;
40
40
  /** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
41
41
  deletedEntries(): IterableIterator<[string, LogTombstone<T>]>;
42
- /** Stores a value as of `time` (floored to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
42
+ /** Stores a value as of `time` (rounded to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
43
43
  set(key: string, value: T, time: number): boolean;
44
44
  /** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
45
45
  delete(key: string, time: number): boolean;
@@ -119,9 +119,9 @@ export class TransactionFile<T> {
119
119
  return this.deleted.entries();
120
120
  }
121
121
 
122
- /** Stores a value as of `time` (floored to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
122
+ /** Stores a value as of `time` (rounded to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
123
123
  public set(key: string, value: T, time: number): boolean {
124
- time = Math.floor(time);
124
+ time = Math.round(time);
125
125
  if (!this.applySet(key, value, time, Date.now())) return false;
126
126
  this.append({ k: key, t: time, v: value });
127
127
  return true;
@@ -129,7 +129,7 @@ export class TransactionFile<T> {
129
129
 
130
130
  /** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
131
131
  public delete(key: string, time: number): boolean {
132
- time = Math.floor(time);
132
+ time = Math.round(time);
133
133
  let live = this.values.get(key);
134
134
  if (!this.applyDelete(key, time, Date.now(), live?.value, live?.time)) return false;
135
135
  if (live) {
@@ -142,7 +142,7 @@ export class TransactionFile<T> {
142
142
 
143
143
  /** Undoes a marked deletion: the kept value becomes live again, as of `time` (a fresh time, so the restore outranks the deletion everywhere it propagated). Returns false when there is no marked value to restore, or something at least as new is already here. */
144
144
  public unmark(key: string, time: number): boolean {
145
- time = Math.floor(time);
145
+ time = Math.round(time);
146
146
  let tombstone = this.deleted.get(key);
147
147
  if (!tombstone || tombstone.value === undefined) return false;
148
148
  if (!this.applySet(key, tombstone.value, time, Date.now())) return false;
@@ -163,13 +163,13 @@ export class TransactionFile<T> {
163
163
  let had = this.values.delete(key);
164
164
  had = this.deleted.delete(key) || had;
165
165
  if (!had) return;
166
- this.append({ k: key, t: Math.floor(Date.now()), p: 1 });
166
+ this.append({ k: key, t: Math.round(Date.now()), p: 1 });
167
167
  }
168
168
 
169
- // Times are floored to whole milliseconds on every path into the maps (load replays through here too, so fractional times persisted by older code heal on startup). Disk mtimes carry fractional milliseconds but utimes round-trips only whole ones, so a fractional time can never be reproduced by propagation - every copy of the value would compare "older" than the original forever, and the same write would be re-pushed and re-copied every round.
169
+ // Times are ROUNDED to whole milliseconds on every path into the maps (load replays through here too, so fractional times persisted by older code heal on startup). Disk mtimes carry fractional milliseconds but utimes round-trips only whole ones, so a fractional time can never be reproduced by propagation - every copy of the value would compare "older" than the original forever, and the same write would be re-pushed and re-copied every round. Round rather than floor, to match ArchivesDisk (see its get2): utimes goes through a seconds double and can land a hair below the stamped millisecond.
170
170
  private applySet(key: string, value: T, time: number, changedAt: number): boolean {
171
- time = Math.floor(time);
172
- changedAt = Math.floor(changedAt);
171
+ time = Math.round(time);
172
+ changedAt = Math.round(changedAt);
173
173
  if (time < this.timeOf(key)) return false;
174
174
  this.deleted.delete(key);
175
175
  this.values.set(key, { value, time, changedAt });
@@ -177,10 +177,10 @@ export class TransactionFile<T> {
177
177
  }
178
178
 
179
179
  private applyDelete(key: string, time: number, changedAt: number, value?: T, valueTime?: number): boolean {
180
- time = Math.floor(time);
181
- changedAt = Math.floor(changedAt);
180
+ time = Math.round(time);
181
+ changedAt = Math.round(changedAt);
182
182
  if (valueTime !== undefined) {
183
- valueTime = Math.floor(valueTime);
183
+ valueTime = Math.round(valueTime);
184
184
  }
185
185
  if (time < this.timeOf(key)) return false;
186
186
  this.values.delete(key);
@@ -1,11 +1,13 @@
1
1
  import type { IArchives } from "./IArchives";
2
- /** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, the destination already had a NEWER file (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
2
+ /** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, and with preserveWriteTime the two guarded cases - the destination already held something NEWER (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
3
3
  export declare function copyArchiveFile(config: {
4
4
  from: IArchives;
5
5
  to: IArchives;
6
6
  path: string;
7
7
  /** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
8
8
  toPath?: string;
9
+ /** Stamps the destination with the SOURCE's write time instead of now, and turns on the ordering guards around it (the newer-destination refusal up front, and the getInfo confirm after). ONLY for synchronization between replicas of the same key, where the higher write time must win and ordering must survive propagation - never for a user-triggered copy: a plain copy is a NEW write, and the source's old stamp would make it LOSE to any newer write or tombstone at the destination, silently (move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely). */
10
+ preserveWriteTime?: boolean;
9
11
  forceSetImmutable?: boolean;
10
12
  noChecks?: boolean;
11
13
  internal?: boolean;
@@ -8,13 +8,15 @@ import { logStorageError } from "./remoteStorage/storageLogs";
8
8
  const LARGE_COPY_THRESHOLD = 64 * 1024 * 1024;
9
9
  const LARGE_COPY_CHUNK = 32 * 1024 * 1024;
10
10
 
11
- /** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, the destination already had a NEWER file (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
11
+ /** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, and with preserveWriteTime the two guarded cases - the destination already held something NEWER (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
12
12
  export async function copyArchiveFile(config: {
13
13
  from: IArchives;
14
14
  to: IArchives;
15
15
  path: string;
16
16
  /** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
17
17
  toPath?: string;
18
+ /** Stamps the destination with the SOURCE's write time instead of now, and turns on the ordering guards around it (the newer-destination refusal up front, and the getInfo confirm after). ONLY for synchronization between replicas of the same key, where the higher write time must win and ordering must survive propagation - never for a user-triggered copy: a plain copy is a NEW write, and the source's old stamp would make it LOSE to any newer write or tombstone at the destination, silently (move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely). */
19
+ preserveWriteTime?: boolean;
18
20
  forceSetImmutable?: boolean;
19
21
  noChecks?: boolean;
20
22
  internal?: boolean;
@@ -25,14 +27,15 @@ export async function copyArchiveFile(config: {
25
27
  let info = await from.getInfo(path, { noFallbacks: config.noFallbacks });
26
28
  if (!info) return undefined;
27
29
  let size = info.size;
28
- // internal (synchronization between replicas of the same key) preserves the source's write time, so ordering survives propagation. A plain copy is a NEW write and is stamped now: the source's old stamp would make it LOSE to any newer write or tombstone at the destination, silently - move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely.
29
- let writeTime = Math.floor(config.internal && info.writeTime || Date.now());
30
- // A destination that already holds a NEWER file must not be overwritten with our older one - and the destination's own only-take-latest would drop the write SILENTLY, leaving the caller believing the copy happened. Refusing here, loudly, is what turns "the remote has something we missed" from a masked bug into a log line the caller can act on.
31
- let destInfo = await to.getInfo(toPath, { noFallbacks: config.noFallbacks });
32
- // Compared at whole-millisecond precision, here and at the confirm below: disk mtimes carry fractional milliseconds, but utimes round-trips only whole ones, so sub-millisecond differences are storage artifacts of the SAME time, not ordering
33
- if (destInfo && Math.floor(destInfo.writeTime) > Math.floor(writeTime)) {
34
- logStorageError(`Copy refused - a newer file exists at the destination. Refusing to copy ${JSON.stringify(path)} from ${from.getDebugName()} to ${to.getDebugName()}${toPath !== path && ` (as ${JSON.stringify(toPath)})` || ""}: ours ${size} bytes at ${formatDateTimeDetailed(writeTime)}, theirs ${destInfo.size} bytes at ${formatDateTimeDetailed(destInfo.writeTime)} - copying would roll it back`);
35
- return undefined;
30
+ let writeTime = Math.round(config.preserveWriteTime && info.writeTime || Date.now());
31
+ if (config.preserveWriteTime) {
32
+ // A destination that already holds a NEWER file must not be overwritten with our older one - and the destination's own only-take-latest would drop the write SILENTLY, leaving the caller believing the copy happened. Refusing here, loudly, is what turns "the remote has something we missed" from a masked bug into a log line the caller can act on. (A fresh-stamped copy needs no such guard - nothing at the destination can be newer than now - so plain copies skip the round trip.)
33
+ let destInfo = await to.getInfo(toPath, { noFallbacks: config.noFallbacks });
34
+ // Compared at whole-millisecond precision (ROUNDED, matching ArchivesDisk - see its get2), here and at the confirm below: disk mtimes carry fractional milliseconds, but utimes round-trips only whole ones, so sub-millisecond differences are storage artifacts of the SAME time, not ordering
35
+ if (destInfo && Math.round(destInfo.writeTime) > Math.round(writeTime)) {
36
+ logStorageError(`Copy refused - a newer file exists at the destination. Refusing to copy ${JSON.stringify(path)} from ${from.getDebugName()} to ${to.getDebugName()}${toPath !== path && ` (as ${JSON.stringify(toPath)})` || ""}: ours ${size} bytes at ${formatDateTimeDetailed(writeTime)}, theirs ${destInfo.size} bytes at ${formatDateTimeDetailed(destInfo.writeTime)} - copying would roll it back`);
37
+ return undefined;
38
+ }
36
39
  }
37
40
  let copiedSize: number;
38
41
  if (size <= LARGE_COPY_THRESHOLD) {
@@ -67,11 +70,13 @@ export async function copyArchiveFile(config: {
67
70
  });
68
71
  copiedSize = totalSize;
69
72
  }
70
- // Every backend drops a superseded write SILENTLY (its only-take-latest is the last line of defense against races the up-front check can't see), so a returned set is not proof the copy landed - only the destination reporting the file at OUR time or newer is. Newer also counts as landed: the destination is at least as new as what we pushed (and b2 always stamps its own, later, upload time).
71
- let confirmed = await to.getInfo(toPath, { noFallbacks: config.noFallbacks });
72
- if (!confirmed || Math.floor(confirmed.writeTime) < Math.floor(writeTime)) {
73
- logStorageError(`Copy was silently dropped by the destination. Copy of ${JSON.stringify(path)} from ${from.getDebugName()} to ${to.getDebugName()}${toPath !== path && ` (as ${JSON.stringify(toPath)})` || ""}: our copy was ${copiedSize} bytes at ${formatDateTimeDetailed(writeTime)}, but the destination reports ${confirmed && `${confirmed.size} bytes at ${formatDateTimeDetailed(confirmed.writeTime)}` || "nothing"} (a newer write or deletion won the race)`);
74
- return undefined;
73
+ if (config.preserveWriteTime) {
74
+ // Every backend drops a superseded write SILENTLY (its only-take-latest is the last line of defense against races the up-front check can't see), so a returned set is not proof the copy landed - only the destination reporting the file at OUR time or newer is. Newer also counts as landed: the destination is at least as new as what we pushed (and b2 always stamps its own, later, upload time). Only for preserved stamps: a fresh stamp cannot lose the race, and moveArchiveFile does its own confirm before deleting anything.
75
+ let confirmed = await to.getInfo(toPath, { noFallbacks: config.noFallbacks });
76
+ if (!confirmed || Math.round(confirmed.writeTime) < Math.round(writeTime)) {
77
+ logStorageError(`Copy was silently dropped by the destination. Copy of ${JSON.stringify(path)} from ${from.getDebugName()} to ${to.getDebugName()}${toPath !== path && ` (as ${JSON.stringify(toPath)})` || ""}: our copy was ${copiedSize} bytes at ${formatDateTimeDetailed(writeTime)}, but the destination reports ${confirmed && `${confirmed.size} bytes at ${formatDateTimeDetailed(confirmed.writeTime)}` || "nothing"} (a newer write or deletion won the race)`);
78
+ return undefined;
79
+ }
75
80
  }
76
81
  return { writeTime, size: copiedSize };
77
82
  }