sliftutils 1.7.115 → 1.7.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +12 -5
- package/package.json +2 -2
- package/spec.txt +4 -5
- package/storage/ArchivesDisk.ts +8 -8
- package/storage/IArchives.d.ts +6 -2
- package/storage/IArchives.ts +7 -3
- package/storage/TransactionFile.d.ts +1 -1
- package/storage/TransactionFile.ts +11 -11
- package/storage/archiveHelpers.d.ts +3 -1
- package/storage/archiveHelpers.ts +19 -14
- package/storage/dist/ArchivesDisk.ts.cache +10 -10
- package/storage/dist/IArchives.ts.cache +2 -2
- package/storage/dist/TransactionFile.ts.cache +13 -13
- package/storage/dist/archiveHelpers.ts.cache +19 -16
- package/storage/remoteStorage/ArchivesDelayed.ts +1 -1
- package/storage/remoteStorage/ArchivesRemote.ts +2 -2
- package/storage/remoteStorage/blobStore.d.ts +1 -1
- package/storage/remoteStorage/blobStore.ts +15 -9
- package/storage/remoteStorage/bucketDisk.ts +2 -2
- package/storage/remoteStorage/chainStartup.ts +1 -1
- package/storage/remoteStorage/createArchives.ts +1 -1
- package/storage/remoteStorage/dist/ArchivesDelayed.ts.cache +3 -3
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +4 -4
- package/storage/remoteStorage/dist/blobStore.ts.cache +20 -11
- package/storage/remoteStorage/dist/bucketDisk.ts.cache +4 -4
- package/storage/remoteStorage/dist/chainStartup.ts.cache +3 -3
- package/storage/remoteStorage/dist/createArchives.ts.cache +3 -3
- package/storage/remoteStorage/dist/storageController.ts.cache +2 -2
- package/storage/remoteStorage/dist/storeSync.ts.cache +8 -8
- package/storage/remoteStorage/storageController.d.ts +1 -0
- package/storage/remoteStorage/storageController.ts +1 -1
- package/storage/remoteStorage/storeSync.ts +6 -6
- package/yarn.lock +4 -4
package/index.d.ts
CHANGED
|
@@ -2166,7 +2166,7 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2166
2166
|
};
|
|
2167
2167
|
/** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
|
|
2168
2168
|
noFallbacks?: boolean;
|
|
2169
|
-
/** Store-to-store call: the serving node
|
|
2169
|
+
/** Store-to-store call: the serving node never consults its OTHER sources - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). That is the flag's ENTIRE meaning: no fallbacks, nothing else. The read is otherwise fully correct - the node's index still gates it (a key its index says is deleted answers as deleted, never as the history bytes still sitting on its disk). No window or route checks on reads. */
|
|
2170
2170
|
internal?: boolean;
|
|
2171
2171
|
/** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
|
|
2172
2172
|
includeTombstones?: boolean;
|
|
@@ -2184,6 +2184,8 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2184
2184
|
includeMarked?: boolean;
|
|
2185
2185
|
/** Listings normally come ONLY from the authoritative sources (the same nodes writes go to - read-your-writes). With fallbacks, a failing shard's routes are covered by the next source holding them (e.g. a wide read replica) instead of the call failing - high availability at the cost of possibly missing just-written data. Single-source archives ignore the flag. */
|
|
2186
2186
|
fallbacks?: boolean;
|
|
2187
|
+
/** Store-to-store listing: only entries whose bytes the node ITSELF holds - never entries its index redirects to its own other sources. A peer reads with GetConfig.internal (which never chases those redirects), so listing a redirect would just make the peer flag the file missing, purge it, re-list it, and loop forever; the peer hears about such files from the source actually holding them instead. */
|
|
2188
|
+
internal?: boolean;
|
|
2187
2189
|
};
|
|
2188
2190
|
export type DelConfig = {
|
|
2189
2191
|
/** Stamps the deletion (its tombstone) with this write time instead of now. Synchronization passes the ORIGINAL deletion time, so deletion ordering survives propagation exactly like any other write's ordering. */
|
|
@@ -2211,9 +2213,11 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2211
2213
|
time: number;
|
|
2212
2214
|
/** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
|
|
2213
2215
|
routes?: [number, number][];
|
|
2216
|
+
/** See FindConfig.internal - the change feed is a listing too, and redirect entries fail a peer's internal reads the same way. Deletions are always reported (they are index-only, there are no bytes to hold). */
|
|
2217
|
+
internal?: boolean;
|
|
2214
2218
|
};
|
|
2215
2219
|
export type SetConfig = {
|
|
2216
|
-
/** The write time to stamp (see IArchives.set).
|
|
2220
|
+
/** The write time to stamp (see IArchives.set). ROUNDED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. Rounded rather than floored because utimes goes through a seconds double and can read back a hair below the stamped millisecond (see ArchivesDisk.get2). */
|
|
2217
2221
|
lastModified?: number;
|
|
2218
2222
|
/** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
|
|
2219
2223
|
forceSetImmutable?: boolean;
|
|
@@ -2773,7 +2777,7 @@ declare module "sliftutils/storage/TransactionFile" {
|
|
|
2773
2777
|
entries(): IterableIterator<[string, LogEntry<T>]>;
|
|
2774
2778
|
/** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
|
|
2775
2779
|
deletedEntries(): IterableIterator<[string, LogTombstone<T>]>;
|
|
2776
|
-
/** Stores a value as of `time` (
|
|
2780
|
+
/** Stores a value as of `time` (rounded to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
|
|
2777
2781
|
set(key: string, value: T, time: number): boolean;
|
|
2778
2782
|
/** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
|
|
2779
2783
|
delete(key: string, time: number): boolean;
|
|
@@ -2856,13 +2860,15 @@ declare module "sliftutils/storage/TransactionStorage" {
|
|
|
2856
2860
|
|
|
2857
2861
|
declare module "sliftutils/storage/archiveHelpers" {
|
|
2858
2862
|
import type { IArchives } from "./IArchives";
|
|
2859
|
-
/** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, the destination already
|
|
2863
|
+
/** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, and with preserveWriteTime the two guarded cases - the destination already held something NEWER (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
|
|
2860
2864
|
export declare function copyArchiveFile(config: {
|
|
2861
2865
|
from: IArchives;
|
|
2862
2866
|
to: IArchives;
|
|
2863
2867
|
path: string;
|
|
2864
2868
|
/** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
|
|
2865
2869
|
toPath?: string;
|
|
2870
|
+
/** Stamps the destination with the SOURCE's write time instead of now, and turns on the ordering guards around it (the newer-destination refusal up front, and the getInfo confirm after). ONLY for synchronization between replicas of the same key, where the higher write time must win and ordering must survive propagation - never for a user-triggered copy: a plain copy is a NEW write, and the source's old stamp would make it LOSE to any newer write or tombstone at the destination, silently (move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely). */
|
|
2871
|
+
preserveWriteTime?: boolean;
|
|
2866
2872
|
forceSetImmutable?: boolean;
|
|
2867
2873
|
noChecks?: boolean;
|
|
2868
2874
|
internal?: boolean;
|
|
@@ -3713,7 +3719,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3713
3719
|
private assertFreshWriteTarget;
|
|
3714
3720
|
private assertMutable;
|
|
3715
3721
|
private assertInternalWriteAccepted;
|
|
3716
|
-
/** Internal (store-to-store) read:
|
|
3722
|
+
/** Internal (store-to-store) read: never goes to OTHER sources - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form - but the INDEX still gates, because it is the source of truth: a marked deletion keeps its bytes on disk as history (see writeToSources), so the disk alone would happily serve a DELETED file as live. Index says live -> the disk provides the bytes (past any write delay, so a fast write still buffered in memory is invisible here; the caller re-finds it once it flushes). Index says deleted -> the tombstone is the answer, never the disk. No window or route checks. */
|
|
3717
3723
|
private getInternal2;
|
|
3718
3724
|
/** Internal (store-to-store) write: the local disk plus our index, with NO downstream fan-out - the pushing store owns propagation, and fanning its pushes back out is how write loops between stores form. Only-take-latest still applies here. */
|
|
3719
3725
|
private setInternal;
|
|
@@ -4482,6 +4488,7 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
|
|
|
4482
4488
|
sourceConfig: SourceConfig;
|
|
4483
4489
|
time: number;
|
|
4484
4490
|
routes?: [number, number][];
|
|
4491
|
+
internal?: boolean;
|
|
4485
4492
|
}) => Promise<ArchiveFileInfo[]>;
|
|
4486
4493
|
getArchivesConfig: (config: {
|
|
4487
4494
|
account: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sliftutils",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.117",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"files": [
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
58
58
|
"preact-old-types": "^10.28.1",
|
|
59
59
|
"shell-quote": "^1.8.3",
|
|
60
|
-
"socket-function": "^1.2.
|
|
60
|
+
"socket-function": "^1.2.33",
|
|
61
61
|
"typenode": "^6.6.1",
|
|
62
62
|
"typesafecss": "*",
|
|
63
63
|
"ws": "^8.18.3",
|
package/spec.txt
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
Refactor
|
|
2
1
|
|
|
3
|
-
createArchives => storageController => storageServerState => BlobStore
|
|
4
|
-
|
|
5
|
-
So create Archives has to decide who we're going to talk to, and then Storage Controller, it just acts like adds security and stuff. and then it gets into load it bucket, and then load it bucket. It then decides what back-end store we're talking to, and the back-end store, it might talk to multiple sources.
|
|
6
|
-
So there's which server we're going to talk to, which bucket on the server it maps to once we get there, and then there's which back-end stores that we talk to. So I guess it's fine.
|
|
7
2
|
|
|
3
|
+
test
|
|
4
|
+
--internal
|
|
5
|
+
--selcert
|
|
8
6
|
|
|
7
|
+
with a test.ts script
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
TODO: Use wildcard certificates, and then expose urls which match up to the shard fraction, to 2 decimal places:
|
package/storage/ArchivesDisk.ts
CHANGED
|
@@ -164,8 +164,8 @@ export class ArchivesDisk implements IArchives {
|
|
|
164
164
|
await this.handles.run(filePath, async () => {
|
|
165
165
|
if (lastModified) {
|
|
166
166
|
let existing = await statOrUndefined(filePath);
|
|
167
|
-
// An older write never overwrites a newer one (see IArchives.set). Both sides
|
|
168
|
-
if (existing && Math.
|
|
167
|
+
// An older write never overwrites a newer one (see IArchives.set). Both sides rounded: sub-millisecond mtime digits are storage artifacts, not ordering (see get2)
|
|
168
|
+
if (existing && Math.round(lastModified) < Math.round(existing.mtimeMs)) return;
|
|
169
169
|
}
|
|
170
170
|
await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
|
|
171
171
|
let handle = await this.handles.getHandle(filePath, fs.constants.O_RDWR | fs.constants.O_CREAT);
|
|
@@ -238,14 +238,14 @@ export class ArchivesDisk implements IArchives {
|
|
|
238
238
|
if (!size && !config?.includeTombstones) return undefined;
|
|
239
239
|
let start = range && Math.min(range.start, size) || 0;
|
|
240
240
|
let end = range && Math.min(range.end, size) || size;
|
|
241
|
-
// mtimeMs is
|
|
242
|
-
if (end <= start) return { data: Buffer.alloc(0), writeTime: Math.
|
|
241
|
+
// mtimeMs is ROUNDED everywhere this disk reports a time: it carries fractional milliseconds (kernel-stamped files have nanosecond mtimes), but utimes round-trips only whole ones - so a fractional time handed to the rest of the system could never be reproduced by propagation, and every copy of the file would compare "older" than the original forever. Round rather than floor, because utimes goes through a seconds DOUBLE and can land a hair BELOW the stamped millisecond - flooring turned that representation error into a full lost millisecond, un-reproducing our own stamps.
|
|
242
|
+
if (end <= start) return { data: Buffer.alloc(0), writeTime: Math.round(stats.mtimeMs), size };
|
|
243
243
|
let buffer = Buffer.alloc(end - start);
|
|
244
244
|
let { bytesRead } = await handle.read(buffer, 0, buffer.length, start);
|
|
245
245
|
if (bytesRead !== buffer.length) {
|
|
246
246
|
throw new Error(`Expected ${buffer.length} bytes at ${filePath}:${start}, read ${bytesRead}`);
|
|
247
247
|
}
|
|
248
|
-
return { data: buffer, writeTime: Math.
|
|
248
|
+
return { data: buffer, writeTime: Math.round(stats.mtimeMs), size };
|
|
249
249
|
});
|
|
250
250
|
}
|
|
251
251
|
|
|
@@ -256,7 +256,7 @@ export class ArchivesDisk implements IArchives {
|
|
|
256
256
|
let stats = await statOrUndefined(filePath);
|
|
257
257
|
if (!stats || !stats.isFile()) return undefined;
|
|
258
258
|
if (!stats.size && !config?.includeTombstones) return undefined;
|
|
259
|
-
return { writeTime: Math.
|
|
259
|
+
return { writeTime: Math.round(stats.mtimeMs), size: stats.size };
|
|
260
260
|
});
|
|
261
261
|
}
|
|
262
262
|
|
|
@@ -297,7 +297,7 @@ export class ArchivesDisk implements IArchives {
|
|
|
297
297
|
let stats = await statOrUndefined(path.join(this.filesDir, relPath));
|
|
298
298
|
// Deleted while we were walking
|
|
299
299
|
if (!stats) continue;
|
|
300
|
-
infos.set(relPath, { path: relPath, createTime: Math.
|
|
300
|
+
infos.set(relPath, { path: relPath, createTime: Math.round(stats.mtimeMs), size: stats.size });
|
|
301
301
|
}
|
|
302
302
|
}
|
|
303
303
|
|
|
@@ -355,7 +355,7 @@ export class ArchivesDisk implements IArchives {
|
|
|
355
355
|
if (lastModified) {
|
|
356
356
|
// An older write never overwrites a newer one (see IArchives.set) - re-checked HERE, not just when the upload started, because a large upload streams for minutes and that window is exactly when a newer write lands
|
|
357
357
|
let existing = await statOrUndefined(filePath);
|
|
358
|
-
if (existing && Math.
|
|
358
|
+
if (existing && Math.round(lastModified) < Math.round(existing.mtimeMs)) {
|
|
359
359
|
await fs.promises.rm(tmpPath, { force: true });
|
|
360
360
|
return;
|
|
361
361
|
}
|
package/storage/IArchives.d.ts
CHANGED
|
@@ -71,7 +71,7 @@ export type GetConfig = {
|
|
|
71
71
|
};
|
|
72
72
|
/** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
|
|
73
73
|
noFallbacks?: boolean;
|
|
74
|
-
/** Store-to-store call: the serving node
|
|
74
|
+
/** Store-to-store call: the serving node never consults its OTHER sources - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). That is the flag's ENTIRE meaning: no fallbacks, nothing else. The read is otherwise fully correct - the node's index still gates it (a key its index says is deleted answers as deleted, never as the history bytes still sitting on its disk). No window or route checks on reads. */
|
|
75
75
|
internal?: boolean;
|
|
76
76
|
/** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
|
|
77
77
|
includeTombstones?: boolean;
|
|
@@ -89,6 +89,8 @@ export type FindConfig = {
|
|
|
89
89
|
includeMarked?: boolean;
|
|
90
90
|
/** Listings normally come ONLY from the authoritative sources (the same nodes writes go to - read-your-writes). With fallbacks, a failing shard's routes are covered by the next source holding them (e.g. a wide read replica) instead of the call failing - high availability at the cost of possibly missing just-written data. Single-source archives ignore the flag. */
|
|
91
91
|
fallbacks?: boolean;
|
|
92
|
+
/** Store-to-store listing: only entries whose bytes the node ITSELF holds - never entries its index redirects to its own other sources. A peer reads with GetConfig.internal (which never chases those redirects), so listing a redirect would just make the peer flag the file missing, purge it, re-list it, and loop forever; the peer hears about such files from the source actually holding them instead. */
|
|
93
|
+
internal?: boolean;
|
|
92
94
|
};
|
|
93
95
|
export type DelConfig = {
|
|
94
96
|
/** Stamps the deletion (its tombstone) with this write time instead of now. Synchronization passes the ORIGINAL deletion time, so deletion ordering survives propagation exactly like any other write's ordering. */
|
|
@@ -116,9 +118,11 @@ export type ChangesAfterConfig = {
|
|
|
116
118
|
time: number;
|
|
117
119
|
/** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
|
|
118
120
|
routes?: [number, number][];
|
|
121
|
+
/** See FindConfig.internal - the change feed is a listing too, and redirect entries fail a peer's internal reads the same way. Deletions are always reported (they are index-only, there are no bytes to hold). */
|
|
122
|
+
internal?: boolean;
|
|
119
123
|
};
|
|
120
124
|
export type SetConfig = {
|
|
121
|
-
/** The write time to stamp (see IArchives.set).
|
|
125
|
+
/** The write time to stamp (see IArchives.set). ROUNDED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. Rounded rather than floored because utimes goes through a seconds double and can read back a hair below the stamped millisecond (see ArchivesDisk.get2). */
|
|
122
126
|
lastModified?: number;
|
|
123
127
|
/** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
|
|
124
128
|
forceSetImmutable?: boolean;
|
package/storage/IArchives.ts
CHANGED
|
@@ -106,7 +106,7 @@ export type GetConfig = {
|
|
|
106
106
|
range?: { start: number; end: number };
|
|
107
107
|
/** Read ONLY from the primary source - the one writes would target - instead of falling back across the redundant sources. Use this when you want your reads and writes to be somewhat atomic: there will still be issues with the round trip, but without it you could talk to a completely different node and get a much older value. Most reads aren't followed by a write though, so for most cases it's better to get a value than to have to wait (or even throw) when the primary node is not available. */
|
|
108
108
|
noFallbacks?: boolean;
|
|
109
|
-
/** Store-to-store call: the serving node
|
|
109
|
+
/** Store-to-store call: the serving node never consults its OTHER sources - chasing its own remote holders while answering another store is how infinite get loops between stores form (A asks B, B's index points back at A, ...). That is the flag's ENTIRE meaning: no fallbacks, nothing else. The read is otherwise fully correct - the node's index still gates it (a key its index says is deleted answers as deleted, never as the history bytes still sitting on its disk). No window or route checks on reads. */
|
|
110
110
|
internal?: boolean;
|
|
111
111
|
/** Also return size-0 results (tombstones - an empty file IS a missing file) instead of treating them as absent. Off by default, matching getInfo's flag of the same name. Synchronization passes this so a DELETED file (with its write time) is distinguishable from a file that never existed. */
|
|
112
112
|
includeTombstones?: boolean;
|
|
@@ -125,6 +125,8 @@ export type FindConfig = {
|
|
|
125
125
|
includeMarked?: boolean;
|
|
126
126
|
/** Listings normally come ONLY from the authoritative sources (the same nodes writes go to - read-your-writes). With fallbacks, a failing shard's routes are covered by the next source holding them (e.g. a wide read replica) instead of the call failing - high availability at the cost of possibly missing just-written data. Single-source archives ignore the flag. */
|
|
127
127
|
fallbacks?: boolean;
|
|
128
|
+
/** Store-to-store listing: only entries whose bytes the node ITSELF holds - never entries its index redirects to its own other sources. A peer reads with GetConfig.internal (which never chases those redirects), so listing a redirect would just make the peer flag the file missing, purge it, re-list it, and loop forever; the peer hears about such files from the source actually holding them instead. */
|
|
129
|
+
internal?: boolean;
|
|
128
130
|
};
|
|
129
131
|
|
|
130
132
|
export type DelConfig = {
|
|
@@ -155,10 +157,12 @@ export type ChangesAfterConfig = {
|
|
|
155
157
|
time: number;
|
|
156
158
|
/** Only keys routing into one of these [start, end) ranges. Only scanning passes this - it lets a store syncing a partial shard ask for just its slice. */
|
|
157
159
|
routes?: [number, number][];
|
|
160
|
+
/** See FindConfig.internal - the change feed is a listing too, and redirect entries fail a peer's internal reads the same way. Deletions are always reported (they are index-only, there are no bytes to hold). */
|
|
161
|
+
internal?: boolean;
|
|
158
162
|
};
|
|
159
163
|
|
|
160
164
|
export type SetConfig = {
|
|
161
|
-
/** The write time to stamp (see IArchives.set).
|
|
165
|
+
/** The write time to stamp (see IArchives.set). ROUNDED to whole milliseconds by every implementation - the disk can't store fractional milliseconds anyway (utimes round-trips whole ms), so a fractional stamp could never be reproduced by propagation and would compare "newer" than its own copies forever. Rounded rather than floored because utimes goes through a seconds double and can read back a hair below the stamped millisecond (see ArchivesDisk.get2). */
|
|
162
166
|
lastModified?: number;
|
|
163
167
|
/** Makes the write acceptable on immutable targets: an existing path is simply kept (immutability wins - nothing is overwritten) instead of the write throwing. Requires lastModified. Synchronization MUST pass this on every push - a plain set throws on immutable targets, which would abort reconciliation whenever one source in a chain is immutable. */
|
|
164
168
|
forceSetImmutable?: boolean;
|
|
@@ -182,7 +186,7 @@ export type SetLargeFileConfig = SetConfig & {
|
|
|
182
186
|
restartStream?(): Promise<void> | void;
|
|
183
187
|
};
|
|
184
188
|
|
|
185
|
-
// createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written". Always WHOLE milliseconds: write times are
|
|
189
|
+
// createTime is a misnomer kept for compatibility — it is really the LAST-WRITE time, same as getInfo's writeTime. Neither Backblaze nor our remote storage tracks a distinct creation date: each write stamps a fresh timestamp on the current version, so both fields are just "when the bytes served by get() were most recently written". Always WHOLE milliseconds: write times are rounded at every producer (see SetConfig.lastModified).
|
|
186
190
|
export type ArchiveFileInfo = { path: string; createTime: number; size: number };
|
|
187
191
|
|
|
188
192
|
// An in-progress background synchronization task (see ArchivesConfig.syncing)
|
|
@@ -39,7 +39,7 @@ export declare class TransactionFile<T> {
|
|
|
39
39
|
entries(): IterableIterator<[string, LogEntry<T>]>;
|
|
40
40
|
/** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
|
|
41
41
|
deletedEntries(): IterableIterator<[string, LogTombstone<T>]>;
|
|
42
|
-
/** Stores a value as of `time` (
|
|
42
|
+
/** Stores a value as of `time` (rounded to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
|
|
43
43
|
set(key: string, value: T, time: number): boolean;
|
|
44
44
|
/** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
|
|
45
45
|
delete(key: string, time: number): boolean;
|
|
@@ -119,9 +119,9 @@ export class TransactionFile<T> {
|
|
|
119
119
|
return this.deleted.entries();
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
/** Stores a value as of `time` (
|
|
122
|
+
/** Stores a value as of `time` (rounded to whole milliseconds - see applySet). Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
|
|
123
123
|
public set(key: string, value: T, time: number): boolean {
|
|
124
|
-
time = Math.
|
|
124
|
+
time = Math.round(time);
|
|
125
125
|
if (!this.applySet(key, value, time, Date.now())) return false;
|
|
126
126
|
this.append({ k: key, t: time, v: value });
|
|
127
127
|
return true;
|
|
@@ -129,7 +129,7 @@ export class TransactionFile<T> {
|
|
|
129
129
|
|
|
130
130
|
/** Deletes as of `time`, keeping the tombstone. A key that had a live value keeps it in the tombstone as MARKED for deletion (readable and restorable until dropValue). Returns false when something at least as new is already here. */
|
|
131
131
|
public delete(key: string, time: number): boolean {
|
|
132
|
-
time = Math.
|
|
132
|
+
time = Math.round(time);
|
|
133
133
|
let live = this.values.get(key);
|
|
134
134
|
if (!this.applyDelete(key, time, Date.now(), live?.value, live?.time)) return false;
|
|
135
135
|
if (live) {
|
|
@@ -142,7 +142,7 @@ export class TransactionFile<T> {
|
|
|
142
142
|
|
|
143
143
|
/** Undoes a marked deletion: the kept value becomes live again, as of `time` (a fresh time, so the restore outranks the deletion everywhere it propagated). Returns false when there is no marked value to restore, or something at least as new is already here. */
|
|
144
144
|
public unmark(key: string, time: number): boolean {
|
|
145
|
-
time = Math.
|
|
145
|
+
time = Math.round(time);
|
|
146
146
|
let tombstone = this.deleted.get(key);
|
|
147
147
|
if (!tombstone || tombstone.value === undefined) return false;
|
|
148
148
|
if (!this.applySet(key, tombstone.value, time, Date.now())) return false;
|
|
@@ -163,13 +163,13 @@ export class TransactionFile<T> {
|
|
|
163
163
|
let had = this.values.delete(key);
|
|
164
164
|
had = this.deleted.delete(key) || had;
|
|
165
165
|
if (!had) return;
|
|
166
|
-
this.append({ k: key, t: Math.
|
|
166
|
+
this.append({ k: key, t: Math.round(Date.now()), p: 1 });
|
|
167
167
|
}
|
|
168
168
|
|
|
169
|
-
// Times are
|
|
169
|
+
// Times are ROUNDED to whole milliseconds on every path into the maps (load replays through here too, so fractional times persisted by older code heal on startup). Disk mtimes carry fractional milliseconds but utimes round-trips only whole ones, so a fractional time can never be reproduced by propagation - every copy of the value would compare "older" than the original forever, and the same write would be re-pushed and re-copied every round. Round rather than floor, to match ArchivesDisk (see its get2): utimes goes through a seconds double and can land a hair below the stamped millisecond.
|
|
170
170
|
private applySet(key: string, value: T, time: number, changedAt: number): boolean {
|
|
171
|
-
time = Math.
|
|
172
|
-
changedAt = Math.
|
|
171
|
+
time = Math.round(time);
|
|
172
|
+
changedAt = Math.round(changedAt);
|
|
173
173
|
if (time < this.timeOf(key)) return false;
|
|
174
174
|
this.deleted.delete(key);
|
|
175
175
|
this.values.set(key, { value, time, changedAt });
|
|
@@ -177,10 +177,10 @@ export class TransactionFile<T> {
|
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
private applyDelete(key: string, time: number, changedAt: number, value?: T, valueTime?: number): boolean {
|
|
180
|
-
time = Math.
|
|
181
|
-
changedAt = Math.
|
|
180
|
+
time = Math.round(time);
|
|
181
|
+
changedAt = Math.round(changedAt);
|
|
182
182
|
if (valueTime !== undefined) {
|
|
183
|
-
valueTime = Math.
|
|
183
|
+
valueTime = Math.round(valueTime);
|
|
184
184
|
}
|
|
185
185
|
if (time < this.timeOf(key)) return false;
|
|
186
186
|
this.values.delete(key);
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { IArchives } from "./IArchives";
|
|
2
|
-
/** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, the destination already
|
|
2
|
+
/** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, and with preserveWriteTime the two guarded cases - the destination already held something NEWER (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
|
|
3
3
|
export declare function copyArchiveFile(config: {
|
|
4
4
|
from: IArchives;
|
|
5
5
|
to: IArchives;
|
|
6
6
|
path: string;
|
|
7
7
|
/** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
|
|
8
8
|
toPath?: string;
|
|
9
|
+
/** Stamps the destination with the SOURCE's write time instead of now, and turns on the ordering guards around it (the newer-destination refusal up front, and the getInfo confirm after). ONLY for synchronization between replicas of the same key, where the higher write time must win and ordering must survive propagation - never for a user-triggered copy: a plain copy is a NEW write, and the source's old stamp would make it LOSE to any newer write or tombstone at the destination, silently (move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely). */
|
|
10
|
+
preserveWriteTime?: boolean;
|
|
9
11
|
forceSetImmutable?: boolean;
|
|
10
12
|
noChecks?: boolean;
|
|
11
13
|
internal?: boolean;
|
|
@@ -8,13 +8,15 @@ import { logStorageError } from "./remoteStorage/storageLogs";
|
|
|
8
8
|
const LARGE_COPY_THRESHOLD = 64 * 1024 * 1024;
|
|
9
9
|
const LARGE_COPY_CHUNK = 32 * 1024 * 1024;
|
|
10
10
|
|
|
11
|
-
/** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, the destination already
|
|
11
|
+
/** Copies one file between two archives. The source's CURRENT size and write time always come from getInfo right here - callers never supply them, because a stale size turns into ranged reads of a file that has changed (failing forever), and a stale write time re-orders history. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, and undefined for every way the copy did NOT land: the source doesn't have the file, and with preserveWriteTime the two guarded cases - the destination already held something NEWER (refused up front rather than roll it back), or the destination silently dropped the write (its own only-take-latest won a race we lost - caught by confirming with getInfo afterward). The refused/dropped cases are logged as errors; a caller that treats undefined as "missing at the source" must getInfo the destination to learn the actual latest value. */
|
|
12
12
|
export async function copyArchiveFile(config: {
|
|
13
13
|
from: IArchives;
|
|
14
14
|
to: IArchives;
|
|
15
15
|
path: string;
|
|
16
16
|
/** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
|
|
17
17
|
toPath?: string;
|
|
18
|
+
/** Stamps the destination with the SOURCE's write time instead of now, and turns on the ordering guards around it (the newer-destination refusal up front, and the getInfo confirm after). ONLY for synchronization between replicas of the same key, where the higher write time must win and ordering must survive propagation - never for a user-triggered copy: a plain copy is a NEW write, and the source's old stamp would make it LOSE to any newer write or tombstone at the destination, silently (move a file back to a folder it was deleted from and the copy is dropped, then the caller deletes the source, and the file is gone entirely). */
|
|
19
|
+
preserveWriteTime?: boolean;
|
|
18
20
|
forceSetImmutable?: boolean;
|
|
19
21
|
noChecks?: boolean;
|
|
20
22
|
internal?: boolean;
|
|
@@ -25,14 +27,15 @@ export async function copyArchiveFile(config: {
|
|
|
25
27
|
let info = await from.getInfo(path, { noFallbacks: config.noFallbacks });
|
|
26
28
|
if (!info) return undefined;
|
|
27
29
|
let size = info.size;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
30
|
+
let writeTime = Math.round(config.preserveWriteTime && info.writeTime || Date.now());
|
|
31
|
+
if (config.preserveWriteTime) {
|
|
32
|
+
// A destination that already holds a NEWER file must not be overwritten with our older one - and the destination's own only-take-latest would drop the write SILENTLY, leaving the caller believing the copy happened. Refusing here, loudly, is what turns "the remote has something we missed" from a masked bug into a log line the caller can act on. (A fresh-stamped copy needs no such guard - nothing at the destination can be newer than now - so plain copies skip the round trip.)
|
|
33
|
+
let destInfo = await to.getInfo(toPath, { noFallbacks: config.noFallbacks });
|
|
34
|
+
// Compared at whole-millisecond precision (ROUNDED, matching ArchivesDisk - see its get2), here and at the confirm below: disk mtimes carry fractional milliseconds, but utimes round-trips only whole ones, so sub-millisecond differences are storage artifacts of the SAME time, not ordering
|
|
35
|
+
if (destInfo && Math.round(destInfo.writeTime) > Math.round(writeTime)) {
|
|
36
|
+
logStorageError(`Copy refused - a newer file exists at the destination. Refusing to copy ${JSON.stringify(path)} from ${from.getDebugName()} to ${to.getDebugName()}${toPath !== path && ` (as ${JSON.stringify(toPath)})` || ""}: ours ${size} bytes at ${formatDateTimeDetailed(writeTime)}, theirs ${destInfo.size} bytes at ${formatDateTimeDetailed(destInfo.writeTime)} - copying would roll it back`);
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
36
39
|
}
|
|
37
40
|
let copiedSize: number;
|
|
38
41
|
if (size <= LARGE_COPY_THRESHOLD) {
|
|
@@ -67,11 +70,13 @@ export async function copyArchiveFile(config: {
|
|
|
67
70
|
});
|
|
68
71
|
copiedSize = totalSize;
|
|
69
72
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
73
|
+
if (config.preserveWriteTime) {
|
|
74
|
+
// Every backend drops a superseded write SILENTLY (its only-take-latest is the last line of defense against races the up-front check can't see), so a returned set is not proof the copy landed - only the destination reporting the file at OUR time or newer is. Newer also counts as landed: the destination is at least as new as what we pushed (and b2 always stamps its own, later, upload time). Only for preserved stamps: a fresh stamp cannot lose the race, and moveArchiveFile does its own confirm before deleting anything.
|
|
75
|
+
let confirmed = await to.getInfo(toPath, { noFallbacks: config.noFallbacks });
|
|
76
|
+
if (!confirmed || Math.round(confirmed.writeTime) < Math.round(writeTime)) {
|
|
77
|
+
logStorageError(`Copy was silently dropped by the destination. Copy of ${JSON.stringify(path)} from ${from.getDebugName()} to ${to.getDebugName()}${toPath !== path && ` (as ${JSON.stringify(toPath)})` || ""}: our copy was ${copiedSize} bytes at ${formatDateTimeDetailed(writeTime)}, but the destination reports ${confirmed && `${confirmed.size} bytes at ${formatDateTimeDetailed(confirmed.writeTime)}` || "nothing"} (a newer write or deletion won the race)`);
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
75
80
|
}
|
|
76
81
|
return { writeTime, size: copiedSize };
|
|
77
82
|
}
|