rac-delta 1.0.0

package/dist/infrastructure/factories/storage-adpater-factory.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StorageAdapterFactory = void 0;
+const azure_blob_storage_adapter_1 = require("../adapters/azure-blob-storage-adapter");
+const url_storage_adapter_1 = require("../adapters/url-storage-adapter");
+const local_storage_adapter_1 = require("../adapters/local-storage-adapter");
+const http_storage_adapter_1 = require("../adapters/http-storage-adapter");
+const gcs_storage_adapter_1 = require("../adapters/gcs-storage-adapter");
+const ssh_storage_adapter_1 = require("../adapters/ssh-storage-adapter");
+const s3_storage_adapter_1 = require("../adapters/s3-storage-adapter");
+class StorageAdapterFactory {
+    static create(config) {
+        switch (config.type) {
+            case 'local':
+                return new local_storage_adapter_1.LocalStorageAdapter(config);
+            case 'http':
+                return new http_storage_adapter_1.HTTPStorageAdapter(config);
+            case 's3':
+                return new s3_storage_adapter_1.S3StorageAdapter(config);
+            case 'azure':
+                return new azure_blob_storage_adapter_1.AzureBlobStorageAdapter(config);
+            case 'gcs':
+                return new gcs_storage_adapter_1.GCSStorageAdapter(config);
+            case 'ssh':
+                return new ssh_storage_adapter_1.SSHStorageAdapter(config);
+            case 'url':
+                return new url_storage_adapter_1.DefaultUrlStorageAdapter();
+            default:
+                throw new Error(`Unsupported storage type: ${config.type}`);
+        }
+    }
+}
+exports.StorageAdapterFactory = StorageAdapterFactory;

package/dist/infrastructure/pipelines/default-hash-download-pipeline.d.ts

@@ -0,0 +1,27 @@
+import { DeltaService, ReconstructionService, ValidationService } from '../../core/services';
+import { DownloadOptions, HashDownloadPipeline, UpdateStrategy } from '../../core/pipelines';
+import { ChunkSource } from '../../core/services/reconstruction-service';
+import { DeltaPlan, RDIndex } from '../../core/models';
+import { HashStorageAdapter } from '../../core/adapters';
+import { RacDeltaConfig } from '../../core/config';
+import { Nullish } from '../../core/types';
+export declare class DefaultHashDownloadPipeline extends HashDownloadPipeline {
+    protected readonly reconstruction: ReconstructionService;
+    protected readonly validation: ValidationService;
+    protected readonly storage: HashStorageAdapter;
+    protected readonly config: RacDeltaConfig;
+    protected readonly delta: DeltaService;
+    constructor(reconstruction: ReconstructionService, validation: ValidationService, storage: HashStorageAdapter, config: RacDeltaConfig, delta: DeltaService);
+    execute(localDir: string, strategy: UpdateStrategy, remoteIndex?: Nullish<RDIndex>, options?: Nullish<DownloadOptions>): Promise<void>;
+    loadLocalIndex(localDir: string): Promise<RDIndex>;
+    downloadAllMissingChunks(plan: DeltaPlan, target: 'memory' | 'disk', options?: Nullish<DownloadOptions>): Promise<ChunkSource>;
+    verifyAndDeleteObsoleteChunks(plan: DeltaPlan, localDir: string, remoteIndex: RDIndex, chunkSource: ChunkSource, options?: Nullish<DownloadOptions>): Promise<{
+        deletedFiles: string[];
+        verifiedFiles: string[];
+        rebuiltFiles: string[];
+    }>;
+    saveLocalIndex(localDir: string, index: RDIndex): Promise<void>;
+    findLocalIndex(localDir: string): Promise<RDIndex | null>;
+    private groupByFile;
+}
+//# sourceMappingURL=default-hash-download-pipeline.d.ts.map

package/dist/infrastructure/pipelines/default-hash-download-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-hash-download-pipeline.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/pipelines/default-hash-download-pipeline.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC7F,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,MAAM,4CAA4C,CAAC;AACzE,OAAO,EAAc,SAAS,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAEnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAM3C,qBAAa,2BAA4B,SAAQ,oBAAoB;IAEjE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,qBAAqB;IACxD,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,iBAAiB;IAChD,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,kBAAkB;IAC9C,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,cAAc;IACzC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY;gBAJnB,cAAc,EAAE,qBAAqB,EACrC,UAAU,EAAE,iBAAiB,EAC7B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,YAAY;IAKlC,OAAO,CACX,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,cAAc,EACxB,WAAW,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,EAC9B,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC;IA8FV,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IASlD,wBAAwB,CAC5B,IAAI,EAAE,SAAS,EACf,MAAM,EAAE,QAAQ,GAAG,MAAM,EACzB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GACjC,OAAO,CAAC,WAAW,CAAC;IAmEjB,6BAA6B,CACjC,IAAI,EAAE,SAAS,EACf,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,OAAO,EACpB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC;;;;;IA8C9B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IAO/C,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAyB/D,OAAO,CAAC,WAAW;CAapB"}

package/dist/infrastructure/pipelines/default-hash-download-pipeline.js

@@ -0,0 +1,211 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DefaultHashDownloadPipeline = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const pipelines_1 = require("../../core/pipelines");
+const stream_to_buffer_1 = require("../../core/utils/stream-to-buffer");
+const storage_chunk_source_1 = require("../chunk-sources/storage-chunk-source");
+const memory_chunk_source_1 = require("../chunk-sources/memory-chunk-source");
+const disk_chunk_source_1 = require("../chunk-sources/disk-chunk-source");
+class DefaultHashDownloadPipeline extends pipelines_1.HashDownloadPipeline {
+    reconstruction;
+    validation;
+    storage;
+    config;
+    delta;
+    constructor(reconstruction, validation, storage, config, delta) {
+        super(storage, delta, reconstruction, validation, config);
+        this.reconstruction = reconstruction;
+        this.validation = validation;
+        this.storage = storage;
+        this.config = config;
+        this.delta = delta;
+    }
+    async execute(localDir, strategy, remoteIndex, options) {
+        try {
+            this.changeState('scanning', options);
+            const localIndex = !options?.force
+                ? options?.useExistingIndex
+                    ? ((await this.findLocalIndex(localDir)) ?? (await this.loadLocalIndex(localDir)))
+                    : await this.loadLocalIndex(localDir)
+                : null;
+            const remoteIndexToUse = remoteIndex ?? (await this.storage.getRemoteIndex());
+            if (!remoteIndexToUse) {
+                throw new Error('Remote rd-index not provided and was not found in storage. Check if storage pathPrefix is correct or provide an index manually');
+            }
+            const plan = await this.delta.compareForDownload(localIndex, remoteIndexToUse);
+            if (!options?.force &&
+                !plan.missingChunks.length &&
+                !plan.obsoleteChunks.length &&
+                !plan.deletedFiles.length) {
+                console.info('No changes to download, you are up to date.');
+                return;
+            }
+            let chunkSource = null;
+            if (strategy === pipelines_1.UpdateStrategy.DownloadAllFirstToMemory) {
+                this.changeState('downloading', options);
+                chunkSource = await this.downloadAllMissingChunks(plan, 'memory', options);
+            }
+            // StorageChunkSource admits streaming, so it will be downloading chunks while reconstructing in service
+            if (strategy === pipelines_1.UpdateStrategy.StreamFromNetwork) {
+                chunkSource = new storage_chunk_source_1.StorageChunkSource(this.storage);
+            }
+            if (strategy === pipelines_1.UpdateStrategy.DownloadAllFirstToDisk) {
+                this.changeState('downloading', options);
+                chunkSource = await this.downloadAllMissingChunks(plan, 'disk', options);
+            }
+            if (!chunkSource) {
+                throw new Error('No chunkSource found');
+            }
+            if (plan.newAndModifiedFiles.length) {
+                this.changeState('reconstructing', options);
+                await this.reconstruction.reconstructAll(plan, localDir, chunkSource, {
+                    forceRebuild: options?.force,
+                    verifyAfterRebuild: true,
+                    fileConcurrency: options?.fileReconstructionConcurrency,
+                    inPlaceReconstructionThreshold: options?.inPlaceReconstructionThreshold,
+                    onProgress: (reconstructProgress, diskSpeed, networkProgress, networkSpeed) => {
+                        this.updateProgress(reconstructProgress, 'reconstructing', diskSpeed, undefined, options);
+                        if (networkProgress) {
+                            this.updateProgress(networkProgress, 'download', 0, networkSpeed, options);
+                        }
+                    },
+                });
+            }
+            if (plan.obsoleteChunks.length || plan.deletedFiles.length) {
+                this.changeState('cleaning', options);
+                await this.verifyAndDeleteObsoleteChunks(plan, localDir, remoteIndexToUse, chunkSource, options);
+            }
+            await this.saveLocalIndex(localDir, remoteIndexToUse);
+            if (chunkSource instanceof disk_chunk_source_1.DiskChunkSource) {
+                chunkSource.clear();
+            }
+        }
+        finally {
+            await this.storage.dispose();
+        }
+    }
+    async loadLocalIndex(localDir) {
+        const localIndex = await this.delta.createIndexFromDirectory(localDir, this.config.chunkSize, this.config.maxConcurrency);
+        return localIndex;
+    }
+    async downloadAllMissingChunks(plan, target, options) {
+        if (target === 'disk' && !options?.chunksSavePath) {
+            throw new Error('Error: chunksSavePath must be provided under options');
+        }
+        let chunksSavePath = undefined;
+        if (target === 'disk' && options?.chunksSavePath) {
+            chunksSavePath = (0, path_1.isAbsolute)(options?.chunksSavePath)
+                ? options?.chunksSavePath
+                : (0, path_1.resolve)(process.cwd(), options?.chunksSavePath);
+        }
+        const chunkSource = target === 'memory' ? new memory_chunk_source_1.MemoryChunkSource() : new disk_chunk_source_1.DiskChunkSource(chunksSavePath);
+        const chunks = plan.missingChunks;
+        let completed = 0;
+        let totalBytes = 0;
+        const concurrency = this.config.maxConcurrency ?? 6;
+        const queue = [...chunks];
+        let lastUpdateTime = Date.now();
+        let lastBytes = 0;
+        let speed = 0;
+        const worker = async () => {
+            while (queue.length) {
+                const chunk = queue.pop();
+                const readable = await this.storage.getChunk(chunk.hash);
+                if (!readable) {
+                    throw new Error(`Chunk missing: ${chunk.hash}`);
+                }
+                const buffer = await (0, stream_to_buffer_1.streamToBuffer)(readable);
+                chunkSource.setChunk(chunk.hash, buffer);
+                completed++;
+                totalBytes += buffer.length;
+                const now = Date.now();
+                const elapsed = now - lastUpdateTime;
+                // each 100ms
+                if (elapsed >= 100) {
+                    const bytesDiff = totalBytes - lastBytes;
+                    speed = bytesDiff / (elapsed / 1000);
+                    lastUpdateTime = now;
+                    lastBytes = totalBytes;
+                    const percent = ((completed / chunks.length) * 100).toFixed(1);
+                    this.updateProgress(Number(percent), 'download', 0, speed, options);
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: concurrency }, () => worker()));
+        const percent = 100;
+        const totalTime = (Date.now() - lastUpdateTime) / 1000;
+        const avgSpeed = totalBytes / (totalTime || 1);
+        this.updateProgress(percent, 'download', 0, avgSpeed, options);
+        return chunkSource;
+    }
+    async verifyAndDeleteObsoleteChunks(plan, localDir, remoteIndex, chunkSource, options) {
+        const dir = (0, path_1.isAbsolute)(localDir) ? localDir : (0, path_1.resolve)(process.cwd(), localDir);
+        const obsoleteByFile = this.groupByFile(plan.obsoleteChunks);
+        const deletedFiles = [];
+        const verifiedFiles = [];
+        const rebuiltFiles = [];
+        const allFiles = new Set([...plan.deletedFiles, ...obsoleteByFile.keys()]);
+        const totalFiles = allFiles.size;
+        let completedFiles = 0;
+        for (const filePath of allFiles) {
+            const absPath = (0, path_1.join)(dir, filePath);
+            const remoteFile = remoteIndex.files.find((file) => file.path === filePath);
+            // Fully removed file
+            if (!remoteFile || plan.deletedFiles.includes(filePath)) {
+                await (0, promises_1.rm)(absPath, { force: true });
+                deletedFiles.push(filePath);
+            }
+            else {
+                const isValid = await this.validation.validateFile(remoteFile, absPath);
+                if (!isValid) {
+                    await this.reconstruction.reconstructFile(remoteFile, absPath, chunkSource);
+                    rebuiltFiles.push(filePath);
+                }
+                else {
+                    verifiedFiles.push(filePath);
+                }
+            }
+            completedFiles++;
+            this.updateProgress((completedFiles / totalFiles) * 100, 'deleting', undefined, undefined, options);
+        }
+        return { deletedFiles, verifiedFiles, rebuiltFiles };
+    }
+    async saveLocalIndex(localDir, index) {
+        const dir = (0, path_1.isAbsolute)(localDir) ? localDir : (0, path_1.resolve)(process.cwd(), localDir);
+        const indexPath = (0, path_1.join)(dir, 'rd-index.json');
+        await (0, promises_1.writeFile)(indexPath, JSON.stringify(index, null, 2), 'utf-8');
+    }
+    async findLocalIndex(localDir) {
+        const dir = (0, path_1.isAbsolute)(localDir) ? localDir : (0, path_1.resolve)(process.cwd(), localDir);
+        const files = await (0, promises_1.readdir)(dir);
+        const indexFile = files.find((f) => f === 'rd-index.json');
+        if (!indexFile) {
+            return null;
+        }
+        const filePath = (0, path_1.join)(localDir, indexFile);
+        const data = await (0, promises_1.readFile)(filePath, 'utf-8');
+        try {
+            const parsed = JSON.parse(data);
+            return parsed;
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                throw new Error(`Invalid RDIndex JSON at ${filePath}: ${error.message}`);
+            }
+            throw error;
+        }
+    }
+    groupByFile(chunks) {
+        const map = new Map();
+        for (const chunk of chunks) {
+            if (!map.has(chunk.filePath)) {
+                map.set(chunk.filePath, []);
+            }
+            map.get(chunk.filePath).push(chunk);
+        }
+        return map;
+    }
+}
+exports.DefaultHashDownloadPipeline = DefaultHashDownloadPipeline;

package/dist/infrastructure/pipelines/default-hash-upload-pipeline.d.ts

@@ -0,0 +1,19 @@
+import { UploadOptions, HashUploadPipeline } from '../../core/pipelines';
+import { DeltaPlan, RDIndex } from '../../core/models';
+import { HashStorageAdapter } from '../../core/adapters';
+import { RacDeltaConfig } from '../../core/config';
+import { DeltaService } from '../../core/services';
+import { Nullish } from '../../core/types';
+export declare class DefaultHashUploadPipeline extends HashUploadPipeline {
+    protected readonly storage: HashStorageAdapter;
+    protected readonly delta: DeltaService;
+    protected readonly config: RacDeltaConfig;
+    constructor(storage: HashStorageAdapter, delta: DeltaService, config: RacDeltaConfig);
+    execute(directory: string, remoteIndex?: Nullish<RDIndex>, options?: Nullish<UploadOptions>): Promise<RDIndex>;
+    scanDirectory(localDir: string, ignorePatterns?: Nullish<string[]>): Promise<RDIndex>;
+    uploadMissingChunks(plan: DeltaPlan, baseDir: string, force: boolean, options?: Nullish<UploadOptions>): Promise<void>;
+    uploadIndex(index: RDIndex): Promise<void>;
+    deleteObsoleteChunks(plan: DeltaPlan, options?: Nullish<UploadOptions>): Promise<void>;
+    private groupChunksByFile;
+}
+//# sourceMappingURL=default-hash-upload-pipeline.d.ts.map

package/dist/infrastructure/pipelines/default-hash-upload-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-hash-upload-pipeline.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/pipelines/default-hash-upload-pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACzE,OAAO,EAAc,SAAS,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,qBAAa,yBAA0B,SAAQ,kBAAkB;IAE7D,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,kBAAkB;IAC9C,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY;IACtC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,cAAc;gBAFtB,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,cAAc;IAKrC,OAAO,CACX,SAAS,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,EAC9B,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAC/B,OAAO,CAAC,OAAO,CAAC;IAyDb,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;IAUrF,mBAAmB,CACvB,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,OAAO,EACd,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAC/B,OAAO,CAAC,IAAI,CAAC;IA8DV,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1C,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAqD5F,OAAO,CAAC,iBAAiB;CAa1B"}

package/dist/infrastructure/pipelines/default-hash-upload-pipeline.js

@@ -0,0 +1,170 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DefaultHashUploadPipeline = void 0;
+const path_1 = require("path");
+const promises_1 = require("fs/promises");
+const stream_1 = require("stream");
+const pipelines_1 = require("../../core/pipelines");
+class DefaultHashUploadPipeline extends pipelines_1.HashUploadPipeline {
+    storage;
+    delta;
+    config;
+    constructor(storage, delta, config) {
+        super(storage, delta, config);
+        this.storage = storage;
+        this.delta = delta;
+        this.config = config;
+    }
+    async execute(directory, remoteIndex, options) {
+        try {
+            this.changeState('scanning', options);
+            const localIndex = await this.scanDirectory(directory, options?.ignorePatterns);
+            let remoteIndexToUse = remoteIndex ?? null;
+            // If no remote index provided and no force upload, we try to get remote index using storageAdapter
+            if (!remoteIndex && !options?.force) {
+                const result = await this.storage.getRemoteIndex();
+                if (!result && options?.requireRemoteIndex) {
+                    throw new Error('Remote rd-index.json could not be found');
+                }
+                remoteIndexToUse = result;
+            }
+            // If force we always upload everything, so we just ignore remote index
+            if (options?.force) {
+                remoteIndexToUse = null;
+            }
+            // If no remoteIndex provided, everything will be uploaded
+            this.changeState('comparing', options);
+            const deltaPlan = await this.delta.compareForUpload(localIndex, remoteIndexToUse);
+            if (!options?.force &&
+                !deltaPlan.missingChunks.length &&
+                !deltaPlan.obsoleteChunks.length &&
+                !deltaPlan.deletedFiles.length) {
+                console.info('No changes to upload or delete, remote is up to date.');
+                return localIndex;
+            }
+            if (deltaPlan.missingChunks.length) {
+                this.changeState('uploading', options);
+                await this.uploadMissingChunks(deltaPlan, directory, !!options?.force, options);
+            }
+            if (remoteIndexToUse && deltaPlan.obsoleteChunks.length) {
+                this.changeState('cleaning', options);
+                await this.deleteObsoleteChunks(deltaPlan, options);
+            }
+            this.changeState('finalizing', options);
+            await this.uploadIndex(localIndex);
+            return localIndex;
+        }
+        finally {
+            await this.storage.dispose();
+        }
+    }
+    async scanDirectory(localDir, ignorePatterns) {
+        const localIndex = await this.delta.createIndexFromDirectory(localDir, this.config.chunkSize, this.config.maxConcurrency, ignorePatterns);
+        return localIndex;
+    }
+    async uploadMissingChunks(plan, baseDir, force, options) {
+        if (!plan.missingChunks?.length) {
+            return;
+        }
+        const dir = (0, path_1.isAbsolute)(baseDir) ? baseDir : (0, path_1.resolve)(process.cwd(), baseDir);
+        const concurrency = this.config.maxConcurrency ?? 5;
+        const files = this.groupChunksByFile(plan.missingChunks);
+        const totalChunks = plan.missingChunks.length;
+        let uploadedChunks = 0;
+        let uploadedBytes = 0;
+        const startTime = Date.now();
+        const queue = [...files];
+        const worker = async () => {
+            while (true) {
+                const next = queue.pop();
+                if (!next) {
+                    break;
+                }
+                const [relativePath, chunks] = next;
+                chunks.sort((a, b) => a.offset - b.offset);
+                const filePath = (0, path_1.join)(dir, relativePath);
+                const fileHandle = await (0, promises_1.open)(filePath, 'r');
+                try {
+                    const buffer = Buffer.alloc(this.config.chunkSize);
+                    for (const { offset, size, hash } of chunks) {
+                        const { bytesRead } = await fileHandle.read(buffer, 0, size, offset);
+                        const data = buffer.subarray(0, bytesRead);
+                        const stream = stream_1.Readable.from(data);
+                        await this.storage.putChunk(hash, stream, {
+                            overwrite: true,
+                            size,
+                        });
+                        uploadedChunks++;
+                        uploadedBytes += bytesRead;
+                        const percent = ((uploadedChunks / totalChunks) * 100).toFixed(1);
+                        const elapsed = (Date.now() - startTime) / 1000; // seconds
+                        const speed = uploadedBytes / elapsed;
+                        this.updateProgress(Number(percent), 'upload', speed, options);
+                    }
+                }
+                finally {
+                    await fileHandle.close();
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: concurrency }, () => worker()));
+    }
+    async uploadIndex(index) {
+        await this.storage.putRemoteIndex(index);
+    }
+    async deleteObsoleteChunks(plan, options) {
+        if (!plan.obsoleteChunks?.length) {
+            return;
+        }
+        const concurrency = this.config.maxConcurrency ?? 5;
+        const queue = [...plan.obsoleteChunks];
+        const total = queue.length;
+        let deleted = 0;
+        if (total === 0) {
+            return;
+        }
+        const failed = [];
+        const maxRetries = 3;
+        const worker = async () => {
+            while (true) {
+                const chunk = queue.pop();
+                if (!chunk) {
+                    break;
+                }
+                let success = false;
+                for (let attempt = 1; attempt <= maxRetries && !success; attempt++) {
+                    try {
+                        await this.storage.deleteChunk(chunk.hash);
+                        success = true;
+                    }
+                    catch (err) {
+                        if (attempt === maxRetries) {
+                            failed.push(chunk.hash);
+                        }
+                        else {
+                            await new Promise((res) => setTimeout(res, 100 * attempt));
+                        }
+                    }
+                }
+                deleted++;
+                const percent = (deleted / total) * 100;
+                this.updateProgress(percent, 'deleting', undefined, options);
+            }
+        };
+        await Promise.all(Array.from({ length: concurrency }, () => worker()));
+        if (failed.length > 0) {
+            throw new Error(`Failed to delete ${failed.length}/${total} chunks: ${failed.join(', ')}`);
+        }
+    }
+    groupChunksByFile(chunks) {
+        const groups = new Map();
+        for (const chunk of chunks) {
+            if (!groups.has(chunk.filePath)) {
+                groups.set(chunk.filePath, []);
+            }
+            groups.get(chunk.filePath).push(chunk);
+        }
+        return Array.from(groups.entries());
+    }
+}
+exports.DefaultHashUploadPipeline = DefaultHashUploadPipeline;

package/dist/infrastructure/pipelines/default-url-download-pipeline.d.ts

@@ -0,0 +1,30 @@
+import { UrlDownloadPipeline, DownloadOptions, UpdateStrategy } from '../../core/pipelines';
+import { ChunkUrlInfo, DeltaPlan, RDIndex } from '../../core/models';
+import { DeltaService, ReconstructionService, ValidationService } from '../../core/services';
+import { ChunkSource } from '../../core/services/reconstruction-service';
+import { UrlStorageAdapter } from '../../core/adapters';
+import { RacDeltaConfig } from '../../core/config';
+import { Nullish } from '../../core/types';
+export declare class DefaultUrlDownloadPipeline extends UrlDownloadPipeline {
+    protected readonly storage: UrlStorageAdapter;
+    protected readonly reconstruction: ReconstructionService;
+    protected readonly validation: ValidationService;
+    protected readonly delta: DeltaService;
+    protected readonly config: RacDeltaConfig;
+    constructor(storage: UrlStorageAdapter, reconstruction: ReconstructionService, validation: ValidationService, delta: DeltaService, config: RacDeltaConfig);
+    execute(localDir: string, urls: {
+        downloadUrls: Record<string, ChunkUrlInfo>;
+        indexUrl: string;
+    }, strategy: UpdateStrategy, plan?: Nullish<DeltaPlan>, options?: Nullish<DownloadOptions>): Promise<void>;
+    findLocalIndex(localDir: string): Promise<RDIndex | null>;
+    loadLocalIndex(localDir: string): Promise<RDIndex>;
+    downloadAllMissingChunks(downloadUrls: Record<string, ChunkUrlInfo>, target: 'memory' | 'disk', options?: Nullish<DownloadOptions>): Promise<ChunkSource>;
+    verifyAndDeleteObsoleteChunks(plan: DeltaPlan, localDir: string, remoteIndex: RDIndex, chunkSource: ChunkSource, options?: Nullish<DownloadOptions>): Promise<{
+        deletedFiles: string[];
+        verifiedFiles: string[];
+        rebuiltFiles: string[];
+    }>;
+    saveLocalIndex(localDir: string, index: RDIndex): Promise<void>;
+    private groupByFile;
+}
+//# sourceMappingURL=default-url-download-pipeline.d.ts.map

package/dist/infrastructure/pipelines/default-url-download-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-url-download-pipeline.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/pipelines/default-url-download-pipeline.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC5F,OAAO,EAAc,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,MAAM,4CAA4C,CAAC;AAEzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAM3C,qBAAa,0BAA2B,SAAQ,mBAAmB;IAE/D,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,iBAAiB;IAC7C,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,qBAAqB;IACxD,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,iBAAiB;IAChD,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY;IACtC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,cAAc;gBAJtB,OAAO,EAAE,iBAAiB,EAC1B,cAAc,EAAE,qBAAqB,EACrC,UAAU,EAAE,iBAAiB,EAC7B,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,cAAc;IAKrC,OAAO,CACX,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE;QACJ,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC3C,QAAQ,EAAE,MAAM,CAAC;KAClB,EACD,QAAQ,EAAE,cAAc,EACxB,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,EACzB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC;IA2EV,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAyBzD,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IASlD,wBAAwB,CAC5B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,EAC1C,MAAM,EAAE,QAAQ,GAAG,MAAM,EACzB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GACjC,OAAO,CAAC,WAAW,CAAC;IAmEjB,6BAA6B,CACjC,IAAI,EAAE,SAAS,EACf,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,OAAO,EACpB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC;;;;;IA8C9B,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAOrE,OAAO,CAAC,WAAW;CAapB"}