sftp-push-sync 2.3.0 → 3.0.0

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [3.0.0] - 2026-03-04
+
+- Switched from JSON-file based hash cache to NDJSON-based Cache-implementation.
+- Disk-based, only active entries in RAM
+- Scales to 100,000+ files without memory issues
+- Auto-persist (no explicit saving required)
+- Auto-migration - Existing JSON cache (.sync-cache.prod.json) is automatically migrated to LevelDB (.sync-cache-prod/)
+
+## [2.5.0] - 2026-03-04
+
+- Parallel remote walker walkers.mjs: scans 8 directories simultaneously
+- Batch analysis with concurrency compare.mjs: 8 file comparisons in parallel
+- Parallel hash calculation: local + remote hash simultaneously
+- Keep-alive: SftpPushSyncApp.mjs prevents server disconnection. A Keep-Alive packet is sent every 10 seconds.
+
 ## [2.1.0] - 2025-11-19
 
 Sync only handles files and creates missing directories during upload.

package/README.md

@@ -26,6 +26,11 @@ Features:
 
 The file `sftp-push-sync.mjs` is pure JavaScript (ESM), not TypeScript. Node.js can execute it directly as long as "type": "module" is specified in package.json or the file has the extension .mjs.
 
+## Breaking changes in 3.0.0
+
+- New Cache Mechanism: NDJSON instead of JSON.
+- The cache can now handle any number of files.
+
 ## Breaking changes in 2.0.0
 
 - The flags `--upload-list` / `--download-list` have been replaced by
@@ -253,12 +258,12 @@ However, it should also manage directories:
 
 ## Which files are created?
 
-- The cache files: `.sync-cache.*.json`
+- The cache files: `.sync-cache.*.ndjson`. The old ones can be deleted: `.sync-cache.*.json`
 - The log file: `.sftp-push-sync.{target}.log` (Optional, overwritten with each run)
 
 You can safely delete the local cache at any time. The first analysis will then take longer, because remote hashes will be streamed again. After that, everything will run fast.
 
-Note 1: The first run always takes a while, especially with lots of media – so be patient! Once the cache is full, it will be faster. 
+Note 1: The first run always takes a while, especially with lots of media – so be patient! Once the cache is full, it will be faster.
 Note 2: Reliability and accuracy are more important to me than speed.
 
 ## Example Output

package/bin/sftp-push-sync.mjs

@@ -80,6 +80,7 @@ let DRY_RUN = false;
 let RUN_UPLOAD_LIST = false;
 let RUN_DOWNLOAD_LIST = false;
 let SKIP_SYNC = false;
+let SIZE_ONLY = false;
 let cliLogLevel = null;
 let configPath = undefined;
 
@@ -92,6 +93,9 @@ for (let i = 0; i < rest.length; i += 1) {
     case "--dry-run":
       DRY_RUN = true;
       break;
+    case "--size-only":
+      SIZE_ONLY = true;
+      break;
     case "--sidecar-upload":
       RUN_UPLOAD_LIST = true;
       break;
@@ -196,4 +200,4 @@ main().catch((err) => {
     console.error(err);
   }
   process.exit(1);
-});
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sftp-push-sync",
-  "version": "2.3.0",
+  "version": "3.0.0",
   "description": "SFTP sync tool for Hugo projects (local to remote, with hash cache)",
   "type": "module",
   "bin": {

package/src/core/SftpPushSyncApp.mjs

@@ -17,7 +17,7 @@ import { SyncLogger } from "./SyncLogger.mjs";
 import { ScanProgressController } from "./ScanProgressController.mjs";
 
 import { toPosix, shortenPathForProgress } from "../helpers/directory.mjs";
-import { createHashCache } from "../helpers/hashing.mjs";
+import { createHashCacheNDJSON, migrateFromJsonCache } from "../helpers/hash-cache-ndjson.mjs";
 import { walkLocal, walkRemote } from "../helpers/walkers.mjs";
 import {
   analyseDifferences,
@@ -188,6 +188,50 @@ export class SftpPushSyncApp {
     this._consoleAndLog("", ...msg);
   }
 
+  // ---------------------------------------------------------
+  // SFTP Connection Helpers
+  // ---------------------------------------------------------
+
+  /**
+   * Check if SFTP connection is still alive
+   */
+  async _isConnected(sftp) {
+    try {
+      // Try a minimal operation to check connection
+      await sftp.cwd();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Reconnect to SFTP server
+   */
+  async _reconnect(sftp) {
+    try {
+      await sftp.end();
+    } catch {
+      // Ignore errors when closing dead connection
+    }
+
+    await sftp.connect({
+      host: this.connection.host,
+      port: this.connection.port,
+      username: this.connection.user,
+      password: this.connection.password,
+      keepaliveInterval: 10000,
+      keepaliveCountMax: 10,
+      readyTimeout: 30000,
+    });
+
+    if (sftp.client) {
+      sftp.client.setMaxListeners(50);
+    }
+
+    this.log(`${TAB_A}${pc.green("✔ Reconnected to SFTP.")}`);
+  }
+
   // ---------------------------------------------------------
   // Pattern-Helper
   // ---------------------------------------------------------
@@ -650,14 +694,20 @@ export class SftpPushSyncApp {
     ];
     this.autoExcluded = new Set();
 
-    // Hash-Cache
-    const syncCacheName =
-      targetConfig.syncCache || `.sync-cache.${target}.json`;
-    const cachePath = path.resolve(syncCacheName);
-    this.hashCache = createHashCache({
-      cachePath,
+    // Hash-Cache (NDJSON - human-readable, scales to 100k+ files)
+    const oldJsonCacheName = targetConfig.syncCache || `.sync-cache.${target}.json`;
+    const oldJsonCachePath = path.resolve(oldJsonCacheName);
+    const ndjsonCachePath = path.resolve(`.sync-cache.${target}.ndjson`);
+
+    // Migrate from old JSON cache if exists
+    const migration = await migrateFromJsonCache(oldJsonCachePath, ndjsonCachePath, target);
+    if (migration.migrated) {
+      console.log(pc.green(`   ✔ Migrated ${migration.localCount + migration.remoteCount} cache entries from JSON to NDJSON`));
+    }
+
+    this.hashCache = await createHashCacheNDJSON({
+      cachePath: ndjsonCachePath,
       namespace: target,
-      flushInterval: 50,
     });
 
     // Logger
@@ -736,6 +786,12 @@ export class SftpPushSyncApp {
         readyTimeout: 30000,       // 30s timeout for initial connection
       });
       connected = true;
+
+      // Increase max listeners for parallel operations
+      if (sftp.client) {
+        sftp.client.setMaxListeners(50);
+      }
+
       this.log(`${TAB_A}${pc.green("✔ Connected to SFTP.")}`);
 
       if (!skipSync && !fs.existsSync(this.connection.localRoot)) {
@@ -844,7 +900,7 @@ export class SftpPushSyncApp {
       // Phase 3 – Analyse Differences (delegiert an Helper)
       this.log(pc.bold(pc.cyan("🔎 Phase 3: Compare & Decide …")));
 
-      const { getLocalHash, getRemoteHash, save: saveCache } = this.hashCache;
+      const { getLocalHash, getRemoteHash } = this.hashCache;
 
       const diffResult = await analyseDifferences({
         local,
@@ -879,6 +935,12 @@ export class SftpPushSyncApp {
       this.log("");
       this.log(pc.bold(pc.cyan("🧹 Phase 4: Removing orphaned remote files …")));
 
+      // Reconnect if connection was lost during analysis
+      if (!await this._isConnected(sftp)) {
+        this.log(`${TAB_A}${pc.yellow("⚠ Connection lost, reconnecting…")}`);
+        await this._reconnect(sftp);
+      }
+
       toDelete = computeRemoteDeletes({ local, remote });
 
       if (toDelete.length === 0) {
@@ -893,6 +955,13 @@ export class SftpPushSyncApp {
       if (!dryRun && (toAdd.length || toUpdate.length)) {
         this.log("");
         this.log(pc.bold(pc.cyan("📁 Preparing remote directories …")));
+
+        // Ensure connection before directory operations
+        if (!await this._isConnected(sftp)) {
+          this.log(`${TAB_A}${pc.yellow("⚠ Connection lost, reconnecting…")}`);
+          await this._reconnect(sftp);
+        }
+
         await this.ensureAllRemoteDirsExist(
           sftp,
           this.connection.remoteRoot,
@@ -906,6 +975,12 @@ export class SftpPushSyncApp {
         this.log("");
         this.log(pc.bold(pc.cyan("🚚 Phase 5: Apply changes …")));
 
+        // Ensure fresh connection before uploads
+        if (!await this._isConnected(sftp)) {
+          this.log(`${TAB_A}${pc.yellow("⚠ Connection lost, reconnecting…")}`);
+          await this._reconnect(sftp);
+        }
+
         // Upload new files
         await this.runTasks(
           toAdd,
@@ -970,13 +1045,21 @@ export class SftpPushSyncApp {
         this.log(
           pc.bold(pc.cyan("🧹 Cleaning up empty remote directories …"))
         );
+
+        // Ensure connection before cleanup
+        if (!await this._isConnected(sftp)) {
+          this.log(`${TAB_A}${pc.yellow("⚠ Connection lost, reconnecting…")}`);
+          await this._reconnect(sftp);
+        }
+
         await this.cleanupEmptyDirs(sftp, this.connection.remoteRoot, dryRun);
       }
 
       const duration = ((Date.now() - start) / 1000).toFixed(2);
 
-      // Cache am Ende sicher schreiben
-      await saveCache(true);
+      // Save cache and close
+      await this.hashCache.save();
+      await this.hashCache.close();
 
       // Summary
       this.log(hr1());
@@ -1033,9 +1116,9 @@ export class SftpPushSyncApp {
       }
       process.exitCode = 1;
       try {
-        // falls hashCache existiert, Cache noch flushen
-        if (this.hashCache?.save) {
-          await this.hashCache.save(true);
+        // falls hashCache existiert, Cache schließen
+        if (this.hashCache?.close) {
+          await this.hashCache.close();
         }
       } catch {
         // ignore
@@ -1061,4 +1144,4 @@ export class SftpPushSyncApp {
       }
     }
   }
-}
+}

package/src/helpers/compare.mjs

@@ -1,15 +1,16 @@
 /**
  * compare.mjs
- * 
+ *
  * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
- * 
- */ 
+ *
+ */
 // src/helpers/compare.mjs
 import fsp from "fs/promises";
 import path from "path";
 
 /**
  * Analysiert Unterschiede zwischen local- und remote-Maps.
+ * Optimiert: Echtes Batch-Processing mit Concurrency-Limit.
  *
  * Erwartete Struktur:
  *  local:  Map<rel, { rel, localPath, size, mtimeMs, isText? }>
@@ -21,6 +22,7 @@ import path from "path";
  *  - getLocalHash / getRemoteHash: from createHashCache
  *  - analyzeChunk: Progress-Schrittgröße
  *  - updateProgress(prefix, current, total, rel): optional
+ *  - concurrency: Max parallele Vergleiche (default: 5)
  */
 export async function analyseDifferences({
   local,
@@ -31,72 +33,104 @@ export async function analyseDifferences({
   getRemoteHash,
   analyzeChunk = 10,
   updateProgress,
+  concurrency = 10,
 }) {
   const toAdd = [];
   const toUpdate = [];
 
-  const localKeys = new Set(local.keys());
-  const totalToCheck = localKeys.size;
+  const localKeys = [...local.keys()];
+  const totalToCheck = localKeys.length;
   let checked = 0;
 
-  for (const rel of localKeys) {
-    checked += 1;
-
-    if (
-      updateProgress &&
-      (checked === 1 || checked % analyzeChunk === 0 || checked === totalToCheck)
-    ) {
-      updateProgress("Analyse: ", checked, totalToCheck, rel);
-    }
+  // Phase 1: Schneller Vorab-Check ohne SFTP
+  // - Dateien nur lokal → direkt zu toAdd
+  // - Size-Vergleich für existierende Dateien
+  const keysNeedContentCompare = [];
 
+  for (const rel of localKeys) {
     const l = local.get(rel);
     const r = remote.get(rel);
     const remotePath = path.posix.join(remoteRoot, rel);
 
-    // Datei existiert nur lokal → New
     if (!r) {
+      // Datei existiert nur lokal → New (kein SFTP-Call nötig)
       toAdd.push({ rel, local: l, remotePath });
-      continue;
+    } else if (l.size !== r.size) {
+      // Size unterschiedlich → Changed (kein SFTP-Call nötig)
+      toUpdate.push({ rel, local: l, remote: r, remotePath });
+    } else {
+      // Size gleich → Content-Vergleich nötig
+      keysNeedContentCompare.push(rel);
     }
 
-    // 1. Size-Vergleich
-    if (l.size !== r.size) {
-      toUpdate.push({ rel, local: l, remote: r, remotePath });
-      continue;
+    checked++;
+    if (updateProgress && checked % analyzeChunk === 0) {
+      updateProgress("Analyse (Size): ", checked, totalToCheck, rel);
     }
+  }
 
-    // 2. Content-Vergleich
-    if (l.isText) {
-      // Text-Datei: vollständiger inhaltlicher Vergleich
-      const [localBuf, remoteBuf] = await Promise.all([
-        fsp.readFile(l.localPath),
-        sftp.get(r.remotePath),
-      ]);
-
-      const localStr = localBuf.toString("utf8");
-      const remoteStr = (
-        Buffer.isBuffer(remoteBuf) ? remoteBuf : Buffer.from(remoteBuf)
-      ).toString("utf8");
-
-      if (localStr !== remoteStr) {
-        toUpdate.push({ rel, local: l, remote: r, remotePath });
-      }
-    } else {
-      // Binary: Hash-Vergleich mit Cache
-      if (!getLocalHash || !getRemoteHash) {
-        // Fallback: wenn kein Hash-Cache übergeben wurde, treat as changed
-        toUpdate.push({ rel, local: l, remote: r, remotePath });
-        continue;
+  // Phase 2: Content-Vergleich in echten Batches
+  // Nur für Dateien mit gleicher Size
+  const totalContentCompare = keysNeedContentCompare.length;
+
+  for (let i = 0; i < totalContentCompare; i += concurrency) {
+    const batch = keysNeedContentCompare.slice(i, i + concurrency);
+
+    const batchResults = await Promise.all(
+      batch.map(async (rel) => {
+        const l = local.get(rel);
+        const r = remote.get(rel);
+        const remotePath = path.posix.join(remoteRoot, rel);
+
+        try {
+          if (l.isText) {
+            // Text-Datei: vollständiger inhaltlicher Vergleich
+            const [localBuf, remoteBuf] = await Promise.all([
+              fsp.readFile(l.localPath),
+              sftp.get(r.remotePath),
+            ]);
+
+            const localStr = localBuf.toString("utf8");
+            const remoteStr = (
+              Buffer.isBuffer(remoteBuf) ? remoteBuf : Buffer.from(remoteBuf)
+            ).toString("utf8");
+
+            return localStr !== remoteStr
+              ? { rel, local: l, remote: r, remotePath, changed: true }
+              : null;
+          } else {
+            // Binary: Hash-Vergleich mit Cache
+            if (!getLocalHash || !getRemoteHash) {
+              return { rel, local: l, remote: r, remotePath, changed: true };
+            }
+
+            const [localHash, remoteHash] = await Promise.all([
+              getLocalHash(rel, l),
+              getRemoteHash(rel, r, sftp),
+            ]);
+
+            return localHash !== remoteHash
+              ? { rel, local: l, remote: r, remotePath, changed: true }
+              : null;
+          }
+        } catch (err) {
+          // Bei Fehler als changed markieren (sicherer)
+          return { rel, local: l, remote: r, remotePath, changed: true };
+        }
+      })
+    );
+
+    // Ergebnisse sammeln
+    for (const result of batchResults) {
+      if (result && result.changed) {
+        toUpdate.push({ rel: result.rel, local: result.local, remote: result.remote, remotePath: result.remotePath });
       }
+    }
 
-      const [localHash, remoteHash] = await Promise.all([
-        getLocalHash(rel, l),
-        getRemoteHash(rel, r, sftp),
-      ]);
-
-      if (localHash !== remoteHash) {
-        toUpdate.push({ rel, local: l, remote: r, remotePath });
-      }
+    // Progress update
+    const progressCount = Math.min(i + batch.length, totalContentCompare);
+    if (updateProgress) {
+      updateProgress("Analyse (Content): ", checked + progressCount, totalToCheck + totalContentCompare, batch[batch.length - 1]);
     }
   }
 
@@ -119,4 +153,4 @@ export function computeRemoteDeletes({ local, remote }) {
   }
 
   return toDelete;
-}
+}

package/src/helpers/hash-cache-leveldb.mjs

@@ -0,0 +1,299 @@
+/**
+ * hash-cache-leveldb.mjs
+ *
+ * LevelDB-based hash cache for efficient storage of large file sets.
+ * Unlike the JSON-based cache, this scales to 100k+ files without memory issues.
+ *
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ */
+import { Level } from "level";
+import fs from "fs";
+import fsp from "fs/promises";
+import { createHash } from "crypto";
+import { Writable } from "stream";
+
+/**
+ * Streaming-SHA256 für lokale Datei
+ */
+export function hashLocalFile(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash("sha256");
+    const stream = fs.createReadStream(filePath);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+/**
+ * Streaming-SHA256 für Remote-Datei via ssh2-sftp-client
+ */
+export async function hashRemoteFile(sftp, remotePath) {
+  const hash = createHash("sha256");
+
+  const writable = new Writable({
+    write(chunk, enc, cb) {
+      hash.update(chunk);
+      cb();
+    },
+  });
+
+  await sftp.get(remotePath, writable);
+  return hash.digest("hex");
+}
+
+/**
+ * Creates a LevelDB-based hash cache.
+ *
+ * Database structure:
+ *   local:<namespace>:<relPath>  → JSON { size, mtimeMs, hash }
+ *   remote:<namespace>:<relPath> → JSON { size, modifyTime, hash }
+ *
+ * @param {Object} options
+ * @param {string} options.cachePath - Path to the LevelDB directory (e.g., ".sync-cache-prod")
+ * @param {string} options.namespace - Namespace for keys (e.g., "prod")
+ */
+export async function createHashCacheLevelDB({ cachePath, namespace }) {
+  const ns = namespace || "default";
+
+  // Open or create the LevelDB database
+  const db = new Level(cachePath, { valueEncoding: "json" });
+  await db.open();
+
+  function localKey(relPath) {
+    return `local:${ns}:${relPath}`;
+  }
+
+  function remoteKey(relPath) {
+    return `remote:${ns}:${relPath}`;
+  }
+
+  /**
+   * Get cached local hash or compute and store it
+   */
+  async function getLocalHash(rel, meta) {
+    const key = localKey(rel);
+
+    try {
+      const cached = await db.get(key);
+
+      // Cache hit: check if still valid (same size + mtime)
+      if (
+        cached &&
+        cached.size === meta.size &&
+        cached.mtimeMs === meta.mtimeMs &&
+        cached.hash
+      ) {
+        return cached.hash;
+      }
+    } catch (err) {
+      // Key not found - that's fine, we'll compute the hash
+      if (err.code !== "LEVEL_NOT_FOUND") {
+        throw err;
+      }
+    }
+
+    // Compute hash and store
+    const hash = await hashLocalFile(meta.localPath);
+    await db.put(key, {
+      size: meta.size,
+      mtimeMs: meta.mtimeMs,
+      hash,
+    });
+
+    return hash;
+  }
+
+  /**
+   * Get cached remote hash or compute and store it
+   */
+  async function getRemoteHash(rel, meta, sftp) {
+    const key = remoteKey(rel);
+
+    try {
+      const cached = await db.get(key);
+
+      // Cache hit: check if still valid (same size + modifyTime)
+      if (
+        cached &&
+        cached.size === meta.size &&
+        cached.modifyTime === meta.modifyTime &&
+        cached.hash
+      ) {
+        return cached.hash;
+      }
+    } catch (err) {
+      // Key not found - compute the hash
+      if (err.code !== "LEVEL_NOT_FOUND") {
+        throw err;
+      }
+    }
+
+    // Compute hash (downloads file content for hashing)
+    const hash = await hashRemoteFile(sftp, meta.remotePath);
+    await db.put(key, {
+      size: meta.size,
+      modifyTime: meta.modifyTime,
+      hash,
+    });
+
+    return hash;
+  }
+
+  /**
+   * Explicitly save (flush) - LevelDB auto-persists, but this ensures sync
+   */
+  async function save() {
+    // LevelDB auto-persists, nothing to do
+  }
+
+  /**
+   * Close the database connection
+   */
+  async function close() {
+    await db.close();
+  }
+
+  /**
+   * Get statistics about cache contents
+   */
+  async function getStats() {
+    let localCount = 0;
+    let remoteCount = 0;
+
+    for await (const key of db.keys()) {
+      if (key.startsWith(`local:${ns}:`)) {
+        localCount++;
+      } else if (key.startsWith(`remote:${ns}:`)) {
+        remoteCount++;
+      }
+    }
+
+    return { localCount, remoteCount };
+  }
+
+  /**
+   * Clean up stale entries (entries for files that no longer exist)
+   * @param {Set<string>} validLocalPaths - Set of currently existing local file paths
+   * @param {Set<string>} validRemotePaths - Set of currently existing remote file paths
+   */
+  async function cleanup(validLocalPaths, validRemotePaths) {
+    const batch = db.batch();
+    let deletedCount = 0;
+
+    for await (const key of db.keys()) {
+      if (key.startsWith(`local:${ns}:`)) {
+        const relPath = key.slice(`local:${ns}:`.length);
+        if (!validLocalPaths.has(relPath)) {
+          batch.del(key);
+          deletedCount++;
+        }
+      } else if (key.startsWith(`remote:${ns}:`)) {
+        const relPath = key.slice(`remote:${ns}:`.length);
+        if (!validRemotePaths.has(relPath)) {
+          batch.del(key);
+          deletedCount++;
+        }
+      }
+    }
+
+    await batch.write();
+    return deletedCount;
+  }
+
+  return {
+    getLocalHash,
+    getRemoteHash,
+    save,
+    close,
+    getStats,
+    cleanup,
+    db, // Expose for advanced usage
+  };
+}
+
+/**
+ * Migrate from old JSON cache to LevelDB
+ * @param {string} jsonCachePath - Path to old .sync-cache.json file
+ * @param {string} levelDbPath - Path to new LevelDB directory
+ * @param {string} namespace - Namespace for keys
+ */
+export async function migrateFromJsonCache(jsonCachePath, levelDbPath, namespace) {
+  const ns = namespace || "default";
+
+  // Check if JSON cache exists
+  try {
+    await fsp.access(jsonCachePath);
+  } catch {
+    return { migrated: false, reason: "No JSON cache found" };
+  }
+
+  // Check if LevelDB already has data (don't re-migrate)
+  const db = new Level(levelDbPath, { valueEncoding: "json" });
+  await db.open();
+
+  let hasData = false;
+  for await (const _ of db.keys({ limit: 1 })) {
+    hasData = true;
+    break;
+  }
+
+  if (hasData) {
+    await db.close();
+    return { migrated: false, reason: "LevelDB already has data" };
+  }
+
+  // Read and parse JSON cache
+  let jsonCache;
+  try {
+    const raw = await fsp.readFile(jsonCachePath, "utf8");
+    jsonCache = JSON.parse(raw);
+  } catch (parseErr) {
+    await db.close();
+    // Rename corrupt file so it doesn't block future runs
+    try {
+      await fsp.rename(jsonCachePath, jsonCachePath + ".corrupt");
+    } catch {
+      // Ignore rename errors
+    }
+    return { migrated: false, reason: `JSON cache corrupt: ${parseErr.message}` };
+  }
+
+  // Migrate entries
+  const batch = db.batch();
+  let localCount = 0;
+  let remoteCount = 0;
+
+  if (jsonCache.local) {
+    for (const [key, value] of Object.entries(jsonCache.local)) {
+      // Keys in JSON were like "namespace:relPath"
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      batch.put(`local:${ns}:${relPath}`, value);
+      localCount++;
+    }
+  }
+
+  if (jsonCache.remote) {
+    for (const [key, value] of Object.entries(jsonCache.remote)) {
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      batch.put(`remote:${ns}:${relPath}`, value);
+      remoteCount++;
+    }
+  }
+
+  await batch.write();
+  await db.close();
+
+  // Optionally rename old cache to .bak
+  try {
+    await fsp.rename(jsonCachePath, jsonCachePath + ".bak");
+  } catch {
+    // Ignore rename errors
+  }
+
+  return {
+    migrated: true,
+    localCount,
+    remoteCount,
+  };
+}

package/src/helpers/hash-cache-ndjson.mjs

@@ -0,0 +1,412 @@
+/**
+ * hash-cache-ndjson.mjs
+ *
+ * NDJSON-based hash cache for efficient storage of large file sets.
+ * Each line is a valid JSON object - human-readable and easy to debug.
+ * Scales to 100k+ files through streaming reads/writes.
+ *
+ * File format:
+ *   {"t":"l","p":"prod:path/file.js","s":1234,"m":1234567890,"h":"sha256..."}
+ *   {"t":"r","p":"prod:path/file.js","s":1234,"m":"2025-01-01","h":"sha256..."}
+ *
+ * Where: t=type (l=local, r=remote), p=path, s=size, m=mtime, h=hash
+ *
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ */
+import fs from "fs";
+import fsp from "fs/promises";
+import { createHash } from "crypto";
+import { createReadStream, createWriteStream } from "fs";
+import { createInterface } from "readline";
+import { Writable } from "stream";
+
+/**
+ * Streaming-SHA256 für lokale Datei
+ */
+export function hashLocalFile(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash("sha256");
+    const stream = fs.createReadStream(filePath);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+/**
+ * Streaming-SHA256 für Remote-Datei via ssh2-sftp-client
+ */
+export async function hashRemoteFile(sftp, remotePath) {
+  const hash = createHash("sha256");
+
+  const writable = new Writable({
+    write(chunk, enc, cb) {
+      hash.update(chunk);
+      cb();
+    },
+  });
+
+  await sftp.get(remotePath, writable);
+  return hash.digest("hex");
+}
+
+/**
+ * Creates an NDJSON-based hash cache.
+ *
+ * Data is stored in Maps during runtime and persisted as NDJSON on save().
+ * Auto-saves every 1000 changes to prevent data loss on crash/abort.
+ *
+ * @param {Object} options
+ * @param {string} options.cachePath - Path to the NDJSON file (e.g., ".sync-cache.prod.ndjson")
+ * @param {string} options.namespace - Namespace for keys (e.g., "prod")
+ * @param {number} options.autoSaveInterval - Save after this many changes (default: 1000)
+ */
+export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInterval = 1000 }) {
+  const ns = namespace || "default";
+
+  // In-memory storage
+  const localCache = new Map();
+  const remoteCache = new Map();
+
+  // Auto-save tracking
+  let changesSinceLastSave = 0;
+  let saveInProgress = false;
+
+  // Load existing cache if present
+  await loadCache();
+
+  /**
+   * Load cache from NDJSON file
+   */
+  async function loadCache() {
+    try {
+      await fsp.access(cachePath);
+    } catch {
+      // File doesn't exist - start fresh
+      return;
+    }
+
+    const fileStream = createReadStream(cachePath, { encoding: "utf8" });
+    const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
+
+    let lineNum = 0;
+    for await (const line of rl) {
+      lineNum++;
+      if (!line.trim()) continue;
+
+      try {
+        const entry = JSON.parse(line);
+        const key = `${ns}:${entry.p}`;
+
+        if (entry.t === "l") {
+          localCache.set(key, {
+            size: entry.s,
+            mtimeMs: entry.m,
+            hash: entry.h,
+          });
+        } else if (entry.t === "r") {
+          remoteCache.set(key, {
+            size: entry.s,
+            modifyTime: entry.m,
+            hash: entry.h,
+          });
+        }
+      } catch (parseErr) {
+        // Skip corrupt lines, log warning
+        console.warn(`  ⚠ Skipping corrupt cache line ${lineNum}: ${parseErr.message}`);
+      }
+    }
+  }
+
+  function localKey(relPath) {
+    return `${ns}:${relPath}`;
+  }
+
+  function remoteKey(relPath) {
+    return `${ns}:${relPath}`;
+  }
+
+  /**
+   * Check if auto-save is needed and perform it
+   */
+  async function checkAutoSave() {
+    changesSinceLastSave++;
+    if (changesSinceLastSave >= autoSaveInterval && !saveInProgress) {
+      saveInProgress = true;
+      try {
+        await save();
+        changesSinceLastSave = 0;
+      } finally {
+        saveInProgress = false;
+      }
+    }
+  }
+
+  /**
+   * Get cached local hash or compute and store it
+   */
+  async function getLocalHash(rel, meta) {
+    const key = localKey(rel);
+    const cached = localCache.get(key);
+
+    // Cache hit: check if still valid (same size + mtime)
+    if (
+      cached &&
+      cached.size === meta.size &&
+      Math.abs(cached.mtimeMs - meta.mtimeMs) < 1000
+    ) {
+      return cached.hash;
+    }
+
+    // Cache miss or stale: compute new hash
+    const filePath = meta.fullPath || meta.localPath;
+    const hash = await hashLocalFile(filePath);
+
+    localCache.set(key, {
+      size: meta.size,
+      mtimeMs: meta.mtimeMs,
+      hash,
+    });
+
+    // Auto-save periodically
+    await checkAutoSave();
+
+    return hash;
+  }
+
+  /**
+   * Get cached remote hash or compute and store it
+   */
+  async function getRemoteHash(rel, meta, sftp) {
+    const key = remoteKey(rel);
+    const cached = remoteCache.get(key);
+
+    // Cache hit: check if still valid (same size + modifyTime)
+    if (
+      cached &&
+      cached.size === meta.size &&
+      cached.modifyTime === meta.modifyTime
+    ) {
+      return cached.hash;
+    }
+
+    // Cache miss or stale: compute new hash
+    const filePath = meta.fullPath || meta.remotePath;
+    const hash = await hashRemoteFile(sftp, filePath);
+
+    remoteCache.set(key, {
+      size: meta.size,
+      modifyTime: meta.modifyTime,
+      hash,
+    });
+
+    // Auto-save periodically
+    await checkAutoSave();
+
+    return hash;
+  }
+
+  /**
+   * Save cache to NDJSON file using streaming writes
+   */
+  async function save() {
+    const tempPath = cachePath + ".tmp";
+    const writeStream = createWriteStream(tempPath, { encoding: "utf8" });
+
+    // Write local entries
+    for (const [fullKey, value] of localCache) {
+      // Extract relPath from "namespace:relPath"
+      const relPath = fullKey.startsWith(`${ns}:`)
+        ? fullKey.slice(ns.length + 1)
+        : fullKey;
+
+      const line = JSON.stringify({
+        t: "l",
+        p: relPath,
+        s: value.size,
+        m: value.mtimeMs,
+        h: value.hash,
+      });
+      writeStream.write(line + "\n");
+    }
+
+    // Write remote entries
+    for (const [fullKey, value] of remoteCache) {
+      const relPath = fullKey.startsWith(`${ns}:`)
+        ? fullKey.slice(ns.length + 1)
+        : fullKey;
+
+      const line = JSON.stringify({
+        t: "r",
+        p: relPath,
+        s: value.size,
+        m: value.modifyTime,
+        h: value.hash,
+      });
+      writeStream.write(line + "\n");
+    }
+
+    // Wait for stream to finish
+    await new Promise((resolve, reject) => {
+      writeStream.on("finish", resolve);
+      writeStream.on("error", reject);
+      writeStream.end();
+    });
+
+    // Atomic rename
+    await fsp.rename(tempPath, cachePath);
+  }
+
+  /**
+   * Close the cache (no-op for NDJSON, but keeps API compatible)
+   */
+  async function close() {
+    // Nothing to do - Maps are garbage collected
+  }
+
+  /**
+   * Get cache statistics
+   */
+  function getStats() {
+    return {
+      localEntries: localCache.size,
+      remoteEntries: remoteCache.size,
+      totalEntries: localCache.size + remoteCache.size,
+    };
+  }
+
+  /**
+   * Remove entries for files that no longer exist
+   * @param {Set<string>} currentLocalFiles - Set of current local relative paths
+   * @param {Set<string>} currentRemoteFiles - Set of current remote relative paths
+   */
+  async function cleanup(currentLocalFiles, currentRemoteFiles) {
+    let deletedCount = 0;
+
+    // Clean local entries
+    for (const key of localCache.keys()) {
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      if (!currentLocalFiles.has(relPath)) {
+        localCache.delete(key);
+        deletedCount++;
+      }
+    }
+
+    // Clean remote entries
+    for (const key of remoteCache.keys()) {
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      if (!currentRemoteFiles.has(relPath)) {
+        remoteCache.delete(key);
+        deletedCount++;
+      }
+    }
+
+    return deletedCount;
+  }
+
+  return {
+    getLocalHash,
+    getRemoteHash,
+    save,
+    close,
+    getStats,
+    cleanup,
+  };
+}
+
+/**
+ * Migrate from old JSON cache to NDJSON
+ * @param {string} jsonCachePath - Path to old .sync-cache.json file
+ * @param {string} ndjsonPath - Path to new .ndjson file
+ * @param {string} namespace - Namespace for keys
+ */
+export async function migrateFromJsonCache(jsonCachePath, ndjsonPath, namespace) {
+  const ns = namespace || "default";
+
+  // Check if JSON cache exists
+  try {
+    await fsp.access(jsonCachePath);
+  } catch {
+    return { migrated: false, reason: "No JSON cache found" };
+  }
+
+  // Check if NDJSON already exists (don't re-migrate)
+  try {
+    const stats = await fsp.stat(ndjsonPath);
+    if (stats.size > 0) {
+      return { migrated: false, reason: "NDJSON cache already exists" };
+    }
+  } catch {
+    // File doesn't exist - proceed with migration
+  }
+
+  // Read and parse JSON cache
+  let jsonCache;
+  try {
+    const raw = await fsp.readFile(jsonCachePath, "utf8");
+    jsonCache = JSON.parse(raw);
+  } catch (parseErr) {
+    // Rename corrupt file
+    try {
+      await fsp.rename(jsonCachePath, jsonCachePath + ".corrupt");
+    } catch {
+      // Ignore rename errors
+    }
+    return { migrated: false, reason: `JSON cache corrupt: ${parseErr.message}` };
+  }
+
+  // Write to NDJSON
+  const writeStream = createWriteStream(ndjsonPath, { encoding: "utf8" });
+  let localCount = 0;
+  let remoteCount = 0;
+
+  if (jsonCache.local) {
+    for (const [key, value] of Object.entries(jsonCache.local)) {
+      // Keys in JSON were like "namespace:relPath"
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      const line = JSON.stringify({
+        t: "l",
+        p: relPath,
+        s: value.size,
+        m: value.mtimeMs,
+        h: value.hash,
+      });
+      writeStream.write(line + "\n");
+      localCount++;
+    }
+  }
+
+  if (jsonCache.remote) {
+    for (const [key, value] of Object.entries(jsonCache.remote)) {
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      const line = JSON.stringify({
+        t: "r",
+        p: relPath,
+        s: value.size,
+        m: value.modifyTime,
+        h: value.hash,
+      });
+      writeStream.write(line + "\n");
+      remoteCount++;
+    }
+  }
+
+  await new Promise((resolve, reject) => {
+    writeStream.on("finish", resolve);
+    writeStream.on("error", reject);
+    writeStream.end();
+  });
+
+  // Rename old cache to .bak
+  try {
+    await fsp.rename(jsonCachePath, jsonCachePath + ".migrated");
+  } catch {
+    // Ignore rename errors
+  }
+
+  return {
+    migrated: true,
+    localCount,
+    remoteCount,
+  };
+}

package/src/helpers/hashing.mjs

@@ -1,9 +1,9 @@
 /**
  * hashing.mjs
- * 
+ *
  * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
- * 
- */ 
+ *
+ */
 // src/helpers/hashing.mjs
 import fs from "fs";
 import fsp from "fs/promises";
@@ -53,7 +53,7 @@ export async function hashRemoteFile(sftp, remotePath) {
 export function createHashCache({
   cachePath,
   namespace,
-  flushInterval = 50,
+  flushInterval = 25,  // Save more frequently to release memory
 }) {
   const ns = namespace || "default";
 
@@ -85,8 +85,40 @@ export function createHashCache({
 
   async function save(force = false) {
     if (!dirty && !force) return;
-    const data = JSON.stringify(cache, null, 2);
-    await fsp.writeFile(cachePath, data, "utf8");
+
+    // Stream-basiertes Schreiben für große Caches
+    // Verwendet for...in statt Object.keys() um Speicher zu sparen
+    const fd = await fsp.open(cachePath, 'w');
+    try {
+      await fd.write('{"version":1,"local":{');
+
+      let firstLocal = true;
+      for (const key in cache.local) {
+        if (Object.prototype.hasOwnProperty.call(cache.local, key)) {
+          const entry = cache.local[key];
+          const line = `${JSON.stringify(key)}:${JSON.stringify(entry)}`;
+          await fd.write(firstLocal ? line : ',' + line);
+          firstLocal = false;
+        }
+      }
+
+      await fd.write('},"remote":{');
+
+      let firstRemote = true;
+      for (const key in cache.remote) {
+        if (Object.prototype.hasOwnProperty.call(cache.remote, key)) {
+          const entry = cache.remote[key];
+          const line = `${JSON.stringify(key)}:${JSON.stringify(entry)}`;
+          await fd.write(firstRemote ? line : ',' + line);
+          firstRemote = false;
+        }
+      }
+
+      await fd.write('}}');
+    } finally {
+      await fd.close();
+    }
+
     dirty = false;
     dirtyCount = 0;
   }
@@ -198,4 +230,4 @@ export async function getRemoteHash(rel, meta, cacheRemote, key, markDirty, sftp
     await markDirty();
   }
   return hash;
-}
+}

package/src/helpers/walkers.mjs

@@ -1,9 +1,9 @@
 /**
  * walkers.mjs
- * 
+ *
  * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
- * 
- */ 
+ *
+ */
 // src/helpers/walkers.mjs
 import fsp from "fs/promises";
 import path from "path";
@@ -116,6 +116,7 @@ export async function walkLocalPlain(root) {
 
 /**
  * Remote-Walker mit INCLUDE/EXCLUDE über filterFn
+ * Optimiert: Parallelisierte Verzeichnis-Traversierung
  */
 export async function walkRemote(
   sftp,
@@ -125,13 +126,43 @@ export async function walkRemote(
     progress = null,
     scanChunk = 100,
     log = null,
+    concurrency = 5,  // Max parallel directory listings
   } = {}
 ) {
   const result = new Map();
   let scanned = 0;
 
+  // Semaphore für Concurrency-Kontrolle
+  let activeCount = 0;
+  const waiting = [];
+
+  async function acquireSemaphore() {
+    if (activeCount < concurrency) {
+      activeCount++;
+      return;
+    }
+    await new Promise((resolve) => waiting.push(resolve));
+    activeCount++;
+  }
+
+  function releaseSemaphore() {
+    activeCount--;
+    if (waiting.length > 0) {
+      const next = waiting.shift();
+      next();
+    }
+  }
+
   async function recurse(remoteDir, prefix) {
-    const items = await sftp.list(remoteDir);
+    await acquireSemaphore();
+    let items;
+    try {
+      items = await sftp.list(remoteDir);
+    } finally {
+      releaseSemaphore();
+    }
+
+    const subdirPromises = [];
 
     for (const item of items) {
       if (!item.name || item.name === "." || item.name === "..") continue;
@@ -142,7 +173,8 @@ export async function walkRemote(
       if (filterFn && !filterFn(rel)) continue;
 
       if (item.type === "d") {
-        await recurse(full, rel);
+        // Parallele Verarbeitung von Unterverzeichnissen
+        subdirPromises.push(recurse(full, rel));
       } else {
         result.set(rel, {
           rel,
@@ -166,6 +198,9 @@ export async function walkRemote(
         }
       }
     }
+
+    // Warte auf alle Unterverzeichnisse parallel
+    await Promise.all(subdirPromises);
   }
 
   await recurse(remoteRoot, "");
@@ -215,4 +250,4 @@ export async function walkRemotePlain(sftp, remoteRoot) {
 
   await recurse(remoteRoot, "");
   return result;
-}
+}