sftp-push-sync 2.1.5 → 2.5.0

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [2.4.0] - 2026-03-04
+
+- Parallel remote walker walkers.mjs: scans 8 directories simultaneously
+- Batch analysis with concurrency compare.mjs: 8 file comparisons in parallel
+- Parallel hash calculation: local + remote hash simultaneously
+- Keep-alive: SftpPushSyncApp.mjs prevents server disconnection
+
+## [2.3.0] - 2026-03-04
+
+- Keep-Alive enabled - a Keep-Alive packet is sent every 10 seconds.
+
 ## [2.1.0] - 2025-11-19
 
 Sync only handles files and creates missing directories during upload.

package/README.md

@@ -157,7 +157,7 @@ There are 7 steps to follow:
 
 - Phase 1: Scan local files
 - Phase 2: Scan remote files
-- Phase 3: Compare & decide
+- Phase 3: Compare & Decide
 - Phase 4: Removing orphaned remote files
 - Phase 5: Preparing remote directories
 - Phase 6: Apply changes
@@ -258,7 +258,8 @@ However, it should also manage directories:
 
 You can safely delete the local cache at any time. The first analysis will then take longer, because remote hashes will be streamed again. After that, everything will run fast.
 
-Note: The first run always takes a while, especially with lots of media – so be patient! Once the cache is full, it will be faster.
+Note 1: The first run always takes a while, especially with lots of media – so be patient! Once the cache is full, it will be faster. 
+Note 2: Reliability and accuracy are more important to me than speed.
 
 ## Example Output
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sftp-push-sync",
-  "version": "2.1.5",
+  "version": "2.5.0",
   "description": "SFTP sync tool for Hugo projects (local to remote, with hash cache)",
   "type": "module",
   "bin": {

package/src/core/SftpPushSyncApp.mjs

@@ -730,8 +730,18 @@ export class SftpPushSyncApp {
         port: this.connection.port,
         username: this.connection.user,
         password: this.connection.password,
+        // Keep-Alive to prevent server disconnection during long operations
+        keepaliveInterval: 10000,  // Send keepalive every 10 seconds
+        keepaliveCountMax: 10,     // Allow up to 10 missed keepalives before disconnect
+        readyTimeout: 30000,       // 30s timeout for initial connection
       });
       connected = true;
+      
+      // Increase max listeners for parallel operations
+      if (sftp.client) {
+        sftp.client.setMaxListeners(50);
+      }
+      
       this.log(`${TAB_A}${pc.green("✔ Connected to SFTP.")}`);
 
       if (!skipSync && !fs.existsSync(this.connection.localRoot)) {
@@ -838,7 +848,7 @@ export class SftpPushSyncApp {
       this.log("");
 
       // Phase 3 – Analyse Differences (delegiert an Helper)
-      this.log(pc.bold(pc.cyan("🔎 Phase 3: Compare & decide …")));
+      this.log(pc.bold(pc.cyan("🔎 Phase 3: Compare & Decide …")));
 
       const { getLocalHash, getRemoteHash, save: saveCache } = this.hashCache;
 

package/src/helpers/compare.mjs

@@ -10,6 +10,7 @@ import path from "path";
 
 /**
  * Analysiert Unterschiede zwischen local- und remote-Maps.
+ * Optimiert: Parallelisierte Analyse mit Concurrency-Limit.
  *
  * Erwartete Struktur:
  *  local:  Map<rel, { rel, localPath, size, mtimeMs, isText? }>
@@ -21,6 +22,7 @@ import path from "path";
  *  - getLocalHash / getRemoteHash: from createHashCache
  *  - analyzeChunk: Progress-Schrittgröße
  *  - updateProgress(prefix, current, total, rel): optional
+ *  - concurrency: Max parallele Vergleiche (default: 8)
  */
 export async function analyseDifferences({
   local,
@@ -31,75 +33,114 @@ export async function analyseDifferences({
   getRemoteHash,
   analyzeChunk = 10,
   updateProgress,
+  concurrency = 5,
 }) {
   const toAdd = [];
   const toUpdate = [];
 
-  const localKeys = new Set(local.keys());
-  const totalToCheck = localKeys.size;
+  const localKeys = [...local.keys()];
+  const totalToCheck = localKeys.length;
   let checked = 0;
 
+  // Schneller Vorab-Check: Dateien nur lokal → direkt zu toAdd
+  const keysToCompare = [];
   for (const rel of localKeys) {
-    checked += 1;
-
-    if (
-      updateProgress &&
-      (checked === 1 || checked % analyzeChunk === 0 || checked === totalToCheck)
-    ) {
-      updateProgress("Analyse: ", checked, totalToCheck, rel);
-    }
-
-    const l = local.get(rel);
     const r = remote.get(rel);
     const remotePath = path.posix.join(remoteRoot, rel);
-
-    // Datei existiert nur lokal → New
+    
     if (!r) {
-      toAdd.push({ rel, local: l, remotePath });
-      continue;
+      // Datei existiert nur lokal → New (kein SFTP-Call nötig)
+      toAdd.push({ rel, local: local.get(rel), remotePath });
+      checked++;
+      if (updateProgress && checked % analyzeChunk === 0) {
+        updateProgress("Analyse: ", checked, totalToCheck, rel);
+      }
+    } else {
+      keysToCompare.push(rel);
     }
+  }
 
-    // 1. Size-Vergleich
-    if (l.size !== r.size) {
-      toUpdate.push({ rel, local: l, remote: r, remotePath });
-      continue;
+  // Parallele Verarbeitung mit Semaphore
+  let activeCount = 0;
+  const waiting = [];
+
+  async function acquireSemaphore() {
+    if (activeCount < concurrency) {
+      activeCount++;
+      return;
     }
+    await new Promise((resolve) => waiting.push(resolve));
+    activeCount++;
+  }
 
-    // 2. Content-Vergleich
-    if (l.isText) {
-      // Text-Datei: vollständiger inhaltlicher Vergleich
-      const [localBuf, remoteBuf] = await Promise.all([
-        fsp.readFile(l.localPath),
-        sftp.get(r.remotePath),
-      ]);
+  function releaseSemaphore() {
+    activeCount--;
+    if (waiting.length > 0) {
+      const next = waiting.shift();
+      next();
+    }
+  }
 
-      const localStr = localBuf.toString("utf8");
-      const remoteStr = (
-        Buffer.isBuffer(remoteBuf) ? remoteBuf : Buffer.from(remoteBuf)
-      ).toString("utf8");
+  async function compareFile(rel) {
+    await acquireSemaphore();
+    try {
+      const l = local.get(rel);
+      const r = remote.get(rel);
+      const remotePath = path.posix.join(remoteRoot, rel);
 
-      if (localStr !== remoteStr) {
-        toUpdate.push({ rel, local: l, remote: r, remotePath });
-      }
-    } else {
-      // Binary: Hash-Vergleich mit Cache
-      if (!getLocalHash || !getRemoteHash) {
-        // Fallback: wenn kein Hash-Cache übergeben wurde, treat as changed
+      // 1. Size-Vergleich (schnell, kein SFTP)
+      if (l.size !== r.size) {
         toUpdate.push({ rel, local: l, remote: r, remotePath });
-        continue;
+        return;
       }
 
-      const [localHash, remoteHash] = await Promise.all([
-        getLocalHash(rel, l),
-        getRemoteHash(rel, r, sftp),
-      ]);
-
-      if (localHash !== remoteHash) {
-        toUpdate.push({ rel, local: l, remote: r, remotePath });
+      // 2. Content-Vergleich
+      if (l.isText) {
+        // Text-Datei: vollständiger inhaltlicher Vergleich
+        const [localBuf, remoteBuf] = await Promise.all([
+          fsp.readFile(l.localPath),
+          sftp.get(r.remotePath),
+        ]);
+
+        const localStr = localBuf.toString("utf8");
+        const remoteStr = (
+          Buffer.isBuffer(remoteBuf) ? remoteBuf : Buffer.from(remoteBuf)
+        ).toString("utf8");
+
+        if (localStr !== remoteStr) {
+          toUpdate.push({ rel, local: l, remote: r, remotePath });
+        }
+      } else {
+        // Binary: Hash-Vergleich mit Cache
+        if (!getLocalHash || !getRemoteHash) {
+          toUpdate.push({ rel, local: l, remote: r, remotePath });
+          return;
+        }
+
+        const [localHash, remoteHash] = await Promise.all([
+          getLocalHash(rel, l),
+          getRemoteHash(rel, r, sftp),
+        ]);
+
+        if (localHash !== remoteHash) {
+          toUpdate.push({ rel, local: l, remote: r, remotePath });
+        }
+      }
+    } finally {
+      releaseSemaphore();
+      checked++;
+      if (
+        updateProgress &&
+        (checked === 1 || checked % analyzeChunk === 0 || checked === totalToCheck)
+      ) {
+        updateProgress("Analyse: ", checked, totalToCheck, rel);
       }
     }
   }
 
+  // Starte alle Vergleiche parallel (mit Concurrency-Limit durch Semaphore)
+  await Promise.all(keysToCompare.map(compareFile));
+
   return { toAdd, toUpdate };
 }
 

package/src/helpers/walkers.mjs

@@ -116,6 +116,7 @@ export async function walkLocalPlain(root) {
 
 /**
  * Remote-Walker mit INCLUDE/EXCLUDE über filterFn
+ * Optimiert: Parallelisierte Verzeichnis-Traversierung
  */
 export async function walkRemote(
   sftp,
@@ -125,13 +126,43 @@ export async function walkRemote(
     progress = null,
     scanChunk = 100,
     log = null,
+    concurrency = 5,  // Max parallel directory listings
   } = {}
 ) {
   const result = new Map();
   let scanned = 0;
 
+  // Semaphore für Concurrency-Kontrolle
+  let activeCount = 0;
+  const waiting = [];
+
+  async function acquireSemaphore() {
+    if (activeCount < concurrency) {
+      activeCount++;
+      return;
+    }
+    await new Promise((resolve) => waiting.push(resolve));
+    activeCount++;
+  }
+
+  function releaseSemaphore() {
+    activeCount--;
+    if (waiting.length > 0) {
+      const next = waiting.shift();
+      next();
+    }
+  }
+
   async function recurse(remoteDir, prefix) {
-    const items = await sftp.list(remoteDir);
+    await acquireSemaphore();
+    let items;
+    try {
+      items = await sftp.list(remoteDir);
+    } finally {
+      releaseSemaphore();
+    }
+
+    const subdirPromises = [];
 
     for (const item of items) {
       if (!item.name || item.name === "." || item.name === "..") continue;
@@ -142,7 +173,8 @@ export async function walkRemote(
       if (filterFn && !filterFn(rel)) continue;
 
       if (item.type === "d") {
-        await recurse(full, rel);
+        // Parallele Verarbeitung von Unterverzeichnissen
+        subdirPromises.push(recurse(full, rel));
       } else {
         result.set(rel, {
           rel,
@@ -166,6 +198,9 @@ export async function walkRemote(
         }
       }
     }
+
+    // Warte auf alle Unterverzeichnisse parallel
+    await Promise.all(subdirPromises);
   }
 
   await recurse(remoteRoot, "");