sftp-push-sync 3.0.0 → 3.0.2

package/CHANGELOG.md

@@ -1,12 +1,16 @@
 # Changelog
 
+## [3.0.2] - 2026-03-05
+
+- stability improvements especialy during large and longtime uploads, error handling, log with datetime.
+
 ## [3.0.0] - 2026-03-04
 
 - Switched from JSON-file based hash cache to NDJSON-based Cache-implementation.
 - Disk-based, only active entries in RAM
 - Scales to 100,000+ files without memory issues
 - Auto-persist (no explicit saving required)
-- Auto-migration - Existing JSON cache (.sync-cache.prod.json) is automatically migrated to LevelDB (.sync-cache-prod/)
+- Auto-migration - Existing JSON cache is automatically migrated
 
 ## [2.5.0] - 2026-03-04
 

package/README.md

@@ -19,7 +19,7 @@ Features:
 - adds, updates, deletes files
 - text diff detection
 - Binary files (images, video, audio, PDF, etc.): SHA-256 hash comparison
-- Hashes are cached in .sync-cache.*.json
+- Hashes are cached in `.sync-cache.*.ndjson`
 - Parallel uploads/deletions via worker pool
 - include/exclude patterns
 - Sidecar uploads / downloads - Bypassing the sync process
@@ -104,6 +104,7 @@ Create a `sync.config.json` in the root folder of your project:
     "analyzeChunk": 1
   },
   "logLevel": "normal",
+  "logTimestamps": false,
   "logFile": ".sftp-push-sync.{target}.log"
 }
 ```
@@ -204,6 +205,7 @@ sftp-push-sync prod --sidecar-download --skip-sync
 Logging can also be configured.
 
 - `logLevel` - normal, verbose, laconic.
+- `logTimestamps` - true/false. When enabled, each log line is prefixed with a timestamp `[YYYY-MM-DD HH:mm:ss.SSS]`.
 - `logFile` - an optional logFile.
 - `scanChunk` - After how many elements should a log output be generated during scanning?
 - `analyzeChunk` - After how many elements should a log output be generated during analysis?
@@ -275,4 +277,4 @@ Note 2: Reliability and accuracy are more important to me than speed.
 - <https://www.npmjs.com/package/sftp-push-sync>
 - <https://github.com/cnichte/sftp-push-sync>
 - <https://www.npmjs.com/package/hugo-toolbox>
-- <https://carsten-nichte.de>
+- <https://carsten-nichte.de>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sftp-push-sync",
-  "version": "3.0.0",
+  "version": "3.0.2",
   "description": "SFTP sync tool for Hugo projects (local to remote, with hash cache)",
   "type": "module",
   "bin": {

package/src/core/SftpPushSyncApp.mjs

@@ -200,36 +200,103 @@ export class SftpPushSyncApp {
       // Try a minimal operation to check connection
       await sftp.cwd();
       return true;
-    } catch {
+    } catch (e) {
+      if (this.isVerbose) {
+        this.vlog(`${TAB_A}${pc.dim(`Connection check failed: ${e?.message || e}`)}`);
+      }
       return false;
     }
   }
 
   /**
-   * Reconnect to SFTP server
+   * Reconnect to SFTP server with retry logic
    */
-  async _reconnect(sftp) {
-    try {
-      await sftp.end();
-    } catch {
-      // Ignore errors when closing dead connection
-    }
+  async _reconnect(sftp, maxRetries = 3) {
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        try {
+          await sftp.end();
+        } catch (e) {
+          // Ignore errors when closing dead connection
+          if (this.isVerbose) {
+            this.vlog(`${TAB_A}${pc.dim(`Closing old connection failed (expected): ${e?.message || e}`)}`);
+          }
+        }
 
-    await sftp.connect({
-      host: this.connection.host,
-      port: this.connection.port,
-      username: this.connection.user,
-      password: this.connection.password,
-      keepaliveInterval: 10000,
-      keepaliveCountMax: 10,
-      readyTimeout: 30000,
-    });
+        // Wait before reconnecting (exponential backoff)
+        if (attempt > 1) {
+          const waitTime = 1000 * Math.pow(2, attempt - 1); // 2s, 4s, 8s
+          this.log(`${TAB_A}${pc.yellow(`⏳ Waiting ${waitTime/1000}s before retry ${attempt}/${maxRetries}…`)}`);
+          await new Promise(r => setTimeout(r, waitTime));
+        }
+
+        await sftp.connect({
+          host: this.connection.host,
+          port: this.connection.port,
+          username: this.connection.user,
+          password: this.connection.password,
+          keepaliveInterval: 5000,     // More frequent keepalive (5s instead of 10s)
+          keepaliveCountMax: 6,        // Disconnect after 30s of no response
+          readyTimeout: 60000,         // 60s timeout for initial connection
+          retries: 2,                  // Internal retries
+          retry_factor: 2,
+          retry_minTimeout: 2000,
+        });
+
+        if (sftp.client) {
+          sftp.client.setMaxListeners(50);
+        }
 
-    if (sftp.client) {
-      sftp.client.setMaxListeners(50);
+        this.log(`${TAB_A}${pc.green("✔ Reconnected to SFTP.")}`);
+        return; // Success
+      } catch (err) {
+        const msg = err?.message || String(err);
+        if (attempt === maxRetries) {
+          this.elog(pc.red(`❌ Failed to reconnect after ${maxRetries} attempts: ${msg}`));
+          throw err;
+        }
+        this.wlog(pc.yellow(`⚠ Reconnect attempt ${attempt} failed: ${msg}`));
+      }
+    }
+  }
+
+  /**
+   * Upload a file with progress reporting for large files.
+   * Uses fastPut for files > threshold, with automatic fallback to put on failure.
+   */
+  async _uploadFile(sftp, localPath, remotePath, rel, size) {
+    const LARGE_FILE_THRESHOLD = 5 * 1024 * 1024; // 5MB
+    const sizeMB = (size / (1024 * 1024)).toFixed(1);
+    
+    // For small files, just use put
+    if (size < LARGE_FILE_THRESHOLD) {
+      await sftp.put(localPath, remotePath);
+      return;
     }
 
-    this.log(`${TAB_A}${pc.green("✔ Reconnected to SFTP.")}`);
+    // For large files, try fastPut with progress
+    let lastReportedPercent = 0;
+    const shortRel = rel.length > 50 ? '...' + rel.slice(-47) : rel;
+
+    try {
+      await sftp.fastPut(localPath, remotePath, {
+        step: (transferred, chunk, total) => {
+          const percent = Math.floor((transferred / total) * 100);
+          // Only log at 25%, 50%, 75%, 100%
+          if (percent >= lastReportedPercent + 25) {
+            lastReportedPercent = Math.floor(percent / 25) * 25;
+            this.log(`${TAB_A}${pc.dim(`  ↑ ${sizeMB}MB ${percent}%: ${shortRel}`)}`);
+          }
+        }
+      });
+    } catch (fastPutErr) {
+      // fastPut not supported by server, fall back to regular put
+      if (this.isVerbose) {
+        this.vlog(`${TAB_A}${pc.dim(`  fastPut failed, using put: ${fastPutErr?.message}`)}`);
+      }
+      this.log(`${TAB_A}${pc.dim(`  Uploading ${sizeMB}MB: ${shortRel}`)}`);
+      await sftp.put(localPath, remotePath);
+    }
   }
 
   // ---------------------------------------------------------
@@ -333,18 +400,23 @@ export class SftpPushSyncApp {
   }
 
   // ---------------------------------------------------------
-  // Worker-Pool
+  // Worker-Pool with auto-reconnect
   // ---------------------------------------------------------
 
-  async runTasks(items, workerCount, handler, label = "Tasks") {
+  async runTasks(items, workerCount, handler, label = "Tasks", sftp = null) {
     if (!items || items.length === 0) return;
 
     const total = items.length;
     let done = 0;
     let index = 0;
+    let failedCount = 0;
     const workers = [];
     const actualWorkers = Math.max(1, Math.min(workerCount, total));
 
+    // Mutex for reconnection (only one worker reconnects at a time)
+    let reconnecting = false;
+    let reconnectWaiters = 0;
+
     const worker = async () => {
       // eslint-disable-next-line no-constant-condition
       while (true) {
@@ -353,13 +425,82 @@ export class SftpPushSyncApp {
         index += 1;
         const item = items[i];
 
-        try {
-          await handler(item);
-        } catch (err) {
-          this.elog(
-            pc.red(`${TAB_A}⚠️ Error in ${label}:`),
-            err?.message || err
-          );
+        let retries = 0;
+        const maxRetries = 5; // Increased from 2 to 5 for unstable servers
+
+        while (retries <= maxRetries) {
+          try {
+            await handler(item);
+            break; // Success, exit retry loop
+          } catch (err) {
+            const msg = err?.message || String(err);
+            const isConnectionError =
+              msg.includes("No SFTP connection") ||
+              msg.includes("ECONNRESET") ||
+              msg.includes("ETIMEDOUT") ||
+              msg.includes("ECONNREFUSED") ||
+              msg.includes("connection") ||
+              msg.includes("Channel open failure") ||
+              msg.includes("socket") ||
+              msg.includes("SSH");
+
+            if (isConnectionError && sftp && retries < maxRetries) {
+              // Wait if another worker is already reconnecting
+              let waitCount = 0;
+              reconnectWaiters++;
+              if (reconnecting && this.isVerbose) {
+                this.log(`${TAB_A}${pc.dim(`Worker waiting for reconnect (${reconnectWaiters} waiting)…`)}`);
+              }
+              while (reconnecting && waitCount < 120) { // Max 60 seconds wait
+                await new Promise(r => setTimeout(r, 500));
+                waitCount++;
+                // Log every 10 seconds while waiting
+                if (waitCount % 20 === 0 && this.isVerbose) {
+                  this.log(`${TAB_A}${pc.dim(`Still waiting for reconnect… (${waitCount / 2}s)`)}`);
+                }
+              }
+              reconnectWaiters--;
+
+              // Check if reconnection is still needed
+              if (!await this._isConnected(sftp)) {
+                reconnecting = true;
+                this.log(`${TAB_A}${pc.yellow("⚠ Connection lost during " + label + ", reconnecting…")}`);
+                try {
+                  await this._reconnect(sftp);
+                  this.log(`${TAB_A}${pc.green("✔ Reconnected, resuming " + label + "…")}`);
+                } catch (reconnectErr) {
+                  this.elog(pc.red(`${TAB_A}❌ Reconnect failed: ${reconnectErr?.message || reconnectErr}`));
+                  reconnecting = false;
+                  // Re-throw to trigger retry
+                  throw reconnectErr;
+                } finally {
+                  reconnecting = false;
+                }
+              }
+
+              retries++;
+              const retryDelay = 500 * retries;
+              if (this.isVerbose) {
+                this.log(`${TAB_A}${pc.dim(`Retry ${retries}/${maxRetries} for: ${item.rel || ''} (waiting ${retryDelay}ms)`)}`);
+              }
+              // Brief pause before retry
+              await new Promise(r => setTimeout(r, retryDelay));
+              // Retry the same item
+              continue;
+            }
+
+            // Log error and move on
+            this.elog(
+              pc.red(`${TAB_A}⚠️ Error in ${label} (attempt ${retries + 1}/${maxRetries + 1}):`),
+              msg
+            );
+
+            if (retries >= maxRetries) {
+              failedCount++;
+              this.elog(pc.red(`${TAB_A}❌ Failed after ${maxRetries + 1} attempts: ${item.rel || item.remotePath || ''}`));
+            }
+            break; // Exit retry loop
+          }
         }
 
         done += 1;
@@ -373,6 +514,9 @@ export class SftpPushSyncApp {
       workers.push(worker());
     }
     await Promise.all(workers);
+
+    // Return statistics
+    return { total, done, failed: failedCount };
   }
 
   // ---------------------------------------------------------
@@ -409,6 +553,7 @@ export class SftpPushSyncApp {
     if (total === 0) return;
 
     let current = 0;
+    let failedDirs = 0;
 
     for (const relDir of dirs) {
       current += 1;
@@ -422,24 +567,59 @@ export class SftpPushSyncApp {
         "Folders"
       );
 
-      try {
-        const exists = await sftp.exists(remoteDir);
-        if (!exists) {
-          await sftp.mkdir(remoteDir, true);
-          this.dirStats.createdDirs += 1;
-          this.vlog(`${TAB_A}${pc.dim("dir created:")} ${remoteDir}`);
-        } else {
-          this.vlog(`${TAB_A}${pc.dim("dir ok:")} ${remoteDir}`);
+      let retries = 0;
+      const maxRetries = 3;
+      let success = false;
+
+      while (retries <= maxRetries && !success) {
+        try {
+          const exists = await sftp.exists(remoteDir);
+          if (!exists) {
+            await sftp.mkdir(remoteDir, true);
+            this.dirStats.createdDirs += 1;
+            this.vlog(`${TAB_A}${pc.dim("dir created:")} ${remoteDir}`);
+          } else {
+            this.vlog(`${TAB_A}${pc.dim("dir ok:")} ${remoteDir}`);
+          }
+          success = true;
+        } catch (e) {
+          const msg = e?.message || String(e);
+          const isConnectionError =
+            msg.includes("No SFTP connection") ||
+            msg.includes("ECONNRESET") ||
+            msg.includes("ETIMEDOUT") ||
+            msg.includes("connection") ||
+            msg.includes("Channel open failure") ||
+            msg.includes("socket") ||
+            msg.includes("SSH");
+
+          if (isConnectionError && retries < maxRetries) {
+            this.log(`${TAB_A}${pc.yellow("⚠ Connection lost during directory preparation, reconnecting…")}`);
+            try {
+              await this._reconnect(sftp);
+              retries++;
+              await new Promise(r => setTimeout(r, 500 * retries));
+              continue; // Retry this directory
+            } catch (reconnectErr) {
+              this.elog(pc.red(`${TAB_A}❌ Reconnect failed: ${reconnectErr?.message || reconnectErr}`));
+            }
+          }
+
+          this.wlog(
+            pc.yellow("⚠️  Could not ensure directory:"),
+            remoteDir,
+            msg
+          );
+          failedDirs++;
+          break; // Move to next directory
         }
-      } catch (e) {
-        this.wlog(
-          pc.yellow("⚠️  Could not ensure directory:"),
-          remoteDir,
-          e?.message || e
-        );
       }
     }
 
+    if (failedDirs > 0) {
+      this.wlog(pc.yellow(`⚠️  ${failedDirs} directories could not be created`));
+    }
+
     this.updateProgress2("Prepare dirs: ", total, total, "done", "Folders");
     process.stdout.write("\n");
     this.progressActive = false;
@@ -450,7 +630,24 @@ export class SftpPushSyncApp {
   // ---------------------------------------------------------
 
   async cleanupEmptyDirs(sftp, rootDir, dryRun) {
-    const recurse = async (dir) => {
+    // Track reconnect state at cleanup level
+    let reconnectNeeded = false;
+
+    const attemptReconnect = async () => {
+      if (reconnectNeeded) return false; // Already tried
+      reconnectNeeded = true;
+      this.log(`${TAB_A}${pc.yellow("⚠ Connection lost during cleanup, reconnecting…")}`);
+      try {
+        await this._reconnect(sftp);
+        reconnectNeeded = false;
+        return true;
+      } catch (err) {
+        this.elog(pc.red(`${TAB_A}❌ Reconnect during cleanup failed: ${err?.message || err}`));
+        return false;
+      }
+    };
+
+    const recurse = async (dir, depth = 0) => {
       this.dirStats.cleanupVisited += 1;
 
       const relForProgress = toPosix(path.relative(rootDir, dir)) || ".";
@@ -467,17 +664,37 @@ export class SftpPushSyncApp {
       const subdirs = [];
       let items;
 
-      try {
-        items = await sftp.list(dir);
-      } catch (e) {
-        this.wlog(
-          pc.yellow("⚠️  Could not list directory during cleanup:"),
-          dir,
-          e?.message || e
-        );
-        return false;
+      // Try to list directory with reconnect on failure
+      let retries = 0;
+      while (retries <= 2) {
+        try {
+          items = await sftp.list(dir);
+          break;
+        } catch (e) {
+          const msg = e?.message || String(e);
+          const isConnectionError = msg.includes("No SFTP connection") ||
+            msg.includes("ECONNRESET") || msg.includes("connection");
+
+          if (isConnectionError && retries < 2) {
+            const reconnected = await attemptReconnect();
+            if (reconnected) {
+              retries++;
+              await new Promise(r => setTimeout(r, 500));
+              continue;
+            }
+          }
+
+          this.wlog(
+            pc.yellow("⚠️  Could not list directory during cleanup:"),
+            dir,
+            msg
+          );
+          return false;
+        }
       }
 
+      if (!items) return false;
+
       for (const item of items) {
         if (!item.name || item.name === "." || item.name === "..") continue;
         if (item.type === "d") {
@@ -490,7 +707,7 @@ export class SftpPushSyncApp {
       let allSubdirsEmpty = true;
       for (const sub of subdirs) {
         const full = path.posix.join(dir, sub.name);
-        const subEmpty = await recurse(full);
+        const subEmpty = await recurse(full, depth + 1);
         if (!subEmpty) {
           allSubdirsEmpty = false;
         }
@@ -507,17 +724,34 @@ export class SftpPushSyncApp {
           );
           this.dirStats.cleanupDeleted += 1;
         } else {
-          try {
-            await sftp.rmdir(dir, false);
-            this.log(`${TAB_A}${DEL} Removed empty directory: ${rel}`);
-            this.dirStats.cleanupDeleted += 1;
-          } catch (e) {
-            this.wlog(
-              pc.yellow("⚠️  Could not remove directory:"),
-              dir,
-              e?.message || e
-            );
-            return false;
+          let deleteRetries = 0;
+          while (deleteRetries <= 2) {
+            try {
+              await sftp.rmdir(dir, false);
+              this.log(`${TAB_A}${DEL} Removed empty directory: ${rel}`);
+              this.dirStats.cleanupDeleted += 1;
+              break;
+            } catch (e) {
+              const msg = e?.message || String(e);
+              const isConnectionError = msg.includes("No SFTP connection") ||
+                msg.includes("ECONNRESET") || msg.includes("connection");
+
+              if (isConnectionError && deleteRetries < 2) {
+                const reconnected = await attemptReconnect();
+                if (reconnected) {
+                  deleteRetries++;
+                  await new Promise(r => setTimeout(r, 500));
+                  continue;
+                }
+              }
+
+              this.wlog(
+                pc.yellow("⚠️  Could not remove directory:"),
+                dir,
+                msg
+              );
+              return false;
+            }
           }
         }
       }
@@ -544,8 +778,44 @@ export class SftpPushSyncApp {
   // Hauptlauf
   // ---------------------------------------------------------
 
+  /**
+   * Format duration in human-readable format (mm:ss or hh:mm:ss)
+   */
+  _formatDuration(seconds) {
+    const totalSec = Math.floor(seconds);
+    const hours = Math.floor(totalSec / 3600);
+    const minutes = Math.floor((totalSec % 3600) / 60);
+    const secs = totalSec % 60;
+
+    if (hours > 0) {
+      return `${hours}:${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
+    }
+    return `${minutes}:${String(secs).padStart(2, '0')}`;
+  }
+
   async run() {
     const start = Date.now();
+
+    // Global error handlers to catch unexpected errors
+    const handleFatalError = (type, error) => {
+      const msg = error?.message || String(error);
+      const logMsg = `❌ FATAL ${type}: ${msg}`;
+      console.error(pc.red(logMsg));
+      if (this.logger) {
+        this.logger.writeLine(logMsg);
+        this.logger.writeLine(error?.stack || "No stack trace available");
+        this.logger.close();
+      }
+      process.exitCode = 1;
+    };
+
+    process.on('unhandledRejection', (reason) => {
+      handleFatalError('Unhandled Promise Rejection', reason);
+    });
+    process.on('uncaughtException', (error) => {
+      handleFatalError('Uncaught Exception', error);
+    });
+
     const {
       target,
       dryRun = false,
@@ -627,6 +897,9 @@ export class SftpPushSyncApp {
     this.isVerbose = logLevel === "verbose";
     this.isLaconic = logLevel === "laconic";
 
+    // Timestamps in Logfile
+    this.logTimestamps = configRaw.logTimestamps ?? false;
+
     // Progress-Konfig
     const PROGRESS = configRaw.progress ?? {};
     this.scanChunk = PROGRESS.scanChunk ?? (this.isVerbose ? 1 : 100);
@@ -708,6 +981,7 @@ export class SftpPushSyncApp {
     this.hashCache = await createHashCacheNDJSON({
       cachePath: ndjsonCachePath,
       namespace: target,
+      vlog: this.isVerbose ? (...m) => console.log(...m) : null,
     });
 
     // Logger
@@ -716,7 +990,7 @@ export class SftpPushSyncApp {
     const logFile = path.resolve(
       rawLogFilePattern.replace("{target}", target)
     );
-    this.logger = new SyncLogger(logFile);
+    this.logger = new SyncLogger(logFile, { enableTimestamps: this.logTimestamps });
     await this.logger.init();
 
     // Header
@@ -726,7 +1000,7 @@ export class SftpPushSyncApp {
         `🔐 SFTP Push-Synchronisation: sftp-push-sync  v${pkg.version}`
       )
     );
-    this.log(`${TAB_A}LogLevel: ${this.logLevel}`);
+    this.log(`${TAB_A}LogLevel: ${this.logLevel}${this.logTimestamps ? " (timestamps enabled)" : ""}`);
     this.log(`${TAB_A}Connection: ${pc.cyan(target)}`);
     this.log(`${TAB_A}Worker: ${this.connection.workers}`);
     this.log(
@@ -781,9 +1055,12 @@ export class SftpPushSyncApp {
         username: this.connection.user,
         password: this.connection.password,
         // Keep-Alive to prevent server disconnection during long operations
-        keepaliveInterval: 10000,  // Send keepalive every 10 seconds
-        keepaliveCountMax: 10,     // Allow up to 10 missed keepalives before disconnect
-        readyTimeout: 30000,       // 30s timeout for initial connection
+        keepaliveInterval: 5000,   // Send keepalive every 5 seconds (more frequent for unstable servers)
+        keepaliveCountMax: 6,      // Allow up to 6 missed keepalives (30s total) before disconnect
+        readyTimeout: 60000,       // 60s timeout for initial connection
+        retries: 2,                // Internal retries
+        retry_factor: 2,
+        retry_minTimeout: 2000,
       });
       connected = true;
 
@@ -818,10 +1095,11 @@ export class SftpPushSyncApp {
           symbols: { ADD, CHA, tab_a: TAB_A },
         });
 
-        const duration = ((Date.now() - start) / 1000).toFixed(2);
+        const durationSec = (Date.now() - start) / 1000;
+        const durationFormatted = this._formatDuration(durationSec);
         this.log("");
         this.log(pc.bold(pc.cyan("📊 Summary (bypass only):")));
-        this.log(`${TAB_A}Duration: ${pc.green(duration + " s")}`);
+        this.log(`${TAB_A}Duration: ${pc.green(durationFormatted)} (${durationSec.toFixed(1)}s)`);
         return;
       }
 
@@ -912,11 +1190,29 @@ export class SftpPushSyncApp {
         analyzeChunk: this.analyzeChunk,
         updateProgress: (prefix, current, total, rel) =>
           this.updateProgress2(prefix, current, total, rel, "Files"),
+        log: this.isVerbose ? (...m) => this.log(...m) : null,
       });
 
       toAdd = diffResult.toAdd;
       toUpdate = diffResult.toUpdate;
 
+      // Report large files that skipped hash comparison
+      if (diffResult.largeFilesSkipped && diffResult.largeFilesSkipped.length > 0 && this.isVerbose) {
+        const totalSizeMB = diffResult.largeFilesSkipped.reduce((sum, f) => sum + f.size, 0) / (1024 * 1024);
+        this.log(`   ℹ ${diffResult.largeFilesSkipped.length} large files (${totalSizeMB.toFixed(0)}MB total) skipped hash compare (same size/date)`);
+      }
+
+      // Report compare errors if any
+      if (diffResult.compareErrors && diffResult.compareErrors.length > 0) {
+        this.log("");
+        this.wlog(pc.yellow(`⚠ ${diffResult.compareErrors.length} files had compare errors (will be re-uploaded):`));
+        if (this.isVerbose) {
+          for (const { rel, error } of diffResult.compareErrors) {
+            this.wlog(pc.yellow(`   - ${rel}: ${error}`));
+          }
+        }
+      }
+
       if (toAdd.length === 0 && toUpdate.length === 0) {
         this.log("");
         this.log(`${TAB_A}No differences found. Everything is up to date.`);
@@ -985,32 +1281,34 @@ export class SftpPushSyncApp {
         await this.runTasks(
           toAdd,
           this.connection.workers,
-          async ({ local: l, remotePath }) => {
+          async ({ local: l, remotePath, rel }) => {
             const remoteDir = path.posix.dirname(remotePath);
             try {
               await sftp.mkdir(remoteDir, true);
             } catch {
               // Directory may already exist
             }
-            await sftp.put(l.localPath, remotePath);
+            await this._uploadFile(sftp, l.localPath, remotePath, rel, l.size);
           },
-          "Uploads (new)"
+          "Uploads (new)",
+          sftp
         );
 
         // Updates
         await this.runTasks(
           toUpdate,
           this.connection.workers,
-          async ({ local: l, remotePath }) => {
+          async ({ local: l, remotePath, rel }) => {
             const remoteDir = path.posix.dirname(remotePath);
             try {
               await sftp.mkdir(remoteDir, true);
             } catch {
               // Directory may already exist
             }
-            await sftp.put(l.localPath, remotePath);
+            await this._uploadFile(sftp, l.localPath, remotePath, rel, l.size);
           },
-          "Uploads (update)"
+          "Uploads (update)",
+          sftp
         );
 
         // Deletes
@@ -1028,7 +1326,8 @@ export class SftpPushSyncApp {
               );
             }
           },
-          "Deletes"
+          "Deletes",
+          sftp
         );
       } else {
         this.log("");
@@ -1055,7 +1354,8 @@ export class SftpPushSyncApp {
         await this.cleanupEmptyDirs(sftp, this.connection.remoteRoot, dryRun);
       }
 
-      const duration = ((Date.now() - start) / 1000).toFixed(2);
+      const durationSec = (Date.now() - start) / 1000;
+      const durationFormatted = this._formatDuration(durationSec);
 
       // Save cache and close
       await this.hashCache.save();
@@ -1065,7 +1365,7 @@ export class SftpPushSyncApp {
       this.log(hr1());
       this.log("");
       this.log(pc.bold(pc.cyan("📊 Summary:")));
-      this.log(`${TAB_A}Duration: ${pc.green(duration + " s")}`);
+      this.log(`${TAB_A}Duration: ${pc.green(durationFormatted)} (${durationSec.toFixed(1)}s)`);
       this.log(`${TAB_A}${ADD} Added  : ${toAdd.length}`);
       this.log(`${TAB_A}${CHA} Changed: ${toUpdate.length}`);
       this.log(`${TAB_A}${DEL} Deleted: ${toDelete.length}`);
@@ -1120,8 +1420,11 @@ export class SftpPushSyncApp {
         if (this.hashCache?.close) {
           await this.hashCache.close();
         }
-      } catch {
-        // ignore
+      } catch (e) {
+        // Cache close failed during error cleanup
+        if (this.isVerbose) {
+          this.vlog(`${TAB_A}${pc.dim(`Cache close during cleanup failed: ${e?.message || e}`)}`)
+        }
       }
     } finally {
       try {

package/src/core/SyncLogger.mjs

@@ -1,9 +1,9 @@
 /**
  * SyncLogger.mjs
- * 
+ *
  * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
- * 
- */ 
+ *
+ */
 // src/core/SyncLogger.mjs
 import fs from "fs";
 import fsp from "fs/promises";
@@ -14,9 +14,10 @@ import path from "path";
  * und entfernt ANSI-Farbcodes.
  */
 export class SyncLogger {
-  constructor(filePath) {
+  constructor(filePath, options = {}) {
     this.filePath = filePath;
     this.stream = null;
+    this.enableTimestamps = options.enableTimestamps ?? false;
   }
 
   async init() {
@@ -31,13 +32,24 @@ export class SyncLogger {
     });
   }
 
+  /**
+   * Returns current timestamp in ISO format: [YYYY-MM-DD HH:mm:ss.SSS]
+   */
+  _getTimestamp() {
+    const now = new Date();
+    const pad = (n, len = 2) => String(n).padStart(len, '0');
+    return `[${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}.${pad(now.getMilliseconds(), 3)}]`;
+  }
+
   writeLine(line) {
     if (!this.stream) return;
     const text = typeof line === "string" ? line : String(line);
     const clean = text.replace(/\x1b\[[0-9;]*m/g, "");
 
+    const prefix = this.enableTimestamps ? this._getTimestamp() + " " : "";
+
     try {
-      this.stream.write(clean + "\n");
+      this.stream.write(prefix + clean + "\n");
     } catch {
       // Stream schon zu → ignorieren
     }
@@ -49,4 +61,4 @@ export class SyncLogger {
       this.stream = null;
     }
   }
-}
+}

package/src/helpers/compare.mjs

@@ -23,6 +23,8 @@ import path from "path";
  *  - analyzeChunk: Progress-Schrittgröße
  *  - updateProgress(prefix, current, total, rel): optional
  *  - concurrency: Max parallele Vergleiche (default: 5)
+ *  - log: optional logging function for errors/warnings
+ *  - maxSizeForHash: Files larger than this skip hash comparison (default: 50MB)
  */
 export async function analyseDifferences({
   local,
@@ -34,7 +36,13 @@ export async function analyseDifferences({
   analyzeChunk = 10,
   updateProgress,
   concurrency = 10,
+  log,
+  maxSizeForHash = 50 * 1024 * 1024, // 50MB default
 }) {
+  // Track errors for summary
+  const compareErrors = [];
+  // Track large files skipped
+  const largeFilesSkipped = [];
   const toAdd = [];
   const toUpdate = [];
 
@@ -45,6 +53,7 @@ export async function analyseDifferences({
   // Phase 1: Schneller Vorab-Check ohne SFTP
   // - Dateien nur lokal → direkt zu toAdd
   // - Size-Vergleich für existierende Dateien
+  // - Große Dateien: nur MTime-Vergleich (kein Hash-Download)
   const keysNeedContentCompare = [];
 
   for (const rel of localKeys) {
@@ -58,21 +67,46 @@ export async function analyseDifferences({
     } else if (l.size !== r.size) {
       // Size unterschiedlich → Changed (kein SFTP-Call nötig)
       toUpdate.push({ rel, local: l, remote: r, remotePath });
+    // } else if (l.size > maxSizeForHash) {
+    //   // Große Datei mit gleicher Size: nur MTime vergleichen
+    //   // Remote modifyTime ist String wie "2026-03-05", local mtimeMs ist Timestamp
+    //   const localDate = new Date(l.mtimeMs).toISOString().split('T')[0];
+    //   const remoteDate = r.modifyTime ? r.modifyTime.split('T')[0] : '';
+    //   
+    //   if (localDate > remoteDate) {
+    //     // Local ist neuer → Changed
+    //     toUpdate.push({ rel, local: l, remote: r, remotePath });
+    //     if (log) {
+    //       const sizeMB = (l.size / (1024 * 1024)).toFixed(1);
+    //       log(`   ℹ Large file (${sizeMB}MB) newer locally: ${rel}`);
+    //     }
+    //   } else {
+    //     largeFilesSkipped.push({ rel, size: l.size });
+    //   }
     } else {
-      // Size gleich → Content-Vergleich nötig
+      // Size gleich, normale Größe → Content-Vergleich nötig
       keysNeedContentCompare.push(rel);
     }
 
     checked++;
     if (updateProgress && checked % analyzeChunk === 0) {
-      updateProgress("Analyse (Size): ", checked, totalToCheck, rel);
+      updateProgress("Analyse (quick): ", checked, totalToCheck, rel);
     }
   }
 
+  // Final progress update for Phase 1
+  if (updateProgress) {
+    updateProgress("Analyse (quick): ", totalToCheck, totalToCheck, "done");
+  }
+
   // Phase 2: Content-Vergleich in echten Batches
-  // Nur für Dateien mit gleicher Size
+  // Nur für Dateien mit gleicher Size (und unter maxSizeForHash)
   const totalContentCompare = keysNeedContentCompare.length;
 
+  if (totalContentCompare > 0 && log) {
+    log(`   → ${totalContentCompare} files need content comparison`);
+  }
+
   for (let i = 0; i < totalContentCompare; i += concurrency) {
     const batch = keysNeedContentCompare.slice(i, i + concurrency);
 
@@ -114,8 +148,14 @@ export async function analyseDifferences({
               : null;
           }
         } catch (err) {
-          // Bei Fehler als changed markieren (sicherer)
-          return { rel, local: l, remote: r, remotePath, changed: true };
+          // Log the error so user can see what's happening
+          const errMsg = err?.message || String(err);
+          compareErrors.push({ rel, error: errMsg });
+          if (log) {
+            log(`   ⚠ Compare error for ${rel}: ${errMsg}`);
+          }
+          // Mark as changed (sicherer) - file will be re-uploaded
+          return { rel, local: l, remote: r, remotePath, changed: true, hadError: true };
         }
       })
     );
@@ -127,14 +167,14 @@ export async function analyseDifferences({
       }
     }
 
-    // Progress update
+    // Progress update - show as separate progress (doesn't jump back)
     const progressCount = Math.min(i + batch.length, totalContentCompare);
     if (updateProgress) {
-      updateProgress("Analyse (Content): ", checked + progressCount, totalToCheck + totalContentCompare, batch[batch.length - 1]);
+      updateProgress("Analyse (hash): ", progressCount, totalContentCompare, batch[batch.length - 1]);
     }
   }
 
-  return { toAdd, toUpdate };
+  return { toAdd, toUpdate, compareErrors, largeFilesSkipped };
 }
 
 /**

package/src/helpers/hash-cache-ndjson.mjs

@@ -35,18 +35,62 @@ export function hashLocalFile(filePath) {
 
 /**
  * Streaming-SHA256 für Remote-Datei via ssh2-sftp-client
+ * Mit IDLE-Timeout: nur wenn keine Daten mehr fließen für X Sekunden.
+ * Große Dateien werden korrekt behandelt - solange Daten ankommen, kein Timeout.
+ *
+ * @param {Object} sftp - SFTP client
+ * @param {string} remotePath - Remote file path
+ * @param {number} idleTimeoutMs - Timeout in ms when NO data is received (default: 60000)
+ * @param {number} fileSizeBytes - File size (for logging)
  */
-export async function hashRemoteFile(sftp, remotePath) {
+export async function hashRemoteFile(sftp, remotePath, idleTimeoutMs = 60000, fileSizeBytes = 0) {
   const hash = createHash("sha256");
+  let lastDataTime = Date.now();
+  let totalReceived = 0;
+  let timeoutId = null;
+  let rejectFn = null;
+
+  // Promise that rejects on idle timeout
+  const idleTimeoutPromise = new Promise((_, reject) => {
+    rejectFn = reject;
+    
+    const checkIdle = () => {
+      const idleTime = Date.now() - lastDataTime;
+      if (idleTime >= idleTimeoutMs) {
+        const receivedMB = (totalReceived / (1024 * 1024)).toFixed(1);
+        reject(new Error(`Idle timeout (${Math.round(idleTimeoutMs/1000)}s no data) at ${receivedMB}MB for ${remotePath}`));
+      } else {
+        // Check again in 5 seconds
+        timeoutId = setTimeout(checkIdle, 5000);
+      }
+    };
+    
+    // Start checking after initial timeout
+    timeoutId = setTimeout(checkIdle, idleTimeoutMs);
+  });
 
   const writable = new Writable({
     write(chunk, enc, cb) {
+      lastDataTime = Date.now(); // Reset idle timer on each chunk
+      totalReceived += chunk.length;
       hash.update(chunk);
       cb();
     },
   });
 
-  await sftp.get(remotePath, writable);
+  try {
+    // Race between download and idle timeout
+    await Promise.race([
+      sftp.get(remotePath, writable),
+      idleTimeoutPromise,
+    ]);
+  } finally {
+    // Clean up timeout
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
+  }
+
   return hash.digest("hex");
 }
 
@@ -60,8 +104,9 @@ export async function hashRemoteFile(sftp, remotePath) {
  * @param {string} options.cachePath - Path to the NDJSON file (e.g., ".sync-cache.prod.ndjson")
  * @param {string} options.namespace - Namespace for keys (e.g., "prod")
  * @param {number} options.autoSaveInterval - Save after this many changes (default: 1000)
+ * @param {Function} options.vlog - Optional verbose logging function
  */
-export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInterval = 1000 }) {
+export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInterval = 1000, vlog }) {
   const ns = namespace || "default";
 
   // In-memory storage
@@ -83,6 +128,7 @@ export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInte
       await fsp.access(cachePath);
     } catch {
       // File doesn't exist - start fresh
+      if (vlog) vlog(`   Cache file not found, starting fresh: ${cachePath}`);
       return;
     }
 
@@ -192,7 +238,8 @@ export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInte
 
     // Cache miss or stale: compute new hash
     const filePath = meta.fullPath || meta.remotePath;
-    const hash = await hashRemoteFile(sftp, filePath);
+    // Pass file size for dynamic timeout calculation
+    const hash = await hashRemoteFile(sftp, filePath, 60000, meta.size || 0);
 
     remoteCache.set(key, {
       size: meta.size,