sftp-push-sync 3.0.1 → 3.0.2

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Changelog
 
-## [3.0.1] - 2026-03-05
+## [3.0.2] - 2026-03-05
 
 - stability improvements especialy during large and longtime uploads, error handling, log with datetime.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sftp-push-sync",
-  "version": "3.0.1",
+  "version": "3.0.2",
   "description": "SFTP sync tool for Hugo projects (local to remote, with hash cache)",
   "type": "module",
   "bin": {

package/src/core/SftpPushSyncApp.mjs

@@ -200,7 +200,10 @@ export class SftpPushSyncApp {
       // Try a minimal operation to check connection
       await sftp.cwd();
       return true;
-    } catch {
+    } catch (e) {
+      if (this.isVerbose) {
+        this.vlog(`${TAB_A}${pc.dim(`Connection check failed: ${e?.message || e}`)}`);
+      }
       return false;
     }
   }
@@ -213,8 +216,11 @@ export class SftpPushSyncApp {
       try {
         try {
           await sftp.end();
-        } catch {
+        } catch (e) {
           // Ignore errors when closing dead connection
+          if (this.isVerbose) {
+            this.vlog(`${TAB_A}${pc.dim(`Closing old connection failed (expected): ${e?.message || e}`)}`);
+          }
         }
 
         // Wait before reconnecting (exponential backoff)
@@ -254,6 +260,45 @@ export class SftpPushSyncApp {
     }
   }
 
+  /**
+   * Upload a file with progress reporting for large files.
+   * Uses fastPut for files > threshold, with automatic fallback to put on failure.
+   */
+  async _uploadFile(sftp, localPath, remotePath, rel, size) {
+    const LARGE_FILE_THRESHOLD = 5 * 1024 * 1024; // 5MB
+    const sizeMB = (size / (1024 * 1024)).toFixed(1);
+    
+    // For small files, just use put
+    if (size < LARGE_FILE_THRESHOLD) {
+      await sftp.put(localPath, remotePath);
+      return;
+    }
+
+    // For large files, try fastPut with progress
+    let lastReportedPercent = 0;
+    const shortRel = rel.length > 50 ? '...' + rel.slice(-47) : rel;
+
+    try {
+      await sftp.fastPut(localPath, remotePath, {
+        step: (transferred, chunk, total) => {
+          const percent = Math.floor((transferred / total) * 100);
+          // Only log at 25%, 50%, 75%, 100%
+          if (percent >= lastReportedPercent + 25) {
+            lastReportedPercent = Math.floor(percent / 25) * 25;
+            this.log(`${TAB_A}${pc.dim(`  ↑ ${sizeMB}MB ${percent}%: ${shortRel}`)}`);
+          }
+        }
+      });
+    } catch (fastPutErr) {
+      // fastPut not supported by server, fall back to regular put
+      if (this.isVerbose) {
+        this.vlog(`${TAB_A}${pc.dim(`  fastPut failed, using put: ${fastPutErr?.message}`)}`);
+      }
+      this.log(`${TAB_A}${pc.dim(`  Uploading ${sizeMB}MB: ${shortRel}`)}`);
+      await sftp.put(localPath, remotePath);
+    }
+  }
+
   // ---------------------------------------------------------
   // Pattern-Helper
   // ---------------------------------------------------------
@@ -936,6 +981,7 @@ export class SftpPushSyncApp {
     this.hashCache = await createHashCacheNDJSON({
       cachePath: ndjsonCachePath,
       namespace: target,
+      vlog: this.isVerbose ? (...m) => console.log(...m) : null,
     });
 
     // Logger
@@ -1144,11 +1190,29 @@ export class SftpPushSyncApp {
         analyzeChunk: this.analyzeChunk,
         updateProgress: (prefix, current, total, rel) =>
           this.updateProgress2(prefix, current, total, rel, "Files"),
+        log: this.isVerbose ? (...m) => this.log(...m) : null,
       });
 
       toAdd = diffResult.toAdd;
       toUpdate = diffResult.toUpdate;
 
+      // Report large files that skipped hash comparison
+      if (diffResult.largeFilesSkipped && diffResult.largeFilesSkipped.length > 0 && this.isVerbose) {
+        const totalSizeMB = diffResult.largeFilesSkipped.reduce((sum, f) => sum + f.size, 0) / (1024 * 1024);
+        this.log(`   ℹ ${diffResult.largeFilesSkipped.length} large files (${totalSizeMB.toFixed(0)}MB total) skipped hash compare (same size/date)`);
+      }
+
+      // Report compare errors if any
+      if (diffResult.compareErrors && diffResult.compareErrors.length > 0) {
+        this.log("");
+        this.wlog(pc.yellow(`⚠ ${diffResult.compareErrors.length} files had compare errors (will be re-uploaded):`));
+        if (this.isVerbose) {
+          for (const { rel, error } of diffResult.compareErrors) {
+            this.wlog(pc.yellow(`   - ${rel}: ${error}`));
+          }
+        }
+      }
+
       if (toAdd.length === 0 && toUpdate.length === 0) {
         this.log("");
         this.log(`${TAB_A}No differences found. Everything is up to date.`);
@@ -1217,14 +1281,14 @@ export class SftpPushSyncApp {
         await this.runTasks(
           toAdd,
           this.connection.workers,
-          async ({ local: l, remotePath }) => {
+          async ({ local: l, remotePath, rel }) => {
             const remoteDir = path.posix.dirname(remotePath);
             try {
               await sftp.mkdir(remoteDir, true);
             } catch {
               // Directory may already exist
             }
-            await sftp.put(l.localPath, remotePath);
+            await this._uploadFile(sftp, l.localPath, remotePath, rel, l.size);
           },
           "Uploads (new)",
           sftp
@@ -1234,14 +1298,14 @@ export class SftpPushSyncApp {
         await this.runTasks(
           toUpdate,
           this.connection.workers,
-          async ({ local: l, remotePath }) => {
+          async ({ local: l, remotePath, rel }) => {
             const remoteDir = path.posix.dirname(remotePath);
             try {
               await sftp.mkdir(remoteDir, true);
             } catch {
               // Directory may already exist
             }
-            await sftp.put(l.localPath, remotePath);
+            await this._uploadFile(sftp, l.localPath, remotePath, rel, l.size);
           },
           "Uploads (update)",
           sftp
@@ -1356,8 +1420,11 @@ export class SftpPushSyncApp {
         if (this.hashCache?.close) {
           await this.hashCache.close();
         }
-      } catch {
-        // ignore
+      } catch (e) {
+        // Cache close failed during error cleanup
+        if (this.isVerbose) {
+          this.vlog(`${TAB_A}${pc.dim(`Cache close during cleanup failed: ${e?.message || e}`)}`)
+        }
       }
     } finally {
       try {

package/src/helpers/compare.mjs

@@ -23,6 +23,8 @@ import path from "path";
  *  - analyzeChunk: Progress-Schrittgröße
  *  - updateProgress(prefix, current, total, rel): optional
  *  - concurrency: Max parallele Vergleiche (default: 5)
+ *  - log: optional logging function for errors/warnings
+ *  - maxSizeForHash: Files larger than this skip hash comparison (default: 50MB)
  */
 export async function analyseDifferences({
   local,
@@ -34,7 +36,13 @@ export async function analyseDifferences({
   analyzeChunk = 10,
   updateProgress,
   concurrency = 10,
+  log,
+  maxSizeForHash = 50 * 1024 * 1024, // 50MB default
 }) {
+  // Track errors for summary
+  const compareErrors = [];
+  // Track large files skipped
+  const largeFilesSkipped = [];
   const toAdd = [];
   const toUpdate = [];
 
@@ -45,6 +53,7 @@ export async function analyseDifferences({
   // Phase 1: Schneller Vorab-Check ohne SFTP
   // - Dateien nur lokal → direkt zu toAdd
   // - Size-Vergleich für existierende Dateien
+  // - Große Dateien: nur MTime-Vergleich (kein Hash-Download)
   const keysNeedContentCompare = [];
 
   for (const rel of localKeys) {
@@ -58,21 +67,46 @@ export async function analyseDifferences({
     } else if (l.size !== r.size) {
       // Size unterschiedlich → Changed (kein SFTP-Call nötig)
       toUpdate.push({ rel, local: l, remote: r, remotePath });
+    // } else if (l.size > maxSizeForHash) {
+    //   // Große Datei mit gleicher Size: nur MTime vergleichen
+    //   // Remote modifyTime ist String wie "2026-03-05", local mtimeMs ist Timestamp
+    //   const localDate = new Date(l.mtimeMs).toISOString().split('T')[0];
+    //   const remoteDate = r.modifyTime ? r.modifyTime.split('T')[0] : '';
+    //   
+    //   if (localDate > remoteDate) {
+    //     // Local ist neuer → Changed
+    //     toUpdate.push({ rel, local: l, remote: r, remotePath });
+    //     if (log) {
+    //       const sizeMB = (l.size / (1024 * 1024)).toFixed(1);
+    //       log(`   ℹ Large file (${sizeMB}MB) newer locally: ${rel}`);
+    //     }
+    //   } else {
+    //     largeFilesSkipped.push({ rel, size: l.size });
+    //   }
     } else {
-      // Size gleich → Content-Vergleich nötig
+      // Size gleich, normale Größe → Content-Vergleich nötig
       keysNeedContentCompare.push(rel);
     }
 
     checked++;
     if (updateProgress && checked % analyzeChunk === 0) {
-      updateProgress("Analyse (Size): ", checked, totalToCheck, rel);
+      updateProgress("Analyse (quick): ", checked, totalToCheck, rel);
     }
   }
 
+  // Final progress update for Phase 1
+  if (updateProgress) {
+    updateProgress("Analyse (quick): ", totalToCheck, totalToCheck, "done");
+  }
+
   // Phase 2: Content-Vergleich in echten Batches
-  // Nur für Dateien mit gleicher Size
+  // Nur für Dateien mit gleicher Size (und unter maxSizeForHash)
   const totalContentCompare = keysNeedContentCompare.length;
 
+  if (totalContentCompare > 0 && log) {
+    log(`   → ${totalContentCompare} files need content comparison`);
+  }
+
   for (let i = 0; i < totalContentCompare; i += concurrency) {
     const batch = keysNeedContentCompare.slice(i, i + concurrency);
 
@@ -114,8 +148,14 @@ export async function analyseDifferences({
               : null;
           }
         } catch (err) {
-          // Bei Fehler als changed markieren (sicherer)
-          return { rel, local: l, remote: r, remotePath, changed: true };
+          // Log the error so user can see what's happening
+          const errMsg = err?.message || String(err);
+          compareErrors.push({ rel, error: errMsg });
+          if (log) {
+            log(`   ⚠ Compare error for ${rel}: ${errMsg}`);
+          }
+          // Mark as changed (sicherer) - file will be re-uploaded
+          return { rel, local: l, remote: r, remotePath, changed: true, hadError: true };
         }
       })
     );
@@ -127,14 +167,14 @@ export async function analyseDifferences({
       }
     }
 
-    // Progress update
+    // Progress update - show as separate progress (doesn't jump back)
     const progressCount = Math.min(i + batch.length, totalContentCompare);
     if (updateProgress) {
-      updateProgress("Analyse (Content): ", checked + progressCount, totalToCheck + totalContentCompare, batch[batch.length - 1]);
+      updateProgress("Analyse (hash): ", progressCount, totalContentCompare, batch[batch.length - 1]);
     }
   }
 
-  return { toAdd, toUpdate };
+  return { toAdd, toUpdate, compareErrors, largeFilesSkipped };
 }
 
 /**

package/src/helpers/hash-cache-ndjson.mjs

@@ -35,28 +35,61 @@ export function hashLocalFile(filePath) {
 
 /**
  * Streaming-SHA256 für Remote-Datei via ssh2-sftp-client
- * Mit Timeout, um hängende Verbindungen zu erkennen.
+ * Mit IDLE-Timeout: nur wenn keine Daten mehr fließen für X Sekunden.
+ * Große Dateien werden korrekt behandelt - solange Daten ankommen, kein Timeout.
+ *
+ * @param {Object} sftp - SFTP client
+ * @param {string} remotePath - Remote file path
+ * @param {number} idleTimeoutMs - Timeout in ms when NO data is received (default: 60000)
+ * @param {number} fileSizeBytes - File size (for logging)
  */
-export async function hashRemoteFile(sftp, remotePath, timeoutMs = 60000) {
+export async function hashRemoteFile(sftp, remotePath, idleTimeoutMs = 60000, fileSizeBytes = 0) {
   const hash = createHash("sha256");
+  let lastDataTime = Date.now();
+  let totalReceived = 0;
+  let timeoutId = null;
+  let rejectFn = null;
+
+  // Promise that rejects on idle timeout
+  const idleTimeoutPromise = new Promise((_, reject) => {
+    rejectFn = reject;
+    
+    const checkIdle = () => {
+      const idleTime = Date.now() - lastDataTime;
+      if (idleTime >= idleTimeoutMs) {
+        const receivedMB = (totalReceived / (1024 * 1024)).toFixed(1);
+        reject(new Error(`Idle timeout (${Math.round(idleTimeoutMs/1000)}s no data) at ${receivedMB}MB for ${remotePath}`));
+      } else {
+        // Check again in 5 seconds
+        timeoutId = setTimeout(checkIdle, 5000);
+      }
+    };
+    
+    // Start checking after initial timeout
+    timeoutId = setTimeout(checkIdle, idleTimeoutMs);
+  });
 
   const writable = new Writable({
     write(chunk, enc, cb) {
+      lastDataTime = Date.now(); // Reset idle timer on each chunk
+      totalReceived += chunk.length;
       hash.update(chunk);
       cb();
     },
   });
 
-  // Timeout-Promise
-  const timeoutPromise = new Promise((_, reject) => {
-    setTimeout(() => reject(new Error(`Timeout downloading ${remotePath}`)), timeoutMs);
-  });
-
-  // Race between download and timeout
-  await Promise.race([
-    sftp.get(remotePath, writable),
-    timeoutPromise,
-  ]);
+  try {
+    // Race between download and idle timeout
+    await Promise.race([
+      sftp.get(remotePath, writable),
+      idleTimeoutPromise,
+    ]);
+  } finally {
+    // Clean up timeout
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
+  }
 
   return hash.digest("hex");
 }
@@ -71,8 +104,9 @@ export async function hashRemoteFile(sftp, remotePath, timeoutMs = 60000) {
  * @param {string} options.cachePath - Path to the NDJSON file (e.g., ".sync-cache.prod.ndjson")
  * @param {string} options.namespace - Namespace for keys (e.g., "prod")
  * @param {number} options.autoSaveInterval - Save after this many changes (default: 1000)
+ * @param {Function} options.vlog - Optional verbose logging function
  */
-export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInterval = 1000 }) {
+export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInterval = 1000, vlog }) {
   const ns = namespace || "default";
 
   // In-memory storage
@@ -94,6 +128,7 @@ export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInte
       await fsp.access(cachePath);
     } catch {
       // File doesn't exist - start fresh
+      if (vlog) vlog(`   Cache file not found, starting fresh: ${cachePath}`);
       return;
     }
 
@@ -203,7 +238,8 @@ export async function createHashCacheNDJSON({ cachePath, namespace, autoSaveInte
 
     // Cache miss or stale: compute new hash
     const filePath = meta.fullPath || meta.remotePath;
-    const hash = await hashRemoteFile(sftp, filePath);
+    // Pass file size for dynamic timeout calculation
+    const hash = await hashRemoteFile(sftp, filePath, 60000, meta.size || 0);
 
     remoteCache.set(key, {
       size: meta.size,