sftp-push-sync 2.5.0 → 3.0.1

package/src/core/SyncLogger.mjs

@@ -1,9 +1,9 @@
 /**
  * SyncLogger.mjs
- * 
+ *
  * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
- * 
- */ 
+ *
+ */
 // src/core/SyncLogger.mjs
 import fs from "fs";
 import fsp from "fs/promises";
@@ -14,9 +14,10 @@ import path from "path";
  * und entfernt ANSI-Farbcodes.
  */
 export class SyncLogger {
-  constructor(filePath) {
+  constructor(filePath, options = {}) {
     this.filePath = filePath;
     this.stream = null;
+    this.enableTimestamps = options.enableTimestamps ?? false;
   }
 
   async init() {
@@ -31,13 +32,24 @@ export class SyncLogger {
     });
   }
 
+  /**
+   * Returns current timestamp in ISO format: [YYYY-MM-DD HH:mm:ss.SSS]
+   */
+  _getTimestamp() {
+    const now = new Date();
+    const pad = (n, len = 2) => String(n).padStart(len, '0');
+    return `[${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}.${pad(now.getMilliseconds(), 3)}]`;
+  }
+
   writeLine(line) {
     if (!this.stream) return;
     const text = typeof line === "string" ? line : String(line);
     const clean = text.replace(/\x1b\[[0-9;]*m/g, "");
 
+    const prefix = this.enableTimestamps ? this._getTimestamp() + " " : "";
+
     try {
-      this.stream.write(clean + "\n");
+      this.stream.write(prefix + clean + "\n");
     } catch {
       // Stream schon zu → ignorieren
     }
@@ -49,4 +61,4 @@ export class SyncLogger {
       this.stream = null;
     }
   }
-}
+}

package/src/helpers/compare.mjs

@@ -1,16 +1,16 @@
 /**
  * compare.mjs
- * 
+ *
  * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
- * 
- */ 
+ *
+ */
 // src/helpers/compare.mjs
 import fsp from "fs/promises";
 import path from "path";
 
 /**
  * Analysiert Unterschiede zwischen local- und remote-Maps.
- * Optimiert: Parallelisierte Analyse mit Concurrency-Limit.
+ * Optimiert: Echtes Batch-Processing mit Concurrency-Limit.
  *
  * Erwartete Struktur:
  *  local:  Map<rel, { rel, localPath, size, mtimeMs, isText? }>
@@ -22,7 +22,7 @@ import path from "path";
  *  - getLocalHash / getRemoteHash: from createHashCache
  *  - analyzeChunk: Progress-Schrittgröße
  *  - updateProgress(prefix, current, total, rel): optional
- *  - concurrency: Max parallele Vergleiche (default: 8)
+ *  - concurrency: Max parallele Vergleiche (default: 5)
  */
 export async function analyseDifferences({
   local,
@@ -33,7 +33,7 @@ export async function analyseDifferences({
   getRemoteHash,
   analyzeChunk = 10,
   updateProgress,
-  concurrency = 5,
+  concurrency = 10,
 }) {
   const toAdd = [];
   const toUpdate = [];
@@ -42,104 +42,97 @@ export async function analyseDifferences({
   const totalToCheck = localKeys.length;
   let checked = 0;
 
-  // Schneller Vorab-Check: Dateien nur lokal → direkt zu toAdd
-  const keysToCompare = [];
+  // Phase 1: Schneller Vorab-Check ohne SFTP
+  // - Dateien nur lokal → direkt zu toAdd
+  // - Size-Vergleich für existierende Dateien
+  const keysNeedContentCompare = [];
+
   for (const rel of localKeys) {
+    const l = local.get(rel);
     const r = remote.get(rel);
     const remotePath = path.posix.join(remoteRoot, rel);
-    
+
     if (!r) {
       // Datei existiert nur lokal → New (kein SFTP-Call nötig)
-      toAdd.push({ rel, local: local.get(rel), remotePath });
-      checked++;
-      if (updateProgress && checked % analyzeChunk === 0) {
-        updateProgress("Analyse: ", checked, totalToCheck, rel);
-      }
+      toAdd.push({ rel, local: l, remotePath });
+    } else if (l.size !== r.size) {
+      // Size unterschiedlich → Changed (kein SFTP-Call nötig)
+      toUpdate.push({ rel, local: l, remote: r, remotePath });
     } else {
-      keysToCompare.push(rel);
+      // Size gleich → Content-Vergleich nötig
+      keysNeedContentCompare.push(rel);
     }
-  }
-
-  // Parallele Verarbeitung mit Semaphore
-  let activeCount = 0;
-  const waiting = [];
-
-  async function acquireSemaphore() {
-    if (activeCount < concurrency) {
-      activeCount++;
-      return;
-    }
-    await new Promise((resolve) => waiting.push(resolve));
-    activeCount++;
-  }
 
-  function releaseSemaphore() {
-    activeCount--;
-    if (waiting.length > 0) {
-      const next = waiting.shift();
-      next();
+    checked++;
+    if (updateProgress && checked % analyzeChunk === 0) {
+      updateProgress("Analyse (Size): ", checked, totalToCheck, rel);
     }
   }
 
-  async function compareFile(rel) {
-    await acquireSemaphore();
-    try {
-      const l = local.get(rel);
-      const r = remote.get(rel);
-      const remotePath = path.posix.join(remoteRoot, rel);
-
-      // 1. Size-Vergleich (schnell, kein SFTP)
-      if (l.size !== r.size) {
-        toUpdate.push({ rel, local: l, remote: r, remotePath });
-        return;
-      }
-
-      // 2. Content-Vergleich
-      if (l.isText) {
-        // Text-Datei: vollständiger inhaltlicher Vergleich
-        const [localBuf, remoteBuf] = await Promise.all([
-          fsp.readFile(l.localPath),
-          sftp.get(r.remotePath),
-        ]);
-
-        const localStr = localBuf.toString("utf8");
-        const remoteStr = (
-          Buffer.isBuffer(remoteBuf) ? remoteBuf : Buffer.from(remoteBuf)
-        ).toString("utf8");
-
-        if (localStr !== remoteStr) {
-          toUpdate.push({ rel, local: l, remote: r, remotePath });
-        }
-      } else {
-        // Binary: Hash-Vergleich mit Cache
-        if (!getLocalHash || !getRemoteHash) {
-          toUpdate.push({ rel, local: l, remote: r, remotePath });
-          return;
+  // Phase 2: Content-Vergleich in echten Batches
+  // Nur für Dateien mit gleicher Size
+  const totalContentCompare = keysNeedContentCompare.length;
+
+  for (let i = 0; i < totalContentCompare; i += concurrency) {
+    const batch = keysNeedContentCompare.slice(i, i + concurrency);
+
+    const batchResults = await Promise.all(
+      batch.map(async (rel) => {
+        const l = local.get(rel);
+        const r = remote.get(rel);
+        const remotePath = path.posix.join(remoteRoot, rel);
+
+        try {
+          if (l.isText) {
+            // Text-Datei: vollständiger inhaltlicher Vergleich
+            const [localBuf, remoteBuf] = await Promise.all([
+              fsp.readFile(l.localPath),
+              sftp.get(r.remotePath),
+            ]);
+
+            const localStr = localBuf.toString("utf8");
+            const remoteStr = (
+              Buffer.isBuffer(remoteBuf) ? remoteBuf : Buffer.from(remoteBuf)
+            ).toString("utf8");
+
+            return localStr !== remoteStr
+              ? { rel, local: l, remote: r, remotePath, changed: true }
+              : null;
+          } else {
+            // Binary: Hash-Vergleich mit Cache
+            if (!getLocalHash || !getRemoteHash) {
+              return { rel, local: l, remote: r, remotePath, changed: true };
+            }
+
+            const [localHash, remoteHash] = await Promise.all([
+              getLocalHash(rel, l),
+              getRemoteHash(rel, r, sftp),
+            ]);
+
+            return localHash !== remoteHash
+              ? { rel, local: l, remote: r, remotePath, changed: true }
+              : null;
+          }
+        } catch (err) {
+          // Bei Fehler als changed markieren (sicherer)
+          return { rel, local: l, remote: r, remotePath, changed: true };
         }
+      })
+    );
 
-        const [localHash, remoteHash] = await Promise.all([
-          getLocalHash(rel, l),
-          getRemoteHash(rel, r, sftp),
-        ]);
-
-        if (localHash !== remoteHash) {
-          toUpdate.push({ rel, local: l, remote: r, remotePath });
-        }
-      }
-    } finally {
-      releaseSemaphore();
-      checked++;
-      if (
-        updateProgress &&
-        (checked === 1 || checked % analyzeChunk === 0 || checked === totalToCheck)
-      ) {
-        updateProgress("Analyse: ", checked, totalToCheck, rel);
+    // Ergebnisse sammeln
+    for (const result of batchResults) {
+      if (result && result.changed) {
+        toUpdate.push({ rel: result.rel, local: result.local, remote: result.remote, remotePath: result.remotePath });
       }
     }
-  }
 
-  // Starte alle Vergleiche parallel (mit Concurrency-Limit durch Semaphore)
-  await Promise.all(keysToCompare.map(compareFile));
+    // Progress update
+    const progressCount = Math.min(i + batch.length, totalContentCompare);
+    if (updateProgress) {
+      updateProgress("Analyse (Content): ", checked + progressCount, totalToCheck + totalContentCompare, batch[batch.length - 1]);
+    }
+  }
 
   return { toAdd, toUpdate };
 }
@@ -160,4 +153,4 @@ export function computeRemoteDeletes({ local, remote }) {
   }
 
   return toDelete;
-}
+}

package/src/helpers/hash-cache-leveldb.mjs

@@ -0,0 +1,299 @@
+/**
+ * hash-cache-leveldb.mjs
+ *
+ * LevelDB-based hash cache for efficient storage of large file sets.
+ * Unlike the JSON-based cache, this scales to 100k+ files without memory issues.
+ *
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ */
+import { Level } from "level";
+import fs from "fs";
+import fsp from "fs/promises";
+import { createHash } from "crypto";
+import { Writable } from "stream";
+
+/**
+ * Streaming-SHA256 für lokale Datei
+ */
+export function hashLocalFile(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash("sha256");
+    const stream = fs.createReadStream(filePath);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+/**
+ * Streaming-SHA256 für Remote-Datei via ssh2-sftp-client
+ */
+export async function hashRemoteFile(sftp, remotePath) {
+  const hash = createHash("sha256");
+
+  const writable = new Writable({
+    write(chunk, enc, cb) {
+      hash.update(chunk);
+      cb();
+    },
+  });
+
+  await sftp.get(remotePath, writable);
+  return hash.digest("hex");
+}
+
+/**
+ * Creates a LevelDB-based hash cache.
+ *
+ * Database structure:
+ *   local:<namespace>:<relPath>  → JSON { size, mtimeMs, hash }
+ *   remote:<namespace>:<relPath> → JSON { size, modifyTime, hash }
+ *
+ * @param {Object} options
+ * @param {string} options.cachePath - Path to the LevelDB directory (e.g., ".sync-cache-prod")
+ * @param {string} options.namespace - Namespace for keys (e.g., "prod")
+ */
+export async function createHashCacheLevelDB({ cachePath, namespace }) {
+  const ns = namespace || "default";
+
+  // Open or create the LevelDB database
+  const db = new Level(cachePath, { valueEncoding: "json" });
+  await db.open();
+
+  function localKey(relPath) {
+    return `local:${ns}:${relPath}`;
+  }
+
+  function remoteKey(relPath) {
+    return `remote:${ns}:${relPath}`;
+  }
+
+  /**
+   * Get cached local hash or compute and store it
+   */
+  async function getLocalHash(rel, meta) {
+    const key = localKey(rel);
+
+    try {
+      const cached = await db.get(key);
+
+      // Cache hit: check if still valid (same size + mtime)
+      if (
+        cached &&
+        cached.size === meta.size &&
+        cached.mtimeMs === meta.mtimeMs &&
+        cached.hash
+      ) {
+        return cached.hash;
+      }
+    } catch (err) {
+      // Key not found - that's fine, we'll compute the hash
+      if (err.code !== "LEVEL_NOT_FOUND") {
+        throw err;
+      }
+    }
+
+    // Compute hash and store
+    const hash = await hashLocalFile(meta.localPath);
+    await db.put(key, {
+      size: meta.size,
+      mtimeMs: meta.mtimeMs,
+      hash,
+    });
+
+    return hash;
+  }
+
+  /**
+   * Get cached remote hash or compute and store it
+   */
+  async function getRemoteHash(rel, meta, sftp) {
+    const key = remoteKey(rel);
+
+    try {
+      const cached = await db.get(key);
+
+      // Cache hit: check if still valid (same size + modifyTime)
+      if (
+        cached &&
+        cached.size === meta.size &&
+        cached.modifyTime === meta.modifyTime &&
+        cached.hash
+      ) {
+        return cached.hash;
+      }
+    } catch (err) {
+      // Key not found - compute the hash
+      if (err.code !== "LEVEL_NOT_FOUND") {
+        throw err;
+      }
+    }
+
+    // Compute hash (downloads file content for hashing)
+    const hash = await hashRemoteFile(sftp, meta.remotePath);
+    await db.put(key, {
+      size: meta.size,
+      modifyTime: meta.modifyTime,
+      hash,
+    });
+
+    return hash;
+  }
+
+  /**
+   * Explicitly save (flush) - LevelDB auto-persists, but this ensures sync
+   */
+  async function save() {
+    // LevelDB auto-persists, nothing to do
+  }
+
+  /**
+   * Close the database connection
+   */
+  async function close() {
+    await db.close();
+  }
+
+  /**
+   * Get statistics about cache contents
+   */
+  async function getStats() {
+    let localCount = 0;
+    let remoteCount = 0;
+
+    for await (const key of db.keys()) {
+      if (key.startsWith(`local:${ns}:`)) {
+        localCount++;
+      } else if (key.startsWith(`remote:${ns}:`)) {
+        remoteCount++;
+      }
+    }
+
+    return { localCount, remoteCount };
+  }
+
+  /**
+   * Clean up stale entries (entries for files that no longer exist)
+   * @param {Set<string>} validLocalPaths - Set of currently existing local file paths
+   * @param {Set<string>} validRemotePaths - Set of currently existing remote file paths
+   */
+  async function cleanup(validLocalPaths, validRemotePaths) {
+    const batch = db.batch();
+    let deletedCount = 0;
+
+    for await (const key of db.keys()) {
+      if (key.startsWith(`local:${ns}:`)) {
+        const relPath = key.slice(`local:${ns}:`.length);
+        if (!validLocalPaths.has(relPath)) {
+          batch.del(key);
+          deletedCount++;
+        }
+      } else if (key.startsWith(`remote:${ns}:`)) {
+        const relPath = key.slice(`remote:${ns}:`.length);
+        if (!validRemotePaths.has(relPath)) {
+          batch.del(key);
+          deletedCount++;
+        }
+      }
+    }
+
+    await batch.write();
+    return deletedCount;
+  }
+
+  return {
+    getLocalHash,
+    getRemoteHash,
+    save,
+    close,
+    getStats,
+    cleanup,
+    db, // Expose for advanced usage
+  };
+}
+
+/**
+ * Migrate from old JSON cache to LevelDB
+ * @param {string} jsonCachePath - Path to old .sync-cache.json file
+ * @param {string} levelDbPath - Path to new LevelDB directory
+ * @param {string} namespace - Namespace for keys
+ */
+export async function migrateFromJsonCache(jsonCachePath, levelDbPath, namespace) {
+  const ns = namespace || "default";
+
+  // Check if JSON cache exists
+  try {
+    await fsp.access(jsonCachePath);
+  } catch {
+    return { migrated: false, reason: "No JSON cache found" };
+  }
+
+  // Check if LevelDB already has data (don't re-migrate)
+  const db = new Level(levelDbPath, { valueEncoding: "json" });
+  await db.open();
+
+  let hasData = false;
+  for await (const _ of db.keys({ limit: 1 })) {
+    hasData = true;
+    break;
+  }
+
+  if (hasData) {
+    await db.close();
+    return { migrated: false, reason: "LevelDB already has data" };
+  }
+
+  // Read and parse JSON cache
+  let jsonCache;
+  try {
+    const raw = await fsp.readFile(jsonCachePath, "utf8");
+    jsonCache = JSON.parse(raw);
+  } catch (parseErr) {
+    await db.close();
+    // Rename corrupt file so it doesn't block future runs
+    try {
+      await fsp.rename(jsonCachePath, jsonCachePath + ".corrupt");
+    } catch {
+      // Ignore rename errors
+    }
+    return { migrated: false, reason: `JSON cache corrupt: ${parseErr.message}` };
+  }
+
+  // Migrate entries
+  const batch = db.batch();
+  let localCount = 0;
+  let remoteCount = 0;
+
+  if (jsonCache.local) {
+    for (const [key, value] of Object.entries(jsonCache.local)) {
+      // Keys in JSON were like "namespace:relPath"
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      batch.put(`local:${ns}:${relPath}`, value);
+      localCount++;
+    }
+  }
+
+  if (jsonCache.remote) {
+    for (const [key, value] of Object.entries(jsonCache.remote)) {
+      const relPath = key.startsWith(`${ns}:`) ? key.slice(ns.length + 1) : key;
+      batch.put(`remote:${ns}:${relPath}`, value);
+      remoteCount++;
+    }
+  }
+
+  await batch.write();
+  await db.close();
+
+  // Optionally rename old cache to .bak
+  try {
+    await fsp.rename(jsonCachePath, jsonCachePath + ".bak");
+  } catch {
+    // Ignore rename errors
+  }
+
+  return {
+    migrated: true,
+    localCount,
+    remoteCount,
+  };
+}