sftp-push-sync 1.0.0

package/bin/sync-sftp.mjs

@@ -0,0 +1,651 @@
+#!/usr/bin/env node
+/**
+ ** sync-sftp.mjs - SFTP Syncronisations Tool
+ * 
+ * SFTP push sync with dry run
+ *  1. Upload new files
+ * 2. Delete remote files that no longer exist locally
+ * 3. Detect changes based on size or modified content and upload them
+ * 
+ * Features:
+ *  - multiple connections in sync.config.json
+ *  - dry-run mode
+ *  - mirrors local → remote
+ *  - adds, updates, deletes files
+ *  - text diff detection
+ *  - Binary files (images, video, audio, PDF, etc.): SHA-256 hash comparison
+ *  - Hashes are cached in .sync-cache.json to save space.
+ *  - Parallel uploads/deletes via worker pool
+ *  - include/exclude patterns
+ * 
+ * The file shell-scripts/sync-sftp.mjs is pure JavaScript (ESM), not TypeScript.
+ * Node.js can execute it directly as long as "type": "module" is specified in package.json 
+ * or the file has the extension .mjs.
+ * 
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de
+ */
+import fs from "fs";
+import fsp from "fs/promises";
+import path from "path";
+import SftpClient from "ssh2-sftp-client";
+import { minimatch } from "minimatch";
+import { diffWords } from "diff";
+import { createHash } from "crypto";
+import { Writable } from "stream";
+import pc from "picocolors";
+
+// ---------------------------------------------------------------------------
+// CLI arguments
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2);
+const TARGET = args[0];
+const DRY_RUN = args.includes("--dry-run");
+const VERBOSE = args.includes("--verbose") || args.includes("-v");
+
+if (!TARGET) {
+  console.error(pc.red("❌ Please specify a connection profile:"));
+  console.error(pc.yellow("   node sync-sftp.mjs staging --dry-run"));
+  process.exit(1);
+}
+
+// ---------------------------------------------------------------------------
+// Load config file
+// ---------------------------------------------------------------------------
+
+const CONFIG_PATH = path.resolve("sync.config.json");
+
+if (!fs.existsSync(CONFIG_PATH)) {
+  console.error(pc.red(`❌ Configuration file missing: ${CONFIG_PATH}`));
+  process.exit(1);
+}
+
+let CONFIG_RAW;
+try {
+  CONFIG_RAW = JSON.parse(await fsp.readFile(CONFIG_PATH, "utf8"));
+} catch (err) {
+  elog(pc.red("❌ Error reading sync.config.json:"), err.message);
+  process.exit(1);
+}
+
+if (!CONFIG_RAW.connections || typeof CONFIG_RAW.connections !== "object") {
+  elog(pc.red("❌ sync.config.json must have a 'connections' field."));
+  process.exit(1);
+}
+
+const TARGET_CONFIG = CONFIG_RAW.connections[TARGET];
+if (!TARGET_CONFIG) {
+  elog(pc.red(`❌ Connection '${TARGET}' not found in sync.config.json.`));
+  process.exit(1);
+}
+
+const CONNECTION = {
+  host: TARGET_CONFIG.host,
+  port: TARGET_CONFIG.port ?? 22,
+  user: TARGET_CONFIG.user,
+  password: TARGET_CONFIG.password,
+  localRoot: path.resolve(TARGET_CONFIG.localRoot),
+  remoteRoot: TARGET_CONFIG.remoteRoot,
+  workers: TARGET_CONFIG.worker ?? 2,
+};
+
+// Shared config from JSON
+const INCLUDE = CONFIG_RAW.include ?? [];
+const EXCLUDE = CONFIG_RAW.exclude ?? [];
+const TEXT_EXT = CONFIG_RAW.textExtensions ?? [
+  ".html",
+  ".htm",
+  ".xml",
+  ".txt",
+  ".json",
+  ".js",
+  ".mjs",
+  ".cjs",
+  ".css",
+  ".md",
+  ".svg",
+];
+
+// Cache file name per connection
+const syncCacheName =
+  TARGET_CONFIG.syncCache || `.sync-cache.${TARGET}.json`;
+const CACHE_PATH = path.resolve(syncCacheName);
+
+// ---------------------------------------------------------------------------
+// Load/initialise hash cache
+// ---------------------------------------------------------------------------
+
+let CACHE = {
+  version: 1,
+  local: {}, // key: "<TARGET>:<relPath>" -> { size, mtimeMs, hash }
+  remote: {}, // key: "<TARGET>:<relPath>" -> { size, modifyTime, hash }
+};
+
+try {
+  if (fs.existsSync(CACHE_PATH)) {
+    const raw = JSON.parse(await fsp.readFile(CACHE_PATH, "utf8"));
+    CACHE.version = raw.version ?? 1;
+    CACHE.local = raw.local ?? {};
+    CACHE.remote = raw.remote ?? {};
+  }
+} catch (err) {
+  wlog(pc.yellow("⚠️  Konnte Cache nicht laden, starte ohne Cache:"), err.message);
+}
+
+function cacheKey(relPath) {
+  return `${TARGET}:${relPath}`;
+}
+
+let cacheDirty = false;
+let cacheDirtyCount = 0;
+const CACHE_FLUSH_INTERVAL = 50; // Write cache to disk after 50 new hashes
+
+async function saveCache(force = false) {
+  if (!cacheDirty && !force) return;
+  const data = JSON.stringify(CACHE, null, 2);
+  await fsp.writeFile(CACHE_PATH, data, "utf8");
+  cacheDirty = false;
+  cacheDirtyCount = 0;
+}
+
+async function markCacheDirty() {
+  cacheDirty = true;
+  cacheDirtyCount += 1;
+  if (cacheDirtyCount >= CACHE_FLUSH_INTERVAL) {
+    await saveCache();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+let progressActive = false;
+
+function clearProgressLine() {
+  if (!process.stdout.isTTY || !progressActive) return;
+  const width = process.stdout.columns || 80;
+  const blank = " ".repeat(width - 1);
+  process.stdout.write("\r" + blank + "\r");
+  progressActive = false;
+}
+
+function toPosix(p) {
+  return p.split(path.sep).join("/");
+}
+
+function log(...msg) {
+  clearProgressLine();
+  console.log(...msg);
+}
+
+function vlog(...msg) {
+  if (!VERBOSE) return;
+  clearProgressLine();
+  console.log(...msg);
+}
+
+function elog(...msg) {
+  clearProgressLine();
+  console.error(...msg);
+}
+
+function wlog(...msg) {
+  clearProgressLine();
+  console.warn(...msg);
+}
+
+function matchesAny(patterns, relPath) {
+  if (!patterns || patterns.length === 0) return false;
+  return patterns.some((pattern) =>
+    minimatch(relPath, pattern, { dot: true })
+  );
+}
+
+function isIncluded(relPath) {
+  if (INCLUDE.length > 0 && !matchesAny(INCLUDE, relPath)) return false;
+  if (EXCLUDE.length > 0 && matchesAny(EXCLUDE, relPath)) return false;
+  return true;
+}
+
+function isTextFile(relPath) {
+  const ext = path.extname(relPath).toLowerCase();
+  return TEXT_EXT.includes(ext);
+}
+
+// Single-line progress bar
+function updateProgress(prefix, current, total) {
+  const percent = total > 0 ? ((current / total) * 100).toFixed(1) : "0.0";
+  const msg = `${prefix}${current}/${total} Dateien (${percent}%)`;
+
+  if (!process.stdout.isTTY) {
+    // Fallback: simply log in
+    console.log("   " + msg);
+    return;
+  }
+
+  const width = process.stdout.columns || 80;
+  const line = msg.padEnd(width - 1);
+
+  progressActive = true;
+  process.stdout.write("\r" + line);
+
+  if (current === total) {
+    process.stdout.write("\n");
+    progressActive = false;
+  }
+}
+
+// Simple worker pool for parallel tasks
+async function runTasks(items, workerCount, handler, label = "Tasks") {
+  if (items.length === 0) return;
+
+  const total = items.length;
+  let done = 0;
+  let index = 0;
+
+  async function worker() {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const i = index;
+      if (i >= total) break;
+      index += 1;
+      const item = items[i];
+      try {
+        await handler(item);
+      } catch (err) {
+        elog(pc.red(`   ⚠️ Fehler in ${label}:`), err.message || err);
+      }
+      done += 1;
+      if (done % 10 === 0 || done === total) {
+        updateProgress(`   ${label}: `, done, total);
+      }
+    }
+  }
+
+  const workers = [];
+  const actualWorkers = Math.max(1, Math.min(workerCount, total));
+  for (let i = 0; i < actualWorkers; i += 1) {
+    workers.push(worker());
+  }
+  await Promise.all(workers);
+}
+
+// ---------------------------------------------------------------------------
+// Local file walker (recursive, all subdirectories)
+// ---------------------------------------------------------------------------
+
+async function walkLocal(root) {
+  const result = new Map();
+
+  async function recurse(current) {
+    const entries = await fsp.readdir(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const full = path.join(current, entry.name);
+      if (entry.isDirectory()) {
+        await recurse(full);
+      } else if (entry.isFile()) {
+        const rel = toPosix(path.relative(root, full));
+        if (!isIncluded(rel)) continue;
+        const stat = await fsp.stat(full);
+        result.set(rel, {
+          rel,
+          localPath: full,
+          size: stat.size,
+          mtimeMs: stat.mtimeMs,
+          isText: isTextFile(rel),
+        });
+      }
+    }
+  }
+
+  await recurse(root);
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Remote walker (recursive, all subdirectories)
+// ---------------------------------------------------------------------------
+
+async function walkRemote(sftp, remoteRoot) {
+  const result = new Map();
+
+  async function recurse(remoteDir, prefix) {
+    const items = await sftp.list(remoteDir);
+
+    for (const item of items) {
+      if (!item.name || item.name === "." || item.name === "..") continue;
+
+      const full = path.posix.join(remoteDir, item.name);
+      const rel = prefix ? `${prefix}/${item.name}` : item.name;
+
+      if (item.type === "d") {
+        await recurse(full, rel);
+      } else {
+        result.set(rel, {
+          rel,
+          remotePath: full,
+          size: Number(item.size),
+          modifyTime: item.modifyTime ?? 0,
+        });
+      }
+    }
+  }
+
+  await recurse(remoteRoot, "");
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Hash helper for binaries (streaming, memory-efficient)
+// ---------------------------------------------------------------------------
+
+function hashLocalFile(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash("sha256");
+    const stream = fs.createReadStream(filePath);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+async function hashRemoteFile(sftp, remotePath) {
+  const hash = createHash("sha256");
+
+  const writable = new Writable({
+    write(chunk, enc, cb) {
+      hash.update(chunk);
+      cb();
+    },
+  });
+
+  await sftp.get(remotePath, writable);
+  return hash.digest("hex");
+}
+
+// Cache-aware Helpers
+async function getLocalHash(rel, meta) {
+  const key = cacheKey(rel);
+  const cached = CACHE.local[key];
+  if (
+    cached &&
+    cached.size === meta.size &&
+    cached.mtimeMs === meta.mtimeMs &&
+    cached.hash
+  ) {
+    return cached.hash;
+  }
+
+  const hash = await hashLocalFile(meta.localPath);
+  CACHE.local[key] = {
+    size: meta.size,
+    mtimeMs: meta.mtimeMs,
+    hash,
+  };
+  await markCacheDirty();
+  return hash;
+}
+
+async function getRemoteHash(rel, meta, sftp) {
+  const key = cacheKey(rel);
+  const cached = CACHE.remote[key];
+  if (
+    cached &&
+    cached.size === meta.size &&
+    cached.modifyTime === meta.modifyTime &&
+    cached.hash
+  ) {
+    return cached.hash;
+  }
+
+  const hash = await hashRemoteFile(sftp, meta.remotePath);
+  CACHE.remote[key] = {
+    size: meta.size,
+    modifyTime: meta.modifyTime,
+    hash,
+  };
+  await markCacheDirty();
+  return hash;
+}
+
+// ---------------------------------------------------------------------------
+// MAIN
+// ---------------------------------------------------------------------------
+
+async function main() {
+  const start = Date.now();
+  
+  log("\n\n==================================================================");
+  log(pc.bold("🔐  SFTP-Synchronisation - shell-scripts/sync-sftp.mjs"));
+  log(`   Connection: ${pc.cyan(TARGET)}  (Worker: ${CONNECTION.workers})`);
+  log(`   Host:Port: ${pc.green(CONNECTION.host)}:${pc.green(CONNECTION.port)}`);
+  log(`   Local:  ${pc.green(CONNECTION.localRoot)}`);
+  log(`   Remote: ${pc.green(CONNECTION.remoteRoot)}`);
+  if (DRY_RUN) log(pc.yellow("   Modus: DRY-RUN (no changes)"));
+  log("-----------------------------------------------------------------\n");
+
+  const sftp = new SftpClient();
+
+  const toAdd = [];
+  const toUpdate = [];
+  const toDelete = [];
+
+  try {
+    await sftp.connect({
+      host: CONNECTION.host,
+      port: CONNECTION.port,
+      username: CONNECTION.user,
+      password: CONNECTION.password,
+    });
+
+    vlog(pc.dim("   Connection established."));
+
+    if (!fs.existsSync(CONNECTION.localRoot)) {
+      console.error(pc.red("❌ Local root does not exist:"), CONNECTION.localRoot);
+      process.exit(1);
+    }
+
+    log(pc.bold(pc.cyan("📥 Phase 1: Scan local files …")));
+    const local = await walkLocal(CONNECTION.localRoot);
+    log(`   → ${local.size} lokale Dateien`);
+
+    log(pc.bold(pc.cyan("📤 Phase 2: Scan remote files …")));
+    const remote = await walkRemote(sftp, CONNECTION.remoteRoot);
+    log(`   → ${remote.size} remote files`);
+
+    const localKeys = new Set(local.keys());
+    const remoteKeys = new Set(remote.keys());
+
+    log(pc.bold(pc.cyan("🔎 Phase 3: Compare & decide …")));
+    const totalToCheck = localKeys.size;
+    let checkedCount = 0;
+
+    // Analysis: just decide, don't upload/delete anything yet
+    for (const rel of localKeys) {
+      checkedCount += 1;
+      if (checkedCount % 500 === 0 || checkedCount === totalToCheck) {
+        updateProgress("   Analyse: ", checkedCount, totalToCheck);
+      }
+
+      const l = local.get(rel);
+      const r = remote.get(rel);
+
+      const remotePath = path.posix.join(CONNECTION.remoteRoot, rel);
+
+      if (!r) {
+        toAdd.push({ rel, local: l, remotePath });
+        log(`${pc.green("➕ New:")} ${rel}`);
+        continue;
+      }
+
+      // 1. size comparison
+      if (l.size !== r.size) {
+        toUpdate.push({ rel, local: l, remote: r, remotePath });
+        log(`${pc.yellow("🔁 Size changed:")} ${rel}`);
+        continue;
+      }
+
+      // 2. content comparison
+      if (l.isText) {
+        // Text file: Read & compare in full
+        const [localBuf, remoteBuf] = await Promise.all([
+          fsp.readFile(l.localPath),
+          sftp.get(r.remotePath),
+        ]);
+
+        const localStr = localBuf.toString("utf8");
+        const remoteStr = (Buffer.isBuffer(remoteBuf)
+          ? remoteBuf
+          : Buffer.from(remoteBuf)
+        ).toString("utf8");
+
+        if (localStr === remoteStr) {
+          vlog(`   ${pc.dim("✓ Unchanged (Text):")} ${rel}`);
+          continue;
+        }
+
+        if (VERBOSE) {
+          const diff = diffWords(remoteStr, localStr);
+          const blocks = diff.filter((d) => d.added || d.removed).length;
+          vlog(`   ↪️ text difference (${blocks} Blocks) in ${rel}`);
+        }
+
+        toUpdate.push({ rel, local: l, remote: r, remotePath });
+        log(`${pc.yellow("   🔁 Content changed(Text):")} ${rel}`);
+      } else {
+        // Binary: Hash comparison with cache
+        const localMeta = l;
+        const remoteMeta = r;
+
+        const [localHash, remoteHash] = await Promise.all([
+          getLocalHash(rel, localMeta),
+          getRemoteHash(rel, remoteMeta, sftp),
+        ]);
+
+        if (localHash === remoteHash) {
+          vlog(`   ${pc.dim("✓ Unchanged (binary, hash):")} ${rel}`);
+          continue;
+        }
+
+        if (VERBOSE) {
+          vlog(`   ↪️ Hash different(binary): ${rel}`);
+          vlog(`      local:  ${localHash}`);
+          vlog(`      remote: ${remoteHash}`);
+        }
+
+        toUpdate.push({ rel, local: l, remote: r, remotePath });
+        log(`${pc.yellow("   🔁 Content changed (Binary):")} ${rel}`);
+      }
+    }
+
+    log("\n" + pc.bold(pc.cyan("🧹 Phase 4: Removing orphaned remote files …")));
+    for (const rel of remoteKeys) {
+      if (!localKeys.has(rel)) {
+        const r = remote.get(rel);
+        toDelete.push({ rel, remotePath: r.remotePath });
+        log(`${pc.red("   🗑 Remove:")} ${rel}`);
+      }
+    }
+
+    // -------------------------------------------------------------------
+    // Phase 5: Execute changes (parallel, worker-based)
+    // -------------------------------------------------------------------
+
+    if (!DRY_RUN) {
+      log("\n" + pc.bold(pc.cyan("🚚 Phase 5: Implement changes …")));
+
+      // Upload new files
+      await runTasks(
+        toAdd,
+        CONNECTION.workers,
+        async ({ rel, local: l, remotePath }) => {
+          const remoteDir = path.posix.dirname(remotePath);
+          try {
+            await sftp.mkdir(remoteDir, true);
+          } catch {
+            // Directory may already exist.
+          }
+          await sftp.put(l.localPath, remotePath);
+        },
+        "Uploads (new)"
+      );
+
+      // Updates
+      await runTasks(
+        toUpdate,
+        CONNECTION.workers,
+        async ({ rel, local: l, remotePath }) => {
+          const remoteDir = path.posix.dirname(remotePath);
+          try {
+            await sftp.mkdir(remoteDir, true);
+          } catch {
+            // Directory may already exist.
+          }
+          await sftp.put(l.localPath, remotePath);
+        },
+        "Uploads (update)"
+      );
+
+      // Deletes
+      await runTasks(
+        toDelete,
+        CONNECTION.workers,
+        async ({ remotePath }) => {
+          try {
+            await sftp.delete(remotePath);
+          } catch (e) {
+            console.error(
+              pc.red("   ⚠️ Error during deletion:"),
+              remotePath,
+              e.message || e
+            );
+          }
+        },
+        "Deletes"
+      );
+    } else {
+      log(pc.yellow("\n💡 DRY-RUN: No files transferred or deleted."));
+    }
+
+    const duration = ((Date.now() - start) / 1000).toFixed(2);
+
+    // Write cache securely at the end
+    await saveCache(true);
+
+    // Summary
+    log("\n" + pc.bold(pc.cyan("📊 Summary:")));
+    log(`   Dauer: ${pc.green(duration + " s")}`);
+    log(`   ➕ Added  : ${toAdd.length}`);
+    log(`   🔁 Changed:   ${toUpdate.length}`);
+    log(`   🗑 Deleted:   ${toDelete.length}`);
+
+    if (toAdd.length || toUpdate.length || toDelete.length) {
+      log("\n📄 Changes:");
+      [...toAdd.map((t) => t.rel)].sort().forEach((f) => console.log(`   ${pc.green("➕")} ${f}`));
+      [...toUpdate.map((t) => t.rel)].sort().forEach((f) => console.log(`   ${pc.yellow("🔁")} ${f}`));
+      [...toDelete.map((t) => t.rel)].sort().forEach((f) => console.log(`   ${pc.red("🗑")} ${f}`));
+    } else {
+      log("\nNo changes.");
+    }
+
+    log("\n" + pc.bold(pc.green("✅ Sync complete.")));
+    log("==================================================================\n\n");
+  } catch (err) {
+    elog(pc.red("❌ Synchronisation error:"), err);
+    process.exitCode = 1;
+    try {
+      await saveCache(true);
+    } catch {
+      // ignore
+    }
+  } finally {
+    try {
+      await sftp.end();
+    } catch {
+      // ignore
+    }
+  }
+}
+
+main();

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "sftp-push-sync",
+  "version": "1.0.0",
+  "description": "SFTP Sync Tool für Hugo-Projekte (local -> remote, mit Hash-Cache)",
+  "type": "module",
+  "bin": {
+    "hugo-sftp-sync": "./bin/sftp-sync.mjs"
+  },
+  "keywords": [
+    "hugo",
+    "sftp",
+    "deploy",
+    "sync"
+  ],
+  "author": "Carsten Nichte",
+  "license": "MIT",
+  "dependencies": {
+    "ssh2-sftp-client": "^10.0.0",
+    "minimatch": "^9.0.3",
+    "diff": "^5.2.0",
+    "picocolors": "^1.0.0"
+  }
+}