sftp-push-sync 2.1.2 → 2.1.4

package/src/core/SyncLogger.mjs

@@ -0,0 +1,52 @@
+/**
+ * SyncLogger.mjs
+ * 
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ * 
+ */ 
+// src/core/SyncLogger.mjs
+import fs from "fs";
+import fsp from "fs/promises";
+import path from "path";
+
+/**
+ * Very small logger: schreibt alles in eine Logdatei
+ * und entfernt ANSI-Farbcodes.
+ */
+export class SyncLogger {
+  constructor(filePath) {
+    this.filePath = filePath;
+    this.stream = null;
+  }
+
+  async init() {
+    if (!this.filePath || this.stream) return;
+
+    const dir = path.dirname(this.filePath);
+    await fsp.mkdir(dir, { recursive: true });
+
+    this.stream = fs.createWriteStream(this.filePath, {
+      flags: "w",
+      encoding: "utf8",
+    });
+  }
+
+  writeLine(line) {
+    if (!this.stream) return;
+    const text = typeof line === "string" ? line : String(line);
+    const clean = text.replace(/\x1b\[[0-9;]*m/g, "");
+
+    try {
+      this.stream.write(clean + "\n");
+    } catch {
+      // Stream schon zu → ignorieren
+    }
+  }
+
+  close() {
+    if (this.stream) {
+      this.stream.end();
+      this.stream = null;
+    }
+  }
+}

package/src/helpers/compare.mjs

@@ -0,0 +1,122 @@
+/**
+ * compare.mjs
+ * 
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ * 
+ */ 
+// src/helpers/compare.mjs
+import fsp from "fs/promises";
+import path from "path";
+
+/**
+ * Analysiert Unterschiede zwischen local- und remote-Maps.
+ *
+ * Erwartete Struktur:
+ *  local:  Map<rel, { rel, localPath, size, mtimeMs, isText? }>
+ *  remote: Map<rel, { rel, remotePath, size, modifyTime }>
+ *
+ * Optionen:
+ *  - remoteRoot: Basis-Pfad auf dem Server
+ *  - sftp: ssh2-sftp-client Instanz
+ *  - getLocalHash / getRemoteHash: from createHashCache
+ *  - analyzeChunk: Progress-Schrittgröße
+ *  - updateProgress(prefix, current, total, rel): optional
+ */
+export async function analyseDifferences({
+  local,
+  remote,
+  remoteRoot,
+  sftp,
+  getLocalHash,
+  getRemoteHash,
+  analyzeChunk = 10,
+  updateProgress,
+}) {
+  const toAdd = [];
+  const toUpdate = [];
+
+  const localKeys = new Set(local.keys());
+  const totalToCheck = localKeys.size;
+  let checked = 0;
+
+  for (const rel of localKeys) {
+    checked += 1;
+
+    if (
+      updateProgress &&
+      (checked === 1 || checked % analyzeChunk === 0 || checked === totalToCheck)
+    ) {
+      updateProgress("Analyse: ", checked, totalToCheck, rel);
+    }
+
+    const l = local.get(rel);
+    const r = remote.get(rel);
+    const remotePath = path.posix.join(remoteRoot, rel);
+
+    // Datei existiert nur lokal → New
+    if (!r) {
+      toAdd.push({ rel, local: l, remotePath });
+      continue;
+    }
+
+    // 1. Size-Vergleich
+    if (l.size !== r.size) {
+      toUpdate.push({ rel, local: l, remote: r, remotePath });
+      continue;
+    }
+
+    // 2. Content-Vergleich
+    if (l.isText) {
+      // Text-Datei: vollständiger inhaltlicher Vergleich
+      const [localBuf, remoteBuf] = await Promise.all([
+        fsp.readFile(l.localPath),
+        sftp.get(r.remotePath),
+      ]);
+
+      const localStr = localBuf.toString("utf8");
+      const remoteStr = (
+        Buffer.isBuffer(remoteBuf) ? remoteBuf : Buffer.from(remoteBuf)
+      ).toString("utf8");
+
+      if (localStr !== remoteStr) {
+        toUpdate.push({ rel, local: l, remote: r, remotePath });
+      }
+    } else {
+      // Binary: Hash-Vergleich mit Cache
+      if (!getLocalHash || !getRemoteHash) {
+        // Fallback: wenn kein Hash-Cache übergeben wurde, treat as changed
+        toUpdate.push({ rel, local: l, remote: r, remotePath });
+        continue;
+      }
+
+      const [localHash, remoteHash] = await Promise.all([
+        getLocalHash(rel, l),
+        getRemoteHash(rel, r, sftp),
+      ]);
+
+      if (localHash !== remoteHash) {
+        toUpdate.push({ rel, local: l, remote: r, remotePath });
+      }
+    }
+  }
+
+  return { toAdd, toUpdate };
+}
+
+/**
+ * Ermittelt zu löschende Dateien (remote-only).
+ *
+ * remote: Map<rel, { rel, remotePath }>
+ */
+export function computeRemoteDeletes({ local, remote }) {
+  const toDelete = [];
+  const localKeys = new Set(local.keys());
+
+  for (const [rel, r] of remote.entries()) {
+    if (!localKeys.has(rel)) {
+      toDelete.push({ rel, remotePath: r.remotePath });
+    }
+  }
+
+  return toDelete;
+}

package/src/helpers/directory.mjs

@@ -0,0 +1,29 @@
+/**
+ * directory.mjs
+ * 
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ * 
+ */
+// src/helpers/directory.mjs
+import path from "path";
+
+/**
+ * Konvertiert einen Pfad in POSIX-Notation (immer /)
+ */
+export function toPosix(p) {
+  return p.split(path.sep).join("/");
+}
+
+/**
+ * Kürzt einen Pfad für die Progressanzeige:
+ * …/parent/file.ext
+ */
+export function shortenPathForProgress(rel) {
+  if (!rel) return "";
+  const parts = rel.split("/");
+  if (parts.length <= 2) return rel;
+
+  const last = parts[parts.length - 1];
+  const prev = parts[parts.length - 2];
+  return `…/${prev}/${last}`;
+}

package/src/helpers/hashing.mjs

@@ -0,0 +1,201 @@
+/**
+ * hashing.mjs
+ * 
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ * 
+ */ 
+// src/helpers/hashing.mjs
+import fs from "fs";
+import fsp from "fs/promises";
+import { createHash } from "crypto";
+import { Writable } from "stream";
+
+/**
+ * Streaming-SHA256 für lokale Datei
+ */
+export function hashLocalFile(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash("sha256");
+    const stream = fs.createReadStream(filePath);
+    stream.on("error", reject);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+/**
+ * Streaming-SHA256 für Remote-Datei via ssh2-sftp-client
+ */
+export async function hashRemoteFile(sftp, remotePath) {
+  const hash = createHash("sha256");
+
+  const writable = new Writable({
+    write(chunk, enc, cb) {
+      hash.update(chunk);
+      cb();
+    },
+  });
+
+  await sftp.get(remotePath, writable);
+  return hash.digest("hex");
+}
+
+/**
+ * Kleiner Helper, der Hash-Cache + Persistenz kapselt.
+ *
+ * Erwartetes Cache-Format:
+ * {
+ *   version: 1,
+ *   local:  { "<ns>:<rel>": { size, mtimeMs, hash } },
+ *   remote: { "<ns>:<rel>": { size, modifyTime, hash } }
+ * }
+ */
+export function createHashCache({
+  cachePath,
+  namespace,
+  flushInterval = 50,
+}) {
+  const ns = namespace || "default";
+
+  let cache = {
+    version: 1,
+    local: {},
+    remote: {},
+  };
+
+  // Versuch: bestehenden Cache laden
+  (async () => {
+    try {
+      const raw = await fsp.readFile(cachePath, "utf8");
+      const parsed = JSON.parse(raw);
+      cache.version = parsed.version ?? 1;
+      cache.local = parsed.local ?? {};
+      cache.remote = parsed.remote ?? {};
+    } catch {
+      // kein Cache oder defekt → einfach neu anfangen
+    }
+  })().catch(() => {});
+
+  let dirty = false;
+  let dirtyCount = 0;
+
+  function cacheKey(relPath) {
+    return `${ns}:${relPath}`;
+  }
+
+  async function save(force = false) {
+    if (!dirty && !force) return;
+    const data = JSON.stringify(cache, null, 2);
+    await fsp.writeFile(cachePath, data, "utf8");
+    dirty = false;
+    dirtyCount = 0;
+  }
+
+  async function markDirty() {
+    dirty = true;
+    dirtyCount += 1;
+    if (dirtyCount >= flushInterval) {
+      await save();
+    }
+  }
+
+  async function getLocalHash(rel, meta) {
+    const key = cacheKey(rel);
+    const cached = cache.local[key];
+
+    if (
+      cached &&
+      cached.size === meta.size &&
+      cached.mtimeMs === meta.mtimeMs &&
+      cached.hash
+    ) {
+      return cached.hash;
+    }
+
+    const hash = await hashLocalFile(meta.localPath);
+    cache.local[key] = {
+      size: meta.size,
+      mtimeMs: meta.mtimeMs,
+      hash,
+    };
+    await markDirty();
+    return hash;
+  }
+
+  async function getRemoteHash(rel, meta, sftp) {
+    const key = cacheKey(rel);
+    const cached = cache.remote[key];
+
+    if (
+      cached &&
+      cached.size === meta.size &&
+      cached.modifyTime === meta.modifyTime &&
+      cached.hash
+    ) {
+      return cached.hash;
+    }
+
+    const hash = await hashRemoteFile(sftp, meta.remotePath);
+    cache.remote[key] = {
+      size: meta.size,
+      modifyTime: meta.modifyTime,
+      hash,
+    };
+    await markDirty();
+    return hash;
+  }
+
+  return {
+    cache,
+    cacheKey,
+    getLocalHash,
+    getRemoteHash,
+    save,
+  };
+}
+
+export async function getLocalHash(rel, meta, cacheLocal, key, markDirty) {
+  const cached = cacheLocal[key];
+  if (
+    cached &&
+    cached.size === meta.size &&
+    cached.mtimeMs === meta.mtimeMs &&
+    cached.hash
+  ) {
+    return cached.hash;
+  }
+
+  const hash = await hashLocalFile(meta.localPath);
+  cacheLocal[key] = {
+    size: meta.size,
+    mtimeMs: meta.mtimeMs,
+    hash,
+  };
+  if (markDirty) {
+    await markDirty();
+  }
+  return hash;
+}
+
+export async function getRemoteHash(rel, meta, cacheRemote, key, markDirty, sftp) {
+  const cached = cacheRemote[key];
+  if (
+    cached &&
+    cached.size === meta.size &&
+    cached.modifyTime === meta.modifyTime &&
+    cached.hash
+  ) {
+    return cached.hash;
+  }
+
+  const hash = await hashRemoteFile(sftp, meta.remotePath);
+  cacheRemote[key] = {
+    size: meta.size,
+    modifyTime: meta.modifyTime,
+    hash,
+  };
+  if (markDirty) {
+    await markDirty();
+  }
+  return hash;
+}

package/src/helpers/progress-constants.mjs

@@ -0,0 +1,28 @@
+/**
+ * progress-constants.mjs
+ * Central constants for progress and output formatting.
+ * 
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ * 
+ */ 
+// src/helpers/progress-constants.mjs
+export const hr1 = () => "─".repeat(65); // horizontal line -
+export const hr2 = () => "=".repeat(65); // horizontal line =
+
+// Einrückungen (Tabs) für konsistente Ausgabe
+export const TAB_A = "   ";   // 3 Spaces
+export const TAB_B = "      "; // 6 Spaces
+
+// Spinner-Frames für Progress-Anzeigen
+export const SPINNER_FRAMES = [
+  "⠋",
+  "⠙",
+  "⠹",
+  "⠸",
+  "⠼",
+  "⠴",
+  "⠦",
+  "⠧",
+  "⠇",
+  "⠏",
+];

package/src/helpers/sidecar.mjs

@@ -0,0 +1,185 @@
+/**
+ * sidecar.mjs
+ * 
+ * @author Carsten Nichte, 2025 / https://carsten-nichte.de/
+ * 
+ */ 
+// src/helpers/sidecar.mjs
+import fs from "fs";
+import fsp from "fs/promises";
+import path from "path";
+import { walkLocalPlain, walkRemotePlain } from "./walkers.mjs";
+
+/** minimales Pattern-Matching à la minimatch (einfach: exakte Strings oder simple "*" am Ende) */
+function matchesAny(patterns, relPath) {
+  if (!patterns || patterns.length === 0) return false;
+  return patterns.some((pattern) => {
+    if (!pattern) return false;
+    if (pattern === relPath) return true;
+    // primitive *-Unterstützung, falls du willst → sonst weglassen
+    if (pattern.endsWith("*")) {
+      const prefix = pattern.slice(0, -1);
+      return relPath.startsWith(prefix);
+    }
+    return false;
+  });
+}
+
+/**
+ * Upload-Targets für Sidecar sammeln
+ */
+export async function collectUploadTargets({
+  sidecarLocalRoot,
+  sidecarRemoteRoot,
+  uploadList,
+}) {
+  const all = await walkLocalPlain(sidecarLocalRoot);
+  const results = [];
+
+  for (const [rel, meta] of all.entries()) {
+    if (matchesAny(uploadList, rel)) {
+      const remotePath = path.posix.join(sidecarRemoteRoot, rel);
+      results.push({
+        rel,
+        localPath: meta.localPath,
+        remotePath,
+      });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Download-Targets für Sidecar sammeln
+ */
+export async function collectDownloadTargets({
+  sftp,
+  sidecarLocalRoot,
+  sidecarRemoteRoot,
+  downloadList,
+}) {
+  const all = await walkRemotePlain(sftp, sidecarRemoteRoot);
+  const results = [];
+
+  for (const [rel, meta] of all.entries()) {
+    if (matchesAny(downloadList, rel)) {
+      const localPath = path.join(sidecarLocalRoot, rel);
+      results.push({
+        rel,
+        remotePath: meta.remotePath,
+        localPath,
+      });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Führt den Bypass-Modus aus (sidecar-upload / sidecar-download)
+ *
+ * Erwartete Parameter:
+ *  - sftp: ssh2-sftp-client
+ *  - connection: { sidecarLocalRoot, sidecarRemoteRoot, workers }
+ *  - uploadList, downloadList: String-Arrays
+ *  - options: { dryRun, runUploadList, runDownloadList }
+ *  - runTasks: Workerpool-Funktion (items, workerCount, handler, label)
+ *  - log, vlog, elog: Logging-Funktionen
+ *  - symbols: { ADD, CHA, tab_a } → damit du deine bestehenden Symbole weiter nutzen kannst
+ */
+export async function performBypassOnly({
+  sftp,
+  connection,
+  uploadList,
+  downloadList,
+  options,
+  runTasks,
+  log,
+  vlog,
+  elog,
+  symbols,
+}) {
+  const { dryRun, runUploadList, runDownloadList } = options;
+  const { sidecarLocalRoot, sidecarRemoteRoot, workers } = connection;
+  const { ADD, CHA, tab_a } = symbols;
+
+  log("");
+  log("🚀 Bypass-Only Mode (skip-sync)");
+  log(`${tab_a}Sidecar Local: ${sidecarLocalRoot}`);
+  log(`${tab_a}Sidecar Remote: ${sidecarRemoteRoot}`);
+
+  if (runUploadList && !fs.existsSync(sidecarLocalRoot)) {
+    const msg = `Sidecar local root does not exist: ${sidecarLocalRoot}`;
+    elog(`❌ ${msg}`);
+    throw new Error(msg);
+  }
+
+  // Upload-Bereich
+  if (runUploadList) {
+    log("");
+    log("⬆️  Upload-Bypass (sidecar-upload) …");
+    const targets = await collectUploadTargets({
+      sidecarLocalRoot,
+      sidecarRemoteRoot,
+      uploadList,
+    });
+    log(`${tab_a}→ ${targets.length} files from uploadList`);
+
+    if (!dryRun) {
+      await runTasks(
+        targets,
+        workers,
+        async ({ localPath, remotePath, rel }) => {
+          const remoteDir = path.posix.dirname(remotePath);
+          try {
+            await sftp.mkdir(remoteDir, true);
+          } catch {
+            // Directory may already exist
+          }
+          await sftp.put(localPath, remotePath);
+          vlog && vlog(`${tab_a}${ADD} Uploaded (bypass): ${rel}`);
+        },
+        "Bypass Uploads"
+      );
+    } else {
+      for (const t of targets) {
+        log(`${tab_a}${ADD} (DRY-RUN) Upload: ${t.rel}`);
+      }
+    }
+  }
+
+  // Download-Bereich
+  if (runDownloadList) {
+    log("");
+    log("⬇️  Download-Bypass (sidecar-download) …");
+    const targets = await collectDownloadTargets({
+      sftp,
+      sidecarLocalRoot,
+      sidecarRemoteRoot,
+      downloadList,
+    });
+    log(`${tab_a}→ ${targets.length} files from downloadList`);
+
+    if (!dryRun) {
+      await runTasks(
+        targets,
+        workers,
+        async ({ remotePath, localPath, rel }) => {
+          const localDir = path.dirname(localPath);
+          await fsp.mkdir(localDir, { recursive: true });
+          await sftp.get(remotePath, localPath);
+          vlog && vlog(`${tab_a}${CHA} Downloaded (bypass): ${rel}`);
+        },
+        "Bypass Downloads"
+      );
+    } else {
+      for (const t of targets) {
+        log(`${tab_a}${CHA} (DRY-RUN) Download: ${t.rel}`);
+      }
+    }
+  }
+
+  log("");
+  log("✅ Bypass-only run finished.");
+}