sftp-push-sync 3.0.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -1
- package/README.md +4 -2
- package/package.json +1 -1
- package/src/core/SftpPushSyncApp.mjs +313 -77
- package/src/core/SyncLogger.mjs +18 -6
- package/src/helpers/hash-cache-ndjson.mjs +13 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [3.0.1] - 2026-03-05
|
|
4
|
+
|
|
5
|
+
- stability improvements especialy during large and longtime uploads, error handling, log with datetime.
|
|
6
|
+
|
|
3
7
|
## [3.0.0] - 2026-03-04
|
|
4
8
|
|
|
5
9
|
- Switched from JSON-file based hash cache to NDJSON-based Cache-implementation.
|
|
6
10
|
- Disk-based, only active entries in RAM
|
|
7
11
|
- Scales to 100,000+ files without memory issues
|
|
8
12
|
- Auto-persist (no explicit saving required)
|
|
9
|
-
- Auto-migration - Existing JSON cache
|
|
13
|
+
- Auto-migration - Existing JSON cache is automatically migrated
|
|
10
14
|
|
|
11
15
|
## [2.5.0] - 2026-03-04
|
|
12
16
|
|
package/README.md
CHANGED
|
@@ -19,7 +19,7 @@ Features:
|
|
|
19
19
|
- adds, updates, deletes files
|
|
20
20
|
- text diff detection
|
|
21
21
|
- Binary files (images, video, audio, PDF, etc.): SHA-256 hash comparison
|
|
22
|
-
- Hashes are cached in
|
|
22
|
+
- Hashes are cached in `.sync-cache.*.ndjson`
|
|
23
23
|
- Parallel uploads/deletions via worker pool
|
|
24
24
|
- include/exclude patterns
|
|
25
25
|
- Sidecar uploads / downloads - Bypassing the sync process
|
|
@@ -104,6 +104,7 @@ Create a `sync.config.json` in the root folder of your project:
|
|
|
104
104
|
"analyzeChunk": 1
|
|
105
105
|
},
|
|
106
106
|
"logLevel": "normal",
|
|
107
|
+
"logTimestamps": false,
|
|
107
108
|
"logFile": ".sftp-push-sync.{target}.log"
|
|
108
109
|
}
|
|
109
110
|
```
|
|
@@ -204,6 +205,7 @@ sftp-push-sync prod --sidecar-download --skip-sync
|
|
|
204
205
|
Logging can also be configured.
|
|
205
206
|
|
|
206
207
|
- `logLevel` - normal, verbose, laconic.
|
|
208
|
+
- `logTimestamps` - true/false. When enabled, each log line is prefixed with a timestamp `[YYYY-MM-DD HH:mm:ss.SSS]`.
|
|
207
209
|
- `logFile` - an optional logFile.
|
|
208
210
|
- `scanChunk` - After how many elements should a log output be generated during scanning?
|
|
209
211
|
- `analyzeChunk` - After how many elements should a log output be generated during analysis?
|
|
@@ -275,4 +277,4 @@ Note 2: Reliability and accuracy are more important to me than speed.
|
|
|
275
277
|
- <https://www.npmjs.com/package/sftp-push-sync>
|
|
276
278
|
- <https://github.com/cnichte/sftp-push-sync>
|
|
277
279
|
- <https://www.npmjs.com/package/hugo-toolbox>
|
|
278
|
-
- <https://carsten-nichte.de>
|
|
280
|
+
- <https://carsten-nichte.de>
|
package/package.json
CHANGED
|
@@ -206,30 +206,52 @@ export class SftpPushSyncApp {
|
|
|
206
206
|
}
|
|
207
207
|
|
|
208
208
|
/**
|
|
209
|
-
* Reconnect to SFTP server
|
|
209
|
+
* Reconnect to SFTP server with retry logic
|
|
210
210
|
*/
|
|
211
|
-
async _reconnect(sftp) {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
211
|
+
async _reconnect(sftp, maxRetries = 3) {
|
|
212
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
213
|
+
try {
|
|
214
|
+
try {
|
|
215
|
+
await sftp.end();
|
|
216
|
+
} catch {
|
|
217
|
+
// Ignore errors when closing dead connection
|
|
218
|
+
}
|
|
217
219
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
keepaliveCountMax: 10,
|
|
225
|
-
readyTimeout: 30000,
|
|
226
|
-
});
|
|
220
|
+
// Wait before reconnecting (exponential backoff)
|
|
221
|
+
if (attempt > 1) {
|
|
222
|
+
const waitTime = 1000 * Math.pow(2, attempt - 1); // 2s, 4s, 8s
|
|
223
|
+
this.log(`${TAB_A}${pc.yellow(`⏳ Waiting ${waitTime/1000}s before retry ${attempt}/${maxRetries}…`)}`);
|
|
224
|
+
await new Promise(r => setTimeout(r, waitTime));
|
|
225
|
+
}
|
|
227
226
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
227
|
+
await sftp.connect({
|
|
228
|
+
host: this.connection.host,
|
|
229
|
+
port: this.connection.port,
|
|
230
|
+
username: this.connection.user,
|
|
231
|
+
password: this.connection.password,
|
|
232
|
+
keepaliveInterval: 5000, // More frequent keepalive (5s instead of 10s)
|
|
233
|
+
keepaliveCountMax: 6, // Disconnect after 30s of no response
|
|
234
|
+
readyTimeout: 60000, // 60s timeout for initial connection
|
|
235
|
+
retries: 2, // Internal retries
|
|
236
|
+
retry_factor: 2,
|
|
237
|
+
retry_minTimeout: 2000,
|
|
238
|
+
});
|
|
231
239
|
|
|
232
|
-
|
|
240
|
+
if (sftp.client) {
|
|
241
|
+
sftp.client.setMaxListeners(50);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
this.log(`${TAB_A}${pc.green("✔ Reconnected to SFTP.")}`);
|
|
245
|
+
return; // Success
|
|
246
|
+
} catch (err) {
|
|
247
|
+
const msg = err?.message || String(err);
|
|
248
|
+
if (attempt === maxRetries) {
|
|
249
|
+
this.elog(pc.red(`❌ Failed to reconnect after ${maxRetries} attempts: ${msg}`));
|
|
250
|
+
throw err;
|
|
251
|
+
}
|
|
252
|
+
this.wlog(pc.yellow(`⚠ Reconnect attempt ${attempt} failed: ${msg}`));
|
|
253
|
+
}
|
|
254
|
+
}
|
|
233
255
|
}
|
|
234
256
|
|
|
235
257
|
// ---------------------------------------------------------
|
|
@@ -333,18 +355,23 @@ export class SftpPushSyncApp {
|
|
|
333
355
|
}
|
|
334
356
|
|
|
335
357
|
// ---------------------------------------------------------
|
|
336
|
-
// Worker-Pool
|
|
358
|
+
// Worker-Pool with auto-reconnect
|
|
337
359
|
// ---------------------------------------------------------
|
|
338
360
|
|
|
339
|
-
async runTasks(items, workerCount, handler, label = "Tasks") {
|
|
361
|
+
async runTasks(items, workerCount, handler, label = "Tasks", sftp = null) {
|
|
340
362
|
if (!items || items.length === 0) return;
|
|
341
363
|
|
|
342
364
|
const total = items.length;
|
|
343
365
|
let done = 0;
|
|
344
366
|
let index = 0;
|
|
367
|
+
let failedCount = 0;
|
|
345
368
|
const workers = [];
|
|
346
369
|
const actualWorkers = Math.max(1, Math.min(workerCount, total));
|
|
347
370
|
|
|
371
|
+
// Mutex for reconnection (only one worker reconnects at a time)
|
|
372
|
+
let reconnecting = false;
|
|
373
|
+
let reconnectWaiters = 0;
|
|
374
|
+
|
|
348
375
|
const worker = async () => {
|
|
349
376
|
// eslint-disable-next-line no-constant-condition
|
|
350
377
|
while (true) {
|
|
@@ -353,13 +380,82 @@ export class SftpPushSyncApp {
|
|
|
353
380
|
index += 1;
|
|
354
381
|
const item = items[i];
|
|
355
382
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
383
|
+
let retries = 0;
|
|
384
|
+
const maxRetries = 5; // Increased from 2 to 5 for unstable servers
|
|
385
|
+
|
|
386
|
+
while (retries <= maxRetries) {
|
|
387
|
+
try {
|
|
388
|
+
await handler(item);
|
|
389
|
+
break; // Success, exit retry loop
|
|
390
|
+
} catch (err) {
|
|
391
|
+
const msg = err?.message || String(err);
|
|
392
|
+
const isConnectionError =
|
|
393
|
+
msg.includes("No SFTP connection") ||
|
|
394
|
+
msg.includes("ECONNRESET") ||
|
|
395
|
+
msg.includes("ETIMEDOUT") ||
|
|
396
|
+
msg.includes("ECONNREFUSED") ||
|
|
397
|
+
msg.includes("connection") ||
|
|
398
|
+
msg.includes("Channel open failure") ||
|
|
399
|
+
msg.includes("socket") ||
|
|
400
|
+
msg.includes("SSH");
|
|
401
|
+
|
|
402
|
+
if (isConnectionError && sftp && retries < maxRetries) {
|
|
403
|
+
// Wait if another worker is already reconnecting
|
|
404
|
+
let waitCount = 0;
|
|
405
|
+
reconnectWaiters++;
|
|
406
|
+
if (reconnecting && this.isVerbose) {
|
|
407
|
+
this.log(`${TAB_A}${pc.dim(`Worker waiting for reconnect (${reconnectWaiters} waiting)…`)}`);
|
|
408
|
+
}
|
|
409
|
+
while (reconnecting && waitCount < 120) { // Max 60 seconds wait
|
|
410
|
+
await new Promise(r => setTimeout(r, 500));
|
|
411
|
+
waitCount++;
|
|
412
|
+
// Log every 10 seconds while waiting
|
|
413
|
+
if (waitCount % 20 === 0 && this.isVerbose) {
|
|
414
|
+
this.log(`${TAB_A}${pc.dim(`Still waiting for reconnect… (${waitCount / 2}s)`)}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
reconnectWaiters--;
|
|
418
|
+
|
|
419
|
+
// Check if reconnection is still needed
|
|
420
|
+
if (!await this._isConnected(sftp)) {
|
|
421
|
+
reconnecting = true;
|
|
422
|
+
this.log(`${TAB_A}${pc.yellow("⚠ Connection lost during " + label + ", reconnecting…")}`);
|
|
423
|
+
try {
|
|
424
|
+
await this._reconnect(sftp);
|
|
425
|
+
this.log(`${TAB_A}${pc.green("✔ Reconnected, resuming " + label + "…")}`);
|
|
426
|
+
} catch (reconnectErr) {
|
|
427
|
+
this.elog(pc.red(`${TAB_A}❌ Reconnect failed: ${reconnectErr?.message || reconnectErr}`));
|
|
428
|
+
reconnecting = false;
|
|
429
|
+
// Re-throw to trigger retry
|
|
430
|
+
throw reconnectErr;
|
|
431
|
+
} finally {
|
|
432
|
+
reconnecting = false;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
retries++;
|
|
437
|
+
const retryDelay = 500 * retries;
|
|
438
|
+
if (this.isVerbose) {
|
|
439
|
+
this.log(`${TAB_A}${pc.dim(`Retry ${retries}/${maxRetries} for: ${item.rel || ''} (waiting ${retryDelay}ms)`)}`);
|
|
440
|
+
}
|
|
441
|
+
// Brief pause before retry
|
|
442
|
+
await new Promise(r => setTimeout(r, retryDelay));
|
|
443
|
+
// Retry the same item
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// Log error and move on
|
|
448
|
+
this.elog(
|
|
449
|
+
pc.red(`${TAB_A}⚠️ Error in ${label} (attempt ${retries + 1}/${maxRetries + 1}):`),
|
|
450
|
+
msg
|
|
451
|
+
);
|
|
452
|
+
|
|
453
|
+
if (retries >= maxRetries) {
|
|
454
|
+
failedCount++;
|
|
455
|
+
this.elog(pc.red(`${TAB_A}❌ Failed after ${maxRetries + 1} attempts: ${item.rel || item.remotePath || ''}`));
|
|
456
|
+
}
|
|
457
|
+
break; // Exit retry loop
|
|
458
|
+
}
|
|
363
459
|
}
|
|
364
460
|
|
|
365
461
|
done += 1;
|
|
@@ -373,6 +469,9 @@ export class SftpPushSyncApp {
|
|
|
373
469
|
workers.push(worker());
|
|
374
470
|
}
|
|
375
471
|
await Promise.all(workers);
|
|
472
|
+
|
|
473
|
+
// Return statistics
|
|
474
|
+
return { total, done, failed: failedCount };
|
|
376
475
|
}
|
|
377
476
|
|
|
378
477
|
// ---------------------------------------------------------
|
|
@@ -409,6 +508,7 @@ export class SftpPushSyncApp {
|
|
|
409
508
|
if (total === 0) return;
|
|
410
509
|
|
|
411
510
|
let current = 0;
|
|
511
|
+
let failedDirs = 0;
|
|
412
512
|
|
|
413
513
|
for (const relDir of dirs) {
|
|
414
514
|
current += 1;
|
|
@@ -422,24 +522,59 @@ export class SftpPushSyncApp {
|
|
|
422
522
|
"Folders"
|
|
423
523
|
);
|
|
424
524
|
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
525
|
+
let retries = 0;
|
|
526
|
+
const maxRetries = 3;
|
|
527
|
+
let success = false;
|
|
528
|
+
|
|
529
|
+
while (retries <= maxRetries && !success) {
|
|
530
|
+
try {
|
|
531
|
+
const exists = await sftp.exists(remoteDir);
|
|
532
|
+
if (!exists) {
|
|
533
|
+
await sftp.mkdir(remoteDir, true);
|
|
534
|
+
this.dirStats.createdDirs += 1;
|
|
535
|
+
this.vlog(`${TAB_A}${pc.dim("dir created:")} ${remoteDir}`);
|
|
536
|
+
} else {
|
|
537
|
+
this.vlog(`${TAB_A}${pc.dim("dir ok:")} ${remoteDir}`);
|
|
538
|
+
}
|
|
539
|
+
success = true;
|
|
540
|
+
} catch (e) {
|
|
541
|
+
const msg = e?.message || String(e);
|
|
542
|
+
const isConnectionError =
|
|
543
|
+
msg.includes("No SFTP connection") ||
|
|
544
|
+
msg.includes("ECONNRESET") ||
|
|
545
|
+
msg.includes("ETIMEDOUT") ||
|
|
546
|
+
msg.includes("connection") ||
|
|
547
|
+
msg.includes("Channel open failure") ||
|
|
548
|
+
msg.includes("socket") ||
|
|
549
|
+
msg.includes("SSH");
|
|
550
|
+
|
|
551
|
+
if (isConnectionError && retries < maxRetries) {
|
|
552
|
+
this.log(`${TAB_A}${pc.yellow("⚠ Connection lost during directory preparation, reconnecting…")}`);
|
|
553
|
+
try {
|
|
554
|
+
await this._reconnect(sftp);
|
|
555
|
+
retries++;
|
|
556
|
+
await new Promise(r => setTimeout(r, 500 * retries));
|
|
557
|
+
continue; // Retry this directory
|
|
558
|
+
} catch (reconnectErr) {
|
|
559
|
+
this.elog(pc.red(`${TAB_A}❌ Reconnect failed: ${reconnectErr?.message || reconnectErr}`));
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
this.wlog(
|
|
564
|
+
pc.yellow("⚠️ Could not ensure directory:"),
|
|
565
|
+
remoteDir,
|
|
566
|
+
msg
|
|
567
|
+
);
|
|
568
|
+
failedDirs++;
|
|
569
|
+
break; // Move to next directory
|
|
433
570
|
}
|
|
434
|
-
} catch (e) {
|
|
435
|
-
this.wlog(
|
|
436
|
-
pc.yellow("⚠️ Could not ensure directory:"),
|
|
437
|
-
remoteDir,
|
|
438
|
-
e?.message || e
|
|
439
|
-
);
|
|
440
571
|
}
|
|
441
572
|
}
|
|
442
573
|
|
|
574
|
+
if (failedDirs > 0) {
|
|
575
|
+
this.wlog(pc.yellow(`⚠️ ${failedDirs} directories could not be created`));
|
|
576
|
+
}
|
|
577
|
+
|
|
443
578
|
this.updateProgress2("Prepare dirs: ", total, total, "done", "Folders");
|
|
444
579
|
process.stdout.write("\n");
|
|
445
580
|
this.progressActive = false;
|
|
@@ -450,7 +585,24 @@ export class SftpPushSyncApp {
|
|
|
450
585
|
// ---------------------------------------------------------
|
|
451
586
|
|
|
452
587
|
async cleanupEmptyDirs(sftp, rootDir, dryRun) {
|
|
453
|
-
|
|
588
|
+
// Track reconnect state at cleanup level
|
|
589
|
+
let reconnectNeeded = false;
|
|
590
|
+
|
|
591
|
+
const attemptReconnect = async () => {
|
|
592
|
+
if (reconnectNeeded) return false; // Already tried
|
|
593
|
+
reconnectNeeded = true;
|
|
594
|
+
this.log(`${TAB_A}${pc.yellow("⚠ Connection lost during cleanup, reconnecting…")}`);
|
|
595
|
+
try {
|
|
596
|
+
await this._reconnect(sftp);
|
|
597
|
+
reconnectNeeded = false;
|
|
598
|
+
return true;
|
|
599
|
+
} catch (err) {
|
|
600
|
+
this.elog(pc.red(`${TAB_A}❌ Reconnect during cleanup failed: ${err?.message || err}`));
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
};
|
|
604
|
+
|
|
605
|
+
const recurse = async (dir, depth = 0) => {
|
|
454
606
|
this.dirStats.cleanupVisited += 1;
|
|
455
607
|
|
|
456
608
|
const relForProgress = toPosix(path.relative(rootDir, dir)) || ".";
|
|
@@ -467,17 +619,37 @@ export class SftpPushSyncApp {
|
|
|
467
619
|
const subdirs = [];
|
|
468
620
|
let items;
|
|
469
621
|
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
622
|
+
// Try to list directory with reconnect on failure
|
|
623
|
+
let retries = 0;
|
|
624
|
+
while (retries <= 2) {
|
|
625
|
+
try {
|
|
626
|
+
items = await sftp.list(dir);
|
|
627
|
+
break;
|
|
628
|
+
} catch (e) {
|
|
629
|
+
const msg = e?.message || String(e);
|
|
630
|
+
const isConnectionError = msg.includes("No SFTP connection") ||
|
|
631
|
+
msg.includes("ECONNRESET") || msg.includes("connection");
|
|
632
|
+
|
|
633
|
+
if (isConnectionError && retries < 2) {
|
|
634
|
+
const reconnected = await attemptReconnect();
|
|
635
|
+
if (reconnected) {
|
|
636
|
+
retries++;
|
|
637
|
+
await new Promise(r => setTimeout(r, 500));
|
|
638
|
+
continue;
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
this.wlog(
|
|
643
|
+
pc.yellow("⚠️ Could not list directory during cleanup:"),
|
|
644
|
+
dir,
|
|
645
|
+
msg
|
|
646
|
+
);
|
|
647
|
+
return false;
|
|
648
|
+
}
|
|
479
649
|
}
|
|
480
650
|
|
|
651
|
+
if (!items) return false;
|
|
652
|
+
|
|
481
653
|
for (const item of items) {
|
|
482
654
|
if (!item.name || item.name === "." || item.name === "..") continue;
|
|
483
655
|
if (item.type === "d") {
|
|
@@ -490,7 +662,7 @@ export class SftpPushSyncApp {
|
|
|
490
662
|
let allSubdirsEmpty = true;
|
|
491
663
|
for (const sub of subdirs) {
|
|
492
664
|
const full = path.posix.join(dir, sub.name);
|
|
493
|
-
const subEmpty = await recurse(full);
|
|
665
|
+
const subEmpty = await recurse(full, depth + 1);
|
|
494
666
|
if (!subEmpty) {
|
|
495
667
|
allSubdirsEmpty = false;
|
|
496
668
|
}
|
|
@@ -507,17 +679,34 @@ export class SftpPushSyncApp {
|
|
|
507
679
|
);
|
|
508
680
|
this.dirStats.cleanupDeleted += 1;
|
|
509
681
|
} else {
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
e?.message || e
|
|
519
|
-
|
|
520
|
-
|
|
682
|
+
let deleteRetries = 0;
|
|
683
|
+
while (deleteRetries <= 2) {
|
|
684
|
+
try {
|
|
685
|
+
await sftp.rmdir(dir, false);
|
|
686
|
+
this.log(`${TAB_A}${DEL} Removed empty directory: ${rel}`);
|
|
687
|
+
this.dirStats.cleanupDeleted += 1;
|
|
688
|
+
break;
|
|
689
|
+
} catch (e) {
|
|
690
|
+
const msg = e?.message || String(e);
|
|
691
|
+
const isConnectionError = msg.includes("No SFTP connection") ||
|
|
692
|
+
msg.includes("ECONNRESET") || msg.includes("connection");
|
|
693
|
+
|
|
694
|
+
if (isConnectionError && deleteRetries < 2) {
|
|
695
|
+
const reconnected = await attemptReconnect();
|
|
696
|
+
if (reconnected) {
|
|
697
|
+
deleteRetries++;
|
|
698
|
+
await new Promise(r => setTimeout(r, 500));
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
this.wlog(
|
|
704
|
+
pc.yellow("⚠️ Could not remove directory:"),
|
|
705
|
+
dir,
|
|
706
|
+
msg
|
|
707
|
+
);
|
|
708
|
+
return false;
|
|
709
|
+
}
|
|
521
710
|
}
|
|
522
711
|
}
|
|
523
712
|
}
|
|
@@ -544,8 +733,44 @@ export class SftpPushSyncApp {
|
|
|
544
733
|
// Hauptlauf
|
|
545
734
|
// ---------------------------------------------------------
|
|
546
735
|
|
|
736
|
+
/**
|
|
737
|
+
* Format duration in human-readable format (mm:ss or hh:mm:ss)
|
|
738
|
+
*/
|
|
739
|
+
_formatDuration(seconds) {
|
|
740
|
+
const totalSec = Math.floor(seconds);
|
|
741
|
+
const hours = Math.floor(totalSec / 3600);
|
|
742
|
+
const minutes = Math.floor((totalSec % 3600) / 60);
|
|
743
|
+
const secs = totalSec % 60;
|
|
744
|
+
|
|
745
|
+
if (hours > 0) {
|
|
746
|
+
return `${hours}:${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
|
|
747
|
+
}
|
|
748
|
+
return `${minutes}:${String(secs).padStart(2, '0')}`;
|
|
749
|
+
}
|
|
750
|
+
|
|
547
751
|
async run() {
|
|
548
752
|
const start = Date.now();
|
|
753
|
+
|
|
754
|
+
// Global error handlers to catch unexpected errors
|
|
755
|
+
const handleFatalError = (type, error) => {
|
|
756
|
+
const msg = error?.message || String(error);
|
|
757
|
+
const logMsg = `❌ FATAL ${type}: ${msg}`;
|
|
758
|
+
console.error(pc.red(logMsg));
|
|
759
|
+
if (this.logger) {
|
|
760
|
+
this.logger.writeLine(logMsg);
|
|
761
|
+
this.logger.writeLine(error?.stack || "No stack trace available");
|
|
762
|
+
this.logger.close();
|
|
763
|
+
}
|
|
764
|
+
process.exitCode = 1;
|
|
765
|
+
};
|
|
766
|
+
|
|
767
|
+
process.on('unhandledRejection', (reason) => {
|
|
768
|
+
handleFatalError('Unhandled Promise Rejection', reason);
|
|
769
|
+
});
|
|
770
|
+
process.on('uncaughtException', (error) => {
|
|
771
|
+
handleFatalError('Uncaught Exception', error);
|
|
772
|
+
});
|
|
773
|
+
|
|
549
774
|
const {
|
|
550
775
|
target,
|
|
551
776
|
dryRun = false,
|
|
@@ -627,6 +852,9 @@ export class SftpPushSyncApp {
|
|
|
627
852
|
this.isVerbose = logLevel === "verbose";
|
|
628
853
|
this.isLaconic = logLevel === "laconic";
|
|
629
854
|
|
|
855
|
+
// Timestamps in Logfile
|
|
856
|
+
this.logTimestamps = configRaw.logTimestamps ?? false;
|
|
857
|
+
|
|
630
858
|
// Progress-Konfig
|
|
631
859
|
const PROGRESS = configRaw.progress ?? {};
|
|
632
860
|
this.scanChunk = PROGRESS.scanChunk ?? (this.isVerbose ? 1 : 100);
|
|
@@ -716,7 +944,7 @@ export class SftpPushSyncApp {
|
|
|
716
944
|
const logFile = path.resolve(
|
|
717
945
|
rawLogFilePattern.replace("{target}", target)
|
|
718
946
|
);
|
|
719
|
-
this.logger = new SyncLogger(logFile);
|
|
947
|
+
this.logger = new SyncLogger(logFile, { enableTimestamps: this.logTimestamps });
|
|
720
948
|
await this.logger.init();
|
|
721
949
|
|
|
722
950
|
// Header
|
|
@@ -726,7 +954,7 @@ export class SftpPushSyncApp {
|
|
|
726
954
|
`🔐 SFTP Push-Synchronisation: sftp-push-sync v${pkg.version}`
|
|
727
955
|
)
|
|
728
956
|
);
|
|
729
|
-
this.log(`${TAB_A}LogLevel: ${this.logLevel}`);
|
|
957
|
+
this.log(`${TAB_A}LogLevel: ${this.logLevel}${this.logTimestamps ? " (timestamps enabled)" : ""}`);
|
|
730
958
|
this.log(`${TAB_A}Connection: ${pc.cyan(target)}`);
|
|
731
959
|
this.log(`${TAB_A}Worker: ${this.connection.workers}`);
|
|
732
960
|
this.log(
|
|
@@ -781,9 +1009,12 @@ export class SftpPushSyncApp {
|
|
|
781
1009
|
username: this.connection.user,
|
|
782
1010
|
password: this.connection.password,
|
|
783
1011
|
// Keep-Alive to prevent server disconnection during long operations
|
|
784
|
-
keepaliveInterval:
|
|
785
|
-
keepaliveCountMax:
|
|
786
|
-
readyTimeout:
|
|
1012
|
+
keepaliveInterval: 5000, // Send keepalive every 5 seconds (more frequent for unstable servers)
|
|
1013
|
+
keepaliveCountMax: 6, // Allow up to 6 missed keepalives (30s total) before disconnect
|
|
1014
|
+
readyTimeout: 60000, // 60s timeout for initial connection
|
|
1015
|
+
retries: 2, // Internal retries
|
|
1016
|
+
retry_factor: 2,
|
|
1017
|
+
retry_minTimeout: 2000,
|
|
787
1018
|
});
|
|
788
1019
|
connected = true;
|
|
789
1020
|
|
|
@@ -818,10 +1049,11 @@ export class SftpPushSyncApp {
|
|
|
818
1049
|
symbols: { ADD, CHA, tab_a: TAB_A },
|
|
819
1050
|
});
|
|
820
1051
|
|
|
821
|
-
const
|
|
1052
|
+
const durationSec = (Date.now() - start) / 1000;
|
|
1053
|
+
const durationFormatted = this._formatDuration(durationSec);
|
|
822
1054
|
this.log("");
|
|
823
1055
|
this.log(pc.bold(pc.cyan("📊 Summary (bypass only):")));
|
|
824
|
-
this.log(`${TAB_A}Duration: ${pc.green(
|
|
1056
|
+
this.log(`${TAB_A}Duration: ${pc.green(durationFormatted)} (${durationSec.toFixed(1)}s)`);
|
|
825
1057
|
return;
|
|
826
1058
|
}
|
|
827
1059
|
|
|
@@ -994,7 +1226,8 @@ export class SftpPushSyncApp {
|
|
|
994
1226
|
}
|
|
995
1227
|
await sftp.put(l.localPath, remotePath);
|
|
996
1228
|
},
|
|
997
|
-
"Uploads (new)"
|
|
1229
|
+
"Uploads (new)",
|
|
1230
|
+
sftp
|
|
998
1231
|
);
|
|
999
1232
|
|
|
1000
1233
|
// Updates
|
|
@@ -1010,7 +1243,8 @@ export class SftpPushSyncApp {
|
|
|
1010
1243
|
}
|
|
1011
1244
|
await sftp.put(l.localPath, remotePath);
|
|
1012
1245
|
},
|
|
1013
|
-
"Uploads (update)"
|
|
1246
|
+
"Uploads (update)",
|
|
1247
|
+
sftp
|
|
1014
1248
|
);
|
|
1015
1249
|
|
|
1016
1250
|
// Deletes
|
|
@@ -1028,7 +1262,8 @@ export class SftpPushSyncApp {
|
|
|
1028
1262
|
);
|
|
1029
1263
|
}
|
|
1030
1264
|
},
|
|
1031
|
-
"Deletes"
|
|
1265
|
+
"Deletes",
|
|
1266
|
+
sftp
|
|
1032
1267
|
);
|
|
1033
1268
|
} else {
|
|
1034
1269
|
this.log("");
|
|
@@ -1055,7 +1290,8 @@ export class SftpPushSyncApp {
|
|
|
1055
1290
|
await this.cleanupEmptyDirs(sftp, this.connection.remoteRoot, dryRun);
|
|
1056
1291
|
}
|
|
1057
1292
|
|
|
1058
|
-
const
|
|
1293
|
+
const durationSec = (Date.now() - start) / 1000;
|
|
1294
|
+
const durationFormatted = this._formatDuration(durationSec);
|
|
1059
1295
|
|
|
1060
1296
|
// Save cache and close
|
|
1061
1297
|
await this.hashCache.save();
|
|
@@ -1065,7 +1301,7 @@ export class SftpPushSyncApp {
|
|
|
1065
1301
|
this.log(hr1());
|
|
1066
1302
|
this.log("");
|
|
1067
1303
|
this.log(pc.bold(pc.cyan("📊 Summary:")));
|
|
1068
|
-
this.log(`${TAB_A}Duration: ${pc.green(
|
|
1304
|
+
this.log(`${TAB_A}Duration: ${pc.green(durationFormatted)} (${durationSec.toFixed(1)}s)`);
|
|
1069
1305
|
this.log(`${TAB_A}${ADD} Added : ${toAdd.length}`);
|
|
1070
1306
|
this.log(`${TAB_A}${CHA} Changed: ${toUpdate.length}`);
|
|
1071
1307
|
this.log(`${TAB_A}${DEL} Deleted: ${toDelete.length}`);
|
package/src/core/SyncLogger.mjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* SyncLogger.mjs
|
|
3
|
-
*
|
|
3
|
+
*
|
|
4
4
|
* @author Carsten Nichte, 2025 / https://carsten-nichte.de/
|
|
5
|
-
*
|
|
6
|
-
*/
|
|
5
|
+
*
|
|
6
|
+
*/
|
|
7
7
|
// src/core/SyncLogger.mjs
|
|
8
8
|
import fs from "fs";
|
|
9
9
|
import fsp from "fs/promises";
|
|
@@ -14,9 +14,10 @@ import path from "path";
|
|
|
14
14
|
* und entfernt ANSI-Farbcodes.
|
|
15
15
|
*/
|
|
16
16
|
export class SyncLogger {
|
|
17
|
-
constructor(filePath) {
|
|
17
|
+
constructor(filePath, options = {}) {
|
|
18
18
|
this.filePath = filePath;
|
|
19
19
|
this.stream = null;
|
|
20
|
+
this.enableTimestamps = options.enableTimestamps ?? false;
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
async init() {
|
|
@@ -31,13 +32,24 @@ export class SyncLogger {
|
|
|
31
32
|
});
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Returns current timestamp in ISO format: [YYYY-MM-DD HH:mm:ss.SSS]
|
|
37
|
+
*/
|
|
38
|
+
_getTimestamp() {
|
|
39
|
+
const now = new Date();
|
|
40
|
+
const pad = (n, len = 2) => String(n).padStart(len, '0');
|
|
41
|
+
return `[${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}.${pad(now.getMilliseconds(), 3)}]`;
|
|
42
|
+
}
|
|
43
|
+
|
|
34
44
|
writeLine(line) {
|
|
35
45
|
if (!this.stream) return;
|
|
36
46
|
const text = typeof line === "string" ? line : String(line);
|
|
37
47
|
const clean = text.replace(/\x1b\[[0-9;]*m/g, "");
|
|
38
48
|
|
|
49
|
+
const prefix = this.enableTimestamps ? this._getTimestamp() + " " : "";
|
|
50
|
+
|
|
39
51
|
try {
|
|
40
|
-
this.stream.write(clean + "\n");
|
|
52
|
+
this.stream.write(prefix + clean + "\n");
|
|
41
53
|
} catch {
|
|
42
54
|
// Stream schon zu → ignorieren
|
|
43
55
|
}
|
|
@@ -49,4 +61,4 @@ export class SyncLogger {
|
|
|
49
61
|
this.stream = null;
|
|
50
62
|
}
|
|
51
63
|
}
|
|
52
|
-
}
|
|
64
|
+
}
|
|
@@ -35,8 +35,9 @@ export function hashLocalFile(filePath) {
|
|
|
35
35
|
|
|
36
36
|
/**
|
|
37
37
|
* Streaming-SHA256 für Remote-Datei via ssh2-sftp-client
|
|
38
|
+
* Mit Timeout, um hängende Verbindungen zu erkennen.
|
|
38
39
|
*/
|
|
39
|
-
export async function hashRemoteFile(sftp, remotePath) {
|
|
40
|
+
export async function hashRemoteFile(sftp, remotePath, timeoutMs = 60000) {
|
|
40
41
|
const hash = createHash("sha256");
|
|
41
42
|
|
|
42
43
|
const writable = new Writable({
|
|
@@ -46,7 +47,17 @@ export async function hashRemoteFile(sftp, remotePath) {
|
|
|
46
47
|
},
|
|
47
48
|
});
|
|
48
49
|
|
|
49
|
-
|
|
50
|
+
// Timeout-Promise
|
|
51
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
52
|
+
setTimeout(() => reject(new Error(`Timeout downloading ${remotePath}`)), timeoutMs);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
// Race between download and timeout
|
|
56
|
+
await Promise.race([
|
|
57
|
+
sftp.get(remotePath, writable),
|
|
58
|
+
timeoutPromise,
|
|
59
|
+
]);
|
|
60
|
+
|
|
50
61
|
return hash.digest("hex");
|
|
51
62
|
}
|
|
52
63
|
|