parrot-blackbox 2.1.0 → 2.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parrot-blackbox",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "Crash-proof, multi-cloud backup & recovery automation for Parrot OS. Daily/weekly off-disk backups with automatic catch-up, smart storage across many MEGA + Google Drive accounts, Timeshift snapshot backups and one-command restore.",
   "type": "module",
   "main": "src/cli.js",

package/src/storage/allocator.js

@@ -122,12 +122,12 @@ export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, c
     entries: [],
   };
 
-  const report = (done, text) => {
-    if (typeof onProgress === 'function') onProgress({ done, total: files.length, text });
+  const emit = (done, text, remote) => {
+    if (typeof onProgress === 'function') onProgress({ done, total: totalSize, text, remote });
   };
 
   // PASS 1: Planning
-  report(0, 'planning allocations...');
+  emit(0, 'planning allocations...');
   const batches = new Map(); // account.remote -> array of relative paths
   const splits = []; // files that need splitting
 
@@ -162,24 +162,30 @@ export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, c
 
   // PASS 2: Batch Uploading (Sequential accounts, but parallel files inside rclone)
   const batchDir = process.env.PBB_STATE_DIR || '/tmp';
-  let placed = 0;
-  
+  const fileSizes = new Map(files.map((f) => [f.rel, f.size]));
+  const batchBytes = (rels) => rels.reduce((s, r) => s + (fileSizes.get(r) ?? 0), 0);
+  let placedBytes = 0;
+
   for (const [remote, batch] of batches.entries()) {
     if (batch.files.length === 0) continue;
     const batchPath = path.join(batchDir, `batch-${remote.replace(/[^a-z0-9]/gi, '_')}-${Date.now()}.txt`);
     fs.writeFileSync(batchPath, batch.files.join('\n') + '\n');
-    
+
     // Ensure the remote directory exists to prevent "directory not found" errors
     // when rclone tries to list the destination for --files-from checking.
     await mkdirRemote(`${remote}:${basePath}`);
-    
-    report(placed, `uploading batch of ${batch.files.length} files to ${remote}...`);
-    const res = await copyBatch(localDir, `${remote}:${basePath}`, batchPath);
+
+    const thisBatchBytes = batchBytes(batch.files);
+    const label = `uploading ${batch.files.length} file(s) → ${remote}`;
+    emit(placedBytes, label, remote);
+    const res = await copyBatch(localDir, `${remote}:${basePath}`, batchPath, {
+      onProgress: (p) => emit(placedBytes + p.done, label, remote),
+    });
     if (!res.ok) throw new Error(`batch upload failed to ${remote}: ${res.error}`);
-    
+
     fs.unlinkSync(batchPath);
-    placed += batch.files.length;
-    report(placed, `placed ${placed}/${files.length}`);
+    placedBytes += thisBatchBytes;
+    emit(placedBytes, `uploaded ${bytesHuman(placedBytes)}`, remote);
   }
 
   // PASS 3: Split large files
@@ -189,27 +195,27 @@ export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, c
     const locs = [];
     let start = 0;
     let partIndex = 0;
-    
-    report(placed, `splitting huge file ${rel}...`);
+
+    emit(placedBytes, `splitting huge file ${rel}...`);
     while (start < entry.size) {
       const len = Math.min(chunkSize, entry.size - start);
       const acc = chooseAccount(len, pool);
       if (!acc) throw outOfSpace();
-      
+
       const partAbs = await makePartFile(entry, start, len);
       const partPath = `${destPath}.part-${String(partIndex).padStart(4, '0')}`;
-      
+
       const res = await copyToFile(partAbs, `${acc.remote}:${partPath}`);
       if (!res.ok) throw new Error(`upload failed for ${rel} part on ${acc.remote}: ${res.error}`);
-      
+
       consume(acc, len);
       locs.push({ remote: acc.remote, path: partPath, start, end: start + len, size: len });
       start += len;
       partIndex += 1;
+      placedBytes += len;
+      emit(placedBytes, `uploaded ${bytesHuman(placedBytes)}`, acc.remote);
     }
     manifest.entries.push({ rel, type: 'file', size: entry.size, split: true, loc: locs });
-    placed += 1;
-    report(placed, `placed ${placed}/${files.length}`);
   }
 
   // Manifest: cloud + local mirror.

package/src/storage/rclone.js

@@ -71,19 +71,59 @@ export async function copyToFile(localFile, remotePath, { ignoreExisting = false
   return { ok: res.exitCode === 0, exitCode: res.exitCode, error: res.stderr?.trim() };
 }
 
-/** Copy a batch of files using --files-from. */
-export async function copyBatch(localDir, remotePath, filesFromPath) {
+/** Copy a batch of files using --files-from. When `onProgress` is supplied we
+ *  ask rclone for live transfer stats and parse its `Transferred: X / Y` frames
+ *  so the caller receives byte-level progress as `{ done, total }` (bytes). */
+export async function copyBatch(localDir, remotePath, filesFromPath, { onProgress } = {}) {
   const args = [
     'copy', localDir, remotePath,
     '--files-from', filesFromPath,
     '--transfers=16',
     '--checkers=16',
-    '--fast-list'
+    '--fast-list',
   ];
-  const res = await rejectFalse(args);
+  if (typeof onProgress === 'function') {
+    // rclone emits periodic frames even when stdout is not a TTY (pipelines).
+    args.push('--progress', '--stats=1s');
+  }
+  const child = rejectFalse(args);
+  if (typeof onProgress === 'function') {
+    await consumeProgress(child, onProgress);
+  }
+  const res = await child;
   return { ok: res.exitCode === 0, exitCode: res.exitCode, error: res.stderr?.trim() };
 }
 
+/** Consume rclone's --progress stdout, emitting parsed byte counts per frame. */
+async function consumeProgress(child, onProgress) {
+  let buf = '';
+  for await (const chunk of child.stdout) {
+    buf += chunk;
+    const frames = buf.split(/[\r\n]+/);
+    buf = frames.pop(); // keep a partial trailing frame for the next chunk
+    for (const frame of frames) {
+      const p = parseTransferFrame(frame);
+      if (p) onProgress(p);
+    }
+  }
+  const tail = parseTransferFrame(buf);
+  if (tail) onProgress(tail);
+}
+
+/**
+ * Extract the BYTE `Transferred: <done> / <total>` from an rclone progress
+ * frame. This deliberately requires a size unit (KiB/MiB/GiB/…) so the separate
+ * FILE-count `Transferred: 3 / 5` line is never mistaken for bytes.
+ * @returns {{done:number,total:number}|null}
+ */
+export function parseTransferFrame(frame) {
+  const m = /\bTransferred:\s+([\d.]+\s*(?:[kmgt]i?)?b)\s*\/\s*([\d.]+\s*(?:[kmgt]i?)?b)/i.exec(frame);
+  if (!m) return null;
+  const done = parseBytes(m[1]);
+  const total = parseBytes(m[2]);
+  return done !== null && total !== null ? { done, total } : null;
+}
+
 /** Download a batch of files using --files-from (reverse of copyBatch). */
 export async function downloadBatch(remotePath, localDir, filesFromPath) {
   const args = [