parrot-blackbox 2.1.0 → 2.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parrot-blackbox",
-  "version": "2.1.0",
+  "version": "2.1.2",
   "description": "Crash-proof, multi-cloud backup & recovery automation for Parrot OS. Daily/weekly off-disk backups with automatic catch-up, smart storage across many MEGA + Google Drive accounts, Timeshift snapshot backups and one-command restore.",
   "type": "module",
   "main": "src/cli.js",

package/src/storage/allocator.js

@@ -65,7 +65,11 @@ export function walkFiles(localDir) {
  *      will Google Drive accounts be used.
  *   2. Within each provider tier the classic water-filling strategy applies:
  *      pick the account that results in the lowest used-percentage after
- *      placing the file (most headroom wins on ties).
+ *      placing the file. Accounts that are within ~0.5% of quota from each
+ *      other are treated as equally full, and the account EARLIEST in the
+ *      configured pool order wins the tie — so an all-empty pool fills in
+ *      account order (mega-1, mega-2, …) instead of chasing `rclone about`
+ *      usage noise.
  */
 export function chooseAccount(needed, accounts) {
   // Tier 0 = mega, Tier 1 = gdrive / everything else.
@@ -79,14 +83,26 @@ export function chooseAccount(needed, accounts) {
   const candidates = eligible.filter((a) => providerTier(a) === bestTier);
 
   // Water-fill within the chosen tier: minimise resulting used-percentage.
+  // SCORE_EPS absorbs `rclone about` usage noise — MEGA inconsistently reports
+  // a few bytes to tens-of-MiB of phantom usage on otherwise-empty accounts.
+  // As long as accounts are effectively equally full, files go onto the account
+  // EARLIEST in the configured pool order.
+  const SCORE_EPS = 0.005; // 0.5 percentage points of quota
   let best = null;
   let bestScore = Infinity;
-  for (const acc of candidates) {
+  let bestIdx = Infinity;
+  for (let i = 0; i < candidates.length; i++) {
+    const acc = candidates[i];
     const quota = acc.total || 0;
     const score = quota ? (acc.used + needed) / quota : needed;
-    if (score < bestScore || (score === bestScore && acc.free > (best?.free ?? -1))) {
+    if (score < bestScore - SCORE_EPS) {
       bestScore = score;
       best = acc;
+      bestIdx = i;
+    } else if (Math.abs(score - bestScore) <= SCORE_EPS && i < bestIdx) {
+      // Effectively a tie → the earlier account in the pool wins.
+      best = acc;
+      bestIdx = i;
     }
   }
   return best;
@@ -122,12 +138,12 @@ export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, c
     entries: [],
   };
 
-  const report = (done, text) => {
-    if (typeof onProgress === 'function') onProgress({ done, total: files.length, text });
+  const emit = (done, text, remote) => {
+    if (typeof onProgress === 'function') onProgress({ done, total: totalSize, text, remote });
   };
 
   // PASS 1: Planning
-  report(0, 'planning allocations...');
+  emit(0, 'planning allocations...');
   const batches = new Map(); // account.remote -> array of relative paths
   const splits = []; // files that need splitting
 
@@ -162,24 +178,30 @@ export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, c
 
   // PASS 2: Batch Uploading (Sequential accounts, but parallel files inside rclone)
   const batchDir = process.env.PBB_STATE_DIR || '/tmp';
-  let placed = 0;
-  
+  const fileSizes = new Map(files.map((f) => [f.rel, f.size]));
+  const batchBytes = (rels) => rels.reduce((s, r) => s + (fileSizes.get(r) ?? 0), 0);
+  let placedBytes = 0;
+
   for (const [remote, batch] of batches.entries()) {
     if (batch.files.length === 0) continue;
     const batchPath = path.join(batchDir, `batch-${remote.replace(/[^a-z0-9]/gi, '_')}-${Date.now()}.txt`);
     fs.writeFileSync(batchPath, batch.files.join('\n') + '\n');
-    
+
     // Ensure the remote directory exists to prevent "directory not found" errors
     // when rclone tries to list the destination for --files-from checking.
     await mkdirRemote(`${remote}:${basePath}`);
-    
-    report(placed, `uploading batch of ${batch.files.length} files to ${remote}...`);
-    const res = await copyBatch(localDir, `${remote}:${basePath}`, batchPath);
+
+    const thisBatchBytes = batchBytes(batch.files);
+    const label = `uploading ${batch.files.length} file(s) → ${remote}`;
+    emit(placedBytes, label, remote);
+    const res = await copyBatch(localDir, `${remote}:${basePath}`, batchPath, {
+      onProgress: (p) => emit(placedBytes + p.done, label, remote),
+    });
     if (!res.ok) throw new Error(`batch upload failed to ${remote}: ${res.error}`);
-    
+
     fs.unlinkSync(batchPath);
-    placed += batch.files.length;
-    report(placed, `placed ${placed}/${files.length}`);
+    placedBytes += thisBatchBytes;
+    emit(placedBytes, `uploaded ${bytesHuman(placedBytes)}`, remote);
   }
 
   // PASS 3: Split large files
@@ -189,27 +211,27 @@ export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, c
     const locs = [];
     let start = 0;
     let partIndex = 0;
-    
-    report(placed, `splitting huge file ${rel}...`);
+
+    emit(placedBytes, `splitting huge file ${rel}...`);
     while (start < entry.size) {
       const len = Math.min(chunkSize, entry.size - start);
       const acc = chooseAccount(len, pool);
       if (!acc) throw outOfSpace();
-      
+
       const partAbs = await makePartFile(entry, start, len);
       const partPath = `${destPath}.part-${String(partIndex).padStart(4, '0')}`;
-      
+
       const res = await copyToFile(partAbs, `${acc.remote}:${partPath}`);
       if (!res.ok) throw new Error(`upload failed for ${rel} part on ${acc.remote}: ${res.error}`);
-      
+
       consume(acc, len);
       locs.push({ remote: acc.remote, path: partPath, start, end: start + len, size: len });
       start += len;
       partIndex += 1;
+      placedBytes += len;
+      emit(placedBytes, `uploaded ${bytesHuman(placedBytes)}`, acc.remote);
     }
     manifest.entries.push({ rel, type: 'file', size: entry.size, split: true, loc: locs });
-    placed += 1;
-    report(placed, `placed ${placed}/${files.length}`);
   }
 
   // Manifest: cloud + local mirror.

package/src/storage/archive.js

@@ -150,7 +150,7 @@ export async function restoreArtifact(manifest, destDir, { onProgress } = {}) {
       for (const entry of batch.entries) {
         const target = path.join(destDir, ...entry.rel.split('/'));
         const loc = entry.loc[0];
-        const r = await copyToFile(`${loc.remote}:${loc.path}`, target);
+        const r = await copyToFile(`${loc.remote}:${loc.path}`, target, { force: true });
         if (!r.ok) throw new Error(`download failed for ${entry.rel}: ${r.error}`);
         files += 1;
         bytes += entry.size;

package/src/storage/rclone.js

@@ -64,26 +64,67 @@ export async function copyDir(localDir, remotePath) {
 }
 
 /** Copy a single local file to an exact remote path. */
-export async function copyToFile(localFile, remotePath, { ignoreExisting = false } = {}) {
+export async function copyToFile(localFile, remotePath, { ignoreExisting = false, force = false } = {}) {
   const args = ['copyto', localFile, remotePath];
   if (ignoreExisting) args.push('--ignore-existing');
+  if (force) args.push('--ignore-times'); // overwrite even a "newer" destination
   const res = await rejectFalse(args);
   return { ok: res.exitCode === 0, exitCode: res.exitCode, error: res.stderr?.trim() };
 }
 
-/** Copy a batch of files using --files-from. */
-export async function copyBatch(localDir, remotePath, filesFromPath) {
+/** Copy a batch of files using --files-from. When `onProgress` is supplied we
+ *  ask rclone for live transfer stats and parse its `Transferred: X / Y` frames
+ *  so the caller receives byte-level progress as `{ done, total }` (bytes). */
+export async function copyBatch(localDir, remotePath, filesFromPath, { onProgress } = {}) {
   const args = [
     'copy', localDir, remotePath,
     '--files-from', filesFromPath,
     '--transfers=16',
     '--checkers=16',
-    '--fast-list'
+    '--fast-list',
   ];
-  const res = await rejectFalse(args);
+  if (typeof onProgress === 'function') {
+    // rclone emits periodic frames even when stdout is not a TTY (pipelines).
+    args.push('--progress', '--stats=1s');
+  }
+  const child = rejectFalse(args);
+  if (typeof onProgress === 'function') {
+    await consumeProgress(child, onProgress);
+  }
+  const res = await child;
   return { ok: res.exitCode === 0, exitCode: res.exitCode, error: res.stderr?.trim() };
 }
 
+/** Consume rclone's --progress stdout, emitting parsed byte counts per frame. */
+async function consumeProgress(child, onProgress) {
+  let buf = '';
+  for await (const chunk of child.stdout) {
+    buf += chunk;
+    const frames = buf.split(/[\r\n]+/);
+    buf = frames.pop(); // keep a partial trailing frame for the next chunk
+    for (const frame of frames) {
+      const p = parseTransferFrame(frame);
+      if (p) onProgress(p);
+    }
+  }
+  const tail = parseTransferFrame(buf);
+  if (tail) onProgress(tail);
+}
+
+/**
+ * Extract the BYTE `Transferred: <done> / <total>` from an rclone progress
+ * frame. This deliberately requires a size unit (KiB/MiB/GiB/…) so the separate
+ * FILE-count `Transferred: 3 / 5` line is never mistaken for bytes.
+ * @returns {{done:number,total:number}|null}
+ */
+export function parseTransferFrame(frame) {
+  const m = /\bTransferred:\s+([\d.]+\s*(?:[kmgt]i?)?b)\s*\/\s*([\d.]+\s*(?:[kmgt]i?)?b)/i.exec(frame);
+  if (!m) return null;
+  const done = parseBytes(m[1]);
+  const total = parseBytes(m[2]);
+  return done !== null && total !== null ? { done, total } : null;
+}
+
 /** Download a batch of files using --files-from (reverse of copyBatch). */
 export async function downloadBatch(remotePath, localDir, filesFromPath) {
   const args = [
@@ -91,7 +132,12 @@ export async function downloadBatch(remotePath, localDir, filesFromPath) {
     '--files-from', filesFromPath,
     '--transfers=16',
     '--checkers=16',
-    '--fast-list'
+    '--fast-list',
+    // Restore MUST reproduce the artifact exactly. rclone's default skips a
+    // destination file whose mtime looks newer (exactly what fresh-install
+    // default files are), which silently leaves "conflicts" — your backed-up
+    // version never comes back. --ignore-times forces the overwrite.
+    '--ignore-times',
   ];
   const res = await rejectFalse(args);
   return { ok: res.exitCode === 0, exitCode: res.exitCode, error: res.stderr?.trim() };