parrot-blackbox 1.0.12 → 1.0.13

package/README.md

@@ -107,6 +107,25 @@ and pending backups.
 
 ---
 
+## 🤖 Background Automation (Set it and forget it)
+
+To make `parrot-blackbox` take weekly snapshots and upload them to your cloud accounts completely automatically in the background, install the background service:
+
+```bash
+parrot-blackbox schedule install
+parrot-blackbox daemon start
+```
+
+Once started, the daemon runs silently as a background process. It wakes up every Saturday at 22:00, takes a Timeshift snapshot, and trickles the upload to your cloud accounts. 
+If your laptop happens to be powered off or completely offline on Saturday night, the daemon is smart enough to immediately catch up and run the missed backup the very next time you connect to WiFi!
+
+You can check on the daemon at any time using:
+```bash
+parrot-blackbox status
+```
+
+---
+
 ## Managing cloud accounts (rclone remotes)
 
 Every MEGA / Google Drive login becomes one rclone remote = one pool account.
@@ -449,6 +468,7 @@ Every account is exactly one rclone remote (`mega-1:`, `gdrive-6:` …).
   percentage** (water-filling), then most free.
 - A file that doesn't fit any single account is split at `chunkSize` boundaries
   and the pieces are spread across accounts — restore reassembles byte-perfect.
+- **BTRFS native support** — BTRFS root-owned snapshot files (e.g. `/etc/credstore`) are securely backed up to the cloud without `EACCES` permission errors via an internal privileged helper that integrates with the familiar interactive sudo prompt.
 - The manifest is written to the cloud **and** mirrored locally, so a wiped
   machine can still find and restore everything.
 
@@ -495,18 +515,27 @@ parrot-blackbox restore files <backup-id> ~/recovered
 ```bash
 # on the fresh Parrot:
 npm install -g parrot-blackbox
-rclone config                       # re-add the same remotes
+rclone config                             # re-add the same remotes
 parrot-blackbox account add mega mega-1   # …add every account
-parrot-blackbox snapshot list       # see what's in the cloud
+parrot-blackbox snapshot list             # see what's in the cloud
 parrot-blackbox restore snapshot <snapshot-id> --yes
 ```
 
-This downloads the snapshot into Timeshift, then runs the interactive restore
-(expect a `sudo` password prompt) — it **overwrites the fresh install's system
-root** with the snapshot. After a reboot you're back to your old system; then
-`sudo apt update && sudo apt upgrade` to move onto a newer Parrot release if
-you installed one. Encryption or not makes no difference — snapshot restore is
-run from inside the OS.
+#### What happens during a snapshot restore on a fresh install?
+When you run the restore command, `parrot-blackbox` does the following:
+1. **Downloads the snapshot:** It pulls the exact system snapshot from your cloud accounts, reassembles the byte-chunks, and places it into your local Timeshift directory.
+2. **Runs Timeshift Restore:** It then hands over control to Timeshift (`sudo timeshift --restore`).
+3. **Overwrites the OS:** Timeshift systematically replaces the system files of your *fresh* install with the files from your *snapshot*. Your programs, configurations, user settings, and installed packages are exactly reverted to how they were when the snapshot was taken.
+4. **Bootloader update:** Finally, Timeshift updates your GRUB bootloader and `/etc/fstab` to match the UUIDs of your new disk partitions, ensuring the system can boot.
+
+#### What about Disk Encryption (Passphrases)?
+**Disk encryption (LUKS) operates at the hardware/disk level, while snapshots operate at the file level.** 
+If your old system had a passphrase (encrypted), but you decide to fresh-install Parrot OS *without* a passphrase (unencrypted), **the restore will still work perfectly.**
+- The snapshot only contains your files and programs, not the LUKS encryption container.
+- When Timeshift restores the files onto your new unencrypted fresh install, it simply places the files into the unencrypted disk.
+- Timeshift automatically detects the new partition layout, updates your boot configurations, and after a reboot, you will have your exact old system back, but running on an unencrypted drive without any password prompts on boot!
+
+*Note: After rebooting into your restored system, it is always a good idea to run `sudo apt update && sudo apt upgrade` to ensure all packages are perfectly aligned with the latest Parrot OS repositories.*
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parrot-blackbox",
-  "version": "1.0.12",
+  "version": "1.0.13",
   "description": "parrot-blackbox — crash-proof, multi-cloud backup & recovery automation for Parrot OS. Daily/weekly off-disk backups with automatic catch-up, smart storage across many MEGA + Google Drive accounts, Timeshift snapshot backups and one-command restore.",
   "type": "module",
   "main": "src/cli.js",

package/src/storage/allocator.js

@@ -18,7 +18,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import streams from 'node:stream/promises';
 import { createWriteStream } from 'node:fs';
-import { copyToFile, mkdirRemote } from './rclone.js';
+import { copyToFile, copyBatch, mkdirRemote } from './rclone.js';
 import { bytesHuman } from '../util/misc.js';
 
 export { bytesHuman };
@@ -69,10 +69,7 @@ export function chooseAccount(needed, accounts) {
 }
 
 /**
- * Place a whole local dir into the pool.
- * @param {string} localDir artifact directory to place
- * @param {object} opts {kind, id, accounts, remoteRoot, chunkSize, onProgress}
- * @returns {Promise<object>} manifest
+ * Place a whole local dir into the pool using two-pass batching for speed.
  */
 export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, chunkSize, onProgress }) {
   if (!accounts || accounts.length === 0) {
@@ -105,57 +102,84 @@ export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, c
     if (typeof onProgress === 'function') onProgress({ done, total: files.length, text });
   };
 
-  let placed = 0;
+  // PASS 1: Planning
+  report(0, 'planning allocations...');
+  const batches = new Map(); // account.remote -> array of relative paths
+  const splits = []; // files that need splitting
+
+  for (const acc of pool) {
+    batches.set(acc.remote, { acc, files: [] });
+  }
+
   for (const entry of entries) {
     if (entry.isDir) {
       const acc = chooseAccount(0, pool);
       if (!acc) throw outOfSpace();
-      const rp = `${basePath}/${entry.rel}`;
-      const res = await mkdirRemote(`${acc.remote}:${rp}`);
-      if (!res.ok) throw new Error(`mkdir failed on ${acc.remote}: ${res.error}`);
-      manifest.entries.push({ rel: entry.rel, type: 'dir', size: 0, loc: [{ remote: acc.remote, path: rp }] });
+      manifest.entries.push({ rel: entry.rel, type: 'dir', size: 0, loc: [{ remote: acc.remote, path: `${basePath}/${entry.rel}` }] });
       continue;
     }
 
-    const rel = entry.rel;
-    const destPath = `${basePath}/${rel}`;
     const freeMax = Math.max(0, ...pool.map((a) => a.free));
     if (entry.size <= freeMax) {
       const acc = chooseAccount(entry.size, pool);
-      if (!acc) { report(placed, `no space for ${rel}`); throw outOfSpace(); }
-      const res = await copyToFile(entry.abs, `${acc.remote}:${destPath}`);
-      if (!res.ok) throw new Error(`upload failed for ${rel} on ${acc.remote}: ${res.error}`);
+      if (!acc) throw outOfSpace();
       consume(acc, entry.size);
+      batches.get(acc.remote).files.push(entry.rel);
       manifest.entries.push({
-        rel,
+        rel: entry.rel,
         type: 'file',
         size: entry.size,
-        loc: [{ remote: acc.remote, path: destPath, start: 0, end: entry.size, size: entry.size }],
+        loc: [{ remote: acc.remote, path: `${basePath}/${entry.rel}`, start: 0, end: entry.size, size: entry.size }],
       });
     } else {
-      // Split across accounts by byte ranges.
-      const locs = [];
-      let start = 0;
-      let partIndex = 0;
-      while (start < entry.size) {
-        const len = Math.min(chunkSize, entry.size - start);
-        const acc = chooseAccount(len, pool);
-        if (process.env.PBB_DEBUG_ALLOC === '1') {
-          console.error(`[alloc] file=${entry.rel} size=${entry.size} start=${start} len=${len} pool=${pool.map((a) => `${a.remote}:free=${a.free}`).join(',')} acc=${acc ? acc.remote : 'NONE'}`);
-        }
-        if (!acc) { report(placed, `no space for part of ${rel}`); throw outOfSpace(); }
-        const partAbs = await makePartFile(entry, start, len);
-        const partPath = `${destPath}.part-${String(partIndex).padStart(4, '0')}`;
-        const res = await copyToFile(partAbs, `${acc.remote}:${partPath}`);
-        if (!res.ok) throw new Error(`upload failed for ${rel} part on ${acc.remote}: ${res.error}`);
-        consume(acc, len);
-        locs.push({ remote: acc.remote, path: partPath, start, end: start + len, size: len });
-        start += len;
-        partIndex += 1;
-      }
-      manifest.entries.push({ rel, type: 'file', size: entry.size, split: true, loc: locs });
-      report(placed, `split ${rel} across ${locs.length} account(s)`);
+      splits.push(entry);
+    }
+  }
+
+  // PASS 2: Batch Uploading (Sequential accounts, but parallel files inside rclone)
+  const batchDir = process.env.PBB_STATE_DIR || '/tmp';
+  let placed = 0;
+  
+  for (const [remote, batch] of batches.entries()) {
+    if (batch.files.length === 0) continue;
+    const batchPath = path.join(batchDir, `batch-${remote.replace(/[^a-z0-9]/gi, '_')}-${Date.now()}.txt`);
+    fs.writeFileSync(batchPath, batch.files.join('\n') + '\n');
+    
+    report(placed, `uploading batch of ${batch.files.length} files to ${remote}...`);
+    const res = await copyBatch(localDir, `${remote}:${basePath}`, batchPath);
+    if (!res.ok) throw new Error(`batch upload failed to ${remote}: ${res.error}`);
+    
+    fs.unlinkSync(batchPath);
+    placed += batch.files.length;
+    report(placed, `placed ${placed}/${files.length}`);
+  }
+
+  // PASS 3: Split large files
+  for (const entry of splits) {
+    const rel = entry.rel;
+    const destPath = `${basePath}/${rel}`;
+    const locs = [];
+    let start = 0;
+    let partIndex = 0;
+    
+    report(placed, `splitting huge file ${rel}...`);
+    while (start < entry.size) {
+      const len = Math.min(chunkSize, entry.size - start);
+      const acc = chooseAccount(len, pool);
+      if (!acc) throw outOfSpace();
+      
+      const partAbs = await makePartFile(entry, start, len);
+      const partPath = `${destPath}.part-${String(partIndex).padStart(4, '0')}`;
+      
+      const res = await copyToFile(partAbs, `${acc.remote}:${partPath}`);
+      if (!res.ok) throw new Error(`upload failed for ${rel} part on ${acc.remote}: ${res.error}`);
+      
+      consume(acc, len);
+      locs.push({ remote: acc.remote, path: partPath, start, end: start + len, size: len });
+      start += len;
+      partIndex += 1;
     }
+    manifest.entries.push({ rel, type: 'file', size: entry.size, split: true, loc: locs });
     placed += 1;
     report(placed, `placed ${placed}/${files.length}`);
   }

package/src/storage/rclone.js

@@ -71,6 +71,19 @@ export async function copyToFile(localFile, remotePath, { ignoreExisting = false
   return { ok: res.exitCode === 0, exitCode: res.exitCode, error: res.stderr?.trim() };
 }
 
+/** Copy a batch of files using --files-from. */
+export async function copyBatch(localDir, remotePath, filesFromPath) {
+  const args = [
+    'copy', localDir, remotePath,
+    '--files-from', filesFromPath,
+    '--transfers=16',
+    '--checkers=16',
+    '--fast-list'
+  ];
+  const res = await rejectFalse(args);
+  return { ok: res.exitCode === 0, exitCode: res.exitCode, error: res.stderr?.trim() };
+}
+
 /** Create a remote directory. */
 export async function mkdirRemote(remotePath) {
   const res = await rejectFalse(['mkdir', remotePath]);