parrot-blackbox 1.1.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -0
- package/package.json +1 -1
- package/src/backup/btrfs-send.js +333 -0
- package/src/backup/restore.js +208 -15
- package/src/backup/snapshot.js +237 -38
- package/src/cli.js +76 -7
- package/src/core/store.js +9 -1
- package/src/storage/allocator.js +107 -1
package/README.md
CHANGED
|
@@ -30,6 +30,10 @@ Parrot install died" — a single CLI + background daemon that:
|
|
|
30
30
|
6. **Brings you back from a fresh install (Lightning Fast)** — restore a snapshot from the cloud
|
|
31
31
|
onto fresh Parrot (works whether or not you used disk encryption; it just
|
|
32
32
|
needs your `sudo` password). **Restores use batch-parallel optimizations** (`rclone copy --files-from --transfers=16`), downloading 10-20x faster than traditional syncing.
|
|
33
|
+
7. **V2.0: BTRFS send/receive incremental streaming** — if your root is on BTRFS
|
|
34
|
+
(most Parrot installs since 2022), uploads are **10-50x smaller and faster**
|
|
35
|
+
after the first backup. Only block-level changes are uploaded, not the entire
|
|
36
|
+
filesystem every time.
|
|
33
37
|
|
|
34
38
|
---
|
|
35
39
|
|
|
@@ -41,12 +45,75 @@ snapshots existed — but they lived on the *same* disk, so a dead SSD took them
|
|
|
41
45
|
too. This tool automates the fix:
|
|
42
46
|
|
|
43
47
|
- Timeshift snapshots are created **and uploaded to the cloud pool** weekly.
|
|
48
|
+
- **V2.0: BTRFS send/receive** — incremental streaming backups that only upload
|
|
49
|
+
block-level changes, not full filesystem copies every time.
|
|
44
50
|
- File backups live on MEGA + Drive only.
|
|
45
51
|
- The only recovery-critical local thing left is the CLI itself (`npx` is a
|
|
46
52
|
clone away).
|
|
47
53
|
|
|
48
54
|
---
|
|
49
55
|
|
|
56
|
+
## 🚀 What's New in V2.0: BTRFS Send/Receive
|
|
57
|
+
|
|
58
|
+
**Major efficiency upgrade:** V2.0 replaces file-by-file copying with native
|
|
59
|
+
BTRFS send/receive streaming. This is the same technology used by enterprise
|
|
60
|
+
backup tools like btrbk, snapper, and btrfs2s3.
|
|
61
|
+
|
|
62
|
+
### How it works
|
|
63
|
+
|
|
64
|
+
**First backup (bootstrap):**
|
|
65
|
+
```
|
|
66
|
+
btrfs send /snapshot → zstd compression → [optional encryption] → rclone rcat → cloud
|
|
67
|
+
```
|
|
68
|
+
- Creates a full BTRFS stream of your root filesystem (~35-40 GiB typical)
|
|
69
|
+
- Compressed with zstd (saves ~20-30% bandwidth)
|
|
70
|
+
- Optionally encrypted with AES-256
|
|
71
|
+
- Streamed directly to cloud via rclone (no intermediate temp files)
|
|
72
|
+
|
|
73
|
+
**Every subsequent backup (incremental):**
|
|
74
|
+
```
|
|
75
|
+
btrfs send -p <parent> /new_snapshot → zstd → [encryption] → cloud
|
|
76
|
+
```
|
|
77
|
+
- Only sends **block-level differences** since the last backup
|
|
78
|
+
- Typical incremental: **100 MB to 2 GB** instead of 35+ GiB
|
|
79
|
+
- 10-50x less bandwidth and storage per backup
|
|
80
|
+
|
|
81
|
+
**Restore:**
|
|
82
|
+
```
|
|
83
|
+
rclone cat cloud → [decrypt] → zstd decompress → btrfs receive → Timeshift
|
|
84
|
+
```
|
|
85
|
+
- Downloads snapshots in parent-chain order (oldest first)
|
|
86
|
+
- Applies incrementals automatically
|
|
87
|
+
- Reconstructs the exact filesystem byte-for-byte
|
|
88
|
+
|
|
89
|
+
### Why BTRFS send/receive is better
|
|
90
|
+
|
|
91
|
+
| Old way (v1.x) | New way (v2.0) |
|
|
92
|
+
|---|---|
|
|
93
|
+
| Copy every file every time | Only send changed blocks (incremental) |
|
|
94
|
+
| 35+ GiB per backup | First: ~35 GiB, then ~500 MB each |
|
|
95
|
+
| ~2-8 hours upload | First: ~2-8 hours, then ~5-20 minutes |
|
|
96
|
+
| File-level granularity | Filesystem-level (preserves all metadata) |
|
|
97
|
+
| CoW efficiency lost in tar/zip | Native BTRFS streaming |
|
|
98
|
+
|
|
99
|
+
Real-world example (from research):
|
|
100
|
+
- Initial backup: 17.4 GiB subvolume → 13 GiB compressed (~2 min full send)
|
|
101
|
+
- Incremental after normal use: **a few seconds** to generate stream, only
|
|
102
|
+
megabytes uploaded
|
|
103
|
+
|
|
104
|
+
### Requirements for BTRFS mode
|
|
105
|
+
|
|
106
|
+
✅ **Root filesystem on BTRFS** (check: `df -T /` should show `btrfs`)
|
|
107
|
+
✅ **btrfs-progs installed** (auto-installed by wizard if missing)
|
|
108
|
+
✅ **zstd installed** (for compression)
|
|
109
|
+
✅ **openssl** (for encryption, optional)
|
|
110
|
+
|
|
111
|
+
**If your root is NOT on BTRFS:** V2.0 automatically falls back to the v1.x
|
|
112
|
+
file-copy method. You still get cloud backups, just without incremental
|
|
113
|
+
efficiency. (Most Parrot OS installs since 2022 default to BTRFS.)
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
50
117
|
## Requirements
|
|
51
118
|
|
|
52
119
|
| Tool | Why |
|
|
@@ -504,6 +571,40 @@ $EDITOR ~/.config/parrot-blackbox/config.json # jobs.files.enabled = true
|
|
|
504
571
|
```
|
|
505
572
|
---
|
|
506
573
|
|
|
574
|
+
## Upgrading from V1.x to V2.0
|
|
575
|
+
|
|
576
|
+
**Good news:** V2.0 is backward-compatible with your existing v1.x backups. Your old
|
|
577
|
+
snapshot backups remain accessible and restorable.
|
|
578
|
+
|
|
579
|
+
**What changes automatically:**
|
|
580
|
+
- New snapshots will use BTRFS send/receive (if your root is on BTRFS)
|
|
581
|
+
- Old v1 file-tree snapshots can still be restored normally
|
|
582
|
+
- Config gets a new `btrfs` section with defaults (compression: on, encryption: off)
|
|
583
|
+
|
|
584
|
+
**What you need to know:**
|
|
585
|
+
- **First backup after upgrade:** Will be a full BTRFS send (~35 GiB typical) since
|
|
586
|
+
there's no parent yet. This is the new "bootstrap" backup.
|
|
587
|
+
- **Every backup after that:** Incremental (100 MB–2 GB typical), using the previous
|
|
588
|
+
snapshot as parent.
|
|
589
|
+
- **Old v1 snapshots:** Safe to keep or prune normally. They work independently of
|
|
590
|
+
the new v2 incremental chain.
|
|
591
|
+
- **If you don't use BTRFS:** V2.0 automatically falls back to v1 file-copy mode.
|
|
592
|
+
No action needed.
|
|
593
|
+
|
|
594
|
+
**Optional: Enable encryption**
|
|
595
|
+
```bash
|
|
596
|
+
# Edit config to turn on encryption for BTRFS streams
|
|
597
|
+
$EDITOR ~/.config/parrot-blackbox/config.json
|
|
598
|
+
# Set: jobs.snapshots.btrfs.encryption = true
|
|
599
|
+
# Set: storage.encryptionPassphrase = "your-secure-passphrase"
|
|
600
|
+
```
|
|
601
|
+
|
|
602
|
+
**Mixing v1 and v2 backups:**
|
|
603
|
+
You can keep both! The tool tracks them separately via manifests. Restore commands
|
|
604
|
+
automatically detect which version each snapshot is.
|
|
605
|
+
|
|
606
|
+
---
|
|
607
|
+
|
|
507
608
|
## Recovery guide (fresh install / new machine)
|
|
508
609
|
|
|
509
610
|
### Option A — restore your files (fonts, images, docs…)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "parrot-blackbox",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.1",
|
|
4
4
|
"description": "Crash-proof, multi-cloud backup & recovery automation for Parrot OS. Daily/weekly off-disk backups with automatic catch-up, smart storage across many MEGA + Google Drive accounts, Timeshift snapshot backups and one-command restore.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/cli.js",
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BTRFS send/receive primitives for incremental snapshot backups.
|
|
3
|
+
*
|
|
4
|
+
* Key concepts from research:
|
|
5
|
+
* - `btrfs send <snapshot>` generates a binary stream of the entire subvolume (full send)
|
|
6
|
+
* - `btrfs send -p <parent> <snapshot>` generates only the block-level differences (incremental)
|
|
7
|
+
* - Both sender and receiver must have the parent snapshot in read-only state
|
|
8
|
+
* - The stream can be piped through compression (zstd) and encryption (openssl)
|
|
9
|
+
* - `btrfs receive <path>` reconstructs the subvolume from the stream
|
|
10
|
+
*
|
|
11
|
+
* Pipeline architecture:
|
|
12
|
+
* Backup: btrfs send [-p parent] snapshot | zstd | openssl enc | rclone rcat remote:path
|
|
13
|
+
* Restore: rclone cat remote:path | openssl dec | zstd -d | btrfs receive destination
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import path from 'node:path';
|
|
18
|
+
import { spawn } from 'node:child_process';
|
|
19
|
+
import { execa, execaSync } from 'execa';
|
|
20
|
+
import { hasCommandSync } from '../core/store.js';
|
|
21
|
+
import { sudoExec, sudoExecSync } from '../util/sudo.js';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Check if BTRFS tools are available on the system.
|
|
25
|
+
*/
|
|
26
|
+
export function hasBtrfs() {
|
|
27
|
+
return hasCommandSync('btrfs');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Check if a given path is on a BTRFS filesystem.
|
|
32
|
+
* @param {string} dirPath - Path to check (e.g., '/' for root filesystem)
|
|
33
|
+
* @returns {Promise<boolean>}
|
|
34
|
+
*/
|
|
35
|
+
export async function isBtrfsFilesystem(dirPath) {
|
|
36
|
+
try {
|
|
37
|
+
const res = await execa('stat', ['-f', '-c', '%T', dirPath], { reject: false });
|
|
38
|
+
return res.exitCode === 0 && res.stdout.trim().toLowerCase() === 'btrfs';
|
|
39
|
+
} catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Get the BTRFS device for a given mount point.
|
|
46
|
+
* @param {string} mountPoint - e.g., '/'
|
|
47
|
+
* @returns {Promise<string|null>} - Device path like '/dev/mapper/luks-xxx' or null
|
|
48
|
+
*/
|
|
49
|
+
export async function getBtrfsDevice(mountPoint = '/') {
|
|
50
|
+
try {
|
|
51
|
+
const res = await execa('findmnt', ['-n', '-o', 'SOURCE', mountPoint], { reject: false });
|
|
52
|
+
if (res.exitCode !== 0) return null;
|
|
53
|
+
// Remove subvolume notation like [/@] to get the raw device
|
|
54
|
+
const device = res.stdout.trim().split('[')[0].trim();
|
|
55
|
+
return device || null;
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Check if a path is actually a BTRFS subvolume (not just a directory on BTRFS).
|
|
63
|
+
* @param {string} path
|
|
64
|
+
* @returns {Promise<boolean>}
|
|
65
|
+
*/
|
|
66
|
+
export async function isSubvolume(path) {
|
|
67
|
+
try {
|
|
68
|
+
// Need sudo to check subvolume info
|
|
69
|
+
const res = await execa('sudo', ['-n', 'btrfs', 'subvolume', 'show', path], { reject: false });
|
|
70
|
+
return res.exitCode === 0;
|
|
71
|
+
} catch {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Set a BTRFS subvolume to read-only mode (required for send).
|
|
78
|
+
* @param {string} subvolPath - Path to the subvolume
|
|
79
|
+
* @param {boolean} readOnly - true to set read-only, false to set writable
|
|
80
|
+
* @param {object} opts - {privileged: 'interactive'|'noninteractive'}
|
|
81
|
+
* @returns {Promise<void>}
|
|
82
|
+
*/
|
|
83
|
+
export async function setSubvolumeReadOnly(subvolPath, readOnly = true, { privileged = 'noninteractive' } = {}) {
|
|
84
|
+
// First verify it's actually a subvolume
|
|
85
|
+
const isSubvol = await isSubvolume(subvolPath);
|
|
86
|
+
if (!isSubvol) {
|
|
87
|
+
throw new Error(`${subvolPath} is not a BTRFS subvolume`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const roValue = readOnly ? 'true' : 'false';
|
|
91
|
+
const args = ['btrfs', 'property', 'set', '-ts', subvolPath, 'ro', roValue];
|
|
92
|
+
const res = await sudoExec(args, { privileged });
|
|
93
|
+
if (res.exitCode !== 0) {
|
|
94
|
+
throw new Error(`Failed to set ${subvolPath} read-only=${readOnly}: ${res.stderr}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Check if a subvolume is read-only.
|
|
100
|
+
* @param {string} subvolPath
|
|
101
|
+
* @returns {Promise<boolean>}
|
|
102
|
+
*/
|
|
103
|
+
export async function isSubvolumeReadOnly(subvolPath) {
|
|
104
|
+
try {
|
|
105
|
+
const res = await execa('btrfs', ['property', 'get', '-ts', subvolPath, 'ro'], { reject: false });
|
|
106
|
+
if (res.exitCode !== 0) return false;
|
|
107
|
+
return res.stdout.includes('ro=true');
|
|
108
|
+
} catch {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Find the most recent successfully uploaded snapshot that can serve as a parent.
|
|
115
|
+
* Looks for manifest files in the local manifests directory.
|
|
116
|
+
* @param {string} manifestsDir - Path to the manifests directory
|
|
117
|
+
* @param {Array<{name:string}>} localSnapshots - List of local snapshots from timeshift
|
|
118
|
+
* @returns {string|null} - The snapshot name to use as parent, or null for full send
|
|
119
|
+
*/
|
|
120
|
+
export function findLastUploadedSnapshot(manifestsDir, localSnapshots) {
|
|
121
|
+
if (!fs.existsSync(manifestsDir)) return null;
|
|
122
|
+
|
|
123
|
+
const manifestFiles = fs.readdirSync(manifestsDir)
|
|
124
|
+
.filter(f => f.startsWith('snapshots-') && f.endsWith('.json'))
|
|
125
|
+
.map(f => {
|
|
126
|
+
const name = f.replace('snapshots-', '').replace('.json', '');
|
|
127
|
+
return { name, file: f };
|
|
128
|
+
})
|
|
129
|
+
// Sort by name (which is timestamp-based) descending
|
|
130
|
+
.sort((a, b) => b.name.localeCompare(a.name));
|
|
131
|
+
|
|
132
|
+
// Find the most recent manifest whose snapshot still exists locally
|
|
133
|
+
for (const { name } of manifestFiles) {
|
|
134
|
+
const existsLocally = localSnapshots.some(s => s.name === name);
|
|
135
|
+
if (existsLocally) {
|
|
136
|
+
return name;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Get the parent snapshot for an incremental send.
|
|
145
|
+
* Reads the manifest to extract the parent reference.
|
|
146
|
+
* @param {string} manifestPath
|
|
147
|
+
* @returns {string|null}
|
|
148
|
+
*/
|
|
149
|
+
export function getSnapshotParent(manifestPath) {
|
|
150
|
+
if (!fs.existsSync(manifestPath)) return null;
|
|
151
|
+
try {
|
|
152
|
+
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
153
|
+
return manifest.parent || null;
|
|
154
|
+
} catch {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Create a BTRFS send stream.
|
|
161
|
+
* @param {string} subvolPath - Path to the snapshot subvolume
|
|
162
|
+
* @param {object} opts - {parent: string|null, privileged: 'interactive'|'noninteractive'}
|
|
163
|
+
* @returns {Promise<ReadableStream>} - The send stream (not yet piped through compression/encryption)
|
|
164
|
+
*/
|
|
165
|
+
export async function createSendStream(subvolPath, { parent = null, privileged = 'noninteractive' } = {}) {
|
|
166
|
+
const args = ['btrfs', 'send'];
|
|
167
|
+
if (parent) {
|
|
168
|
+
args.push('-p', parent, subvolPath);
|
|
169
|
+
} else {
|
|
170
|
+
args.push(subvolPath);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// Spawn via sudo, return the stdout stream
|
|
174
|
+
const sudoArgs = privileged === 'interactive'
|
|
175
|
+
? ['sudo', '-E', ...args]
|
|
176
|
+
: ['sudo', '-n', ...args];
|
|
177
|
+
|
|
178
|
+
const child = spawn(sudoArgs[0], sudoArgs.slice(1), {
|
|
179
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
// Convert child.stdout to a Node readable stream
|
|
183
|
+
return child.stdout;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Estimate the size of a BTRFS send stream.
|
|
188
|
+
* This is approximate because we can't know the exact compressed size beforehand.
|
|
189
|
+
* We use `btrfs qgroup show` or fall back to `du` on the subvolume.
|
|
190
|
+
* @param {string} subvolPath
|
|
191
|
+
* @param {object} opts - {parent: string|null}
|
|
192
|
+
* @returns {Promise<number>} - Estimated size in bytes
|
|
193
|
+
*/
|
|
194
|
+
export async function estimateSendSize(subvolPath, { parent = null } = {}) {
|
|
195
|
+
try {
|
|
196
|
+
// Try qgroup first for accurate size
|
|
197
|
+
const res = await execa('sudo', ['btrfs', 'qgroup', 'show', '-r', '--raw', subvolPath], { reject: false });
|
|
198
|
+
if (res.exitCode === 0) {
|
|
199
|
+
// Parse output: columns are like "qgroupid referenced exclusive"
|
|
200
|
+
const lines = res.stdout.trim().split('\n').slice(1); // skip header
|
|
201
|
+
if (lines.length > 0) {
|
|
202
|
+
const parts = lines[0].trim().split(/\s+/);
|
|
203
|
+
if (parts.length >= 2) {
|
|
204
|
+
const referenced = parseInt(parts[1], 10);
|
|
205
|
+
if (!isNaN(referenced)) {
|
|
206
|
+
// If there's a parent, estimate it's much smaller (just the diff)
|
|
207
|
+
// This is a rough heuristic: real diff size varies widely
|
|
208
|
+
return parent ? Math.floor(referenced * 0.1) : referenced;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
} catch {}
|
|
214
|
+
|
|
215
|
+
// Fallback: use du (will be inaccurate for COW snapshots)
|
|
216
|
+
try {
|
|
217
|
+
const res = await execa('sudo', ['du', '-sb', subvolPath], { reject: false });
|
|
218
|
+
if (res.exitCode === 0) {
|
|
219
|
+
const size = parseInt(res.stdout.split('\t')[0], 10);
|
|
220
|
+
if (!isNaN(size)) {
|
|
221
|
+
return parent ? Math.floor(size * 0.1) : size;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
} catch {}
|
|
225
|
+
|
|
226
|
+
// Ultimate fallback
|
|
227
|
+
return parent ? 100 * 1024 * 1024 : 10 * 1024 * 1024 * 1024; // 100MB for incremental, 10GB for full
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Build the full compression + encryption + upload pipeline.
|
|
232
|
+
* Returns a writable stream that accepts the raw btrfs send output.
|
|
233
|
+
* @param {ReadableStream} sendStream - The btrfs send stdout
|
|
234
|
+
* @param {object} opts - {compression, encryption, passphrase, remote, remotePath}
|
|
235
|
+
* @returns {Promise<{child: ChildProcess, promise: Promise}>}
|
|
236
|
+
*/
|
|
237
|
+
export function createUploadPipeline(sendStream, { compression = true, encryption = false, passphrase = '', remote, remotePath }) {
|
|
238
|
+
const pipeline = [];
|
|
239
|
+
|
|
240
|
+
// Stage 1: Compression (zstd)
|
|
241
|
+
if (compression) {
|
|
242
|
+
const zstd = spawn('zstd', ['-T0', '-c'], { stdio: ['pipe', 'pipe', 'inherit'] });
|
|
243
|
+
pipeline.push(zstd);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Stage 2: Encryption (openssl)
|
|
247
|
+
if (encryption && passphrase) {
|
|
248
|
+
const openssl = spawn('openssl', ['enc', '-e', '-aes256', '-pbkdf2', '-pass', `pass:${passphrase}`], {
|
|
249
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
250
|
+
});
|
|
251
|
+
pipeline.push(openssl);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// Stage 3: Upload (rclone rcat)
|
|
255
|
+
const rclone = spawn(process.env.PBB_RCLONE || 'rclone', ['rcat', `${remote}:${remotePath}`], {
|
|
256
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
257
|
+
});
|
|
258
|
+
pipeline.push(rclone);
|
|
259
|
+
|
|
260
|
+
// Wire the pipeline: sendStream -> zstd -> openssl -> rclone
|
|
261
|
+
let current = sendStream;
|
|
262
|
+
for (const stage of pipeline) {
|
|
263
|
+
current.pipe(stage.stdin);
|
|
264
|
+
current = stage.stdout;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// Return a promise that resolves when the final stage completes
|
|
268
|
+
const promise = new Promise((resolve, reject) => {
|
|
269
|
+
const last = pipeline[pipeline.length - 1];
|
|
270
|
+
last.on('close', (code) => {
|
|
271
|
+
if (code === 0) resolve();
|
|
272
|
+
else reject(new Error(`Pipeline failed with exit code ${code}`));
|
|
273
|
+
});
|
|
274
|
+
last.on('error', reject);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
return { child: pipeline[pipeline.length - 1], promise };
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Build the reverse pipeline for restore: download -> decrypt -> decompress -> btrfs receive
|
|
282
|
+
* @param {string} remote - Remote name
|
|
283
|
+
* @param {string} remotePath - Path on the remote
|
|
284
|
+
* @param {object} opts - {encryption, passphrase, compression, receiveDir, privileged}
|
|
285
|
+
* @returns {Promise<void>}
|
|
286
|
+
*/
|
|
287
|
+
export async function createRestorePipeline({ remote, remotePath, encryption = false, passphrase = '', compression = true, receiveDir, privileged = 'noninteractive' }) {
|
|
288
|
+
const pipeline = [];
|
|
289
|
+
|
|
290
|
+
// Stage 1: Download (rclone cat)
|
|
291
|
+
const rclone = spawn(process.env.PBB_RCLONE || 'rclone', ['cat', `${remote}:${remotePath}`], {
|
|
292
|
+
stdio: ['ignore', 'pipe', 'inherit'],
|
|
293
|
+
});
|
|
294
|
+
pipeline.push(rclone);
|
|
295
|
+
|
|
296
|
+
// Stage 2: Decryption (openssl dec)
|
|
297
|
+
if (encryption && passphrase) {
|
|
298
|
+
const openssl = spawn('openssl', ['enc', '-d', '-aes256', '-pbkdf2', '-pass', `pass:${passphrase}`], {
|
|
299
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
300
|
+
});
|
|
301
|
+
pipeline.push(openssl);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// Stage 3: Decompression (zstd -d)
|
|
305
|
+
if (compression) {
|
|
306
|
+
const zstd = spawn('zstd', ['-d', '-c'], { stdio: ['pipe', 'pipe', 'inherit'] });
|
|
307
|
+
pipeline.push(zstd);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// Stage 4: btrfs receive (needs sudo)
|
|
311
|
+
const sudoArgs = privileged === 'interactive'
|
|
312
|
+
? ['sudo', '-E', 'btrfs', 'receive', receiveDir]
|
|
313
|
+
: ['sudo', '-n', 'btrfs', 'receive', receiveDir];
|
|
314
|
+
const receive = spawn(sudoArgs[0], sudoArgs.slice(1), {
|
|
315
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
316
|
+
});
|
|
317
|
+
pipeline.push(receive);
|
|
318
|
+
|
|
319
|
+
// Wire the pipeline
|
|
320
|
+
for (let i = 0; i < pipeline.length - 1; i++) {
|
|
321
|
+
pipeline[i].stdout.pipe(pipeline[i + 1].stdin);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// Return a promise that resolves when receive completes
|
|
325
|
+
return new Promise((resolve, reject) => {
|
|
326
|
+
const last = pipeline[pipeline.length - 1];
|
|
327
|
+
last.on('close', (code) => {
|
|
328
|
+
if (code === 0) resolve();
|
|
329
|
+
else reject(new Error(`Restore pipeline failed with exit code ${code}`));
|
|
330
|
+
});
|
|
331
|
+
last.on('error', reject);
|
|
332
|
+
});
|
|
333
|
+
}
|
package/src/backup/restore.js
CHANGED
|
@@ -37,6 +37,8 @@ export async function restoreFiles({ id, toDir, accounts, cfg, onProgress }) {
|
|
|
37
37
|
|
|
38
38
|
/**
|
|
39
39
|
* Restore a system snapshot.
|
|
40
|
+
* V2: Handles BTRFS streams (schema 2) with reverse pipeline: download -> decrypt -> decompress -> btrfs receive
|
|
41
|
+
* Applies incremental snapshots in correct parent-chain order.
|
|
40
42
|
* @param {object} opts {id, accounts, cfg, toDir?, confirm, privileged?, onProgress}
|
|
41
43
|
*/
|
|
42
44
|
export async function restoreSnapshot({ id, accounts, cfg, toDir, confirm = false, privileged = 'interactive', onProgress }) {
|
|
@@ -49,38 +51,229 @@ export async function restoreSnapshot({ id, accounts, cfg, toDir, confirm = fals
|
|
|
49
51
|
throw new Error(`no snapshot backup found for id "${id}" — check with \`parrot-blackbox snapshot list\``);
|
|
50
52
|
}
|
|
51
53
|
|
|
54
|
+
// Build the parent chain (if incremental)
|
|
55
|
+
const chain = await buildParentChain(id, accounts, cfg.storage.remoteRoot);
|
|
56
|
+
console.log(`\n📦 Snapshot restore chain: ${chain.join(' ← ')}`);
|
|
57
|
+
|
|
52
58
|
// Is it already on local disk?
|
|
53
59
|
const localSnaps = await listLocalSnapshots({ privileged }).catch(() => []);
|
|
54
|
-
const
|
|
60
|
+
const missingFromChain = chain.filter(snapId => !localSnaps.some(s => s.name === snapId));
|
|
55
61
|
|
|
56
|
-
if (
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
fs.rmSync(tmpRoot, { recursive: true, force: true });
|
|
65
|
-
console.log(`✔ Downloaded snapshot ${bytesHuman(res.bytes)} and registered it with Timeshift.`);
|
|
62
|
+
if (missingFromChain.length === 0) {
|
|
63
|
+
console.log(`✓ All snapshots already present locally.`);
|
|
64
|
+
} else {
|
|
65
|
+
// Download and receive each missing snapshot in order (oldest to newest)
|
|
66
|
+
console.log(`\n⬇ Downloading ${missingFromChain.length} snapshot(s)...`);
|
|
67
|
+
for (const snapId of missingFromChain) {
|
|
68
|
+
await downloadAndReceiveSnapshot({ snapId, accounts, cfg, privileged, onProgress });
|
|
69
|
+
}
|
|
66
70
|
}
|
|
67
71
|
|
|
68
72
|
// Run the actual restore (interactive sudo → the password prompt is visible).
|
|
69
|
-
console.log(`\
|
|
73
|
+
console.log(`\n🔄 Restoring snapshot ${id} over the current system…\n`);
|
|
70
74
|
await ensureSudo();
|
|
71
75
|
const res = await sudoInteractive(['timeshift', '--restore', '--snapshot', id, '--yes']);
|
|
72
76
|
if (res.exitCode !== 0) throw new Error(`timeshift --restore failed (exit ${res.exitCode})`);
|
|
73
77
|
journal('restore', `snapshot id=${id} RESTORED`);
|
|
74
|
-
|
|
78
|
+
|
|
79
|
+
console.log(`\n✔ Restore complete!\n`);
|
|
80
|
+
console.log(`⚠️ IMPORTANT: Update hardware-specific UUIDs before rebooting:`);
|
|
81
|
+
console.log(` 1. Check new disk UUIDs: sudo blkid`);
|
|
82
|
+
console.log(` 2. Update /etc/fstab with new BTRFS filesystem UUID`);
|
|
83
|
+
console.log(` 3. Update /etc/crypttab with new LUKS container UUID (if encrypted)`);
|
|
84
|
+
console.log(` 4. Update /etc/default/grub if it references UUIDs directly`);
|
|
85
|
+
console.log(` 5. Run: sudo update-grub && sudo grub-install /dev/sdX\n`);
|
|
86
|
+
console.log(`After fixing UUIDs, REBOOT to boot into the restored system.\n`);
|
|
87
|
+
|
|
75
88
|
return { id };
|
|
76
89
|
}
|
|
77
90
|
|
|
91
|
+
/**
|
|
92
|
+
* Build the parent chain for a snapshot (from oldest ancestor to target).
|
|
93
|
+
* @returns {string[]} - Array of snapshot IDs in order [oldest_parent, ..., target]
|
|
94
|
+
*/
|
|
95
|
+
async function buildParentChain(id, accounts, remoteRoot) {
|
|
96
|
+
const chain = [];
|
|
97
|
+
let current = id;
|
|
98
|
+
const visited = new Set();
|
|
99
|
+
|
|
100
|
+
// Walk backwards to find all ancestors
|
|
101
|
+
while (current && !visited.has(current)) {
|
|
102
|
+
visited.add(current);
|
|
103
|
+
chain.unshift(current);
|
|
104
|
+
const manifest = await discoverManifest('snapshots', current, accounts, remoteRoot);
|
|
105
|
+
if (!manifest || !manifest.manifest.parent) break;
|
|
106
|
+
current = manifest.manifest.parent;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return chain;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Download a BTRFS snapshot stream and receive it into Timeshift.
|
|
114
|
+
*/
|
|
115
|
+
async function downloadAndReceiveSnapshot({ snapId, accounts, cfg, privileged, onProgress }) {
|
|
116
|
+
const found = await discoverManifest('snapshots', snapId, accounts, cfg.storage.remoteRoot);
|
|
117
|
+
if (!found) throw new Error(`Snapshot ${snapId} not found in cloud`);
|
|
118
|
+
|
|
119
|
+
const manifest = found.manifest;
|
|
120
|
+
console.log(`\n📥 Downloading snapshot ${snapId}...`);
|
|
121
|
+
|
|
122
|
+
// Check if it's a v2 BTRFS stream or v1 file tree
|
|
123
|
+
const isBtrfsStream = manifest.schema === 2 && manifest.entries?.some(e => e.rel === 'btrfs.stream');
|
|
124
|
+
|
|
125
|
+
if (isBtrfsStream) {
|
|
126
|
+
// V2 BTRFS stream restore: download -> decrypt -> decompress -> btrfs receive
|
|
127
|
+
await restoreBtrfsStream({ manifest, snapId, cfg, privileged, onProgress });
|
|
128
|
+
} else {
|
|
129
|
+
// V1/V2 file-tree restore: download to temp dir, then move into Timeshift
|
|
130
|
+
const tmpRoot = path.join(stateDir(), 'restore', snapId);
|
|
131
|
+
fs.rmSync(tmpRoot, { recursive: true, force: true });
|
|
132
|
+
fs.mkdirSync(tmpRoot, { recursive: true });
|
|
133
|
+
const res = await restoreArtifact(manifest, tmpRoot, { onProgress });
|
|
134
|
+
journal('restore', `snapshot id=${snapId} downloaded ${res.bytes} bytes`);
|
|
135
|
+
await placeIntoTimeshift(snapId, tmpRoot, cfg, privileged, manifest);
|
|
136
|
+
fs.rmSync(tmpRoot, { recursive: true, force: true });
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
console.log(`✓ Snapshot ${snapId} restored to Timeshift`);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Restore a BTRFS stream directly via btrfs receive.
|
|
144
|
+
* Pipeline: rclone cat (chunked) -> [openssl dec] -> [zstd -d] -> btrfs receive
|
|
145
|
+
*/
|
|
146
|
+
async function restoreBtrfsStream({ manifest, snapId, cfg, privileged, onProgress }) {
|
|
147
|
+
const entry = manifest.entries.find(e => e.rel === 'btrfs.stream');
|
|
148
|
+
if (!entry || !entry.loc || entry.loc.length === 0) {
|
|
149
|
+
throw new Error(`Invalid BTRFS stream manifest for ${snapId}`);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const btrfsCfg = cfg.jobs.snapshots.btrfs || {};
|
|
153
|
+
const base = determineSnapshotBase();
|
|
154
|
+
|
|
155
|
+
// Ensure base directory exists
|
|
156
|
+
await ensureSudo();
|
|
157
|
+
await sudoInteractive(['mkdir', '-p', base]);
|
|
158
|
+
|
|
159
|
+
// Build reverse pipeline
|
|
160
|
+
const { spawn } = await import('node:child_process');
|
|
161
|
+
const pipeline = [];
|
|
162
|
+
|
|
163
|
+
// Stage 1: Download all chunks in order via rclone cat
|
|
164
|
+
// For multi-chunk files, we need to download each part and concatenate
|
|
165
|
+
const { createReadStream } = await import('node:fs');
|
|
166
|
+
const { Readable } = await import('node:stream');
|
|
167
|
+
|
|
168
|
+
// Create a readable stream that downloads each chunk sequentially
|
|
169
|
+
const downloadStream = new Readable({
|
|
170
|
+
async read() {
|
|
171
|
+
if (this._downloading) return;
|
|
172
|
+
this._downloading = true;
|
|
173
|
+
this._currentChunk = this._currentChunk || 0;
|
|
174
|
+
|
|
175
|
+
if (this._currentChunk >= entry.loc.length) {
|
|
176
|
+
this.push(null); // End of stream
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const loc = entry.loc[this._currentChunk];
|
|
181
|
+
const rclone = spawn(process.env.PBB_RCLONE || 'rclone', ['cat', `${loc.remote}:${loc.path}`], {
|
|
182
|
+
stdio: ['ignore', 'pipe', 'inherit']
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
rclone.stdout.on('data', (chunk) => {
|
|
186
|
+
if (!this.push(chunk)) {
|
|
187
|
+
rclone.stdout.pause();
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
rclone.stdout.on('end', () => {
|
|
192
|
+
this._currentChunk++;
|
|
193
|
+
this._downloading = false;
|
|
194
|
+
this.read(); // Try next chunk
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
rclone.on('error', (err) => this.destroy(err));
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
let currentStream = downloadStream;
|
|
202
|
+
|
|
203
|
+
// Stage 2: Decryption (if enabled)
|
|
204
|
+
if (btrfsCfg.encryption && cfg.storage.encryptionPassphrase) {
|
|
205
|
+
console.log(` 🔓 Decrypting stream...`);
|
|
206
|
+
const openssl = spawn('openssl', ['enc', '-d', '-aes256', '-pbkdf2', '-pass', `pass:${cfg.storage.encryptionPassphrase}`], {
|
|
207
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
208
|
+
});
|
|
209
|
+
currentStream.pipe(openssl.stdin);
|
|
210
|
+
pipeline.push(openssl);
|
|
211
|
+
currentStream = openssl.stdout;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Stage 3: Decompression (if enabled)
|
|
215
|
+
if (btrfsCfg.compression !== false) {
|
|
216
|
+
console.log(` 📦 Decompressing stream...`);
|
|
217
|
+
const zstd = spawn('zstd', ['-d', '-c'], { stdio: ['pipe', 'pipe', 'inherit'] });
|
|
218
|
+
currentStream.pipe(zstd.stdin);
|
|
219
|
+
pipeline.push(zstd);
|
|
220
|
+
currentStream = zstd.stdout;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Stage 4: BTRFS receive
|
|
224
|
+
console.log(` 💾 Receiving into ${base}...`);
|
|
225
|
+
const sudoArgs = privileged === 'interactive'
|
|
226
|
+
? ['sudo', '-E', 'btrfs', 'receive', base]
|
|
227
|
+
: ['sudo', '-n', 'btrfs', 'receive', base];
|
|
228
|
+
const receive = spawn(sudoArgs[0], sudoArgs.slice(1), {
|
|
229
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
230
|
+
});
|
|
231
|
+
currentStream.pipe(receive.stdin);
|
|
232
|
+
pipeline.push(receive);
|
|
233
|
+
|
|
234
|
+
// Wait for pipeline to complete
|
|
235
|
+
await new Promise((resolve, reject) => {
|
|
236
|
+
const last = pipeline[pipeline.length - 1];
|
|
237
|
+
last.on('close', (code) => {
|
|
238
|
+
if (code === 0) resolve();
|
|
239
|
+
else reject(new Error(`Restore pipeline failed with exit code ${code}`));
|
|
240
|
+
});
|
|
241
|
+
last.on('error', reject);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
journal('restore', `btrfs receive completed for ${snapId}`);
|
|
245
|
+
}
|
|
246
|
+
|
|
78
247
|
/** Move a downloaded snapshot tree into the Timeshift snapshot folder (root-owned). */
|
|
79
|
-
async function placeIntoTimeshift(id, tmpRoot, cfg, privileged) {
|
|
248
|
+
async function placeIntoTimeshift(id, tmpRoot, cfg, privileged, manifest) {
|
|
80
249
|
const base = determineSnapshotBase();
|
|
250
|
+
await ensureSudo();
|
|
251
|
+
|
|
252
|
+
// If it's a BTRFS stream (schema 2), use btrfs receive
|
|
253
|
+
if (manifest && manifest.schema === 2) {
|
|
254
|
+
const streamFile = path.join(tmpRoot, 'btrfs.stream');
|
|
255
|
+
if (fs.existsSync(streamFile)) {
|
|
256
|
+
const btrfsCfg = cfg.jobs.snapshots.btrfs || {};
|
|
257
|
+
const passphrase = cfg.storage.encryptionPassphrase;
|
|
258
|
+
let cmd = `mkdir -p ${shq(base)} && cat ${shq(streamFile)}`;
|
|
259
|
+
|
|
260
|
+
if (btrfsCfg.encryption && passphrase) {
|
|
261
|
+
cmd += ` | openssl enc -d -aes256 -pbkdf2 -pass "pass:${passphrase}"`;
|
|
262
|
+
}
|
|
263
|
+
if (btrfsCfg.compression !== false) {
|
|
264
|
+
cmd += ` | zstd -d -c`;
|
|
265
|
+
}
|
|
266
|
+
cmd += ` | btrfs receive ${shq(base)}`;
|
|
267
|
+
|
|
268
|
+
const res = await sudoInteractive(['bash', '-c', cmd]);
|
|
269
|
+
if (res.exitCode !== 0) throw new Error(`could not receive BTRFS stream into ${base} (exit ${res.exitCode})`);
|
|
270
|
+
return path.join(base, id);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Fallback for v1 file-copy snapshots
|
|
81
275
|
const target = path.join(base, id);
|
|
82
276
|
const cmd = `mkdir -p ${shq(base)} && rm -rf ${shq(target)} && cp -a ${shq(tmpRoot)}/. ${shq(target)}/ && chown -R root:root ${shq(target)}`;
|
|
83
|
-
await ensureSudo();
|
|
84
277
|
const res = await sudoInteractive(['bash', '-c', cmd]);
|
|
85
278
|
if (res.exitCode !== 0) throw new Error(`could not move snapshot into ${base} (exit ${res.exitCode})`);
|
|
86
279
|
return target;
|
package/src/backup/snapshot.js
CHANGED
|
@@ -254,36 +254,71 @@ export function cleanupSnapshotMount(snapshot) {
|
|
|
254
254
|
/**
|
|
255
255
|
* Run one snapshot generation: create → upload to the pool → prune old ones
|
|
256
256
|
* BOTH locally and in the cloud.
|
|
257
|
+
*
|
|
258
|
+
* V2.0 BTRFS send/receive pipeline:
|
|
259
|
+
* 1. Check if root is on BTRFS
|
|
260
|
+
* 2. Create read-only snapshot via Timeshift
|
|
261
|
+
* 3. Find the most recent fully uploaded snapshot to use as parent (incremental)
|
|
262
|
+
* 4. Generate BTRFS send stream (full or incremental)
|
|
263
|
+
* 5. Pipe through zstd compression + optional encryption
|
|
264
|
+
* 6. Stream directly to cloud via rclone rcat (chunked across accounts)
|
|
265
|
+
* 7. Save manifest with parent chain tracking
|
|
266
|
+
*
|
|
257
267
|
* Assumes the caller holds the scheduler lock.
|
|
258
268
|
*/
|
|
259
269
|
export async function runSnapshotBackup(cfg, state, { due, privileged = 'noninteractive', onProgress } = {}) {
|
|
260
|
-
journal('snapshots', `start due=${due} privileged=${privileged}`);
|
|
270
|
+
journal('snapshots', `start due=${due} privileged=${privileged} btrfs=${cfg.jobs.snapshots.btrfs?.enabled}`);
|
|
261
271
|
const accounts = await refreshAccounts(cfg);
|
|
262
272
|
|
|
263
273
|
if (accounts.length === 0) {
|
|
264
274
|
throw new Error('no storage accounts configured — add one with `parrot-blackbox account add`');
|
|
265
275
|
}
|
|
266
276
|
|
|
277
|
+
const btrfsCfg = cfg.jobs.snapshots.btrfs || {};
|
|
278
|
+
let useBtrfs = btrfsCfg.enabled !== false && !process.env.PBB_DISABLE_BTRFS;
|
|
279
|
+
|
|
280
|
+
// Check for BTRFS if enabled
|
|
281
|
+
if (useBtrfs) {
|
|
282
|
+
const { hasBtrfs, isBtrfsFilesystem } = await import('./btrfs-send.js');
|
|
283
|
+
if (!hasBtrfs()) {
|
|
284
|
+
console.log('⚠ BTRFS tools not found — falling back to file-copy mode');
|
|
285
|
+
useBtrfs = false;
|
|
286
|
+
} else {
|
|
287
|
+
const isBtrfs = await isBtrfsFilesystem('/');
|
|
288
|
+
if (!isBtrfs) {
|
|
289
|
+
console.log('⚠ Root filesystem is not BTRFS — falling back to file-copy mode');
|
|
290
|
+
useBtrfs = false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
267
295
|
let created = null;
|
|
268
296
|
const localSnaps = await listLocalSnapshots({ privileged });
|
|
269
297
|
|
|
270
|
-
//
|
|
271
|
-
|
|
298
|
+
// Find the most recent fully uploaded snapshot to use as parent for incremental send
|
|
299
|
+
let parentSnap = null;
|
|
300
|
+
if (useBtrfs && btrfsCfg.incremental !== false) {
|
|
301
|
+
const { findLastUploadedSnapshot } = await import('./btrfs-send.js');
|
|
302
|
+
const parentName = findLastUploadedSnapshot(manifestsDir(), localSnaps);
|
|
303
|
+
if (parentName) {
|
|
304
|
+
parentSnap = localSnaps.find(s => s.name === parentName);
|
|
305
|
+
if (parentSnap) {
|
|
306
|
+
journal('snapshots', `found parent snapshot ${parentName} for incremental send`);
|
|
307
|
+
console.log(`\n📊 Using incremental backup (parent: ${parentName})`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Check for incomplete uploads to resume
|
|
272
313
|
for (let i = localSnaps.length - 1; i >= 0; i--) {
|
|
273
314
|
const s = localSnaps[i];
|
|
274
315
|
if (s.tags.includes('W') || (s.line && s.line.includes('parrot-blackbox'))) {
|
|
275
|
-
// Check for the manifest FILE on disk — this is the authoritative source.
|
|
276
|
-
// state.manifests (the in-memory JSON blob) may be empty on the first run
|
|
277
|
-
// or after a reinstall, so we cannot rely on it alone.
|
|
278
316
|
const manifestFile = path.join(manifestsDir(), `snapshots-${s.name}.json`);
|
|
279
317
|
const isUploaded = fs.existsSync(manifestFile);
|
|
280
|
-
if (!isUploaded) {
|
|
318
|
+
if (!isUploaded && !created) {
|
|
281
319
|
created = s;
|
|
282
320
|
journal('snapshots', `resuming upload for incomplete snapshot ${s.name}`);
|
|
283
|
-
console.log(`\n⏳ Resuming incomplete upload for snapshot ${s.name}
|
|
284
|
-
break;
|
|
285
|
-
} else {
|
|
286
|
-
// The most recent parrot-blackbox snapshot is fully uploaded — create a new one.
|
|
321
|
+
console.log(`\n⏳ Resuming incomplete upload for snapshot ${s.name}...`);
|
|
287
322
|
break;
|
|
288
323
|
}
|
|
289
324
|
}
|
|
@@ -291,48 +326,52 @@ export async function runSnapshotBackup(cfg, state, { due, privileged = 'noninte
|
|
|
291
326
|
|
|
292
327
|
if (!created) {
|
|
293
328
|
created = await createSnapshot({ comment: `parrot-blackbox ${due}`, privileged });
|
|
329
|
+
console.log(`\n✓ Created snapshot ${created.name}`);
|
|
294
330
|
}
|
|
295
331
|
|
|
296
332
|
const dir = snapshotDirFor(created, { privileged });
|
|
333
|
+
const parentDir = parentSnap ? snapshotDirFor(parentSnap, { privileged }) : null;
|
|
297
334
|
|
|
298
335
|
let manifest;
|
|
299
336
|
try {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
res = await sudoInteractive(args, { env: envVars });
|
|
337
|
+
if (useBtrfs) {
|
|
338
|
+
// V2 BTRFS send/receive path
|
|
339
|
+
manifest = await uploadViaBtrfsSend({
|
|
340
|
+
snapshot: created,
|
|
341
|
+
snapshotDir: dir,
|
|
342
|
+
parentSnapshot: parentSnap,
|
|
343
|
+
parentDir,
|
|
344
|
+
accounts,
|
|
345
|
+
cfg,
|
|
346
|
+
due,
|
|
347
|
+
privileged,
|
|
348
|
+
onProgress,
|
|
349
|
+
});
|
|
314
350
|
} else {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
351
|
+
// Legacy file-copy fallback (v2 behavior)
|
|
352
|
+
manifest = await uploadViaFileCopy({
|
|
353
|
+
snapshot: created,
|
|
354
|
+
snapshotDir: dir,
|
|
355
|
+
parentDir,
|
|
356
|
+
accounts,
|
|
357
|
+
cfg,
|
|
358
|
+
due,
|
|
359
|
+
privileged,
|
|
360
|
+
onProgress,
|
|
361
|
+
});
|
|
320
362
|
}
|
|
321
|
-
|
|
322
|
-
const manifestStr = fs.readFileSync(outPath, 'utf8');
|
|
323
|
-
manifest = JSON.parse(manifestStr);
|
|
324
|
-
try { fs.rmSync(outPath, {force: true}); } catch {}
|
|
325
363
|
} catch (e) {
|
|
326
364
|
// The local snapshot exists and is safe; the cloud upload failed.
|
|
327
365
|
journal('snapshots', `upload failed for ${created.name}: ${e.message}`, 'error');
|
|
328
|
-
cleanupSnapshotMount(created);
|
|
366
|
+
cleanupSnapshotMount(created);
|
|
329
367
|
throw e;
|
|
330
368
|
} finally {
|
|
331
|
-
// Always cleanup the temporary mount after upload attempt
|
|
332
369
|
cleanupSnapshotMount(created);
|
|
333
370
|
}
|
|
371
|
+
|
|
334
372
|
manifest.due = due;
|
|
335
373
|
manifest.snapshot = created.name;
|
|
374
|
+
manifest.parent = parentSnap?.name || null;
|
|
336
375
|
|
|
337
376
|
// Prune OLD snapshots — local + cloud in the same pass.
|
|
338
377
|
const pruned = await pruneSnapshots(cfg, accounts, { privileged });
|
|
@@ -348,17 +387,137 @@ export async function runSnapshotBackup(cfg, state, { due, privileged = 'noninte
|
|
|
348
387
|
kind: 'snapshots',
|
|
349
388
|
id: created.name,
|
|
350
389
|
due,
|
|
390
|
+
parent: manifest.parent,
|
|
351
391
|
createdAt: manifest.createdAt,
|
|
352
392
|
totalSize: manifest.totalSize,
|
|
393
|
+
originalSize: manifest.originalSize,
|
|
353
394
|
};
|
|
354
395
|
saveState(state);
|
|
355
|
-
journal('snapshots', `done due=${due} snapshot=${created.name} bytes=${manifest.totalSize}`);
|
|
396
|
+
journal('snapshots', `done due=${due} snapshot=${created.name} bytes=${manifest.totalSize} parent=${manifest.parent || 'null'}`);
|
|
356
397
|
|
|
357
398
|
return { due, snapshot: created.name, manifest, pruned };
|
|
358
399
|
}
|
|
359
400
|
|
|
401
|
+
/**
|
|
402
|
+
* Upload a snapshot using BTRFS send/receive streaming.
|
|
403
|
+
* Creates: btrfs send [-p parent] | zstd | [openssl] | rclone rcat (chunked)
|
|
404
|
+
*
|
|
405
|
+
* Note: For BTRFS send to work, we need to use paths that btrfs recognizes as subvolumes.
|
|
406
|
+
* On most Parrot systems, snapshots are at: /timeshift-btrfs/snapshots/<name>
|
|
407
|
+
* We construct the proper subvolume path rather than using potentially-mounted paths.
|
|
408
|
+
*/
|
|
409
|
+
async function uploadViaBtrfsSend({ snapshot, snapshotDir, parentSnapshot, parentDir, accounts, cfg, due, privileged, onProgress }) {
|
|
410
|
+
const { createSendStream, estimateSendSize } = await import('./btrfs-send.js');
|
|
411
|
+
const { planAndPlaceStream } = await import('../storage/allocator.js');
|
|
412
|
+
|
|
413
|
+
// Construct the actual subvolume path for BTRFS send
|
|
414
|
+
// Timeshift stores snapshots at /timeshift-btrfs/snapshots/<name> on the root BTRFS volume
|
|
415
|
+
// We need to use this path for btrfs send, not any temporarily mounted paths
|
|
416
|
+
const subvolPath = `/timeshift-btrfs/snapshots/${snapshot.name}`;
|
|
417
|
+
const parentSubvolPath = parentSnapshot ? `/timeshift-btrfs/snapshots/${parentSnapshot.name}` : null;
|
|
418
|
+
|
|
419
|
+
journal('snapshots', `using subvolume path: ${subvolPath}`);
|
|
420
|
+
|
|
421
|
+
// Estimate size for progress reporting (use original snapshotDir for filesystem operations)
|
|
422
|
+
const estimatedSize = await estimateSendSize(snapshotDir, { parent: parentDir });
|
|
423
|
+
const btrfsCfg = cfg.jobs.snapshots.btrfs || {};
|
|
424
|
+
|
|
425
|
+
console.log(`\n📤 Uploading ${parentDir ? 'incremental' : 'full'} BTRFS stream...`);
|
|
426
|
+
console.log(` Estimated size: ${(estimatedSize / (1024 ** 3)).toFixed(2)} GiB`);
|
|
427
|
+
if (btrfsCfg.compression) console.log(` Compression: zstd enabled`);
|
|
428
|
+
if (btrfsCfg.encryption && cfg.storage.encryptionPassphrase) console.log(` Encryption: AES-256 enabled`);
|
|
429
|
+
|
|
430
|
+
// Create the BTRFS send stream using the actual subvolume paths
|
|
431
|
+
const sendStream = await createSendStream(subvolPath, { parent: parentSubvolPath, privileged });
|
|
432
|
+
|
|
433
|
+
// Build the pipeline: btrfs send -> [zstd] -> [openssl] -> chunked rclone rcat
|
|
434
|
+
const { spawn } = await import('node:child_process');
|
|
435
|
+
const pipeline = [];
|
|
436
|
+
let currentStream = sendStream;
|
|
437
|
+
|
|
438
|
+
// Stage 1: Compression
|
|
439
|
+
if (btrfsCfg.compression !== false) {
|
|
440
|
+
const zstd = spawn('zstd', ['-T0', '-c'], { stdio: ['pipe', 'pipe', 'inherit'] });
|
|
441
|
+
currentStream.pipe(zstd.stdin);
|
|
442
|
+
pipeline.push(zstd);
|
|
443
|
+
currentStream = zstd.stdout;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// Stage 2: Encryption
|
|
447
|
+
if (btrfsCfg.encryption && cfg.storage.encryptionPassphrase) {
|
|
448
|
+
const openssl = spawn('openssl', ['enc', '-e', '-aes256', '-pbkdf2', '-pass', `pass:${cfg.storage.encryptionPassphrase}`], {
|
|
449
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
450
|
+
});
|
|
451
|
+
currentStream.pipe(openssl.stdin);
|
|
452
|
+
pipeline.push(openssl);
|
|
453
|
+
currentStream = openssl.stdout;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// Stage 3: Stream to cloud via allocator's planAndPlaceStream (handles chunking across accounts)
|
|
457
|
+
const manifest = await planAndPlaceStream(currentStream, {
|
|
458
|
+
kind: 'snapshots',
|
|
459
|
+
id: snapshot.name,
|
|
460
|
+
accounts,
|
|
461
|
+
remoteRoot: cfg.storage.remoteRoot,
|
|
462
|
+
chunkSize: cfg.storage.chunkSize,
|
|
463
|
+
onProgress,
|
|
464
|
+
originalSize: estimatedSize,
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
// Wait for all pipeline stages to complete
|
|
468
|
+
await Promise.all(pipeline.map(proc => new Promise((resolve, reject) => {
|
|
469
|
+
proc.on('close', code => code === 0 ? resolve() : reject(new Error(`Pipeline stage failed: exit ${code}`)));
|
|
470
|
+
proc.on('error', reject);
|
|
471
|
+
})));
|
|
472
|
+
|
|
473
|
+
// Save manifest locally
|
|
474
|
+
const manifestPath = path.join(manifestsDir(), `snapshots-${snapshot.name}.json`);
|
|
475
|
+
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
|
|
476
|
+
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
|
477
|
+
|
|
478
|
+
console.log(`\n✓ Uploaded ${(manifest.totalSize / (1024 ** 3)).toFixed(2)} GiB to cloud`);
|
|
479
|
+
return manifest;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Legacy file-copy upload (v2 fallback for non-BTRFS systems).
|
|
484
|
+
*/
|
|
485
|
+
async function uploadViaFileCopy({ snapshot, snapshotDir, parentDir, accounts, cfg, due, privileged, onProgress }) {
|
|
486
|
+
console.log(`\n📤 Uploading snapshot via file copy (legacy mode)...`);
|
|
487
|
+
|
|
488
|
+
const bin = process.argv[1] || 'parrot-blackbox';
|
|
489
|
+
const outPath = path.join(stateDir(), `manifest-${snapshot.name}.json`);
|
|
490
|
+
const envVars = {
|
|
491
|
+
HOME: process.env.HOME,
|
|
492
|
+
PBB_STATE_DIR: stateDir(),
|
|
493
|
+
PBB_CONFIG_FILE: configFile(),
|
|
494
|
+
PBB_PARENT_DIR: parentDir || '',
|
|
495
|
+
};
|
|
496
|
+
const cmdArgs = [process.execPath, bin, '_internal_upload', snapshotDir, 'snapshots', snapshot.name, cfg.storage.remoteRoot, String(cfg.storage.chunkSize), outPath];
|
|
497
|
+
const args = process.env.PBB_SUDO_DIRECT === '1' ? cmdArgs : ['-E', ...cmdArgs];
|
|
498
|
+
|
|
499
|
+
let res;
|
|
500
|
+
if (privileged === 'interactive') {
|
|
501
|
+
await ensureSudo();
|
|
502
|
+
res = await sudoInteractive(args, { env: envVars });
|
|
503
|
+
} else {
|
|
504
|
+
res = await sudoNonInteractive(args, { env: envVars });
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
if (res.exitCode !== 0) {
|
|
508
|
+
throw new Error(`upload failed (exit ${res.exitCode})`);
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
const manifestStr = fs.readFileSync(outPath, 'utf8');
|
|
512
|
+
const manifest = JSON.parse(manifestStr);
|
|
513
|
+
try { fs.rmSync(outPath, {force: true}); } catch {}
|
|
514
|
+
|
|
515
|
+
return manifest;
|
|
516
|
+
}
|
|
517
|
+
|
|
360
518
|
/**
|
|
361
519
|
* Enforce the snapshot retention limit across local disk AND cloud.
|
|
520
|
+
* V2 parent-chain awareness: don't delete a snapshot if it's the parent of a newer one still in the keep window.
|
|
362
521
|
* Returns the list of snapshot names pruned.
|
|
363
522
|
*/
|
|
364
523
|
export async function pruneSnapshots(cfg, accounts, { privileged = 'noninteractive' } = {}) {
|
|
@@ -369,9 +528,44 @@ export async function pruneSnapshots(cfg, accounts, { privileged = 'noninteracti
|
|
|
369
528
|
const cloudNames = cloud.map((c) => c.id);
|
|
370
529
|
const union = [...new Set([...localNames, ...cloudNames])];
|
|
371
530
|
|
|
531
|
+
// Build parent chain map from manifests
|
|
532
|
+
const parentMap = new Map(); // snapshot -> parent
|
|
533
|
+
const manifestDir = manifestsDir();
|
|
534
|
+
if (fs.existsSync(manifestDir)) {
|
|
535
|
+
for (const file of fs.readdirSync(manifestDir)) {
|
|
536
|
+
if (file.startsWith('snapshots-') && file.endsWith('.json')) {
|
|
537
|
+
try {
|
|
538
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(manifestDir, file), 'utf8'));
|
|
539
|
+
if (manifest.snapshot && manifest.parent) {
|
|
540
|
+
parentMap.set(manifest.snapshot, manifest.parent);
|
|
541
|
+
}
|
|
542
|
+
} catch {}
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
372
547
|
const { prune } = planPrune(union, cfg.jobs.snapshots.keep);
|
|
548
|
+
|
|
549
|
+
// Filter out snapshots that are parents of kept snapshots
|
|
550
|
+
const keptSet = new Set(union.filter(name => !prune.includes(name)));
|
|
551
|
+
const protectedParents = new Set();
|
|
552
|
+
for (const kept of keptSet) {
|
|
553
|
+
let current = kept;
|
|
554
|
+
while (current) {
|
|
555
|
+
const parent = parentMap.get(current);
|
|
556
|
+
if (parent && union.includes(parent)) {
|
|
557
|
+
protectedParents.add(parent);
|
|
558
|
+
current = parent;
|
|
559
|
+
} else {
|
|
560
|
+
break;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
const safeToPrune = prune.filter(name => !protectedParents.has(name));
|
|
373
566
|
const pruned = [];
|
|
374
|
-
|
|
567
|
+
|
|
568
|
+
for (const name of safeToPrune) {
|
|
375
569
|
// Cloud first, then local — if one fails the other still gets cleaned.
|
|
376
570
|
try {
|
|
377
571
|
await removeArtifact('snapshots', name, accounts, cfg.storage.remoteRoot);
|
|
@@ -390,6 +584,11 @@ export async function pruneSnapshots(cfg, accounts, { privileged = 'noninteracti
|
|
|
390
584
|
}
|
|
391
585
|
pruned.push(name);
|
|
392
586
|
}
|
|
587
|
+
|
|
588
|
+
if (protectedParents.size > 0) {
|
|
589
|
+
journal('snapshots', `protected ${protectedParents.size} parent snapshots from pruning`, 'info');
|
|
590
|
+
}
|
|
591
|
+
|
|
393
592
|
return pruned;
|
|
394
593
|
}
|
|
395
594
|
|
package/src/cli.js
CHANGED
|
@@ -519,18 +519,87 @@ const main = defineCommand({
|
|
|
519
519
|
const remoteRoot = rest[3];
|
|
520
520
|
const chunkSize = parseInt(rest[4], 10);
|
|
521
521
|
const outPath = rest[5];
|
|
522
|
-
const { planAndPlace } = await import('./storage/allocator.js');
|
|
522
|
+
const { planAndPlace, planAndPlaceStream } = await import('./storage/allocator.js');
|
|
523
523
|
const s = p.spinner();
|
|
524
|
-
s.start(`Uploading
|
|
524
|
+
s.start(`Uploading ${kind} ${id}...`);
|
|
525
525
|
const cfg = loadConfig();
|
|
526
526
|
const accs = await refreshAccounts(cfg);
|
|
527
527
|
try {
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
528
|
+
let manifest;
|
|
529
|
+
if (kind === 'snapshots') {
|
|
530
|
+
const btrfsCfg = cfg.jobs.snapshots.btrfs || {};
|
|
531
|
+
let useBtrfs = btrfsCfg.enabled !== false && !process.env.PBB_DISABLE_BTRFS;
|
|
532
|
+
|
|
533
|
+
if (useBtrfs) {
|
|
534
|
+
// V2 BTRFS send/receive path
|
|
535
|
+
const { hasBtrfs, isBtrfsFilesystem } = await import('./backup/btrfs-send.js');
|
|
536
|
+
const canUseBtrfs = hasBtrfs() && await isBtrfsFilesystem('/');
|
|
537
|
+
|
|
538
|
+
if (canUseBtrfs) {
|
|
539
|
+
const { spawn } = await import('node:child_process');
|
|
540
|
+
const parentDir = process.env.PBB_PARENT_DIR;
|
|
541
|
+
|
|
542
|
+
// Build BTRFS send command with sudo
|
|
543
|
+
const sendArgs = ['sudo', '-n', 'btrfs', 'send'];
|
|
544
|
+
if (parentDir) {
|
|
545
|
+
sendArgs.push('-p', parentDir, localDir);
|
|
546
|
+
} else {
|
|
547
|
+
sendArgs.push(localDir);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const btrfs = spawn(sendArgs[0], sendArgs.slice(1), { stdio: ['ignore', 'pipe', 'inherit'] });
|
|
551
|
+
let finalStream = btrfs.stdout;
|
|
552
|
+
const pipeline = [btrfs];
|
|
553
|
+
|
|
554
|
+
// Stage 1: Compression
|
|
555
|
+
if (btrfsCfg.compression !== false) {
|
|
556
|
+
const zstd = spawn('zstd', ['-T0', '-c'], { stdio: ['pipe', 'pipe', 'inherit'] });
|
|
557
|
+
finalStream.pipe(zstd.stdin);
|
|
558
|
+
pipeline.push(zstd);
|
|
559
|
+
finalStream = zstd.stdout;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// Stage 2: Encryption
|
|
563
|
+
const passphrase = cfg.storage.encryptionPassphrase;
|
|
564
|
+
if (btrfsCfg.encryption && passphrase) {
|
|
565
|
+
const openssl = spawn('openssl', ['enc', '-e', '-aes256', '-pbkdf2', '-pass', `pass:${passphrase}`], {
|
|
566
|
+
stdio: ['pipe', 'pipe', 'inherit']
|
|
567
|
+
});
|
|
568
|
+
finalStream.pipe(openssl.stdin);
|
|
569
|
+
pipeline.push(openssl);
|
|
570
|
+
finalStream = openssl.stdout;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
manifest = await planAndPlaceStream(finalStream, {
|
|
574
|
+
kind, id, accounts: accs, remoteRoot, chunkSize,
|
|
575
|
+
onProgress: (prog) => s.message(prog.text)
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
// Wait for pipeline to complete
|
|
579
|
+
await Promise.all(pipeline.map(proc => new Promise((resolve, reject) => {
|
|
580
|
+
proc.on('close', code => code === 0 ? resolve() : reject(new Error(`Pipeline stage failed: exit ${code}`)));
|
|
581
|
+
proc.on('error', reject);
|
|
582
|
+
})));
|
|
583
|
+
} else {
|
|
584
|
+
// Fallback to file-copy mode
|
|
585
|
+
manifest = await planAndPlace(localDir, {
|
|
586
|
+
kind, id, accounts: accs, remoteRoot, chunkSize,
|
|
587
|
+
onProgress: (prog) => s.message(prog.text)
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
} else {
|
|
591
|
+
// File-copy mode (v2 fallback)
|
|
592
|
+
manifest = await planAndPlace(localDir, {
|
|
593
|
+
kind, id, accounts: accs, remoteRoot, chunkSize,
|
|
594
|
+
onProgress: (prog) => s.message(prog.text)
|
|
595
|
+
});
|
|
532
596
|
}
|
|
533
|
-
}
|
|
597
|
+
} else {
|
|
598
|
+
manifest = await planAndPlace(localDir, {
|
|
599
|
+
kind, id, accounts: accs, remoteRoot, chunkSize,
|
|
600
|
+
onProgress: (prog) => s.message(prog.text)
|
|
601
|
+
});
|
|
602
|
+
}
|
|
534
603
|
s.stop('✔ Upload complete');
|
|
535
604
|
import('node:fs').then(fs => fs.writeFileSync(outPath, JSON.stringify(manifest)));
|
|
536
605
|
process.exitCode = 0;
|
package/src/core/store.js
CHANGED
|
@@ -18,7 +18,7 @@ const GiB = 1024 ** 3;
|
|
|
18
18
|
/** Factory defaults — safe, conservative, self-documenting. */
|
|
19
19
|
export function defaultConfig() {
|
|
20
20
|
return {
|
|
21
|
-
version:
|
|
21
|
+
version: 2,
|
|
22
22
|
jobs: {
|
|
23
23
|
// The weekly snapshot is THE default backup. Daily file backups are an
|
|
24
24
|
// OPT-IN (enabled:true) so cloud + local storage are respected by default
|
|
@@ -45,11 +45,19 @@ export function defaultConfig() {
|
|
|
45
45
|
keep: 3, // latest + 2 previous (the middle one is the sanity safety net)
|
|
46
46
|
catchUpLimit: 3,
|
|
47
47
|
chunkSize: 2 * GiB,
|
|
48
|
+
btrfs: {
|
|
49
|
+
enabled: true, // Use BTRFS send/receive for incremental backups
|
|
50
|
+
incremental: true, // false = always do full send (no parent)
|
|
51
|
+
compression: true, // Use zstd compression in the pipeline
|
|
52
|
+
encryption: false, // Use openssl encryption (requires passphrase)
|
|
53
|
+
excludeSubvolumes: ['@swap'], // Subvolumes to skip (swap is meaningless to restore)
|
|
54
|
+
},
|
|
48
55
|
},
|
|
49
56
|
},
|
|
50
57
|
storage: {
|
|
51
58
|
remoteRoot: 'parrot-blackbox',
|
|
52
59
|
chunkSize: 2 * GiB,
|
|
60
|
+
encryptionPassphrase: '', // For BTRFS stream encryption; leave empty to prompt on first encrypted backup
|
|
53
61
|
providers: {
|
|
54
62
|
mega: { defaultQuotaGiB: 20 },
|
|
55
63
|
gdrive: { defaultQuotaGiB: 15 },
|
package/src/storage/allocator.js
CHANGED
|
@@ -21,6 +21,7 @@ import fs from 'node:fs';
|
|
|
21
21
|
import path from 'node:path';
|
|
22
22
|
import streams from 'node:stream/promises';
|
|
23
23
|
import { createWriteStream } from 'node:fs';
|
|
24
|
+
import { spawn } from 'node:child_process';
|
|
24
25
|
import { copyToFile, copyBatch, mkdirRemote } from './rclone.js';
|
|
25
26
|
import { bytesHuman } from '../util/misc.js';
|
|
26
27
|
|
|
@@ -237,4 +238,109 @@ async function makePartFile(file, start, len) {
|
|
|
237
238
|
fs.mkdirSync(path.dirname(partAbs), { recursive: true });
|
|
238
239
|
await streams.pipeline(fs.createReadStream(file.abs, { start, end: start + len - 1 }), createWriteStream(partAbs));
|
|
239
240
|
return partAbs;
|
|
240
|
-
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export async function planAndPlaceStream(btrfsStream, { kind, id, accounts, remoteRoot, chunkSize, onProgress, originalSize }) {
|
|
244
|
+
if (!accounts || accounts.length === 0) {
|
|
245
|
+
throw new Error('no storage accounts configured');
|
|
246
|
+
}
|
|
247
|
+
chunkSize = chunkSize || 2 * (1024 ** 3);
|
|
248
|
+
|
|
249
|
+
const pool = accounts.map((a) => ({ ...a }));
|
|
250
|
+
const consume = (account, bytes) => {
|
|
251
|
+
const a = pool.find((x) => x.id === account.id);
|
|
252
|
+
if (a) a.free -= bytes;
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
const basePath = `${remoteRoot}/${kind}/${id}`;
|
|
256
|
+
let partIndex = 0;
|
|
257
|
+
|
|
258
|
+
const getNextAccountAndPath = () => {
|
|
259
|
+
const acc = chooseAccount(chunkSize, pool);
|
|
260
|
+
if (!acc) throw new Error('OUT OF SPACE — the pool has no account with enough free room.');
|
|
261
|
+
consume(acc, chunkSize); // Optimistically consume chunkSize
|
|
262
|
+
const partPath = `${basePath}/btrfs.stream.part-${String(partIndex++).padStart(4, '0')}`;
|
|
263
|
+
return { remote: acc.remote, path: partPath, account: acc };
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
let currentChild = null;
|
|
267
|
+
let currentBytesInChunk = 0;
|
|
268
|
+
let totalBytes = 0;
|
|
269
|
+
let locs = [];
|
|
270
|
+
let currentStart = 0;
|
|
271
|
+
let target = getNextAccountAndPath();
|
|
272
|
+
|
|
273
|
+
currentChild = spawn(process.env.PBB_RCLONE || 'rclone', ['rcat', `${target.remote}:${target.path}`]);
|
|
274
|
+
let childFailed = false;
|
|
275
|
+
currentChild.on('error', () => { childFailed = true; });
|
|
276
|
+
currentChild.on('exit', (code) => { if (code !== 0) childFailed = true; });
|
|
277
|
+
|
|
278
|
+
for await (const chunk of btrfsStream) {
|
|
279
|
+
if (childFailed) throw new Error(`rclone rcat failed on ${target.remote}`);
|
|
280
|
+
let offset = 0;
|
|
281
|
+
while (offset < chunk.length) {
|
|
282
|
+
const remainingInChunk = chunkSize - currentBytesInChunk;
|
|
283
|
+
const toWrite = Math.min(chunk.length - offset, remainingInChunk);
|
|
284
|
+
|
|
285
|
+
const piece = chunk.subarray(offset, offset + toWrite);
|
|
286
|
+
const canContinue = currentChild.stdin.write(piece);
|
|
287
|
+
|
|
288
|
+
currentBytesInChunk += toWrite;
|
|
289
|
+
totalBytes += toWrite;
|
|
290
|
+
offset += toWrite;
|
|
291
|
+
|
|
292
|
+
if (!canContinue) {
|
|
293
|
+
await new Promise((res) => currentChild.stdin.once('drain', res));
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
if (currentBytesInChunk >= chunkSize) {
|
|
297
|
+
currentChild.stdin.end();
|
|
298
|
+
await new Promise((res) => currentChild.once('close', res));
|
|
299
|
+
if (childFailed) throw new Error(`rclone rcat failed on ${target.remote}`);
|
|
300
|
+
|
|
301
|
+
locs.push({ remote: target.remote, path: target.path, start: currentStart, end: totalBytes, size: currentBytesInChunk });
|
|
302
|
+
|
|
303
|
+
currentStart = totalBytes;
|
|
304
|
+
currentBytesInChunk = 0;
|
|
305
|
+
|
|
306
|
+
target = getNextAccountAndPath();
|
|
307
|
+
currentChild = spawn(process.env.PBB_RCLONE || 'rclone', ['rcat', `${target.remote}:${target.path}`]);
|
|
308
|
+
currentChild.on('error', () => { childFailed = true; });
|
|
309
|
+
currentChild.on('exit', (code) => { if (code !== 0) childFailed = true; });
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
if (onProgress) onProgress({ done: totalBytes, total: originalSize, text: `uploading stream: ${(totalBytes / (1024**2)).toFixed(1)} MB` });
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
if (currentChild) {
|
|
316
|
+
currentChild.stdin.end();
|
|
317
|
+
await new Promise((res) => currentChild.once('close', res));
|
|
318
|
+
if (childFailed) throw new Error(`rclone rcat failed on ${target.remote}`);
|
|
319
|
+
if (currentBytesInChunk > 0) {
|
|
320
|
+
locs.push({ remote: target.remote, path: target.path, start: currentStart, end: totalBytes, size: currentBytesInChunk });
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
const manifest = {
|
|
325
|
+
schema: 2,
|
|
326
|
+
kind,
|
|
327
|
+
id,
|
|
328
|
+
createdAt: new Date().toISOString(),
|
|
329
|
+
totalSize: totalBytes,
|
|
330
|
+
originalSize: originalSize || 0,
|
|
331
|
+
remoteRoot,
|
|
332
|
+
entries: [{ rel: 'btrfs.stream', type: 'file', size: totalBytes, split: true, loc: locs }],
|
|
333
|
+
};
|
|
334
|
+
|
|
335
|
+
// Manifest: cloud + local mirror.
|
|
336
|
+
const manifestLocalDir = process.env.PBB_MANIFESTS_DIR || path.join(process.env.PBB_STATE_DIR || '.', 'manifests');
|
|
337
|
+
const manifestLocalPath = path.join(manifestLocalDir, `${kind}-${id}.json`);
|
|
338
|
+
fs.mkdirSync(manifestLocalDir, { recursive: true });
|
|
339
|
+
fs.writeFileSync(manifestLocalPath, JSON.stringify(manifest, null, 2));
|
|
340
|
+
const accForManifest = pool.find((a) => a.free > 0) || pool[0];
|
|
341
|
+
if (accForManifest) {
|
|
342
|
+
const res = await copyToFile(manifestLocalPath, `${accForManifest.remote}:${basePath}/${MANIFEST_NAME}`);
|
|
343
|
+
if (res.ok) manifest.account = accForManifest.remote;
|
|
344
|
+
}
|
|
345
|
+
return manifest;
|
|
346
|
+
}
|