telstore 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -3
- package/bin/telstore.js +18 -4
- package/package.json +1 -1
- package/src/caption.js +64 -2
- package/src/cipher.js +211 -0
- package/src/cli.js +56 -3
- package/src/commands/join.js +273 -0
- package/src/commands/list.js +11 -4
- package/src/commands/restore-stream.js +28 -1
- package/src/commands/restore.js +46 -21
- package/src/commands/status.js +6 -1
- package/src/commands/upload-stream.js +51 -3
- package/src/commands/upload.js +175 -4
- package/src/commands/verify.js +8 -1
- package/src/manifest.js +108 -4
- package/src/password.js +101 -0
- package/src/state.js +24 -2
- package/src/uploader.js +6 -1
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { promises as fs } from 'node:fs'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
|
|
5
|
+
import { askConfirm } from '../confirm.js'
|
|
6
|
+
import { chunkCipher } from '../cipher.js'
|
|
7
|
+
import { chunkFileName, isEncrypted, parseManifest, safeOutName } from '../manifest.js'
|
|
8
|
+
import { askPassword as realAskPassword, unlockManifest } from '../password.js'
|
|
9
|
+
import { createProgress, formatBytes } from '../progress.js'
|
|
10
|
+
|
|
11
|
+
// Big enough that a 1800MB chunk is a couple of hundred reads, small enough that one buffer
|
|
12
|
+
// reused for every chunk is nothing next to the file being written.
|
|
13
|
+
const COPY_SIZE = 8 * 1024 * 1024
|
|
14
|
+
|
|
15
|
+
// A backup id becomes part of a path below, and it comes out of a file a person downloaded
|
|
16
|
+
// and can edit. Every id telstore writes is `telstore-YYYYMMDD-<hex>`; anything carrying a
|
|
17
|
+
// separator or a leading dot would have join read chunks from somewhere other than beside
|
|
18
|
+
// the manifest, so it is refused rather than followed.
|
|
19
|
+
function assertPlainId(id) {
|
|
20
|
+
if (typeof id !== 'string' || !/^[\w.-]+$/.test(id) || id.startsWith('.')) {
|
|
21
|
+
throw new Error(
|
|
22
|
+
`The manifest gives ${JSON.stringify(id)} as its backup id, which is not a plain name, ` +
|
|
23
|
+
'so telstore cannot tell which files beside it are its chunks.',
|
|
24
|
+
)
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function readManifest(manifestPath) {
|
|
29
|
+
let bytes
|
|
30
|
+
|
|
31
|
+
try {
|
|
32
|
+
bytes = await fs.readFile(manifestPath)
|
|
33
|
+
} catch (err) {
|
|
34
|
+
throw new Error(`Cannot read the manifest ${manifestPath}: ${err.message}`)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return parseManifest(bytes)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Every chunk is checked for presence and length before a byte is written, and every fault
|
|
41
|
+
// is named in one message: a download by hand that missed three files should cost one more
|
|
42
|
+
// trip to the chat, not three.
|
|
43
|
+
async function checkChunkFiles(manifest, files, dir) {
|
|
44
|
+
const missing = []
|
|
45
|
+
const wrong = []
|
|
46
|
+
|
|
47
|
+
for (const [index, chunk] of manifest.chunks.entries()) {
|
|
48
|
+
const name = path.basename(files[index])
|
|
49
|
+
let stat
|
|
50
|
+
|
|
51
|
+
try {
|
|
52
|
+
stat = await fs.stat(files[index])
|
|
53
|
+
} catch (err) {
|
|
54
|
+
if (err.code !== 'ENOENT') throw err
|
|
55
|
+
missing.push(name)
|
|
56
|
+
continue
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (!stat.isFile()) {
|
|
60
|
+
wrong.push(`${name} is not a file.`)
|
|
61
|
+
} else if (stat.size !== chunk.size) {
|
|
62
|
+
wrong.push(`${name} has ${stat.size} bytes, the manifest records ${chunk.size} bytes.`)
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (missing.length === 0 && wrong.length === 0) return
|
|
67
|
+
|
|
68
|
+
const lines = [`Cannot join ${manifest.id}:`]
|
|
69
|
+
|
|
70
|
+
if (missing.length > 0) {
|
|
71
|
+
lines.push(`${missing.length} of ${manifest.chunks.length} chunks are not in ${dir}:`)
|
|
72
|
+
lines.push(...missing.map((name) => ` ${name}`))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
lines.push(...wrong)
|
|
76
|
+
lines.push(
|
|
77
|
+
'Download them from the chat into that folder under exactly these names and run join ' +
|
|
78
|
+
'again. A browser that saved a name twice may have added " (1)" to it.',
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
throw new Error(lines.join('\n'))
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function writeAll(handle, buffer, length, position) {
|
|
85
|
+
let written = 0
|
|
86
|
+
|
|
87
|
+
while (written < length) {
|
|
88
|
+
const { bytesWritten } = await handle.write(buffer, written, length - written, position + written)
|
|
89
|
+
written += bytesWritten
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Copies one chunk file into place and hashes exactly the bytes it copied. The length was
|
|
94
|
+
// checked up front, and is checked again here because the file is read now, not then: a
|
|
95
|
+
// download still being written, or replaced in between, is a different file from the one
|
|
96
|
+
// that was stat'd.
|
|
97
|
+
//
|
|
98
|
+
// For an encrypted backup the file holds ciphertext: that is what is hashed against the
|
|
99
|
+
// manifest, and the plaintext written into place is hashed against the seal.
|
|
100
|
+
async function copyChunk({ file, handle, offset, chunk, count, buffer, advance, cipher = null, plainSha256 = null }) {
|
|
101
|
+
const name = path.basename(file)
|
|
102
|
+
const source = await fs.open(file, 'r')
|
|
103
|
+
const hash = createHash('sha256')
|
|
104
|
+
const plain = cipher ? createHash('sha256') : null
|
|
105
|
+
let copied = 0
|
|
106
|
+
|
|
107
|
+
try {
|
|
108
|
+
for (;;) {
|
|
109
|
+
const { bytesRead } = await source.read(buffer, 0, buffer.length, copied)
|
|
110
|
+
|
|
111
|
+
if (bytesRead === 0) break
|
|
112
|
+
|
|
113
|
+
if (copied + bytesRead > chunk.size) {
|
|
114
|
+
throw new Error(`${name} grew past the ${chunk.size} bytes the manifest records while it was being read.`)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const read = buffer.subarray(0, bytesRead)
|
|
118
|
+
const out = cipher ? cipher.apply(read, copied) : read
|
|
119
|
+
|
|
120
|
+
hash.update(read)
|
|
121
|
+
plain?.update(out)
|
|
122
|
+
await writeAll(handle, out, bytesRead, offset + copied)
|
|
123
|
+
copied += bytesRead
|
|
124
|
+
advance(bytesRead)
|
|
125
|
+
}
|
|
126
|
+
} finally {
|
|
127
|
+
await source.close()
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (copied !== chunk.size) {
|
|
131
|
+
throw new Error(`${name} has ${copied} bytes, the manifest records ${chunk.size} bytes.`)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (hash.digest('hex') !== chunk.sha256) {
|
|
135
|
+
throw new Error(
|
|
136
|
+
`${name} does not match the sha256 the manifest records for chunk ${chunk.i + 1}/${count}. ` +
|
|
137
|
+
'It is damaged or belongs to another backup — download it again.',
|
|
138
|
+
)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (plain && plain.digest('hex') !== plainSha256) {
|
|
142
|
+
throw new Error(
|
|
143
|
+
`${name} matched its encrypted sha256 but decrypted to bytes that do not match the ` +
|
|
144
|
+
`manifest for chunk ${chunk.i + 1}/${count}. That points at telstore rather than at the download.`,
|
|
145
|
+
)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// The offline half of restore: the chunks and the manifest were downloaded by hand, from
|
|
150
|
+
// Telegram web or anywhere else, and only the reassembly is left. Nothing here talks to
|
|
151
|
+
// the network, so nothing here needs a login or a message id — but every guarantee restore
|
|
152
|
+
// makes about the bytes still holds, because the manifest is what makes them, not the chat.
|
|
153
|
+
export async function runJoin(manifestPath, options = {}, deps = {}) {
|
|
154
|
+
const {
|
|
155
|
+
confirm = askConfirm,
|
|
156
|
+
cwd = process.cwd(),
|
|
157
|
+
writeErr = (line) => process.stderr.write(line),
|
|
158
|
+
log: writeLog = (line) => console.log(line),
|
|
159
|
+
silent = false,
|
|
160
|
+
onTempChunk = () => {},
|
|
161
|
+
askPassword = realAskPassword,
|
|
162
|
+
interactive = () => Boolean(process.stdin.isTTY),
|
|
163
|
+
} = deps
|
|
164
|
+
|
|
165
|
+
const log = silent ? () => {} : writeLog
|
|
166
|
+
const warn = silent ? () => {} : writeErr
|
|
167
|
+
|
|
168
|
+
const manifest = await readManifest(manifestPath)
|
|
169
|
+
assertPlainId(manifest.id)
|
|
170
|
+
|
|
171
|
+
const dir = path.dirname(path.resolve(cwd, manifestPath))
|
|
172
|
+
const files = manifest.chunks.map((chunk) => path.join(dir, chunkFileName(manifest.id, chunk.i)))
|
|
173
|
+
|
|
174
|
+
await checkChunkFiles(manifest, files, dir)
|
|
175
|
+
|
|
176
|
+
// After the files are known to be there — that costs nothing and needs no password — and
|
|
177
|
+
// before the overwrite question, for restore's reason: nobody should answer [y/N] about a
|
|
178
|
+
// file telstore then cannot decrypt.
|
|
179
|
+
const opened = isEncrypted(manifest)
|
|
180
|
+
? await unlockManifest(manifest, { askPassword, interactive, say: log })
|
|
181
|
+
: null
|
|
182
|
+
|
|
183
|
+
const target = path.resolve(cwd, options.out ?? safeOutName(manifest.name))
|
|
184
|
+
|
|
185
|
+
// Not `.partial`: that name belongs to restore, which resumes from it, and a join aimed at
|
|
186
|
+
// the same file would otherwise truncate hours of somebody's download without asking.
|
|
187
|
+
const joining = `${target}.joining`
|
|
188
|
+
|
|
189
|
+
// Only ENOENT means "no file yet". Treating a permission or I/O error as absence would have
|
|
190
|
+
// telstore overwrite the user's file without asking.
|
|
191
|
+
let exists = true
|
|
192
|
+
try {
|
|
193
|
+
await fs.stat(target)
|
|
194
|
+
} catch (err) {
|
|
195
|
+
if (err.code !== 'ENOENT') throw err
|
|
196
|
+
exists = false
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (exists && !(await confirm(`${target} already exists. Overwrite? [y/N] `))) {
|
|
200
|
+
throw new Error('Cancelled on request.')
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
log(`Backup ${manifest.id}`)
|
|
204
|
+
if (opened) log('Lock encrypted')
|
|
205
|
+
log(`File ${target} (${formatBytes(manifest.size)}, ${manifest.chunks.length} chunks)`)
|
|
206
|
+
log(`From ${dir}\n`)
|
|
207
|
+
|
|
208
|
+
// Said before the open, so a Ctrl-C that lands between the two still has the name to remove.
|
|
209
|
+
onTempChunk(joining)
|
|
210
|
+
|
|
211
|
+
let handle = null
|
|
212
|
+
|
|
213
|
+
try {
|
|
214
|
+
handle = await fs.open(joining, 'w')
|
|
215
|
+
|
|
216
|
+
const buffer = Buffer.allocUnsafe(COPY_SIZE)
|
|
217
|
+
const count = manifest.chunks.length
|
|
218
|
+
const progress = createProgress({ total: manifest.size, label: `Chunk 1/${count}`, write: warn })
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
for (const [index, chunk] of manifest.chunks.entries()) {
|
|
222
|
+
progress.setLabel(`Chunk ${chunk.i + 1}/${count}`)
|
|
223
|
+
|
|
224
|
+
await copyChunk({
|
|
225
|
+
file: files[index],
|
|
226
|
+
handle,
|
|
227
|
+
offset: chunk.i * manifest.chunkSize,
|
|
228
|
+
chunk,
|
|
229
|
+
count,
|
|
230
|
+
buffer,
|
|
231
|
+
advance: progress.advance,
|
|
232
|
+
cipher: opened ? chunkCipher(opened.keys.chunkKey, chunk.iv) : null,
|
|
233
|
+
plainSha256: opened ? opened.plainSha256[chunk.i] : null,
|
|
234
|
+
})
|
|
235
|
+
}
|
|
236
|
+
} finally {
|
|
237
|
+
progress.finish()
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
await handle.close()
|
|
241
|
+
handle = null
|
|
242
|
+
|
|
243
|
+
// Last line of defence, as in restore: every chunk matched and the file is still the
|
|
244
|
+
// wrong length means the layout went wrong somewhere, and the rename must not happen.
|
|
245
|
+
const written = await fs.stat(joining)
|
|
246
|
+
|
|
247
|
+
if (written.size !== manifest.size) {
|
|
248
|
+
throw new Error(
|
|
249
|
+
`The joined file has ${written.size} bytes, the manifest records ${manifest.size} bytes — mismatch.`,
|
|
250
|
+
)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
await fs.rename(joining, target)
|
|
254
|
+
} catch (err) {
|
|
255
|
+
if (handle) await handle.close().catch(() => {})
|
|
256
|
+
|
|
257
|
+
// Unlike restore's .partial there is nothing here worth keeping: the chunks are still on
|
|
258
|
+
// disk, and a half-joined file is only somewhere a wrong file could be picked up from.
|
|
259
|
+
try {
|
|
260
|
+
await fs.rm(joining, { force: true })
|
|
261
|
+
} catch (rmErr) {
|
|
262
|
+
warn(`\nCould not remove ${joining} (${rmErr.message}). It is incomplete — remove it by hand.\n`)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
throw err
|
|
266
|
+
} finally {
|
|
267
|
+
onTempChunk(null)
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
log(`\nDone. Wrote ${formatBytes(manifest.size)} to ${target}`)
|
|
271
|
+
|
|
272
|
+
return { path: target, size: manifest.size }
|
|
273
|
+
}
|
package/src/commands/list.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { parseManifestCaption } from '../caption.js'
|
|
1
|
+
import { parseManifestCaption, terminalSafe } from '../caption.js'
|
|
2
2
|
import { chatName, describeChat } from '../chat.js'
|
|
3
3
|
import {
|
|
4
4
|
closeQuietly,
|
|
@@ -22,6 +22,7 @@ const COLUMNS = [
|
|
|
22
22
|
{ header: 'CHUNKS', key: 'chunks', right: true },
|
|
23
23
|
{ header: 'CREATED', key: 'created' },
|
|
24
24
|
{ header: 'NOTE', key: 'note' },
|
|
25
|
+
{ header: 'LOCK', key: 'lock' },
|
|
25
26
|
]
|
|
26
27
|
|
|
27
28
|
// The table is read at a glance, and the note is the one field with no shape at all — 500
|
|
@@ -97,6 +98,7 @@ function toRow(message) {
|
|
|
97
98
|
chunks: UNKNOWN,
|
|
98
99
|
created: utcDay(message.date),
|
|
99
100
|
note: UNKNOWN,
|
|
101
|
+
lock: UNKNOWN,
|
|
100
102
|
}
|
|
101
103
|
}
|
|
102
104
|
|
|
@@ -107,6 +109,9 @@ function toRow(message) {
|
|
|
107
109
|
chunks: String(card.chunks),
|
|
108
110
|
created: card.createdAt.slice(0, 10),
|
|
109
111
|
note: card.note ? shorten(card.note) : UNKNOWN,
|
|
112
|
+
// The hint is here because this is where someone who forgot a password looks first. It comes
|
|
113
|
+
// off a caption anyone in the chat can edit, so it is made safe to print before it is printed.
|
|
114
|
+
lock: card.encrypted ? (card.hint ? `🔒 ${shorten(terminalSafe(card.hint))}` : '🔒') : UNKNOWN,
|
|
110
115
|
}
|
|
111
116
|
}
|
|
112
117
|
|
|
@@ -150,10 +155,12 @@ function matchesTerm(message, term) {
|
|
|
150
155
|
}
|
|
151
156
|
|
|
152
157
|
function renderTable(rows) {
|
|
153
|
-
// Most people never write a note, and a column of dashes tells them nothing they
|
|
154
|
-
// already know
|
|
158
|
+
// Most people never write a note or encrypt, and a column of dashes tells them nothing they
|
|
159
|
+
// did not already know.
|
|
155
160
|
const columns = COLUMNS.filter(
|
|
156
|
-
(column) =>
|
|
161
|
+
(column) =>
|
|
162
|
+
(column.key !== 'note' || rows.some((row) => row.note !== UNKNOWN)) &&
|
|
163
|
+
(column.key !== 'lock' || rows.some((row) => row.lock !== UNKNOWN)),
|
|
157
164
|
)
|
|
158
165
|
|
|
159
166
|
const widths = columns.map((column) =>
|
|
@@ -9,7 +9,9 @@ import {
|
|
|
9
9
|
readMessageBytes as realReadMessageBytes,
|
|
10
10
|
} from '../client.js'
|
|
11
11
|
import { configFile, defaultConfigDir, loadConfig } from '../config.js'
|
|
12
|
-
import {
|
|
12
|
+
import { chunkCipher, decryptInPlace } from '../cipher.js'
|
|
13
|
+
import { isEncrypted, parseManifest } from '../manifest.js'
|
|
14
|
+
import { askPassword as realAskPassword, unlockManifest } from '../password.js'
|
|
13
15
|
import { createProgress, formatBytes, plural } from '../progress.js'
|
|
14
16
|
import { assertLoggedIn } from '../session.js'
|
|
15
17
|
import { requireChat, resolveSettings } from '../settings.js'
|
|
@@ -50,6 +52,8 @@ export async function runRestoreStream(backupId, childArgv, options = {}, deps =
|
|
|
50
52
|
// tarx only. The alias promises gzip, so it checks the claim before spending a gigabyte
|
|
51
53
|
// finding out; `restore <id> -- tar xf -` promises nothing and is asked nothing.
|
|
52
54
|
requireGzipName = false,
|
|
55
|
+
askPassword = realAskPassword,
|
|
56
|
+
interactive = () => Boolean(process.stdin.isTTY),
|
|
53
57
|
} = deps
|
|
54
58
|
|
|
55
59
|
const config = await loadConfig(configDir)
|
|
@@ -109,8 +113,16 @@ export async function runRestoreStream(backupId, childArgv, options = {}, deps =
|
|
|
109
113
|
)
|
|
110
114
|
}
|
|
111
115
|
|
|
116
|
+
// Before the command is started, for the reason the gzip check above runs before the
|
|
117
|
+
// download: a restore that cannot be decrypted should cost nothing, and a command started
|
|
118
|
+
// for it would be a tar waiting on a pipe that is never going to carry anything.
|
|
119
|
+
const opened = isEncrypted(manifest)
|
|
120
|
+
? await unlockManifest(manifest, { askPassword, interactive, say: log })
|
|
121
|
+
: null
|
|
122
|
+
|
|
112
123
|
log(`Backup ${backupId}`)
|
|
113
124
|
log(`Name ${manifest.name} (${plural(manifest.chunks.length, 'chunk')}, ${formatBytes(manifest.size)})`)
|
|
125
|
+
if (opened) log('Lock encrypted')
|
|
114
126
|
log(`From ${describeChat(chat)}`)
|
|
115
127
|
log(`Into ${childArgv.join(' ')}\n`)
|
|
116
128
|
|
|
@@ -240,6 +252,21 @@ export async function runRestoreStream(backupId, childArgv, options = {}, deps =
|
|
|
240
252
|
)
|
|
241
253
|
}
|
|
242
254
|
|
|
255
|
+
// The rule this file keeps — no byte reaches the command before its chunk is verified —
|
|
256
|
+
// now means verified as plaintext: decrypted in the temp file, hashed against the seal,
|
|
257
|
+
// and only then pumped.
|
|
258
|
+
if (opened) {
|
|
259
|
+
const clear = await decryptInPlace(handle, 0, size, chunkCipher(opened.keys.chunkKey, chunk.iv))
|
|
260
|
+
|
|
261
|
+
if (clear !== opened.plainSha256[chunk.i]) {
|
|
262
|
+
throw new Error(
|
|
263
|
+
`Chunk ${chunk.i + 1} matched its encrypted sha256 but decrypted to bytes that do ` +
|
|
264
|
+
'not match the manifest. That points at telstore rather than at the backup. ' +
|
|
265
|
+
`${received(childArgv, written)}`,
|
|
266
|
+
)
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
243
270
|
// Only now, and this line is the guarantee: everything above it is what makes the
|
|
244
271
|
// difference between handing a command the backup and handing it whatever arrived.
|
|
245
272
|
//
|
package/src/commands/restore.js
CHANGED
|
@@ -11,8 +11,10 @@ import { askConfirm } from '../confirm.js'
|
|
|
11
11
|
import { configFile, defaultConfigDir, loadConfig } from '../config.js'
|
|
12
12
|
import { assertLoggedIn } from '../session.js'
|
|
13
13
|
import { requireChat, resolveSettings } from '../settings.js'
|
|
14
|
+
import { chunkCipher, decryptInPlace } from '../cipher.js'
|
|
14
15
|
import { downloadToFile, hashRange } from '../downloader.js'
|
|
15
|
-
import { parseManifest } from '../manifest.js'
|
|
16
|
+
import { isEncrypted, parseManifest, safeOutName } from '../manifest.js'
|
|
17
|
+
import { askPassword as realAskPassword, unlockManifest } from '../password.js'
|
|
16
18
|
import { createProgress, formatBytes, formatDuration } from '../progress.js'
|
|
17
19
|
import { clearRestore, pruneRestores, restoreKey, saveRestore } from '../state.js'
|
|
18
20
|
|
|
@@ -59,29 +61,12 @@ export async function realDownloadChunk(client, message, handle, offset, onProgr
|
|
|
59
61
|
return await downloadToFile(client, message, handle.fd, { offset, onProgress, ...options })
|
|
60
62
|
}
|
|
61
63
|
|
|
62
|
-
// manifest.name comes from data downloaded off Telegram — don't trust it when picking
|
|
63
|
-
// a path ourselves. path.basename stops "../../x" but still returns "..", "." or "" for
|
|
64
|
-
// a few pathological names: path.resolve('..') is the parent directory, so a multi-GB
|
|
65
|
-
// .partial file would land outside the current directory and only blow up at rename.
|
|
66
|
-
function safeOutName(name) {
|
|
67
|
-
const base = path.basename(String(name ?? ''))
|
|
68
|
-
|
|
69
|
-
if (base === '' || base === '.' || base === '..') {
|
|
70
|
-
throw new Error(
|
|
71
|
-
`The name in the manifest ("${name}") cannot be used as a file name. ` +
|
|
72
|
-
'Run again with --out <path> to choose where to write.',
|
|
73
|
-
)
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
return base
|
|
77
|
-
}
|
|
78
|
-
|
|
79
64
|
// How many chunks at the front of a .partial already hold what the manifest says they
|
|
80
65
|
// should. The evidence is the file, never a record: a record makes claims about a local
|
|
81
66
|
// file anyone can edit between runs, and a claim that is wrong here renames a corrupt file
|
|
82
67
|
// into place. Every chunk in the finished file was hashed against the manifest by the run
|
|
83
68
|
// that renamed it, whether this run downloaded it or found it already there.
|
|
84
|
-
async function scanPartial(handle, manifest, log) {
|
|
69
|
+
async function scanPartial(handle, manifest, log, expected) {
|
|
85
70
|
let done = 0
|
|
86
71
|
|
|
87
72
|
for (const chunk of manifest.chunks) {
|
|
@@ -90,7 +75,7 @@ async function scanPartial(handle, manifest, log) {
|
|
|
90
75
|
// Downloads run in order, so what is already present is a prefix. The first chunk that
|
|
91
76
|
// does not match is where this run starts, and reading past it would hash gigabytes
|
|
92
77
|
// nobody has written yet.
|
|
93
|
-
if (digest !== chunk
|
|
78
|
+
if (digest !== expected(chunk)) break
|
|
94
79
|
|
|
95
80
|
done += 1
|
|
96
81
|
log(`Chunk ${chunk.i + 1}/${manifest.chunks.length} already restored, skipping.`)
|
|
@@ -114,6 +99,10 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
114
99
|
log: writeLog = (line) => console.log(line),
|
|
115
100
|
silent = false,
|
|
116
101
|
onBackupId = () => {},
|
|
102
|
+
askPassword = realAskPassword,
|
|
103
|
+
interactive = () => Boolean(process.stdin.isTTY),
|
|
104
|
+
// Passwords that opened an earlier backup in the same batch, tried before asking again.
|
|
105
|
+
knownPasswords = [],
|
|
117
106
|
} = deps
|
|
118
107
|
|
|
119
108
|
const config = await loadConfig(configDir)
|
|
@@ -140,6 +129,17 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
140
129
|
}
|
|
141
130
|
|
|
142
131
|
const manifest = parseManifest(await readMessageBytes(client, manifestMessage))
|
|
132
|
+
|
|
133
|
+
// Before the overwrite question and before the .partial: a password that cannot be had must
|
|
134
|
+
// cost nothing, and nobody should answer [y/N] about a file telstore then cannot write.
|
|
135
|
+
const opened = isEncrypted(manifest)
|
|
136
|
+
? await unlockManifest(manifest, { askPassword, interactive, known: knownPasswords, say: log })
|
|
137
|
+
: null
|
|
138
|
+
|
|
139
|
+
// What each finished chunk hashes to in the .partial. A decrypted chunk is plaintext there,
|
|
140
|
+
// and only the sealed hash says what that plaintext must be.
|
|
141
|
+
const onDisk = (chunk) => (opened ? opened.plainSha256[chunk.i] : chunk.sha256)
|
|
142
|
+
|
|
143
143
|
// When the user passes --out, respect that path verbatim.
|
|
144
144
|
const target = path.resolve(options.out ?? safeOutName(manifest.name))
|
|
145
145
|
const partial = `${target}.partial`
|
|
@@ -194,6 +194,7 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
194
194
|
}
|
|
195
195
|
|
|
196
196
|
log(`Backup ${manifest.id}`)
|
|
197
|
+
if (opened) log('Lock encrypted')
|
|
197
198
|
log(`File ${target} (${formatBytes(manifest.size)}, ${manifest.chunks.length} chunks)\n`)
|
|
198
199
|
|
|
199
200
|
let handle
|
|
@@ -226,7 +227,7 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
226
227
|
// that long is the hang this project refuses everywhere: the heading lands before
|
|
227
228
|
// the first read and a line per chunk arrives as the scan advances.
|
|
228
229
|
log(`Checking what is already in ${partial}...`)
|
|
229
|
-
done = await scanPartial(handle, manifest, log)
|
|
230
|
+
done = await scanPartial(handle, manifest, log, onDisk)
|
|
230
231
|
if (done === 0) log(`Nothing in ${partial} matches this backup, starting over.`)
|
|
231
232
|
log('')
|
|
232
233
|
}
|
|
@@ -302,6 +303,26 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
302
303
|
)
|
|
303
304
|
}
|
|
304
305
|
|
|
306
|
+
// Only after the ciphertext has matched: that match is what proves these are the bytes
|
|
307
|
+
// that went up, and the manifest's seal is what proves the hash itself. The plaintext
|
|
308
|
+
// check that follows is the second one, and it exists for telstore's own mistakes.
|
|
309
|
+
if (opened) {
|
|
310
|
+
const clear = await decryptInPlace(
|
|
311
|
+
handle,
|
|
312
|
+
chunk.i * manifest.chunkSize,
|
|
313
|
+
chunk.size,
|
|
314
|
+
chunkCipher(opened.keys.chunkKey, chunk.iv),
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
if (clear !== opened.plainSha256[chunk.i]) {
|
|
318
|
+
throw new Error(
|
|
319
|
+
`Chunk ${chunk.i + 1} matched its encrypted sha256 but decrypted to bytes that do ` +
|
|
320
|
+
'not match the manifest. That points at telstore rather than at the backup; ' +
|
|
321
|
+
`nothing was renamed, and the download is kept at ${partial} for inspection.`,
|
|
322
|
+
)
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
305
326
|
await note(chunk.i + 1)
|
|
306
327
|
}
|
|
307
328
|
} finally {
|
|
@@ -402,11 +423,15 @@ export async function runRestores(backupIds, options = {}, deps = {}) {
|
|
|
402
423
|
const warn = silent ? () => {} : writeErr
|
|
403
424
|
|
|
404
425
|
let shared = null
|
|
426
|
+
// Shared across every id in the batch, so a password that opened the first backup is tried
|
|
427
|
+
// silently on the rest before asking again.
|
|
428
|
+
const passwords = []
|
|
405
429
|
const perId = {
|
|
406
430
|
...deps,
|
|
407
431
|
connect: async (theirConfig, connectOptions) =>
|
|
408
432
|
(shared ??= await connect(theirConfig, connectOptions)),
|
|
409
433
|
disconnect: async () => {},
|
|
434
|
+
knownPasswords: passwords,
|
|
410
435
|
}
|
|
411
436
|
|
|
412
437
|
const results = []
|
package/src/commands/status.js
CHANGED
|
@@ -39,7 +39,11 @@ function field(label, value) {
|
|
|
39
39
|
function resumeCommand(state, destination) {
|
|
40
40
|
const matches = destination !== null && state.chat === String(destination)
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
// An encrypted record refuses a run without --encrypt, so a line without it would be a command
|
|
43
|
+
// telstore prints to be pasted and then turns away.
|
|
44
|
+
const encrypt = state.enc ? ' --encrypt' : ''
|
|
45
|
+
|
|
46
|
+
return `npx telstore ${shellArg(state.path)}${encrypt}${matches ? '' : ` --chat ${shellArg(state.chat)}`}`
|
|
43
47
|
}
|
|
44
48
|
|
|
45
49
|
// status is the command someone runs *because* something is wrong, so a record a truncated
|
|
@@ -83,6 +87,7 @@ const NO_RESUME = {
|
|
|
83
87
|
changed: 'the file has changed since the backup started',
|
|
84
88
|
'not-a-file': 'that path is no longer a file',
|
|
85
89
|
unreadable: 'the record does not name a file that can be read',
|
|
90
|
+
damaged: 'the record is damaged, and upload refuses to carry on from it',
|
|
86
91
|
}
|
|
87
92
|
|
|
88
93
|
// Why the .partial cannot be resumed from, in the same spirit as NO_RESUME above: an
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
1
2
|
import { promises as fs } from 'node:fs'
|
|
2
3
|
import path from 'node:path'
|
|
3
4
|
|
|
4
5
|
import { MAX_CHUNKS, PART_SIZE } from '../chunking.js'
|
|
5
|
-
import { chunkCaption, manifestCaption, parseNote } from '../caption.js'
|
|
6
|
+
import { chunkCaption, manifestCaption, parseNote, terminalSafe } from '../caption.js'
|
|
6
7
|
import { chatName, describeChat } from '../chat.js'
|
|
8
|
+
import { chunkCipher, deriveKeys, newIv, newSalt, sealManifest } from '../cipher.js'
|
|
7
9
|
import {
|
|
8
10
|
MESSAGE_BATCH_SIZE,
|
|
9
11
|
closeQuietly,
|
|
@@ -18,6 +20,7 @@ import {
|
|
|
18
20
|
newBackupId,
|
|
19
21
|
serializeManifest,
|
|
20
22
|
} from '../manifest.js'
|
|
23
|
+
import { askNewPassword as realAskNewPassword } from '../password.js'
|
|
21
24
|
import { createStreamProgress, formatBytes, plural } from '../progress.js'
|
|
22
25
|
import { requireChat, resolveSettings } from '../settings.js'
|
|
23
26
|
import { deleteCommand } from '../shell.js'
|
|
@@ -67,6 +70,8 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
67
70
|
// `discardChunkFile` can still take it with it. Said before the file is opened and unsaid
|
|
68
71
|
// after it is removed, so the caller's copy is never narrower than what is actually on disk.
|
|
69
72
|
onTempChunk = () => {},
|
|
73
|
+
askNewPassword = realAskNewPassword,
|
|
74
|
+
secret = null,
|
|
70
75
|
} = deps
|
|
71
76
|
|
|
72
77
|
// Before the command is started, let alone connected to Telegram: the note is the one thing
|
|
@@ -80,6 +85,20 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
80
85
|
const chunkSize = settings.chunkSize
|
|
81
86
|
const concurrency = settings.uploadConcurrency
|
|
82
87
|
|
|
88
|
+
// Before the record, the connection and the command: a run that cannot get a password has
|
|
89
|
+
// nothing to unwind, and nobody should watch pg_dump start before being asked for one. No
|
|
90
|
+
// check value is kept — a stream is never resumed, so there is no second run to compare with.
|
|
91
|
+
let keys = null
|
|
92
|
+
let enc = null
|
|
93
|
+
|
|
94
|
+
if (options.encrypt) {
|
|
95
|
+
const chosen = secret ?? (await askNewPassword())
|
|
96
|
+
const salt = newSalt()
|
|
97
|
+
|
|
98
|
+
keys = await deriveKeys(chosen.password, salt)
|
|
99
|
+
enc = { salt, ...(chosen.hint ? { hint: chosen.hint } : {}) }
|
|
100
|
+
}
|
|
101
|
+
|
|
83
102
|
const id = newBackupId()
|
|
84
103
|
const key = streamKey(id)
|
|
85
104
|
|
|
@@ -210,6 +229,7 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
210
229
|
|
|
211
230
|
log(`Backup ${id}`)
|
|
212
231
|
log(`Name ${name} (chunks of ${formatBytes(chunkSize)})`)
|
|
232
|
+
if (enc) log(`Lock encrypted${enc.hint ? ` (hint: ${terminalSafe(enc.hint)})` : ''}`)
|
|
213
233
|
log(`From ${childArgv.join(' ')}`)
|
|
214
234
|
log(`To ${describeChat(chat)}\n`)
|
|
215
235
|
|
|
@@ -292,6 +312,12 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
292
312
|
progress.setLabel(`Chunk ${count + 1}`)
|
|
293
313
|
}
|
|
294
314
|
|
|
315
|
+
// A fresh iv per chunk; see newIv in src/cipher.js for why it is never derived
|
|
316
|
+
// from the index.
|
|
317
|
+
const iv = keys ? newIv() : null
|
|
318
|
+
const cipher = keys ? chunkCipher(keys.chunkKey, iv) : null
|
|
319
|
+
const plain = keys ? createHash('sha256') : null
|
|
320
|
+
|
|
295
321
|
// Offset 0 of a file holding exactly this chunk: uploadRange neither knows nor
|
|
296
322
|
// cares that the bytes arrived through a pipe rather than off a disk.
|
|
297
323
|
const { inputFile, sha256 } = await uploadRange(client, handle.fd, {
|
|
@@ -302,6 +328,12 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
302
328
|
partSize,
|
|
303
329
|
onProgress: (bytes) => progress.advance(bytes),
|
|
304
330
|
retryOptions: { ...retryOptions, onRetry },
|
|
331
|
+
transform: cipher
|
|
332
|
+
? (bytes, at) => {
|
|
333
|
+
plain.update(bytes)
|
|
334
|
+
return cipher.apply(bytes, at)
|
|
335
|
+
}
|
|
336
|
+
: undefined,
|
|
305
337
|
})
|
|
306
338
|
|
|
307
339
|
const message = await sendChunk(client, chat, {
|
|
@@ -323,7 +355,12 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
323
355
|
key,
|
|
324
356
|
state,
|
|
325
357
|
count,
|
|
326
|
-
{
|
|
358
|
+
{
|
|
359
|
+
msgId: message.id,
|
|
360
|
+
size: filled.bytes,
|
|
361
|
+
sha256,
|
|
362
|
+
...(cipher ? { iv, plainSha256: plain.digest('hex') } : {}),
|
|
363
|
+
},
|
|
327
364
|
configDir,
|
|
328
365
|
)
|
|
329
366
|
|
|
@@ -371,15 +408,24 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
371
408
|
throw new Error(`${childArgv[0]} wrote nothing, so there is no backup to make.`)
|
|
372
409
|
}
|
|
373
410
|
|
|
374
|
-
|
|
411
|
+
let manifest = buildManifest({
|
|
375
412
|
id,
|
|
376
413
|
name,
|
|
377
414
|
size,
|
|
378
415
|
chunkSize,
|
|
379
416
|
note,
|
|
417
|
+
enc,
|
|
380
418
|
chunks: Array.from({ length: count }, (_, i) => ({ i, ...state.done[String(i)] })),
|
|
381
419
|
})
|
|
382
420
|
|
|
421
|
+
if (keys) {
|
|
422
|
+
manifest = sealManifest(
|
|
423
|
+
manifest,
|
|
424
|
+
keys,
|
|
425
|
+
Array.from({ length: count }, (_, i) => state.done[String(i)].plainSha256),
|
|
426
|
+
)
|
|
427
|
+
}
|
|
428
|
+
|
|
383
429
|
const card = await sendManifest(client, chat, {
|
|
384
430
|
bytes: serializeManifest(manifest),
|
|
385
431
|
fileName: manifestFileName(id),
|
|
@@ -390,6 +436,8 @@ export async function runStreamUpload(name, childArgv, options = {}, deps = {})
|
|
|
390
436
|
chunks: manifest.chunks.length,
|
|
391
437
|
createdAt: manifest.createdAt,
|
|
392
438
|
note: manifest.note ?? null,
|
|
439
|
+
encrypted: Boolean(keys),
|
|
440
|
+
hint: manifest.enc?.hint ?? null,
|
|
393
441
|
}),
|
|
394
442
|
})
|
|
395
443
|
|