telstore 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -5
- package/bin/telstore.js +271 -4
- package/package.json +1 -1
- package/src/caption.js +70 -3
- package/src/cipher.js +211 -0
- package/src/cli.js +321 -13
- package/src/client.js +7 -1
- package/src/commands/delete.js +333 -41
- package/src/commands/down.js +311 -0
- package/src/commands/join.js +273 -0
- package/src/commands/list.js +12 -35
- package/src/commands/restore-stream.js +434 -0
- package/src/commands/restore.js +73 -41
- package/src/commands/status.js +196 -20
- package/src/commands/upload-stream.js +507 -0
- package/src/commands/upload.js +212 -29
- package/src/commands/verify.js +9 -6
- package/src/manifest.js +161 -5
- package/src/password.js +101 -0
- package/src/progress.js +86 -0
- package/src/shell.js +33 -0
- package/src/spawn.js +38 -0
- package/src/state.js +102 -2
- package/src/stream.js +295 -0
- package/src/tar.js +23 -0
- package/src/uploader.js +6 -1
package/src/commands/upload.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
1
2
|
import { promises as fs } from 'node:fs'
|
|
2
3
|
import path from 'node:path'
|
|
3
4
|
|
|
@@ -5,11 +6,20 @@ import { Api } from 'teleproto'
|
|
|
5
6
|
import { CustomFile } from 'teleproto/client/uploads.js'
|
|
6
7
|
|
|
7
8
|
import { PART_SIZE, planChunks } from '../chunking.js'
|
|
8
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
MAX_HINT_LENGTH,
|
|
11
|
+
chunkCaption,
|
|
12
|
+
hasControlCharacter,
|
|
13
|
+
manifestCaption,
|
|
14
|
+
parseNote,
|
|
15
|
+
terminalSafe,
|
|
16
|
+
} from '../caption.js'
|
|
17
|
+
import { chunkCipher, deriveKeys, newIv, newSalt, passwordCheck, sealManifest } from '../cipher.js'
|
|
9
18
|
import { describeChat } from '../chat.js'
|
|
10
19
|
import { closeQuietly, connect as realConnect } from '../client.js'
|
|
11
20
|
import { askConfirm } from '../confirm.js'
|
|
12
21
|
import { configFile, defaultConfigDir, loadConfig } from '../config.js'
|
|
22
|
+
import { askNewPassword as realAskNewPassword, askPassword as realAskPassword } from '../password.js'
|
|
13
23
|
import { expandSources } from '../sources.js'
|
|
14
24
|
import { assertLoggedIn } from '../session.js'
|
|
15
25
|
import { requireChat, resolveSettings } from '../settings.js'
|
|
@@ -24,6 +34,7 @@ import { createProgress, formatBytes, formatDuration } from '../progress.js'
|
|
|
24
34
|
import {
|
|
25
35
|
MAX_STATES,
|
|
26
36
|
clearState,
|
|
37
|
+
encryptedStateKey,
|
|
27
38
|
loadState,
|
|
28
39
|
markChunkDone,
|
|
29
40
|
pruneStates,
|
|
@@ -34,12 +45,40 @@ import {
|
|
|
34
45
|
import { uploadRange } from '../uploader.js'
|
|
35
46
|
|
|
36
47
|
// Above this threshold the wait must be spelled out, per spec §8.
|
|
37
|
-
const LONG_WAIT_MS = 60_000
|
|
48
|
+
export const LONG_WAIT_MS = 60_000
|
|
38
49
|
|
|
39
50
|
// A transient error that resolves itself on the next try is not news, and one line per
|
|
40
51
|
// occurrence buries the progress bar in a wall of text. Stay quiet until the third retry:
|
|
41
52
|
// by then the trouble has outlived two backoffs and is worth saying out loud.
|
|
42
|
-
const ANNOUNCE_AFTER_ATTEMPT = 3
|
|
53
|
+
export const ANNOUNCE_AFTER_ATTEMPT = 3
|
|
54
|
+
|
|
55
|
+
// Retries and FLOOD_WAIT must be announced: a silent FLOOD_WAIT_3600 leaves the user
|
|
56
|
+
// staring at a frozen progress bar for an hour, assuming the process has hung.
|
|
57
|
+
//
|
|
58
|
+
// Exported because a stream upload waits on the same Telegram and has to say the same
|
|
59
|
+
// things about it. Two copies of this wording would drift, and the one that drifted would
|
|
60
|
+
// be the one nobody was reading at the time.
|
|
61
|
+
export function createOnRetry(warn) {
|
|
62
|
+
return function onRetry(err, attempt, delayMs, elapsedMs = 0) {
|
|
63
|
+
if (delayMs > LONG_WAIT_MS) {
|
|
64
|
+
warn(
|
|
65
|
+
`\nTelegram wants ${formatDuration(delayMs / 1000)} of waiting before the next send ` +
|
|
66
|
+
`(${err.message}). telstore is waiting and will carry on by itself, leave it running.\n`,
|
|
67
|
+
)
|
|
68
|
+
return
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// The exception to staying quiet: an attempt that took a minute to fail spent that
|
|
72
|
+
// minute with the bar frozen, which is exactly what a hang looks like. Those are worth
|
|
73
|
+
// a line the first time, whatever the attempt number.
|
|
74
|
+
if (attempt < ANNOUNCE_AFTER_ATTEMPT && elapsedMs < LONG_WAIT_MS) return
|
|
75
|
+
|
|
76
|
+
warn(
|
|
77
|
+
`\nTemporary error (${err.message}), retry ${attempt} in ` +
|
|
78
|
+
`${formatDuration(delayMs / 1000)}.\n`,
|
|
79
|
+
)
|
|
80
|
+
}
|
|
81
|
+
}
|
|
43
82
|
|
|
44
83
|
// Chunks and manifests differ only in where the bytes come from. Everything Telegram is
|
|
45
84
|
// told about them — document, not preview; this exact file name — is decided once.
|
|
@@ -52,11 +91,15 @@ async function sendDocument(client, peer, { file, fileName, caption }) {
|
|
|
52
91
|
})
|
|
53
92
|
}
|
|
54
93
|
|
|
55
|
-
|
|
94
|
+
// The defaults behind runUpload's `sendChunk` and `sendManifest` deps. Exported because a
|
|
95
|
+
// stream upload sends the same two kinds of document to the same Telegram, and a second copy
|
|
96
|
+
// of "document, not preview; this exact file name" is how the two start disagreeing about
|
|
97
|
+
// what telstore actually put in the chat.
|
|
98
|
+
export async function realSendChunk(client, peer, { inputFile, fileName, caption }) {
|
|
56
99
|
return await sendDocument(client, peer, { file: inputFile, fileName, caption })
|
|
57
100
|
}
|
|
58
101
|
|
|
59
|
-
async function realSendManifest(client, peer, { bytes, fileName, caption }) {
|
|
102
|
+
export async function realSendManifest(client, peer, { bytes, fileName, caption }) {
|
|
60
103
|
return await sendDocument(client, peer, {
|
|
61
104
|
file: new CustomFile(fileName, bytes.length, '', bytes),
|
|
62
105
|
fileName,
|
|
@@ -107,6 +150,31 @@ async function statSource(absPath, { note = null, filesAfterNote = false } = {})
|
|
|
107
150
|
return stat
|
|
108
151
|
}
|
|
109
152
|
|
|
153
|
+
// The plaintext hashes the seal carries, read from the record that collected them one chunk at
|
|
154
|
+
// a time — possibly across several runs. A record that lost one cannot produce a manifest that
|
|
155
|
+
// decrypts, and sending one anyway would be a backup that restores to nothing.
|
|
156
|
+
function plainHashesOf(state, chunks, file) {
|
|
157
|
+
return chunks.map((chunk) => {
|
|
158
|
+
const entry = state.done[String(chunk.i)]
|
|
159
|
+
|
|
160
|
+
if (
|
|
161
|
+
typeof entry?.iv !== 'string' ||
|
|
162
|
+
!/^[0-9a-f]{16}$/.test(entry.iv) ||
|
|
163
|
+
typeof entry.plainSha256 !== 'string' ||
|
|
164
|
+
!/^[0-9a-f]{64}$/.test(entry.plainSha256)
|
|
165
|
+
) {
|
|
166
|
+
throw new Error(
|
|
167
|
+
`The record of this unfinished backup has no encryption details for chunk ${chunk.i + 1}, ` +
|
|
168
|
+
`so telstore cannot write a manifest that decrypts it. ${file} is damaged — delete it ` +
|
|
169
|
+
'and run again to start a new backup, which leaves the chunks already sent sitting in ' +
|
|
170
|
+
'the chat with nothing to point at them.',
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return entry.plainSha256
|
|
175
|
+
})
|
|
176
|
+
}
|
|
177
|
+
|
|
110
178
|
export async function runUpload(filePath, options = {}, deps = {}) {
|
|
111
179
|
const {
|
|
112
180
|
connect = realConnect,
|
|
@@ -123,6 +191,10 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
123
191
|
// Where on the command line the note sat, as `route` saw it. Nothing else can know, and
|
|
124
192
|
// a run that never says leaves the advice unsaid rather than guessed at.
|
|
125
193
|
filesAfterNote = false,
|
|
194
|
+
askNewPassword = realAskNewPassword,
|
|
195
|
+
askPassword = realAskPassword,
|
|
196
|
+
// A batch asks once and hands the answer to every file, rather than once per file.
|
|
197
|
+
secret = null,
|
|
126
198
|
} = deps
|
|
127
199
|
|
|
128
200
|
const absPath = path.resolve(filePath)
|
|
@@ -142,9 +214,57 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
142
214
|
const chat = requireChat(settings)
|
|
143
215
|
const concurrency = settings.uploadConcurrency
|
|
144
216
|
|
|
145
|
-
const
|
|
217
|
+
const encrypt = Boolean(options.encrypt)
|
|
218
|
+
|
|
219
|
+
// An encrypted record lives under a key of its own so an older telstore cannot find it (see
|
|
220
|
+
// encryptedStateKey). This run reads and writes its own key only, and looks under the other one
|
|
221
|
+
// just to refuse: a record there is this same file, unfinished the other way round.
|
|
222
|
+
const plainKey = stateKey(absPath, stat.size, stat.mtimeMs)
|
|
223
|
+
const encryptedKey = encryptedStateKey(absPath, stat.size, stat.mtimeMs)
|
|
224
|
+
const key = encrypt ? encryptedKey : plainKey
|
|
225
|
+
const otherKey = encrypt ? plainKey : encryptedKey
|
|
146
226
|
|
|
147
227
|
let state = await loadState(key, configDir)
|
|
228
|
+
const other = await loadState(otherKey, configDir)
|
|
229
|
+
|
|
230
|
+
// A record whose kind disagrees with the key it is filed under was not written by this build —
|
|
231
|
+
// a hand edit, or this branch before encrypted records moved. One carrying `enc` under the plain
|
|
232
|
+
// key is exactly what an older telstore would resume in plain, so neither kind is trusted to say
|
|
233
|
+
// which way it goes: both are refused as damaged, whichever way this run was asked to go.
|
|
234
|
+
const filed = [
|
|
235
|
+
{ filedUnder: plainKey, record: encrypt ? other : state, encrypted: false },
|
|
236
|
+
{ filedUnder: encryptedKey, record: encrypt ? state : other, encrypted: true },
|
|
237
|
+
]
|
|
238
|
+
|
|
239
|
+
for (const { filedUnder, record, encrypted } of filed) {
|
|
240
|
+
if (record && Boolean(record.enc) !== encrypted) {
|
|
241
|
+
throw new Error(
|
|
242
|
+
`The record of this unfinished backup says it is ${record.enc ? '' : 'not '}encrypted but is ` +
|
|
243
|
+
`filed where ${record.enc ? 'a plain' : 'an encrypted'} one belongs, so telstore cannot ` +
|
|
244
|
+
`tell how the chunks already sent went up. ${stateFile(filedUnder, configDir)} is damaged — ` +
|
|
245
|
+
'delete it and run again to start a new backup, which leaves the chunks already sent ' +
|
|
246
|
+
'sitting in the chat with nothing to point at them.',
|
|
247
|
+
)
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// An unfinished backup is encrypted or it is not, and the chunks already in the chat decide
|
|
252
|
+
// which. Carrying on the other way would mix plaintext and ciphertext in one backup no manifest
|
|
253
|
+
// could describe, so a run that disagrees is refused the way a disagreeing --chunk-size is.
|
|
254
|
+
if (other) {
|
|
255
|
+
const file = stateFile(otherKey, configDir)
|
|
256
|
+
|
|
257
|
+
throw new Error(
|
|
258
|
+
other.enc
|
|
259
|
+
? `This unfinished backup is encrypted, and this run has no --encrypt. Run again with ` +
|
|
260
|
+
`--encrypt to carry on, or delete ${file} and run again to start a new backup, which ` +
|
|
261
|
+
'leaves the chunks already sent sitting in the chat with nothing to point at them.'
|
|
262
|
+
: `This unfinished backup is not encrypted, and this run asks for --encrypt — the chunks ` +
|
|
263
|
+
`already in ${other.chat} went up as they are. Run again without --encrypt to carry ` +
|
|
264
|
+
`on, or delete ${file} and run again to start a new, encrypted backup, which leaves ` +
|
|
265
|
+
'the chunks already sent sitting in the chat with nothing to point at them.',
|
|
266
|
+
)
|
|
267
|
+
}
|
|
148
268
|
|
|
149
269
|
// The chunks already in the chat were cut at the size this backup started with, and
|
|
150
270
|
// nothing can re-cut them. Carrying on at a different size would abandon every one of
|
|
@@ -195,6 +315,59 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
195
315
|
)
|
|
196
316
|
}
|
|
197
317
|
|
|
318
|
+
let keys = null
|
|
319
|
+
let enc = null
|
|
320
|
+
|
|
321
|
+
if (encrypt && resuming) {
|
|
322
|
+
if (
|
|
323
|
+
typeof state.enc?.salt !== 'string' ||
|
|
324
|
+
!/^[0-9a-f]{32}$/.test(state.enc.salt) ||
|
|
325
|
+
typeof state.enc.check !== 'string' ||
|
|
326
|
+
// The record's hint goes into the manifest, and parseManifest refuses one that is not text,
|
|
327
|
+
// too long, or carrying a control character. Caught here it is a damaged record; caught at
|
|
328
|
+
// restore it would be a backup that uploaded cleanly and can never be restored.
|
|
329
|
+
(state.enc.hint !== undefined &&
|
|
330
|
+
(typeof state.enc.hint !== 'string' ||
|
|
331
|
+
state.enc.hint.length > MAX_HINT_LENGTH ||
|
|
332
|
+
hasControlCharacter(state.enc.hint)))
|
|
333
|
+
) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
'The record of this unfinished backup says it is encrypted but does not carry what is ' +
|
|
336
|
+
`needed to carry on encrypting it. ${stateFile(key, configDir)} is damaged — delete it ` +
|
|
337
|
+
'and run again to start a new backup, which leaves the chunks already sent sitting in ' +
|
|
338
|
+
'the chat with nothing to point at them.',
|
|
339
|
+
)
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// The password is asked for again rather than kept: nothing on this disk can decrypt the
|
|
343
|
+
// chunks already in the chat. The check refuses a different one, which would put two keys
|
|
344
|
+
// into a single backup.
|
|
345
|
+
const password =
|
|
346
|
+
secret?.password ??
|
|
347
|
+
(await askPassword(`Password for ${state.id}${state.enc.hint ? ` (hint: ${terminalSafe(state.enc.hint)})` : ''}: `))
|
|
348
|
+
|
|
349
|
+
keys = await deriveKeys(password, state.enc.salt)
|
|
350
|
+
|
|
351
|
+
if (passwordCheck(keys) !== state.enc.check) {
|
|
352
|
+
throw new Error(
|
|
353
|
+
`That is not the password backup ${state.id} was started with, and the chunks already in ` +
|
|
354
|
+
`${state.chat} are encrypted with that one. Run again and type the first password, or ` +
|
|
355
|
+
`delete ${stateFile(key, configDir)} and run again to start a new backup, which leaves ` +
|
|
356
|
+
'the chunks already sent sitting in the chat with nothing to point at them.',
|
|
357
|
+
)
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
enc = state.enc
|
|
361
|
+
} else if (encrypt) {
|
|
362
|
+
// Before the record and before the connection: a run that cannot get a password has written
|
|
363
|
+
// nothing and opened nothing.
|
|
364
|
+
const chosen = secret ?? (await askNewPassword())
|
|
365
|
+
const salt = newSalt()
|
|
366
|
+
|
|
367
|
+
keys = await deriveKeys(chosen.password, salt)
|
|
368
|
+
enc = { salt, check: passwordCheck(keys), ...(chosen.hint ? { hint: chosen.hint } : {}) }
|
|
369
|
+
}
|
|
370
|
+
|
|
198
371
|
if (!resuming) {
|
|
199
372
|
state = {
|
|
200
373
|
id: newBackupId(),
|
|
@@ -203,6 +376,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
203
376
|
size: stat.size,
|
|
204
377
|
mtimeMs: stat.mtimeMs,
|
|
205
378
|
chunkSize,
|
|
379
|
+
...(enc ? { enc } : {}),
|
|
206
380
|
done: {},
|
|
207
381
|
}
|
|
208
382
|
await saveState(key, state, configDir)
|
|
@@ -224,30 +398,11 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
224
398
|
const log = silent ? () => {} : writeLog
|
|
225
399
|
const warn = silent ? () => {} : writeErr
|
|
226
400
|
|
|
227
|
-
|
|
228
|
-
// staring at a frozen progress bar for an hour, assuming the process has hung.
|
|
229
|
-
function onRetry(err, attempt, delayMs, elapsedMs = 0) {
|
|
230
|
-
if (delayMs > LONG_WAIT_MS) {
|
|
231
|
-
warn(
|
|
232
|
-
`\nTelegram wants ${formatDuration(delayMs / 1000)} of waiting before the next send ` +
|
|
233
|
-
`(${err.message}). telstore is waiting and will carry on by itself, leave it running.\n`,
|
|
234
|
-
)
|
|
235
|
-
return
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
// The exception to staying quiet: an attempt that took a minute to fail spent that
|
|
239
|
-
// minute with the bar frozen, which is exactly what a hang looks like. Those are worth
|
|
240
|
-
// a line the first time, whatever the attempt number.
|
|
241
|
-
if (attempt < ANNOUNCE_AFTER_ATTEMPT && elapsedMs < LONG_WAIT_MS) return
|
|
242
|
-
|
|
243
|
-
warn(
|
|
244
|
-
`\nTemporary error (${err.message}), retry ${attempt} in ` +
|
|
245
|
-
`${formatDuration(delayMs / 1000)}.\n`,
|
|
246
|
-
)
|
|
247
|
-
}
|
|
401
|
+
const onRetry = createOnRetry(warn)
|
|
248
402
|
|
|
249
403
|
log(`Backup ${state.id}`)
|
|
250
404
|
log(`File ${absPath} (${formatBytes(stat.size)}, ${chunks.length} chunks)`)
|
|
405
|
+
if (enc) log(`Lock encrypted${enc.hint ? ` (hint: ${terminalSafe(enc.hint)})` : ''}`)
|
|
251
406
|
log(`To ${describeChat(chat)}\n`)
|
|
252
407
|
|
|
253
408
|
const client = await connect(config, { verbose: settings.verbose })
|
|
@@ -291,6 +446,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
291
446
|
const handle = await fs.open(absPath, 'r')
|
|
292
447
|
|
|
293
448
|
try {
|
|
449
|
+
// A fresh iv per attempt at this chunk; see newIv in src/cipher.js for why it is never
|
|
450
|
+
// derived from the index.
|
|
451
|
+
const iv = keys ? newIv() : null
|
|
452
|
+
const cipher = keys ? chunkCipher(keys.chunkKey, iv) : null
|
|
453
|
+
const plain = keys ? createHash('sha256') : null
|
|
454
|
+
|
|
294
455
|
const { inputFile, sha256 } = await uploadRange(client, handle.fd, {
|
|
295
456
|
offset: chunk.offset,
|
|
296
457
|
length: chunk.length,
|
|
@@ -299,6 +460,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
299
460
|
partSize,
|
|
300
461
|
onProgress: (bytes) => progress.advance(bytes),
|
|
301
462
|
retryOptions: { ...retryOptions, onRetry },
|
|
463
|
+
transform: cipher
|
|
464
|
+
? (bytes, at) => {
|
|
465
|
+
plain.update(bytes)
|
|
466
|
+
return cipher.apply(bytes, at)
|
|
467
|
+
}
|
|
468
|
+
: undefined,
|
|
302
469
|
})
|
|
303
470
|
|
|
304
471
|
const message = await sendChunk(client, chat, {
|
|
@@ -311,7 +478,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
311
478
|
key,
|
|
312
479
|
state,
|
|
313
480
|
chunk.i,
|
|
314
|
-
{
|
|
481
|
+
{
|
|
482
|
+
msgId: message.id,
|
|
483
|
+
size: chunk.length,
|
|
484
|
+
sha256,
|
|
485
|
+
...(cipher ? { iv, plainSha256: plain.digest('hex') } : {}),
|
|
486
|
+
},
|
|
315
487
|
configDir,
|
|
316
488
|
)
|
|
317
489
|
} finally {
|
|
@@ -339,15 +511,18 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
339
511
|
)
|
|
340
512
|
}
|
|
341
513
|
|
|
342
|
-
|
|
514
|
+
let manifest = buildManifest({
|
|
343
515
|
id: state.id,
|
|
344
516
|
name: path.basename(absPath),
|
|
345
517
|
size: stat.size,
|
|
346
518
|
chunkSize,
|
|
347
519
|
note,
|
|
520
|
+
enc: enc ? { salt: enc.salt, ...(enc.hint ? { hint: enc.hint } : {}) } : null,
|
|
348
521
|
chunks: chunks.map((chunk) => ({ i: chunk.i, ...state.done[String(chunk.i)] })),
|
|
349
522
|
})
|
|
350
523
|
|
|
524
|
+
if (keys) manifest = sealManifest(manifest, keys, plainHashesOf(state, chunks, stateFile(key, configDir)))
|
|
525
|
+
|
|
351
526
|
await sendManifest(client, chat, {
|
|
352
527
|
bytes: serializeManifest(manifest),
|
|
353
528
|
fileName: manifestFileName(state.id),
|
|
@@ -358,6 +533,8 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
358
533
|
chunks: manifest.chunks.length,
|
|
359
534
|
createdAt: manifest.createdAt,
|
|
360
535
|
note: manifest.note ?? null,
|
|
536
|
+
encrypted: Boolean(keys),
|
|
537
|
+
hint: manifest.enc?.hint ?? null,
|
|
361
538
|
}),
|
|
362
539
|
})
|
|
363
540
|
|
|
@@ -390,6 +567,7 @@ export async function runUploads(filePaths, options = {}, deps = {}) {
|
|
|
390
567
|
confirm = askConfirm,
|
|
391
568
|
interactive = () => Boolean(process.stdin.isTTY),
|
|
392
569
|
filesAfterNote = false,
|
|
570
|
+
askNewPassword = realAskNewPassword,
|
|
393
571
|
} = deps
|
|
394
572
|
|
|
395
573
|
const log = silent ? () => {} : writeLog
|
|
@@ -476,12 +654,17 @@ export async function runUploads(filePaths, options = {}, deps = {}) {
|
|
|
476
654
|
}
|
|
477
655
|
}
|
|
478
656
|
|
|
657
|
+
// Once, after the list has been confirmed: nobody should type a password for a batch they are
|
|
658
|
+
// about to cancel. Every file still gets a salt of its own inside runUpload.
|
|
659
|
+
const secret = options.encrypt ? await askNewPassword() : null
|
|
660
|
+
|
|
479
661
|
let shared = null
|
|
480
662
|
const perFile = {
|
|
481
663
|
...deps,
|
|
482
664
|
connect: async (theirConfig, connectOptions) =>
|
|
483
665
|
(shared ??= await connect(theirConfig, connectOptions)),
|
|
484
666
|
disconnect: async () => {},
|
|
667
|
+
secret,
|
|
485
668
|
}
|
|
486
669
|
|
|
487
670
|
const results = []
|
package/src/commands/verify.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { terminalSafe } from '../caption.js'
|
|
1
2
|
import { chatName, describeChat } from '../chat.js'
|
|
2
3
|
import {
|
|
3
4
|
MESSAGE_BATCH_SIZE,
|
|
@@ -10,8 +11,8 @@ import {
|
|
|
10
11
|
readMessageBytes as realReadMessageBytes,
|
|
11
12
|
} from '../client.js'
|
|
12
13
|
import { configFile, defaultConfigDir, loadConfig } from '../config.js'
|
|
13
|
-
import { chunkFileName, manifestFileName, parseManifest } from '../manifest.js'
|
|
14
|
-
import { formatBytes, formatDuration } from '../progress.js'
|
|
14
|
+
import { chunkFileName, isEncrypted, manifestFileName, parseManifest } from '../manifest.js'
|
|
15
|
+
import { formatBytes, formatDuration, plural } from '../progress.js'
|
|
15
16
|
import { assertLoggedIn } from '../session.js'
|
|
16
17
|
import { requireChat, resolveSettings } from '../settings.js'
|
|
17
18
|
|
|
@@ -21,10 +22,6 @@ function describeName(name) {
|
|
|
21
22
|
return typeof name === 'string' && name.trim() !== '' ? name : '—'
|
|
22
23
|
}
|
|
23
24
|
|
|
24
|
-
function plural(n, word) {
|
|
25
|
-
return `${n} ${word}${n === 1 ? '' : 's'}`
|
|
26
|
-
}
|
|
27
|
-
|
|
28
25
|
// What is wrong with one chunk, or null when nothing is. The first failing check wins: a
|
|
29
26
|
// chunk is damaged or it is not, and listing three complaints about one message would make
|
|
30
27
|
// "2 damaged" mean something other than two chunks.
|
|
@@ -135,6 +132,12 @@ export async function runVerify(backupId, options = {}, deps = {}) {
|
|
|
135
132
|
`(${formatBytes(manifest.size)}, ${plural(total, 'chunk')})`,
|
|
136
133
|
)
|
|
137
134
|
log(`In ${describeChat(chat)}`)
|
|
135
|
+
|
|
136
|
+
// Said, not checked: verify never has the password and never needs it. What it asks the chat
|
|
137
|
+
// about — presence, file name, length — is the same for ciphertext.
|
|
138
|
+
if (isEncrypted(manifest)) {
|
|
139
|
+
log(`Lock encrypted${manifest.enc.hint ? ` (hint: ${terminalSafe(manifest.enc.hint)})` : ''}`)
|
|
140
|
+
}
|
|
138
141
|
log('')
|
|
139
142
|
|
|
140
143
|
// A backup at the 10000-chunk ceiling is a hundred requests, sent one at a time. Silence
|
package/src/manifest.js
CHANGED
|
@@ -1,9 +1,24 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
|
+
import path from 'node:path'
|
|
2
3
|
|
|
4
|
+
import { MAX_HINT_LENGTH, hasControlCharacter } from './caption.js'
|
|
3
5
|
import { countChunks } from './chunking.js'
|
|
4
6
|
|
|
5
7
|
export const MANIFEST_VERSION = 1
|
|
6
8
|
|
|
9
|
+
// An encrypted backup's manifest, and only that. The bump is what stops an older telstore, which
|
|
10
|
+
// checks `v` and nothing else it does not know: handed a version 1 manifest with an extra `enc`
|
|
11
|
+
// in it, it would download the ciphertext, match every sha256 (they are the ciphertext's), match
|
|
12
|
+
// the length (CTR keeps it), rename, and print Done over a file of random bytes.
|
|
13
|
+
export const ENCRYPTED_MANIFEST_VERSION = 2
|
|
14
|
+
|
|
15
|
+
const SALT_HEX = /^[0-9a-f]{32}$/
|
|
16
|
+
const IV_HEX = /^[0-9a-f]{16}$/
|
|
17
|
+
|
|
18
|
+
export function isEncrypted(manifest) {
|
|
19
|
+
return manifest?.v === ENCRYPTED_MANIFEST_VERSION
|
|
20
|
+
}
|
|
21
|
+
|
|
7
22
|
export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).toString('hex')) {
|
|
8
23
|
const yyyy = now.getUTCFullYear()
|
|
9
24
|
const mm = String(now.getUTCMonth() + 1).padStart(2, '0')
|
|
@@ -11,8 +26,60 @@ export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).t
|
|
|
11
26
|
return `telstore-${yyyy}${mm}${dd}-${randomHex()}`
|
|
12
27
|
}
|
|
13
28
|
|
|
29
|
+
// The day a backup id carries, as the UTC second that day began. newBackupId stamps it above
|
|
30
|
+
// from the clock of the machine making the backup, so it is that machine's idea of the day
|
|
31
|
+
// rather than Telegram's — which is why the one reader of this (delete's walk of the chat)
|
|
32
|
+
// gives it a day of slack and only ever uses it as a floor.
|
|
33
|
+
//
|
|
34
|
+
// A date that does not exist is not a day: `telstore-20269999-abc` would otherwise roll over
|
|
35
|
+
// into a year's time and read as a floor above everything in the chat, which is an early stop
|
|
36
|
+
// nobody would see. Null instead, and the caller falls back to a bound it can prove.
|
|
37
|
+
const BACKUP_ID_DAY = /^telstore-(\d{4})(\d{2})(\d{2})-[0-9a-f]+$/
|
|
38
|
+
|
|
39
|
+
export function backupIdDay(id) {
|
|
40
|
+
const match = BACKUP_ID_DAY.exec(String(id))
|
|
41
|
+
|
|
42
|
+
if (!match) return null
|
|
43
|
+
|
|
44
|
+
const [year, month, day] = match.slice(1).map(Number)
|
|
45
|
+
const at = new Date(Date.UTC(year, month - 1, day))
|
|
46
|
+
|
|
47
|
+
if (at.getUTCFullYear() !== year || at.getUTCMonth() !== month - 1 || at.getUTCDate() !== day) {
|
|
48
|
+
return null
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return Math.floor(at.getTime() / 1000)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// The infix in every chunk's file name. A constant rather than a literal for the same reason
|
|
55
|
+
// MANIFEST_SUFFIX is one: there are two readers of that name now — the writer below and
|
|
56
|
+
// isChunkFileName — and a reader that disagrees with the writer by one character finds
|
|
57
|
+
// nothing at all.
|
|
58
|
+
const CHUNK_INFIX = '.part'
|
|
59
|
+
|
|
14
60
|
export function chunkFileName(id, i) {
|
|
15
|
-
return `${id}
|
|
61
|
+
return `${id}${CHUNK_INFIX}${String(i + 1).padStart(4, '0')}`
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Whether a document in a chat is a chunk of this backup, decided by the file name telstore
|
|
65
|
+
// itself wrote and not by the caption beside it — the rule findManifestMessage already keeps,
|
|
66
|
+
// for the same reason: a caption is text a person can edit and a file name is not.
|
|
67
|
+
//
|
|
68
|
+
// The number is checked but never read back. What the caller needs is which backup a document
|
|
69
|
+
// belongs to, and a chunk whose index says something impossible is still that backup's chunk.
|
|
70
|
+
// What the check is for is the other direction: without it `<id>.partial` or `<id>.part.bak`
|
|
71
|
+
// — names telstore never writes, but names a person can give a file they upload themselves —
|
|
72
|
+
// would be read as chunks of a backup and destroyed along with it.
|
|
73
|
+
export function isChunkFileName(id, fileName) {
|
|
74
|
+
if (typeof fileName !== 'string') return false
|
|
75
|
+
|
|
76
|
+
const prefix = `${id}${CHUNK_INFIX}`
|
|
77
|
+
|
|
78
|
+
if (!fileName.startsWith(prefix)) return false
|
|
79
|
+
|
|
80
|
+
const number = fileName.slice(prefix.length)
|
|
81
|
+
|
|
82
|
+
return number.length > 0 && /^[0-9]+$/.test(number)
|
|
16
83
|
}
|
|
17
84
|
|
|
18
85
|
// The suffix telstore has written on every manifest since version 1, and what `list` picks
|
|
@@ -24,6 +91,24 @@ export function manifestFileName(id) {
|
|
|
24
91
|
return `${id}${MANIFEST_SUFFIX}`
|
|
25
92
|
}
|
|
26
93
|
|
|
94
|
+
// manifest.name comes from data downloaded off Telegram — don't trust it when picking
|
|
95
|
+
// a path ourselves. path.basename stops "../../x" but still returns "..", "." or "" for
|
|
96
|
+
// a few pathological names: path.resolve('..') is the parent directory, so a multi-GB
|
|
97
|
+
// file would land outside the current directory and only blow up at rename. Here rather
|
|
98
|
+
// than in restore because join needs it too, and join must not import the network side.
|
|
99
|
+
export function safeOutName(name) {
|
|
100
|
+
const base = path.basename(String(name ?? ''))
|
|
101
|
+
|
|
102
|
+
if (base === '' || base === '.' || base === '..') {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`The name in the manifest ("${name}") cannot be used as a file name. ` +
|
|
105
|
+
'Run again with --out <path> to choose where to write.',
|
|
106
|
+
)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return base
|
|
110
|
+
}
|
|
111
|
+
|
|
27
112
|
export function buildManifest({
|
|
28
113
|
id,
|
|
29
114
|
name,
|
|
@@ -32,9 +117,10 @@ export function buildManifest({
|
|
|
32
117
|
chunks,
|
|
33
118
|
createdAt = new Date().toISOString(),
|
|
34
119
|
note = null,
|
|
120
|
+
enc = null,
|
|
35
121
|
}) {
|
|
36
122
|
return {
|
|
37
|
-
v: MANIFEST_VERSION,
|
|
123
|
+
v: enc ? ENCRYPTED_MANIFEST_VERSION : MANIFEST_VERSION,
|
|
38
124
|
id,
|
|
39
125
|
name,
|
|
40
126
|
size,
|
|
@@ -43,9 +129,19 @@ export function buildManifest({
|
|
|
43
129
|
// Absent rather than null when there is none: a manifest without a note has to be the
|
|
44
130
|
// same file telstore wrote before the flag existed, down to the bytes.
|
|
45
131
|
...(note ? { note } : {}),
|
|
132
|
+
// The same rule for encryption: a plain backup's manifest is today's manifest exactly.
|
|
133
|
+
...(enc ? { enc } : {}),
|
|
134
|
+
// Picked field by field, which is also what keeps a chunk's plaintext hash out: it travels
|
|
135
|
+
// in the state record and in the seal, never in the open.
|
|
46
136
|
chunks: [...chunks]
|
|
47
137
|
.sort((a, b) => a.i - b.i)
|
|
48
|
-
.map(({ i, msgId, size: chunkBytes, sha256 }) => ({
|
|
138
|
+
.map(({ i, msgId, size: chunkBytes, sha256, iv }) => ({
|
|
139
|
+
i,
|
|
140
|
+
msgId,
|
|
141
|
+
size: chunkBytes,
|
|
142
|
+
sha256,
|
|
143
|
+
...(enc ? { iv } : {}),
|
|
144
|
+
})),
|
|
49
145
|
}
|
|
50
146
|
}
|
|
51
147
|
|
|
@@ -96,12 +192,70 @@ export function manifestMessageIds(manifest) {
|
|
|
96
192
|
})
|
|
97
193
|
}
|
|
98
194
|
|
|
195
|
+
// Structure only. Whether the seal opens is a question for the password, which verify never
|
|
196
|
+
// has and never needs — so everything here is answerable from the file alone.
|
|
197
|
+
function checkEncryption(manifest) {
|
|
198
|
+
const { enc } = manifest
|
|
199
|
+
|
|
200
|
+
if (typeof enc !== 'object' || enc === null || Array.isArray(enc)) {
|
|
201
|
+
throw new Error('Manifest is version 2, which is encrypted, but carries no encryption details.')
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (typeof enc.salt !== 'string' || !SALT_HEX.test(enc.salt)) {
|
|
205
|
+
throw new Error(`Manifest records ${JSON.stringify(enc.salt)} as its salt, which is not 32 hex characters.`)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (enc.hint !== undefined && typeof enc.hint !== 'string') {
|
|
209
|
+
throw new Error(`Manifest records a hint of ${JSON.stringify(enc.hint)}, which is not text.`)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// The hint is printed above the password prompt before the seal can be checked, so it is the one
|
|
213
|
+
// field here a stranger's edit reaches a terminal through. telstore never writes one longer than
|
|
214
|
+
// the card allows or one carrying a control character, so either is an edit, and refused.
|
|
215
|
+
if (typeof enc.hint === 'string' && enc.hint.length > MAX_HINT_LENGTH) {
|
|
216
|
+
throw new Error(
|
|
217
|
+
`Manifest records a hint of ${enc.hint.length} characters, and telstore never writes one ` +
|
|
218
|
+
`longer than ${MAX_HINT_LENGTH}.`,
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (typeof enc.hint === 'string' && hasControlCharacter(enc.hint)) {
|
|
223
|
+
throw new Error(
|
|
224
|
+
'Manifest records a hint carrying a control character, which telstore never writes and ' +
|
|
225
|
+
'which would reach the terminal as an instruction rather than as text.',
|
|
226
|
+
)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (typeof enc.sealed !== 'string' || enc.sealed === '') {
|
|
230
|
+
throw new Error('Manifest is encrypted but carries no sealed part, so nothing can check what it decrypts to.')
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
manifest.chunks.forEach((chunk, index) => {
|
|
234
|
+
if (typeof chunk.iv !== 'string' || !IV_HEX.test(chunk.iv)) {
|
|
235
|
+
throw new Error(
|
|
236
|
+
`Manifest records ${JSON.stringify(chunk.iv)} as the iv of chunk ${index + 1}, which is not 16 hex characters.`,
|
|
237
|
+
)
|
|
238
|
+
}
|
|
239
|
+
})
|
|
240
|
+
}
|
|
241
|
+
|
|
99
242
|
export function parseManifest(input) {
|
|
100
243
|
const manifest = parseManifestJson(input)
|
|
101
244
|
|
|
102
|
-
if (manifest.v !== MANIFEST_VERSION) {
|
|
245
|
+
if (manifest.v !== MANIFEST_VERSION && manifest.v !== ENCRYPTED_MANIFEST_VERSION) {
|
|
103
246
|
throw new Error(
|
|
104
|
-
`Manifest uses version ${manifest.v}, this build of telstore only understands
|
|
247
|
+
`Manifest uses version ${manifest.v}, this build of telstore only understands versions ` +
|
|
248
|
+
`${MANIFEST_VERSION} and ${ENCRYPTED_MANIFEST_VERSION}.`,
|
|
249
|
+
)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Version 1 is never encrypted. A manifest claiming both would be restored as plain bytes by
|
|
253
|
+
// every telstore that reads version 1, which is the silent wrong file the bump exists to stop.
|
|
254
|
+
if (manifest.v === MANIFEST_VERSION && manifest.enc !== undefined) {
|
|
255
|
+
throw new Error(
|
|
256
|
+
'Manifest says version 1, which is never encrypted, and carries encryption fields anyway. ' +
|
|
257
|
+
'Restoring it as version 1 would hand over encrypted bytes as the file, so telstore is ' +
|
|
258
|
+
'not reading it.',
|
|
105
259
|
)
|
|
106
260
|
}
|
|
107
261
|
|
|
@@ -156,6 +310,8 @@ export function parseManifest(input) {
|
|
|
156
310
|
}
|
|
157
311
|
})
|
|
158
312
|
|
|
313
|
+
if (manifest.v === ENCRYPTED_MANIFEST_VERSION) checkEncryption(manifest)
|
|
314
|
+
|
|
159
315
|
const expectedChunks = countChunks(manifest.size, manifest.chunkSize)
|
|
160
316
|
if (manifest.chunks.length !== expectedChunks) {
|
|
161
317
|
throw new Error(`Manifest is missing ${expectedChunks - manifest.chunks.length} chunk(s).`)
|