telstore 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -3
- package/bin/telstore.js +18 -4
- package/package.json +1 -1
- package/src/caption.js +64 -2
- package/src/cipher.js +211 -0
- package/src/cli.js +56 -3
- package/src/commands/join.js +273 -0
- package/src/commands/list.js +11 -4
- package/src/commands/restore-stream.js +28 -1
- package/src/commands/restore.js +46 -21
- package/src/commands/status.js +6 -1
- package/src/commands/upload-stream.js +51 -3
- package/src/commands/upload.js +175 -4
- package/src/commands/verify.js +8 -1
- package/src/manifest.js +108 -4
- package/src/password.js +101 -0
- package/src/state.js +24 -2
- package/src/uploader.js +6 -1
package/src/commands/upload.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
1
2
|
import { promises as fs } from 'node:fs'
|
|
2
3
|
import path from 'node:path'
|
|
3
4
|
|
|
@@ -5,11 +6,20 @@ import { Api } from 'teleproto'
|
|
|
5
6
|
import { CustomFile } from 'teleproto/client/uploads.js'
|
|
6
7
|
|
|
7
8
|
import { PART_SIZE, planChunks } from '../chunking.js'
|
|
8
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
MAX_HINT_LENGTH,
|
|
11
|
+
chunkCaption,
|
|
12
|
+
hasControlCharacter,
|
|
13
|
+
manifestCaption,
|
|
14
|
+
parseNote,
|
|
15
|
+
terminalSafe,
|
|
16
|
+
} from '../caption.js'
|
|
17
|
+
import { chunkCipher, deriveKeys, newIv, newSalt, passwordCheck, sealManifest } from '../cipher.js'
|
|
9
18
|
import { describeChat } from '../chat.js'
|
|
10
19
|
import { closeQuietly, connect as realConnect } from '../client.js'
|
|
11
20
|
import { askConfirm } from '../confirm.js'
|
|
12
21
|
import { configFile, defaultConfigDir, loadConfig } from '../config.js'
|
|
22
|
+
import { askNewPassword as realAskNewPassword, askPassword as realAskPassword } from '../password.js'
|
|
13
23
|
import { expandSources } from '../sources.js'
|
|
14
24
|
import { assertLoggedIn } from '../session.js'
|
|
15
25
|
import { requireChat, resolveSettings } from '../settings.js'
|
|
@@ -24,6 +34,7 @@ import { createProgress, formatBytes, formatDuration } from '../progress.js'
|
|
|
24
34
|
import {
|
|
25
35
|
MAX_STATES,
|
|
26
36
|
clearState,
|
|
37
|
+
encryptedStateKey,
|
|
27
38
|
loadState,
|
|
28
39
|
markChunkDone,
|
|
29
40
|
pruneStates,
|
|
@@ -139,6 +150,31 @@ async function statSource(absPath, { note = null, filesAfterNote = false } = {})
|
|
|
139
150
|
return stat
|
|
140
151
|
}
|
|
141
152
|
|
|
153
|
+
// The plaintext hashes the seal carries, read from the record that collected them one chunk at
|
|
154
|
+
// a time — possibly across several runs. A record that lost one cannot produce a manifest that
|
|
155
|
+
// decrypts, and sending one anyway would be a backup that restores to nothing.
|
|
156
|
+
function plainHashesOf(state, chunks, file) {
|
|
157
|
+
return chunks.map((chunk) => {
|
|
158
|
+
const entry = state.done[String(chunk.i)]
|
|
159
|
+
|
|
160
|
+
if (
|
|
161
|
+
typeof entry?.iv !== 'string' ||
|
|
162
|
+
!/^[0-9a-f]{16}$/.test(entry.iv) ||
|
|
163
|
+
typeof entry.plainSha256 !== 'string' ||
|
|
164
|
+
!/^[0-9a-f]{64}$/.test(entry.plainSha256)
|
|
165
|
+
) {
|
|
166
|
+
throw new Error(
|
|
167
|
+
`The record of this unfinished backup has no encryption details for chunk ${chunk.i + 1}, ` +
|
|
168
|
+
`so telstore cannot write a manifest that decrypts it. ${file} is damaged — delete it ` +
|
|
169
|
+
'and run again to start a new backup, which leaves the chunks already sent sitting in ' +
|
|
170
|
+
'the chat with nothing to point at them.',
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return entry.plainSha256
|
|
175
|
+
})
|
|
176
|
+
}
|
|
177
|
+
|
|
142
178
|
export async function runUpload(filePath, options = {}, deps = {}) {
|
|
143
179
|
const {
|
|
144
180
|
connect = realConnect,
|
|
@@ -155,6 +191,10 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
155
191
|
// Where on the command line the note sat, as `route` saw it. Nothing else can know, and
|
|
156
192
|
// a run that never says leaves the advice unsaid rather than guessed at.
|
|
157
193
|
filesAfterNote = false,
|
|
194
|
+
askNewPassword = realAskNewPassword,
|
|
195
|
+
askPassword = realAskPassword,
|
|
196
|
+
// A batch asks once and hands the answer to every file, rather than once per file.
|
|
197
|
+
secret = null,
|
|
158
198
|
} = deps
|
|
159
199
|
|
|
160
200
|
const absPath = path.resolve(filePath)
|
|
@@ -174,9 +214,57 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
174
214
|
const chat = requireChat(settings)
|
|
175
215
|
const concurrency = settings.uploadConcurrency
|
|
176
216
|
|
|
177
|
-
const
|
|
217
|
+
const encrypt = Boolean(options.encrypt)
|
|
218
|
+
|
|
219
|
+
// An encrypted record lives under a key of its own so an older telstore cannot find it (see
|
|
220
|
+
// encryptedStateKey). This run reads and writes its own key only, and looks under the other one
|
|
221
|
+
// just to refuse: a record there is this same file, unfinished the other way round.
|
|
222
|
+
const plainKey = stateKey(absPath, stat.size, stat.mtimeMs)
|
|
223
|
+
const encryptedKey = encryptedStateKey(absPath, stat.size, stat.mtimeMs)
|
|
224
|
+
const key = encrypt ? encryptedKey : plainKey
|
|
225
|
+
const otherKey = encrypt ? plainKey : encryptedKey
|
|
178
226
|
|
|
179
227
|
let state = await loadState(key, configDir)
|
|
228
|
+
const other = await loadState(otherKey, configDir)
|
|
229
|
+
|
|
230
|
+
// A record whose kind disagrees with the key it is filed under was not written by this build —
|
|
231
|
+
// a hand edit, or this branch before encrypted records moved. One carrying `enc` under the plain
|
|
232
|
+
// key is exactly what an older telstore would resume in plain, so neither kind is trusted to say
|
|
233
|
+
// which way it goes: both are refused as damaged, whichever way this run was asked to go.
|
|
234
|
+
const filed = [
|
|
235
|
+
{ filedUnder: plainKey, record: encrypt ? other : state, encrypted: false },
|
|
236
|
+
{ filedUnder: encryptedKey, record: encrypt ? state : other, encrypted: true },
|
|
237
|
+
]
|
|
238
|
+
|
|
239
|
+
for (const { filedUnder, record, encrypted } of filed) {
|
|
240
|
+
if (record && Boolean(record.enc) !== encrypted) {
|
|
241
|
+
throw new Error(
|
|
242
|
+
`The record of this unfinished backup says it is ${record.enc ? '' : 'not '}encrypted but is ` +
|
|
243
|
+
`filed where ${record.enc ? 'a plain' : 'an encrypted'} one belongs, so telstore cannot ` +
|
|
244
|
+
`tell how the chunks already sent went up. ${stateFile(filedUnder, configDir)} is damaged — ` +
|
|
245
|
+
'delete it and run again to start a new backup, which leaves the chunks already sent ' +
|
|
246
|
+
'sitting in the chat with nothing to point at them.',
|
|
247
|
+
)
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// An unfinished backup is encrypted or it is not, and the chunks already in the chat decide
|
|
252
|
+
// which. Carrying on the other way would mix plaintext and ciphertext in one backup no manifest
|
|
253
|
+
// could describe, so a run that disagrees is refused the way a disagreeing --chunk-size is.
|
|
254
|
+
if (other) {
|
|
255
|
+
const file = stateFile(otherKey, configDir)
|
|
256
|
+
|
|
257
|
+
throw new Error(
|
|
258
|
+
other.enc
|
|
259
|
+
? `This unfinished backup is encrypted, and this run has no --encrypt. Run again with ` +
|
|
260
|
+
`--encrypt to carry on, or delete ${file} and run again to start a new backup, which ` +
|
|
261
|
+
'leaves the chunks already sent sitting in the chat with nothing to point at them.'
|
|
262
|
+
: `This unfinished backup is not encrypted, and this run asks for --encrypt — the chunks ` +
|
|
263
|
+
`already in ${other.chat} went up as they are. Run again without --encrypt to carry ` +
|
|
264
|
+
`on, or delete ${file} and run again to start a new, encrypted backup, which leaves ` +
|
|
265
|
+
'the chunks already sent sitting in the chat with nothing to point at them.',
|
|
266
|
+
)
|
|
267
|
+
}
|
|
180
268
|
|
|
181
269
|
// The chunks already in the chat were cut at the size this backup started with, and
|
|
182
270
|
// nothing can re-cut them. Carrying on at a different size would abandon every one of
|
|
@@ -227,6 +315,59 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
227
315
|
)
|
|
228
316
|
}
|
|
229
317
|
|
|
318
|
+
let keys = null
|
|
319
|
+
let enc = null
|
|
320
|
+
|
|
321
|
+
if (encrypt && resuming) {
|
|
322
|
+
if (
|
|
323
|
+
typeof state.enc?.salt !== 'string' ||
|
|
324
|
+
!/^[0-9a-f]{32}$/.test(state.enc.salt) ||
|
|
325
|
+
typeof state.enc.check !== 'string' ||
|
|
326
|
+
// The record's hint goes into the manifest, and parseManifest refuses one that is not text,
|
|
327
|
+
// too long, or carrying a control character. Caught here it is a damaged record; caught at
|
|
328
|
+
// restore it would be a backup that uploaded cleanly and can never be restored.
|
|
329
|
+
(state.enc.hint !== undefined &&
|
|
330
|
+
(typeof state.enc.hint !== 'string' ||
|
|
331
|
+
state.enc.hint.length > MAX_HINT_LENGTH ||
|
|
332
|
+
hasControlCharacter(state.enc.hint)))
|
|
333
|
+
) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
'The record of this unfinished backup says it is encrypted but does not carry what is ' +
|
|
336
|
+
`needed to carry on encrypting it. ${stateFile(key, configDir)} is damaged — delete it ` +
|
|
337
|
+
'and run again to start a new backup, which leaves the chunks already sent sitting in ' +
|
|
338
|
+
'the chat with nothing to point at them.',
|
|
339
|
+
)
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// The password is asked for again rather than kept: nothing on this disk can decrypt the
|
|
343
|
+
// chunks already in the chat. The check refuses a different one, which would put two keys
|
|
344
|
+
// into a single backup.
|
|
345
|
+
const password =
|
|
346
|
+
secret?.password ??
|
|
347
|
+
(await askPassword(`Password for ${state.id}${state.enc.hint ? ` (hint: ${terminalSafe(state.enc.hint)})` : ''}: `))
|
|
348
|
+
|
|
349
|
+
keys = await deriveKeys(password, state.enc.salt)
|
|
350
|
+
|
|
351
|
+
if (passwordCheck(keys) !== state.enc.check) {
|
|
352
|
+
throw new Error(
|
|
353
|
+
`That is not the password backup ${state.id} was started with, and the chunks already in ` +
|
|
354
|
+
`${state.chat} are encrypted with that one. Run again and type the first password, or ` +
|
|
355
|
+
`delete ${stateFile(key, configDir)} and run again to start a new backup, which leaves ` +
|
|
356
|
+
'the chunks already sent sitting in the chat with nothing to point at them.',
|
|
357
|
+
)
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
enc = state.enc
|
|
361
|
+
} else if (encrypt) {
|
|
362
|
+
// Before the record and before the connection: a run that cannot get a password has written
|
|
363
|
+
// nothing and opened nothing.
|
|
364
|
+
const chosen = secret ?? (await askNewPassword())
|
|
365
|
+
const salt = newSalt()
|
|
366
|
+
|
|
367
|
+
keys = await deriveKeys(chosen.password, salt)
|
|
368
|
+
enc = { salt, check: passwordCheck(keys), ...(chosen.hint ? { hint: chosen.hint } : {}) }
|
|
369
|
+
}
|
|
370
|
+
|
|
230
371
|
if (!resuming) {
|
|
231
372
|
state = {
|
|
232
373
|
id: newBackupId(),
|
|
@@ -235,6 +376,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
235
376
|
size: stat.size,
|
|
236
377
|
mtimeMs: stat.mtimeMs,
|
|
237
378
|
chunkSize,
|
|
379
|
+
...(enc ? { enc } : {}),
|
|
238
380
|
done: {},
|
|
239
381
|
}
|
|
240
382
|
await saveState(key, state, configDir)
|
|
@@ -260,6 +402,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
260
402
|
|
|
261
403
|
log(`Backup ${state.id}`)
|
|
262
404
|
log(`File ${absPath} (${formatBytes(stat.size)}, ${chunks.length} chunks)`)
|
|
405
|
+
if (enc) log(`Lock encrypted${enc.hint ? ` (hint: ${terminalSafe(enc.hint)})` : ''}`)
|
|
263
406
|
log(`To ${describeChat(chat)}\n`)
|
|
264
407
|
|
|
265
408
|
const client = await connect(config, { verbose: settings.verbose })
|
|
@@ -303,6 +446,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
303
446
|
const handle = await fs.open(absPath, 'r')
|
|
304
447
|
|
|
305
448
|
try {
|
|
449
|
+
// A fresh iv per attempt at this chunk; see newIv in src/cipher.js for why it is never
|
|
450
|
+
// derived from the index.
|
|
451
|
+
const iv = keys ? newIv() : null
|
|
452
|
+
const cipher = keys ? chunkCipher(keys.chunkKey, iv) : null
|
|
453
|
+
const plain = keys ? createHash('sha256') : null
|
|
454
|
+
|
|
306
455
|
const { inputFile, sha256 } = await uploadRange(client, handle.fd, {
|
|
307
456
|
offset: chunk.offset,
|
|
308
457
|
length: chunk.length,
|
|
@@ -311,6 +460,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
311
460
|
partSize,
|
|
312
461
|
onProgress: (bytes) => progress.advance(bytes),
|
|
313
462
|
retryOptions: { ...retryOptions, onRetry },
|
|
463
|
+
transform: cipher
|
|
464
|
+
? (bytes, at) => {
|
|
465
|
+
plain.update(bytes)
|
|
466
|
+
return cipher.apply(bytes, at)
|
|
467
|
+
}
|
|
468
|
+
: undefined,
|
|
314
469
|
})
|
|
315
470
|
|
|
316
471
|
const message = await sendChunk(client, chat, {
|
|
@@ -323,7 +478,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
323
478
|
key,
|
|
324
479
|
state,
|
|
325
480
|
chunk.i,
|
|
326
|
-
{
|
|
481
|
+
{
|
|
482
|
+
msgId: message.id,
|
|
483
|
+
size: chunk.length,
|
|
484
|
+
sha256,
|
|
485
|
+
...(cipher ? { iv, plainSha256: plain.digest('hex') } : {}),
|
|
486
|
+
},
|
|
327
487
|
configDir,
|
|
328
488
|
)
|
|
329
489
|
} finally {
|
|
@@ -351,15 +511,18 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
351
511
|
)
|
|
352
512
|
}
|
|
353
513
|
|
|
354
|
-
|
|
514
|
+
let manifest = buildManifest({
|
|
355
515
|
id: state.id,
|
|
356
516
|
name: path.basename(absPath),
|
|
357
517
|
size: stat.size,
|
|
358
518
|
chunkSize,
|
|
359
519
|
note,
|
|
520
|
+
enc: enc ? { salt: enc.salt, ...(enc.hint ? { hint: enc.hint } : {}) } : null,
|
|
360
521
|
chunks: chunks.map((chunk) => ({ i: chunk.i, ...state.done[String(chunk.i)] })),
|
|
361
522
|
})
|
|
362
523
|
|
|
524
|
+
if (keys) manifest = sealManifest(manifest, keys, plainHashesOf(state, chunks, stateFile(key, configDir)))
|
|
525
|
+
|
|
363
526
|
await sendManifest(client, chat, {
|
|
364
527
|
bytes: serializeManifest(manifest),
|
|
365
528
|
fileName: manifestFileName(state.id),
|
|
@@ -370,6 +533,8 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
370
533
|
chunks: manifest.chunks.length,
|
|
371
534
|
createdAt: manifest.createdAt,
|
|
372
535
|
note: manifest.note ?? null,
|
|
536
|
+
encrypted: Boolean(keys),
|
|
537
|
+
hint: manifest.enc?.hint ?? null,
|
|
373
538
|
}),
|
|
374
539
|
})
|
|
375
540
|
|
|
@@ -402,6 +567,7 @@ export async function runUploads(filePaths, options = {}, deps = {}) {
|
|
|
402
567
|
confirm = askConfirm,
|
|
403
568
|
interactive = () => Boolean(process.stdin.isTTY),
|
|
404
569
|
filesAfterNote = false,
|
|
570
|
+
askNewPassword = realAskNewPassword,
|
|
405
571
|
} = deps
|
|
406
572
|
|
|
407
573
|
const log = silent ? () => {} : writeLog
|
|
@@ -488,12 +654,17 @@ export async function runUploads(filePaths, options = {}, deps = {}) {
|
|
|
488
654
|
}
|
|
489
655
|
}
|
|
490
656
|
|
|
657
|
+
// Once, after the list has been confirmed: nobody should type a password for a batch they are
|
|
658
|
+
// about to cancel. Every file still gets a salt of its own inside runUpload.
|
|
659
|
+
const secret = options.encrypt ? await askNewPassword() : null
|
|
660
|
+
|
|
491
661
|
let shared = null
|
|
492
662
|
const perFile = {
|
|
493
663
|
...deps,
|
|
494
664
|
connect: async (theirConfig, connectOptions) =>
|
|
495
665
|
(shared ??= await connect(theirConfig, connectOptions)),
|
|
496
666
|
disconnect: async () => {},
|
|
667
|
+
secret,
|
|
497
668
|
}
|
|
498
669
|
|
|
499
670
|
const results = []
|
package/src/commands/verify.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { terminalSafe } from '../caption.js'
|
|
1
2
|
import { chatName, describeChat } from '../chat.js'
|
|
2
3
|
import {
|
|
3
4
|
MESSAGE_BATCH_SIZE,
|
|
@@ -10,7 +11,7 @@ import {
|
|
|
10
11
|
readMessageBytes as realReadMessageBytes,
|
|
11
12
|
} from '../client.js'
|
|
12
13
|
import { configFile, defaultConfigDir, loadConfig } from '../config.js'
|
|
13
|
-
import { chunkFileName, manifestFileName, parseManifest } from '../manifest.js'
|
|
14
|
+
import { chunkFileName, isEncrypted, manifestFileName, parseManifest } from '../manifest.js'
|
|
14
15
|
import { formatBytes, formatDuration, plural } from '../progress.js'
|
|
15
16
|
import { assertLoggedIn } from '../session.js'
|
|
16
17
|
import { requireChat, resolveSettings } from '../settings.js'
|
|
@@ -131,6 +132,12 @@ export async function runVerify(backupId, options = {}, deps = {}) {
|
|
|
131
132
|
`(${formatBytes(manifest.size)}, ${plural(total, 'chunk')})`,
|
|
132
133
|
)
|
|
133
134
|
log(`In ${describeChat(chat)}`)
|
|
135
|
+
|
|
136
|
+
// Said, not checked: verify never has the password and never needs it. What it asks the chat
|
|
137
|
+
// about — presence, file name, length — is the same for ciphertext.
|
|
138
|
+
if (isEncrypted(manifest)) {
|
|
139
|
+
log(`Lock encrypted${manifest.enc.hint ? ` (hint: ${terminalSafe(manifest.enc.hint)})` : ''}`)
|
|
140
|
+
}
|
|
134
141
|
log('')
|
|
135
142
|
|
|
136
143
|
// A backup at the 10000-chunk ceiling is a hundred requests, sent one at a time. Silence
|
package/src/manifest.js
CHANGED
|
@@ -1,9 +1,24 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
|
+
import path from 'node:path'
|
|
2
3
|
|
|
4
|
+
import { MAX_HINT_LENGTH, hasControlCharacter } from './caption.js'
|
|
3
5
|
import { countChunks } from './chunking.js'
|
|
4
6
|
|
|
5
7
|
export const MANIFEST_VERSION = 1
|
|
6
8
|
|
|
9
|
+
// An encrypted backup's manifest, and only that. The bump is what stops an older telstore, which
|
|
10
|
+
// checks `v` and nothing else it does not know: handed a version 1 manifest with an extra `enc`
|
|
11
|
+
// in it, it would download the ciphertext, match every sha256 (they are the ciphertext's), match
|
|
12
|
+
// the length (CTR keeps it), rename, and print Done over a file of random bytes.
|
|
13
|
+
export const ENCRYPTED_MANIFEST_VERSION = 2
|
|
14
|
+
|
|
15
|
+
const SALT_HEX = /^[0-9a-f]{32}$/
|
|
16
|
+
const IV_HEX = /^[0-9a-f]{16}$/
|
|
17
|
+
|
|
18
|
+
export function isEncrypted(manifest) {
|
|
19
|
+
return manifest?.v === ENCRYPTED_MANIFEST_VERSION
|
|
20
|
+
}
|
|
21
|
+
|
|
7
22
|
export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).toString('hex')) {
|
|
8
23
|
const yyyy = now.getUTCFullYear()
|
|
9
24
|
const mm = String(now.getUTCMonth() + 1).padStart(2, '0')
|
|
@@ -76,6 +91,24 @@ export function manifestFileName(id) {
|
|
|
76
91
|
return `${id}${MANIFEST_SUFFIX}`
|
|
77
92
|
}
|
|
78
93
|
|
|
94
|
+
// manifest.name comes from data downloaded off Telegram — don't trust it when picking
|
|
95
|
+
// a path ourselves. path.basename stops "../../x" but still returns "..", "." or "" for
|
|
96
|
+
// a few pathological names: path.resolve('..') is the parent directory, so a multi-GB
|
|
97
|
+
// file would land outside the current directory and only blow up at rename. Here rather
|
|
98
|
+
// than in restore because join needs it too, and join must not import the network side.
|
|
99
|
+
export function safeOutName(name) {
|
|
100
|
+
const base = path.basename(String(name ?? ''))
|
|
101
|
+
|
|
102
|
+
if (base === '' || base === '.' || base === '..') {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`The name in the manifest ("${name}") cannot be used as a file name. ` +
|
|
105
|
+
'Run again with --out <path> to choose where to write.',
|
|
106
|
+
)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return base
|
|
110
|
+
}
|
|
111
|
+
|
|
79
112
|
export function buildManifest({
|
|
80
113
|
id,
|
|
81
114
|
name,
|
|
@@ -84,9 +117,10 @@ export function buildManifest({
|
|
|
84
117
|
chunks,
|
|
85
118
|
createdAt = new Date().toISOString(),
|
|
86
119
|
note = null,
|
|
120
|
+
enc = null,
|
|
87
121
|
}) {
|
|
88
122
|
return {
|
|
89
|
-
v: MANIFEST_VERSION,
|
|
123
|
+
v: enc ? ENCRYPTED_MANIFEST_VERSION : MANIFEST_VERSION,
|
|
90
124
|
id,
|
|
91
125
|
name,
|
|
92
126
|
size,
|
|
@@ -95,9 +129,19 @@ export function buildManifest({
|
|
|
95
129
|
// Absent rather than null when there is none: a manifest without a note has to be the
|
|
96
130
|
// same file telstore wrote before the flag existed, down to the bytes.
|
|
97
131
|
...(note ? { note } : {}),
|
|
132
|
+
// The same rule for encryption: a plain backup's manifest is today's manifest exactly.
|
|
133
|
+
...(enc ? { enc } : {}),
|
|
134
|
+
// Picked field by field, which is also what keeps a chunk's plaintext hash out: it travels
|
|
135
|
+
// in the state record and in the seal, never in the open.
|
|
98
136
|
chunks: [...chunks]
|
|
99
137
|
.sort((a, b) => a.i - b.i)
|
|
100
|
-
.map(({ i, msgId, size: chunkBytes, sha256 }) => ({
|
|
138
|
+
.map(({ i, msgId, size: chunkBytes, sha256, iv }) => ({
|
|
139
|
+
i,
|
|
140
|
+
msgId,
|
|
141
|
+
size: chunkBytes,
|
|
142
|
+
sha256,
|
|
143
|
+
...(enc ? { iv } : {}),
|
|
144
|
+
})),
|
|
101
145
|
}
|
|
102
146
|
}
|
|
103
147
|
|
|
@@ -148,12 +192,70 @@ export function manifestMessageIds(manifest) {
|
|
|
148
192
|
})
|
|
149
193
|
}
|
|
150
194
|
|
|
195
|
+
// Structure only. Whether the seal opens is a question for the password, which verify never
|
|
196
|
+
// has and never needs — so everything here is answerable from the file alone.
|
|
197
|
+
function checkEncryption(manifest) {
|
|
198
|
+
const { enc } = manifest
|
|
199
|
+
|
|
200
|
+
if (typeof enc !== 'object' || enc === null || Array.isArray(enc)) {
|
|
201
|
+
throw new Error('Manifest is version 2, which is encrypted, but carries no encryption details.')
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (typeof enc.salt !== 'string' || !SALT_HEX.test(enc.salt)) {
|
|
205
|
+
throw new Error(`Manifest records ${JSON.stringify(enc.salt)} as its salt, which is not 32 hex characters.`)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (enc.hint !== undefined && typeof enc.hint !== 'string') {
|
|
209
|
+
throw new Error(`Manifest records a hint of ${JSON.stringify(enc.hint)}, which is not text.`)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// The hint is printed above the password prompt before the seal can be checked, so it is the one
|
|
213
|
+
// field here a stranger's edit reaches a terminal through. telstore never writes one longer than
|
|
214
|
+
// the card allows or one carrying a control character, so either is an edit, and refused.
|
|
215
|
+
if (typeof enc.hint === 'string' && enc.hint.length > MAX_HINT_LENGTH) {
|
|
216
|
+
throw new Error(
|
|
217
|
+
`Manifest records a hint of ${enc.hint.length} characters, and telstore never writes one ` +
|
|
218
|
+
`longer than ${MAX_HINT_LENGTH}.`,
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (typeof enc.hint === 'string' && hasControlCharacter(enc.hint)) {
|
|
223
|
+
throw new Error(
|
|
224
|
+
'Manifest records a hint carrying a control character, which telstore never writes and ' +
|
|
225
|
+
'which would reach the terminal as an instruction rather than as text.',
|
|
226
|
+
)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (typeof enc.sealed !== 'string' || enc.sealed === '') {
|
|
230
|
+
throw new Error('Manifest is encrypted but carries no sealed part, so nothing can check what it decrypts to.')
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
manifest.chunks.forEach((chunk, index) => {
|
|
234
|
+
if (typeof chunk.iv !== 'string' || !IV_HEX.test(chunk.iv)) {
|
|
235
|
+
throw new Error(
|
|
236
|
+
`Manifest records ${JSON.stringify(chunk.iv)} as the iv of chunk ${index + 1}, which is not 16 hex characters.`,
|
|
237
|
+
)
|
|
238
|
+
}
|
|
239
|
+
})
|
|
240
|
+
}
|
|
241
|
+
|
|
151
242
|
export function parseManifest(input) {
|
|
152
243
|
const manifest = parseManifestJson(input)
|
|
153
244
|
|
|
154
|
-
if (manifest.v !== MANIFEST_VERSION) {
|
|
245
|
+
if (manifest.v !== MANIFEST_VERSION && manifest.v !== ENCRYPTED_MANIFEST_VERSION) {
|
|
155
246
|
throw new Error(
|
|
156
|
-
`Manifest uses version ${manifest.v}, this build of telstore only understands
|
|
247
|
+
`Manifest uses version ${manifest.v}, this build of telstore only understands versions ` +
|
|
248
|
+
`${MANIFEST_VERSION} and ${ENCRYPTED_MANIFEST_VERSION}.`,
|
|
249
|
+
)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Version 1 is never encrypted. A manifest claiming both would be restored as plain bytes by
|
|
253
|
+
// every telstore that reads version 1, which is the silent wrong file the bump exists to stop.
|
|
254
|
+
if (manifest.v === MANIFEST_VERSION && manifest.enc !== undefined) {
|
|
255
|
+
throw new Error(
|
|
256
|
+
'Manifest says version 1, which is never encrypted, and carries encryption fields anyway. ' +
|
|
257
|
+
'Restoring it as version 1 would hand over encrypted bytes as the file, so telstore is ' +
|
|
258
|
+
'not reading it.',
|
|
157
259
|
)
|
|
158
260
|
}
|
|
159
261
|
|
|
@@ -208,6 +310,8 @@ export function parseManifest(input) {
|
|
|
208
310
|
}
|
|
209
311
|
})
|
|
210
312
|
|
|
313
|
+
if (manifest.v === ENCRYPTED_MANIFEST_VERSION) checkEncryption(manifest)
|
|
314
|
+
|
|
211
315
|
const expectedChunks = countChunks(manifest.size, manifest.chunkSize)
|
|
212
316
|
if (manifest.chunks.length !== expectedChunks) {
|
|
213
317
|
throw new Error(`Manifest is missing ${expectedChunks - manifest.chunks.length} chunk(s).`)
|
package/src/password.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { stderr, stdin } from 'node:process'
|
|
2
|
+
|
|
3
|
+
import { parseHint, terminalSafe } from './caption.js'
|
|
4
|
+
import { openManifest } from './cipher.js'
|
|
5
|
+
import { createPrompts, readSecret } from './prompt.js'
|
|
6
|
+
|
|
7
|
+
export const PASSWORD_ATTEMPTS = 3
|
|
8
|
+
|
|
9
|
+
// A password that came from an environment variable, a flag or a file came from somewhere that
|
|
10
|
+
// kept a copy of it, which is the rule src/token.js keeps for a passphrase. Unattended encrypted
|
|
11
|
+
// backups stay on the `--` pipeline with a key-based tool.
|
|
12
|
+
const NO_TERMINAL_UPLOAD =
|
|
13
|
+
'--encrypt needs a terminal to type the password in, and there is none here. Run the upload ' +
|
|
14
|
+
'where you can type it; telstore does not read a password from anywhere else.'
|
|
15
|
+
|
|
16
|
+
// One readline for the whole exchange, as docs/design/terminal-prompts.md requires: two over one
|
|
17
|
+
// stdin do not take turns, and the second question would read nothing. On stderr, so a stdout
|
|
18
|
+
// someone redirected still carries only what telstore reports.
|
|
19
|
+
export async function askNewPassword({ input = stdin, output = stderr } = {}) {
|
|
20
|
+
if (!input.isTTY) throw new Error(NO_TERMINAL_UPLOAD)
|
|
21
|
+
|
|
22
|
+
const prompts = createPrompts({ input, output })
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
const password = await prompts.askSecret('Password: ')
|
|
26
|
+
|
|
27
|
+
if (password === '') {
|
|
28
|
+
throw new Error('The password is empty. An encrypted backup needs one — run again and type it.')
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const again = await prompts.askSecret('Password again: ')
|
|
32
|
+
|
|
33
|
+
if (again !== password) {
|
|
34
|
+
throw new Error(
|
|
35
|
+
'The two passwords are different, so telstore does not know which one you meant. ' +
|
|
36
|
+
'Nothing was sent — run again.',
|
|
37
|
+
)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const hint = parseHint(await prompts.ask('Hint (optional, shown in the chat as plain text): '), password)
|
|
41
|
+
|
|
42
|
+
return { password, hint }
|
|
43
|
+
} finally {
|
|
44
|
+
prompts.close()
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function askPassword(question) {
|
|
49
|
+
return readSecret(question)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Opens an encrypted manifest or says why it cannot. Passwords this run has already seen work
|
|
53
|
+
// are tried first and silently, so a batch of backups under one password asks once. A failed
|
|
54
|
+
// tag cannot tell a wrong password from an altered manifest, so the last refusal names both,
|
|
55
|
+
// likelier first, as src/token.js does for a token.
|
|
56
|
+
export async function unlockManifest(
|
|
57
|
+
manifest,
|
|
58
|
+
{ askPassword: ask, interactive = () => Boolean(stdin.isTTY), known = [], say = () => {} },
|
|
59
|
+
) {
|
|
60
|
+
for (const password of known) {
|
|
61
|
+
const opened = await openManifest(manifest, password)
|
|
62
|
+
if (opened) return { ...opened, password }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (!interactive()) {
|
|
66
|
+
throw new Error(
|
|
67
|
+
`${manifest.id} is encrypted, and there is no terminal here to type its password in. ` +
|
|
68
|
+
'Run this where you can type it; telstore does not read a password from anywhere else.',
|
|
69
|
+
)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
say(`Backup ${manifest.id} is encrypted.`)
|
|
73
|
+
|
|
74
|
+
// Printed before any key exists, so the seal that covers it cannot have been checked yet: at
|
|
75
|
+
// this moment it is as trustworthy as the chat it came from, and is made safe to print as such.
|
|
76
|
+
const hint = manifest.enc.hint ? terminalSafe(manifest.enc.hint) : ''
|
|
77
|
+
|
|
78
|
+
if (hint) say(`Hint ${hint}`)
|
|
79
|
+
|
|
80
|
+
for (let attempt = 1; attempt <= PASSWORD_ATTEMPTS; attempt += 1) {
|
|
81
|
+
const password = await ask('Password: ')
|
|
82
|
+
const opened = password === '' ? null : await openManifest(manifest, password)
|
|
83
|
+
|
|
84
|
+
if (opened) {
|
|
85
|
+
known.push(password)
|
|
86
|
+
return { ...opened, password }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (attempt < PASSWORD_ATTEMPTS) say('That password does not open it. Try again.')
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
throw new Error(
|
|
93
|
+
`Could not open ${manifest.id} after ${PASSWORD_ATTEMPTS} attempts: either the password is ` +
|
|
94
|
+
'wrong or the manifest was altered. Encryption cannot tell those two apart, so telstore ' +
|
|
95
|
+
'will not guess — check the password first.' +
|
|
96
|
+
(hint
|
|
97
|
+
? ' The hint shown comes from the chat and is only checked once the password opens the ' +
|
|
98
|
+
'backup, so a hint that does not help may itself have been altered.'
|
|
99
|
+
: ''),
|
|
100
|
+
)
|
|
101
|
+
}
|
package/src/state.js
CHANGED
|
@@ -75,6 +75,18 @@ export function stateKey(absPath, size, mtimeMs) {
|
|
|
75
75
|
return createHash('sha1').update(`${absPath}:${size}:${mtimeMs}`).digest('hex')
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
// Where an encrypted upload's record is filed, and the reason it is not filed under stateKey is
|
|
79
|
+
// the reason the manifest went to version 2: an older telstore. That build computes stateKey
|
|
80
|
+
// exactly as this one does and never reads `enc`, so given an unfinished encrypted record under
|
|
81
|
+
// that key it would resume it without --encrypt, send the remaining chunks in plain, write a
|
|
82
|
+
// version 1 manifest with no ivs in it — and a later restore would match every sha256 and print
|
|
83
|
+
// Done over a file that is half ciphertext. A lookup that misses is the only refusal an older
|
|
84
|
+
// build can be made to give. Same 40-hex shape, so every listing, prune and status report that
|
|
85
|
+
// already handles upload records handles this one without learning a new name.
|
|
86
|
+
export function encryptedStateKey(absPath, size, mtimeMs) {
|
|
87
|
+
return createHash('sha1').update(`enc:${absPath}:${size}:${mtimeMs}`).digest('hex')
|
|
88
|
+
}
|
|
89
|
+
|
|
78
90
|
export function stateFile(key, configDir = defaultConfigDir()) {
|
|
79
91
|
return path.join(stateDir(configDir), `${key}.json`)
|
|
80
92
|
}
|
|
@@ -282,7 +294,8 @@ export async function findStates(backupId, configDir = defaultConfigDir()) {
|
|
|
282
294
|
// Whether a record can still be resumed, which is not a question about the record alone:
|
|
283
295
|
// runUpload hashes the file it finds on disk and looks the result up, so a backup is
|
|
284
296
|
// resumable exactly when that hash is still the key this record is filed under. Recomputing
|
|
285
|
-
// through stateKey rather than comparing size and
|
|
297
|
+
// through stateKey (encryptedStateKey for an encrypted record) rather than comparing size and
|
|
298
|
+
// mtime by hand is the point — a second way
|
|
286
299
|
// of asking is a second way to drift, and status would end up promising a resume that upload
|
|
287
300
|
// turns into a brand new backup, stranding every chunk already sent.
|
|
288
301
|
//
|
|
@@ -306,7 +319,16 @@ export async function canResume(key, state) {
|
|
|
306
319
|
}
|
|
307
320
|
|
|
308
321
|
if (!stat.isFile()) return { ok: false, reason: 'not-a-file' }
|
|
309
|
-
|
|
322
|
+
|
|
323
|
+
// Two keys now, and the record says which one it belongs under: runUpload files an encrypted
|
|
324
|
+
// record under encryptedStateKey and a plain one under stateKey. A record sitting under the
|
|
325
|
+
// other kind's key for a file that has not changed is not a changed file — runUpload refuses
|
|
326
|
+
// it as damaged rather than resume it, and the report has to say the same thing.
|
|
327
|
+
const own = (state.enc ? encryptedStateKey : stateKey)(state.path, stat.size, stat.mtimeMs)
|
|
328
|
+
const other = (state.enc ? stateKey : encryptedStateKey)(state.path, stat.size, stat.mtimeMs)
|
|
329
|
+
|
|
330
|
+
if (key === other) return { ok: false, reason: 'damaged' }
|
|
331
|
+
if (key !== own) return { ok: false, reason: 'changed' }
|
|
310
332
|
|
|
311
333
|
return { ok: true }
|
|
312
334
|
}
|
package/src/uploader.js
CHANGED
|
@@ -44,6 +44,11 @@ export async function uploadRange(client, fd, options) {
|
|
|
44
44
|
onProgress,
|
|
45
45
|
retryOptions,
|
|
46
46
|
stallMs = DEFAULT_STALL_MS,
|
|
47
|
+
// Applied to each part as it is read, in order, before it is hashed or sent — so the sha256
|
|
48
|
+
// this returns is of the bytes Telegram holds, and a retry resends the same transformed
|
|
49
|
+
// buffer rather than transforming again. It is what encryption rides on, and this file knows
|
|
50
|
+
// nothing else about it.
|
|
51
|
+
transform = (bytes) => bytes,
|
|
47
52
|
} = options
|
|
48
53
|
|
|
49
54
|
// A pool of fewer than one worker does no work. In the download path that means
|
|
@@ -94,7 +99,7 @@ export async function uploadRange(client, fd, options) {
|
|
|
94
99
|
for (let part = start; part < end; part += 1) {
|
|
95
100
|
const partOffset = part * partSize
|
|
96
101
|
const partLength = Math.min(partSize, length - partOffset)
|
|
97
|
-
const bytes = await readExactly(fd, partLength, offset + partOffset)
|
|
102
|
+
const bytes = transform(await readExactly(fd, partLength, offset + partOffset), partOffset)
|
|
98
103
|
|
|
99
104
|
hash.update(bytes)
|
|
100
105
|
|