telstore 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -5
- package/bin/telstore.js +272 -4
- package/package.json +1 -1
- package/src/caption.js +6 -1
- package/src/cli.js +279 -12
- package/src/client.js +168 -9
- package/src/commands/delete.js +335 -43
- package/src/commands/down.js +311 -0
- package/src/commands/list.js +170 -16
- package/src/commands/restore-stream.js +407 -0
- package/src/commands/restore.js +27 -20
- package/src/commands/status.js +190 -19
- package/src/commands/upload-stream.js +459 -0
- package/src/commands/upload.js +37 -25
- package/src/commands/verify.js +294 -0
- package/src/manifest.js +59 -2
- package/src/progress.js +86 -0
- package/src/shell.js +33 -0
- package/src/spawn.js +38 -0
- package/src/state.js +78 -0
- package/src/stream.js +295 -0
- package/src/tar.js +23 -0
package/src/cli.js
CHANGED
|
@@ -1,19 +1,36 @@
|
|
|
1
1
|
import { basename } from 'node:path'
|
|
2
2
|
import { parseArgs } from 'node:util'
|
|
3
3
|
|
|
4
|
+
import { deleteCommand } from './shell.js'
|
|
5
|
+
import { archiveName } from './tar.js'
|
|
6
|
+
|
|
7
|
+
// The shortcuts dispatch by name, not through SUBCOMMANDS: what they return is not a command
|
|
8
|
+
// called `tarc`, it is an upload (or, for `tarx`, a restore) with the line already
|
|
9
|
+
// rewritten into the form that command understands — there is no `runTarc` for SUBCOMMANDS to
|
|
10
|
+
// route to. They still belong in the set below all the same, because SUBCOMMANDS is the list of
|
|
11
|
+
// words telstore will not read as a file name, and a shortcut claims one exactly as a
|
|
12
|
+
// subcommand does.
|
|
13
|
+
const SHORTCUTS = new Map([
|
|
14
|
+
['tarc', tarcLine],
|
|
15
|
+
['tarx', tarxLine],
|
|
16
|
+
])
|
|
17
|
+
|
|
4
18
|
const SUBCOMMANDS = new Set([
|
|
5
19
|
'login',
|
|
6
20
|
'logout',
|
|
21
|
+
'down',
|
|
7
22
|
'list',
|
|
8
23
|
'restore',
|
|
24
|
+
'verify',
|
|
9
25
|
'delete',
|
|
10
26
|
'status',
|
|
11
27
|
'config',
|
|
12
28
|
'token',
|
|
13
29
|
'help',
|
|
30
|
+
...SHORTCUTS.keys(),
|
|
14
31
|
])
|
|
15
32
|
|
|
16
|
-
const OPTIONS = {
|
|
33
|
+
export const OPTIONS = {
|
|
17
34
|
chat: { type: 'string' },
|
|
18
35
|
'chunk-size': { type: 'string' },
|
|
19
36
|
'upload-concurrency': { type: 'string' },
|
|
@@ -21,6 +38,7 @@ const OPTIONS = {
|
|
|
21
38
|
out: { type: 'string' },
|
|
22
39
|
note: { type: 'string' },
|
|
23
40
|
limit: { type: 'string' },
|
|
41
|
+
search: { type: 'string' },
|
|
24
42
|
verbose: { type: 'boolean' },
|
|
25
43
|
unset: { type: 'boolean' },
|
|
26
44
|
yes: { type: 'boolean' },
|
|
@@ -33,12 +51,19 @@ export const HELP = `telstore — split large files into chunks and store them o
|
|
|
33
51
|
Usage:
|
|
34
52
|
npx telstore login Log in to Telegram, only needed once
|
|
35
53
|
npx telstore <file|folder|pattern>... Split files and upload them to Telegram
|
|
54
|
+
npx telstore <name> -- <command>... Store what a command writes, under <name>
|
|
55
|
+
npx telstore tarc <name> <path>... Archive paths with tar and store the archive
|
|
56
|
+
npx telstore tarx <backup-id> Restore a backup and extract it with tar
|
|
57
|
+
npx telstore restore <id> -- <cmd>... Restore onto a command instead of a file
|
|
36
58
|
npx telstore list List the backups stored in the destination
|
|
59
|
+
npx telstore list --search <text> List only the backups that text appears in
|
|
37
60
|
npx telstore restore <backup-id>... Download the chunks and reassemble the files
|
|
61
|
+
npx telstore verify <backup-id>... Check that a backup's chunks are all still in the chat
|
|
38
62
|
npx telstore delete <backup-id>... Remove backups' chunks and manifests from the chat
|
|
39
63
|
npx telstore status Show the account, the destination and unfinished uploads and restores
|
|
40
64
|
npx telstore config Show every setting and where its value comes from
|
|
41
65
|
npx telstore logout Remove the saved session
|
|
66
|
+
npx telstore down Remove everything telstore keeps on this machine
|
|
42
67
|
|
|
43
68
|
Running on a machine you do not trust:
|
|
44
69
|
npx telstore token Print a session token for another machine
|
|
@@ -50,8 +75,36 @@ backup. A folder means the files one level inside it, and a pattern means the na
|
|
|
50
75
|
one file is listed and confirmed before the first byte goes out. Run telstore again with only
|
|
51
76
|
the files that are left to carry on after an interruption.
|
|
52
77
|
|
|
53
|
-
|
|
54
|
-
|
|
78
|
+
A name followed by -- makes the backup out of what a command writes, so nothing has to be on
|
|
79
|
+
disk first: npx telstore a.tar -- tar cf - ./a. The manifest goes out only if that command's
|
|
80
|
+
output ended and the command exited 0 — an end after a crash looks exactly like an end after
|
|
81
|
+
success, and running the command is how telstore tells them apart. A backup made this way
|
|
82
|
+
cannot be resumed, so a run that fails, and a Ctrl-C, remove the chunks already sent rather
|
|
83
|
+
than keeping them for a second run there will never be. No shell stands in between: a pipeline
|
|
84
|
+
goes in as -- bash -c 'set -o pipefail; ...', which is also where compression or encryption
|
|
85
|
+
belongs. The pipefail is not decoration — a shell reports the last command's exit status, so
|
|
86
|
+
without it a producer that dies halfway through a pipeline still exits 0 and the manifest goes
|
|
87
|
+
out for a truncated backup.
|
|
88
|
+
|
|
89
|
+
tarc and tarx are the common case written out once: "npx telstore tarc a.tar.gz ./dir"
|
|
90
|
+
is "npx telstore a.tar.gz -- tar czf - ./dir", and tarx is the same for "restore <id> --
|
|
91
|
+
tar xzf -". telstore prints the long form as it runs, so the shortcut teaches what it is
|
|
92
|
+
short for. tarc always compresses, so it makes the name say so: a.tar becomes a.tar.gz,
|
|
93
|
+
and a name with no tar in it at all gets .tar.gz. --verbose adds tar's own file listing
|
|
94
|
+
to both. For tarx, --out is the directory it extracts into, and tar's own semantics apply
|
|
95
|
+
there: it overwrites files already in it without asking, unlike restore's one-file [y/N]
|
|
96
|
+
prompt — --out is how you aim it somewhere empty instead. Anything beyond archiving
|
|
97
|
+
the paths — -C, --exclude, a pipeline, another compressor — is what -- is still for.
|
|
98
|
+
|
|
99
|
+
down is logout taken all the way: it removes ~/.telstore entirely — the session, the api_id
|
|
100
|
+
and api_hash, every setting and every resume record — and asks once before it does. It opens
|
|
101
|
+
no connection and deletes nothing from Telegram: the backups stay in the chat, and the session
|
|
102
|
+
stays alive on Telegram's side until you terminate it under Settings → Devices.
|
|
103
|
+
|
|
104
|
+
restore, verify and delete take several ids the same way: one connection, one line each, and
|
|
105
|
+
an exit code that reports any that failed. delete shows everything it is about to destroy and
|
|
106
|
+
asks once. verify downloads nothing: it asks the chat whether every chunk message is still
|
|
107
|
+
there at the length the manifest records, which is what restore would need.
|
|
55
108
|
|
|
56
109
|
Settings:
|
|
57
110
|
npx telstore config <name> Print one setting's value
|
|
@@ -80,11 +133,18 @@ Options apply to one run and are never saved. Use config to change a setting for
|
|
|
80
133
|
shell hands the words after the first to telstore as more files
|
|
81
134
|
to upload.
|
|
82
135
|
--limit <n> How many backups list shows this run.
|
|
136
|
+
--search <text> List only the backups whose file name, note, backup id or
|
|
137
|
+
creation day contains this text. Telegram's own index does
|
|
138
|
+
the looking, so the whole chat is reached without reading it
|
|
139
|
+
message by message — but it matches whole words only:
|
|
140
|
+
"projex" finds projex.zip and "proj" finds nothing. A term
|
|
141
|
+
with spaces has to be quoted, as --note does.
|
|
83
142
|
--token Log in by pasting a session token. It takes no value on
|
|
84
143
|
purpose: a token written on the command line would sit in
|
|
85
144
|
"ps" for the whole life of the command, and stay in that
|
|
86
145
|
machine's shell history afterwards.
|
|
87
|
-
--yes Upload a batch,
|
|
146
|
+
--yes Upload a batch, delete, or wipe this machine with down,
|
|
147
|
+
without being asked to confirm.
|
|
88
148
|
--verbose Show Telegram connection logs for this run.
|
|
89
149
|
-h, --help Show this help.
|
|
90
150
|
`
|
|
@@ -93,7 +153,54 @@ Options apply to one run and are never saved. Use config to change a setting for
|
|
|
93
153
|
// finished chunk to a state file, restore has not. Naming the backup matters because the
|
|
94
154
|
// id is what `status` lists and what a later `restore` needs — the chunks are already in
|
|
95
155
|
// the chat under that id, whether or not this run ever finishes.
|
|
96
|
-
export function interruptMessage(
|
|
156
|
+
export function interruptMessage(
|
|
157
|
+
command,
|
|
158
|
+
{ backupId, done = [], stream = false, again = false, chat = null } = {},
|
|
159
|
+
) {
|
|
160
|
+
// A backup made from a command is the one upload Ctrl-C cannot leave where it is. The bytes
|
|
161
|
+
// have gone past and the next run cuts them differently, so a chunk already in the chat is
|
|
162
|
+
// a chunk no manifest will ever name — which is why this run is asked to remove them and
|
|
163
|
+
// the process waits, rather than promising the resume the file wording promises.
|
|
164
|
+
if (command === 'upload' && stream) {
|
|
165
|
+
// Nothing is in the chat until there is an id to put it under, and a run stopped before
|
|
166
|
+
// that has nothing for anyone to clean up.
|
|
167
|
+
if (!backupId) {
|
|
168
|
+
return '\nStopped before anything was sent.\n'
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
if (again) {
|
|
172
|
+
// "may still" because this is said while the removal is halfway through and nobody
|
|
173
|
+
// knows how far it got. `deleteCommand` is what names the chat, and why: a later
|
|
174
|
+
// `delete` resolves its destination from config, and these ids fired at the wrong peer
|
|
175
|
+
// destroy whatever happens to carry them there. A null chat here is a Ctrl-C that
|
|
176
|
+
// landed before the run said where it was sending — the chatless branch documented
|
|
177
|
+
// beside that function is for exactly this caller.
|
|
178
|
+
const removal = deleteCommand(backupId, chat)
|
|
179
|
+
|
|
180
|
+
return (
|
|
181
|
+
`\nLeaving now. Backup ${backupId} may still have chunks in the chat with no manifest ` +
|
|
182
|
+
`pointing at them — run "${removal}" to remove them.\n`
|
|
183
|
+
)
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return (
|
|
187
|
+
`\nStopping. Backup ${backupId} was made from a command and cannot be resumed, so ` +
|
|
188
|
+
'telstore is removing the chunks it already sent. This takes a moment — press Ctrl-C ' +
|
|
189
|
+
'again to leave now and clean up by hand.\n'
|
|
190
|
+
)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// The restore direction of the same idea, and the difference is the whole message: a stream
|
|
194
|
+
// upload has to unwind what it put in the chat, while this one put nothing there. What it
|
|
195
|
+
// cannot put back is what the command already did with the bytes it was given.
|
|
196
|
+
if (command === 'restore' && stream) {
|
|
197
|
+
return (
|
|
198
|
+
'\nStopped. Nothing in the chat changed and nothing was kept on this machine, but the ' +
|
|
199
|
+
'command had already been given part of the backup, so whatever it wrote from that is ' +
|
|
200
|
+
'incomplete. A restore into a command cannot be resumed — run it again from the start.\n'
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
|
|
97
204
|
if (command === 'upload') {
|
|
98
205
|
const backup = backupId ? `Backup ${backupId} is saved` : 'Progress is saved'
|
|
99
206
|
|
|
@@ -167,6 +274,18 @@ export function interruptMessage(command, { backupId, done = [] } = {}) {
|
|
|
167
274
|
return '\nStopped.\n'
|
|
168
275
|
}
|
|
169
276
|
|
|
277
|
+
// The one `--` this file did not write. protectNegativeChatIds, below, inserts one of its
|
|
278
|
+
// own to rescue a negative chat id from parseArgs, so the position has to be taken off the
|
|
279
|
+
// argv as typed — afterwards the two are indistinguishable, and `config chat -100123` would
|
|
280
|
+
// become a command telstore tries to run.
|
|
281
|
+
function splitAtTerminator(argv) {
|
|
282
|
+
const at = argv.indexOf('--')
|
|
283
|
+
|
|
284
|
+
if (at === -1) return { head: argv, childArgv: null }
|
|
285
|
+
|
|
286
|
+
return { head: argv.slice(0, at), childArgv: argv.slice(at + 1) }
|
|
287
|
+
}
|
|
288
|
+
|
|
170
289
|
// A channel id is negative, and typing it separated by a space is the natural reflex — but
|
|
171
290
|
// parseArgs rejects anything starting with a dash as an option, and reports it as one:
|
|
172
291
|
// `config chat -100123` fails with "Unknown option '-1'", naming a flag nobody typed.
|
|
@@ -221,9 +340,80 @@ function filesNamedAfterNote(tokens) {
|
|
|
221
340
|
return tokens.some((token) => token.kind === 'positional' && token.index > note.index)
|
|
222
341
|
}
|
|
223
342
|
|
|
343
|
+
// tarc is the long form with the three decisions that never change already made: `c` for
|
|
344
|
+
// create, `z` for gzip, `f -` for "write it to stdout, which is where telstore is listening".
|
|
345
|
+
// The missing `-` is not a hypothetical mistake — this project's own help text and README
|
|
346
|
+
// shipped exactly that omission once, and paid for it with an example that exited 2 instead
|
|
347
|
+
// of writing a backup.
|
|
348
|
+
//
|
|
349
|
+
// An expansion rather than a command of its own: `runStreamUpload` is reached with exactly the
|
|
350
|
+
// argv the `--` form reaches it with, so there is no second upload path, no second rollback
|
|
351
|
+
// and no second guarantee. It also prints that argv, so the shortcut teaches the long form
|
|
352
|
+
// instead of hiding it.
|
|
353
|
+
function tarcLine(rest, values, filesAfterNote) {
|
|
354
|
+
const [name, ...paths] = rest
|
|
355
|
+
|
|
356
|
+
if (name === undefined) {
|
|
357
|
+
throw new Error(
|
|
358
|
+
'Missing a name for the backup. tarc stores the archive under a name you choose. ' +
|
|
359
|
+
'Example: npx telstore tarc a.tar.gz ./a',
|
|
360
|
+
)
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// Refused rather than answered with a guess: a rule that read one positional as a name and
|
|
364
|
+
// two as a name plus a path would make `telstore tarc ./x ./y` archive ./y under the name
|
|
365
|
+
// ./x, which is the silent wrong answer this project exists to refuse.
|
|
366
|
+
if (paths.length === 0) {
|
|
367
|
+
throw new Error(
|
|
368
|
+
`Nothing to archive: tarc needs the paths to put in ${name}. ` +
|
|
369
|
+
'Example: npx telstore tarc a.tar.gz ./a',
|
|
370
|
+
)
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
return {
|
|
374
|
+
command: 'upload',
|
|
375
|
+
args: [archiveName(name)],
|
|
376
|
+
options: values,
|
|
377
|
+
filesAfterNote,
|
|
378
|
+
childArgv: ['tar', values.verbose ? 'czvf' : 'czf', '-', ...paths],
|
|
379
|
+
shortcut: 'tarc',
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// The mirror of tarcLine. `x` for extract, `z` because tarc always compressed, `f -` because
|
|
384
|
+
// the bytes arrive on stdin.
|
|
385
|
+
function tarxLine(rest, values, filesAfterNote) {
|
|
386
|
+
requireOneBackupId(rest, 'tarx')
|
|
387
|
+
|
|
388
|
+
const childArgv = ['tar', values.verbose ? 'xzvf' : 'xzf', '-']
|
|
389
|
+
|
|
390
|
+
// Pushed after `-` on purpose, which is the order measured to work on GNU tar 1.35:
|
|
391
|
+
// `tar xzf - -C ./here`. See the probe table in the spec.
|
|
392
|
+
if (values.out !== undefined) childArgv.push('-C', values.out)
|
|
393
|
+
|
|
394
|
+
return { command: 'restore', args: rest, options: values, filesAfterNote, childArgv, shortcut: 'tarx' }
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// One command reads one stream, so a line that names two backups is a line with no answer:
|
|
398
|
+
// extracting two archives into one working directory in sequence is a question nobody asked.
|
|
399
|
+
function requireOneBackupId(ids, what) {
|
|
400
|
+
if (ids.length === 0) {
|
|
401
|
+
throw new Error(`Missing backup id. Example: npx telstore ${what} telstore-20260905-7f3a91`)
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
if (ids.length > 1) {
|
|
405
|
+
throw new Error(
|
|
406
|
+
`One command reads one stream, so ${what} takes one backup id and got ${ids.length}: ` +
|
|
407
|
+
`${ids.join(', ')}. Run telstore once per backup.`,
|
|
408
|
+
)
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
224
412
|
export function route(argv) {
|
|
413
|
+
const { head, childArgv } = splitAtTerminator(argv)
|
|
414
|
+
|
|
225
415
|
const { values, positionals, tokens } = parseArgs({
|
|
226
|
-
args: protectNegativeChatIds(
|
|
416
|
+
args: protectNegativeChatIds(head),
|
|
227
417
|
options: OPTIONS,
|
|
228
418
|
allowPositionals: true,
|
|
229
419
|
tokens: true,
|
|
@@ -232,26 +422,103 @@ export function route(argv) {
|
|
|
232
422
|
const [first, ...rest] = positionals
|
|
233
423
|
const filesAfterNote = filesNamedAfterNote(tokens)
|
|
234
424
|
|
|
425
|
+
// --help (or -h, or the `help` subcommand) always wins, terminator or not: someone typing
|
|
426
|
+
// `telstore --help -- tar cf - ./a` is asking what telstore does, not making a mistake for
|
|
427
|
+
// one of the checks below to catch.
|
|
428
|
+
if (values.help || first === 'help') {
|
|
429
|
+
return { command: 'help', args: [], options: values, filesAfterNote, childArgv, shortcut: null }
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
if (childArgv !== null && childArgv.length === 0) {
|
|
433
|
+
throw new Error(
|
|
434
|
+
'Missing the command after --: telstore has nothing to run and store. ' +
|
|
435
|
+
'Example: npx telstore a.tar -- tar cf - ./a',
|
|
436
|
+
)
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// A terminator changes what "no name" and "which command" mean, so it is read before the
|
|
440
|
+
// ordinary help/chat fallbacks get a chance to answer for it — those apply to a line that
|
|
441
|
+
// never named a command to run at all.
|
|
442
|
+
if (childArgv !== null) {
|
|
443
|
+
// Reached before the generic "takes no command after --" below, because for these two the
|
|
444
|
+
// reason is different and so is the way out: they are not a subcommand that happens not to
|
|
445
|
+
// run commands, they are a command already.
|
|
446
|
+
if (SHORTCUTS.has(first)) {
|
|
447
|
+
throw new Error(
|
|
448
|
+
`${first} already is the command it runs, so it cannot be followed by another one. ` +
|
|
449
|
+
`Drop the -- to use ${first}, or drop ${first} to write the command out yourself.`,
|
|
450
|
+
)
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
if (first !== undefined && SUBCOMMANDS.has(first)) {
|
|
454
|
+
// restore is let through rather than refused here, and the binary is what turns it away.
|
|
455
|
+
// The shape is the spec's stage 2, so the parser keeps it whole — but a refusal that
|
|
456
|
+
// says "not built yet, restore to a file and pipe that" belongs where the command runs,
|
|
457
|
+
// beside the alternative it is offering, not in an argument parser.
|
|
458
|
+
if (first !== 'restore') {
|
|
459
|
+
throw new Error(
|
|
460
|
+
`${first} takes no command after --. An upload (npx telstore a.tar -- tar cf - ./a) ` +
|
|
461
|
+
'and a restore (npx telstore restore <id> -- tar xf -) are the two that run one.',
|
|
462
|
+
)
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
requireOneBackupId(rest, 'restore')
|
|
466
|
+
|
|
467
|
+
// --out places a file, and this path writes none: the bytes go to the command on its
|
|
468
|
+
// stdin. Left to pass silently it would read as "restore into the command AND write
|
|
469
|
+
// the file over there", which is not what happens.
|
|
470
|
+
if (values.out !== undefined) {
|
|
471
|
+
throw new Error(
|
|
472
|
+
'A restore into a command writes no file, so --out has nothing to place: the bytes ' +
|
|
473
|
+
'go to the command on its stdin. Tell the command where to put them instead ' +
|
|
474
|
+
'(npx telstore restore <id> -- tar xf - -C ./here).',
|
|
475
|
+
)
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
return { command: first, args: rest, options: values, filesAfterNote, childArgv, shortcut: null }
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
if (positionals.length === 0) {
|
|
482
|
+
throw new Error(
|
|
483
|
+
'Missing a name before --. telstore stores what the command writes under a name you ' +
|
|
484
|
+
'choose, and there is nothing to take one from. Example: npx telstore a.tar -- tar cf - ./a',
|
|
485
|
+
)
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
if (positionals.length > 1) {
|
|
489
|
+
throw new Error(
|
|
490
|
+
`One command produces one stream, so telstore takes one name before -- and got ` +
|
|
491
|
+
`${positionals.length}: ${positionals.join(', ')}. Run telstore once per backup.`,
|
|
492
|
+
)
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
return { command: 'upload', args: positionals, options: values, filesAfterNote, childArgv, shortcut: null }
|
|
496
|
+
}
|
|
497
|
+
|
|
235
498
|
// `telstore --chat @chan` with no file used to mean "remember this destination". Flags no
|
|
236
499
|
// longer write anything, so that line now asks for a run that has nothing to upload —
|
|
237
500
|
// say where the destination actually lives instead of printing help at someone who was
|
|
238
|
-
// perfectly clear about what they wanted.
|
|
239
|
-
|
|
501
|
+
// perfectly clear about what they wanted. (values.help already returned above, so reaching
|
|
502
|
+
// here means it was never set.)
|
|
503
|
+
if (first === undefined && values.chat) {
|
|
240
504
|
throw new Error(
|
|
241
505
|
`Nothing to upload. To change the destination for good, run "npx telstore config chat ${values.chat}". ` +
|
|
242
506
|
'To use it for one run, pass --chat alongside a file or a command.',
|
|
243
507
|
)
|
|
244
508
|
}
|
|
245
509
|
|
|
246
|
-
if (
|
|
247
|
-
return { command: 'help', args: [], options: values, filesAfterNote }
|
|
510
|
+
if (first === undefined) {
|
|
511
|
+
return { command: 'help', args: [], options: values, filesAfterNote, childArgv, shortcut: null }
|
|
248
512
|
}
|
|
249
513
|
|
|
514
|
+
const shortcut = SHORTCUTS.get(first)
|
|
515
|
+
if (shortcut) return shortcut(rest, values, filesAfterNote)
|
|
516
|
+
|
|
250
517
|
if (SUBCOMMANDS.has(first)) {
|
|
251
|
-
return { command: first, args: rest, options: values, filesAfterNote }
|
|
518
|
+
return { command: first, args: rest, options: values, filesAfterNote, childArgv, shortcut: null }
|
|
252
519
|
}
|
|
253
520
|
|
|
254
521
|
// Every positional, not just the first: `telstore a b c` used to upload `a` and drop the
|
|
255
522
|
// rest without a word, which is the one thing this project never does.
|
|
256
|
-
return { command: 'upload', args: positionals, options: values, filesAfterNote }
|
|
523
|
+
return { command: 'upload', args: positionals, options: values, filesAfterNote, childArgv, shortcut: null }
|
|
257
524
|
}
|
package/src/client.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Api, TelegramClient } from 'teleproto'
|
|
2
2
|
import { Logger } from 'teleproto/extensions/index.js'
|
|
3
3
|
import { LogLevel } from 'teleproto/extensions/Logger.js'
|
|
4
|
+
import { returnBigInt } from 'teleproto/Helpers.js'
|
|
4
5
|
import { StringSession } from 'teleproto/sessions/index.js'
|
|
5
6
|
|
|
7
|
+
import { MANIFEST_TAG } from './caption.js'
|
|
6
8
|
import { manifestFileName } from './manifest.js'
|
|
7
9
|
import { withRetry } from './retry.js'
|
|
8
10
|
import { assertLoggedIn, unlockConfig } from './session.js'
|
|
@@ -23,6 +25,26 @@ export function documentFileName(message) {
|
|
|
23
25
|
return named?.fileName ?? null
|
|
24
26
|
}
|
|
25
27
|
|
|
28
|
+
// Telegram records a document's length as a BigInteger, and comparing that to the plain
|
|
29
|
+
// number a manifest carries with === is false for every size there is.
|
|
30
|
+
export function documentSize(message) {
|
|
31
|
+
const size = message?.media?.document?.size
|
|
32
|
+
|
|
33
|
+
return size === undefined || size === null ? null : returnBigInt(size).toJSNumber()
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// The flat shape both readers of a chat hand back. The raw message is kept alongside it
|
|
37
|
+
// because downloading needs it whole.
|
|
38
|
+
function toDocument(message) {
|
|
39
|
+
return {
|
|
40
|
+
id: message.id,
|
|
41
|
+
fileName: documentFileName(message),
|
|
42
|
+
caption: message.message ?? '',
|
|
43
|
+
date: message.date,
|
|
44
|
+
message,
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
26
48
|
// The one place telstore searches a chat. Both callers want documents and nothing else,
|
|
27
49
|
// and getMessages is preferred over a raw Api.messages.Search because it handles offsets,
|
|
28
50
|
// hashes and pagination itself, so we don't hand-build easily mistyped fields. The raw
|
|
@@ -34,13 +56,99 @@ export async function searchDocuments(client, peer, { search, limit }) {
|
|
|
34
56
|
limit,
|
|
35
57
|
})
|
|
36
58
|
|
|
37
|
-
return messages.map(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
59
|
+
return messages.map(toDocument)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// The paging both readers share. It is ours rather than iterMessages', for the reasons
|
|
63
|
+
// deleteMessages does not use teleproto's: every page then carries the retry policy and the
|
|
64
|
+
// stall deadline, and a page that fails is retried by itself instead of restarting from the
|
|
65
|
+
// newest message. offsetId is the id of the last message of the page before, and Telegram
|
|
66
|
+
// answers with the messages older than it — a reader that forgot to advance it would fetch
|
|
67
|
+
// the newest page over and over and never reach an older backup.
|
|
68
|
+
//
|
|
69
|
+
// A generator because the caller stops when it has what it wants: a chat of ten thousand
|
|
70
|
+
// chunks costs one request to list the backups at the top of it.
|
|
71
|
+
export const DOCUMENT_PAGE_SIZE = 100
|
|
72
|
+
|
|
73
|
+
async function* iterMessagePages(client, peer, { search, what, options }) {
|
|
74
|
+
const {
|
|
75
|
+
pageSize = DOCUMENT_PAGE_SIZE,
|
|
76
|
+
max = Infinity,
|
|
77
|
+
retryOptions = {},
|
|
78
|
+
stallMs = DEFAULT_STALL_MS,
|
|
79
|
+
// Where the walk begins, as the id of the message just above the first one wanted. 0 is
|
|
80
|
+
// "the newest in the chat", which is what list asks for. delete starts at a backup's own
|
|
81
|
+
// manifest instead when the chat has shown it one: the manifest is the last message a
|
|
82
|
+
// backup's run sends, so nothing of that backup is newer, and everything posted since is
|
|
83
|
+
// a page of documents read for nothing.
|
|
84
|
+
offsetId: startId = 0,
|
|
85
|
+
} = options
|
|
86
|
+
|
|
87
|
+
let offsetId = startId
|
|
88
|
+
let read = 0
|
|
89
|
+
|
|
90
|
+
while (read < max) {
|
|
91
|
+
const limit = Math.min(pageSize, max - read)
|
|
92
|
+
|
|
93
|
+
const messages = await withRetry(
|
|
94
|
+
() =>
|
|
95
|
+
withStallTimeout(
|
|
96
|
+
client.getMessages(peer, {
|
|
97
|
+
...(search === undefined ? {} : { search }),
|
|
98
|
+
filter: new Api.InputMessagesFilterDocument(),
|
|
99
|
+
limit,
|
|
100
|
+
offsetId,
|
|
101
|
+
}),
|
|
102
|
+
stallMs,
|
|
103
|
+
() =>
|
|
104
|
+
`Telegram stopped answering while reading the ${what} older than message ` +
|
|
105
|
+
`${offsetId}: nothing back for ${Math.round(stallMs / 1000)}s.`,
|
|
106
|
+
),
|
|
107
|
+
retryOptions,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
if (!messages || messages.length === 0) return
|
|
111
|
+
|
|
112
|
+
for (const message of messages) yield toDocument(message)
|
|
113
|
+
|
|
114
|
+
read += messages.length
|
|
115
|
+
offsetId = messages[messages.length - 1].id
|
|
116
|
+
|
|
117
|
+
// A short page is the end of the results. Asking again would cost a request to be told
|
|
118
|
+
// the same thing.
|
|
119
|
+
if (messages.length < limit) return
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// What `list` reads the chat with. searchDocuments asks Telegram's text index a question;
|
|
124
|
+
// this asks for the documents themselves, newest first, which is the only answer that was
|
|
125
|
+
// right every time it was measured — a chat's text index can come back empty while the chat
|
|
126
|
+
// is full of backups, and did for a whole day in a channel that had just been created
|
|
127
|
+
// (docs/design/captions.md carries the measurements).
|
|
128
|
+
export async function* iterDocuments(client, peer, options = {}) {
|
|
129
|
+
yield* iterMessagePages(client, peer, { what: 'documents', options })
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// What `list --search` reads the chat with, and the one place that knows how to ask the index
|
|
133
|
+
// for backups. Two things about the query are load-bearing, both measured against a real chat
|
|
134
|
+
// on 2026-09-08 (docs/design/captions.md carries the numbers):
|
|
135
|
+
//
|
|
136
|
+
// The tag is ANDed in because a term alone brings the backup's chunk messages back too — a
|
|
137
|
+
// chunk caption carries the id — and on a backup of a few hundred chunks those would fill
|
|
138
|
+
// every page before a single manifest appeared. `#telstore` rides on the manifest alone, so
|
|
139
|
+
// the results come back manifests only: searching one backup id returned 1 manifest and 1
|
|
140
|
+
// chunk, and the same id with the tag returned the manifest by itself.
|
|
141
|
+
//
|
|
142
|
+
// The tag goes *after* the term, never before. A query that starts with the hash is read as
|
|
143
|
+
// a hashtag lookup and stops ANDing the rest: "#telstore projex" came back empty while
|
|
144
|
+
// "projex #telstore" returned the one manifest. Leading it would turn every search into
|
|
145
|
+
// "no backups found", which is the sentence this project must never say wrongly.
|
|
146
|
+
export async function* iterManifestSearch(client, peer, term, options = {}) {
|
|
147
|
+
yield* iterMessagePages(client, peer, {
|
|
148
|
+
search: `${term} ${MANIFEST_TAG}`,
|
|
149
|
+
what: 'search results',
|
|
150
|
+
options,
|
|
151
|
+
})
|
|
44
152
|
}
|
|
45
153
|
|
|
46
154
|
// How telstore finds a backup's manifest, in one place because restore and delete must not
|
|
@@ -70,11 +178,15 @@ export async function readMessageBytes(client, message) {
|
|
|
70
178
|
//
|
|
71
179
|
// Telegram does not complain about an id that is no longer there, so sending a batch twice
|
|
72
180
|
// costs nothing: a delete interrupted halfway is finished by running it again.
|
|
73
|
-
|
|
181
|
+
//
|
|
182
|
+
// The hundred is Telegram's own limit on how many message ids one request may name, and it
|
|
183
|
+
// is the same limit whether the request removes them or asks about them — so getDocuments
|
|
184
|
+
// below counts in the same batches rather than keeping a second opinion about one number.
|
|
185
|
+
export const MESSAGE_BATCH_SIZE = 100
|
|
74
186
|
|
|
75
187
|
export async function deleteMessages(client, peer, ids, options = {}) {
|
|
76
188
|
const {
|
|
77
|
-
batchSize =
|
|
189
|
+
batchSize = MESSAGE_BATCH_SIZE,
|
|
78
190
|
retryOptions = {},
|
|
79
191
|
stallMs = DEFAULT_STALL_MS,
|
|
80
192
|
onBatch,
|
|
@@ -110,6 +222,53 @@ export async function deleteMessages(client, peer, ids, options = {}) {
|
|
|
110
222
|
return deleted
|
|
111
223
|
}
|
|
112
224
|
|
|
225
|
+
// What verify asks the chat, and the read-only mirror of deleteMessages above: our own
|
|
226
|
+
// batching, one request in flight at a time, under the same retry policy and the same stall
|
|
227
|
+
// deadline as every other network wait in telstore.
|
|
228
|
+
//
|
|
229
|
+
// The answer is a Map rather than a list because the question is "which of these are still
|
|
230
|
+
// there". Telegram reports a message that is gone as MessageEmpty — an object carrying the
|
|
231
|
+
// id it was asked about — so anything that is not a real message is left out here, where the
|
|
232
|
+
// shape is understood, rather than passed on to a caller that would read an empty as a chunk
|
|
233
|
+
// still sitting in the chat.
|
|
234
|
+
export async function getDocuments(client, peer, ids, options = {}) {
|
|
235
|
+
const {
|
|
236
|
+
batchSize = MESSAGE_BATCH_SIZE,
|
|
237
|
+
retryOptions = {},
|
|
238
|
+
stallMs = DEFAULT_STALL_MS,
|
|
239
|
+
onBatch,
|
|
240
|
+
} = options
|
|
241
|
+
|
|
242
|
+
const found = new Map()
|
|
243
|
+
|
|
244
|
+
for (let start = 0; start < ids.length; start += batchSize) {
|
|
245
|
+
const batch = ids.slice(start, start + batchSize)
|
|
246
|
+
|
|
247
|
+
const messages = await withRetry(
|
|
248
|
+
() =>
|
|
249
|
+
withStallTimeout(
|
|
250
|
+
client.getMessages(peer, { ids: batch }),
|
|
251
|
+
stallMs,
|
|
252
|
+
() =>
|
|
253
|
+
`Telegram stopped answering while looking up messages ${start + 1}-` +
|
|
254
|
+
`${start + batch.length} of ${ids.length}: nothing back for ` +
|
|
255
|
+
`${Math.round(stallMs / 1000)}s.`,
|
|
256
|
+
),
|
|
257
|
+
retryOptions,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
for (const message of messages ?? []) {
|
|
261
|
+
if (!message || message instanceof Api.MessageEmpty) continue
|
|
262
|
+
|
|
263
|
+
found.set(message.id, message)
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
onBatch?.(Math.min(start + batch.length, ids.length), ids.length)
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return found
|
|
270
|
+
}
|
|
271
|
+
|
|
113
272
|
// Every command ends by putting the connection down, and a failure there must never
|
|
114
273
|
// swallow the real error already on its way up. Commands that print progress hand in an
|
|
115
274
|
// onWarn to say so; the quieter ones let it pass, because a connection that will not
|