telstore 0.1.8 โ†’ 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -28,6 +28,7 @@ npx telstore restore telstore-20260905-7f3a91
28
28
  | `telstore <file\|folder\|pattern>...` | Split each file and upload it. Prints the `backupId` you restore with. |
29
29
  | `telstore list` | The backups stored in the destination, newest first. |
30
30
  | `telstore restore <backup-id>...` | Download every chunk and reassemble the file. Several ids run one after another. |
31
+ | `telstore verify <backup-id>...` | Check that every chunk of a backup is still in the chat. Downloads nothing. |
31
32
  | `telstore delete <backup-id>...` | Remove a backup's chunks and manifest from the chat, for good. Several ids are listed and confirmed once. |
32
33
  | `telstore status` | Account, destination, and unfinished uploads and restores. |
33
34
  | `telstore config` | Show or change settings. |
@@ -65,7 +66,7 @@ a prompt that does not echo.
65
66
 
66
67
  Every chunk goes up as a document captioned `๐Ÿ“ฆ <backupId> ยท 3/12`, followed by a manifest
67
68
  carrying a summary card โ€” file name, size, id, date and the restore command. `list` reads
68
- those cards straight out of the chat, one search and no downloads:
69
+ those cards straight out of the chat, no downloads:
69
70
 
70
71
  ```
71
72
  Destination https://web.telegram.org/k/#@my_backups
@@ -117,13 +118,44 @@ it, and the run ends with a line per file and a non-zero exit code:
117
118
  c.tar telstore-20260905-9de447 (1 chunk)
118
119
  ```
119
120
 
121
+ ## Checking a backup is still there
122
+
123
+ A backup is a set of messages in a chat, and messages can be deleted by hand. `list` reads
124
+ the manifest's card and would happily show a backup whose chunks are long gone; `verify` asks
125
+ the chat about every chunk the manifest names:
126
+
127
+ ```
128
+ $ npx telstore verify telstore-20260905-7f3a91
129
+ Backup telstore-20260905-7f3a91
130
+ File data.tar (21.4 GB, 12 chunks)
131
+ In https://web.telegram.org/k/#@my_backups
132
+
133
+ 12 chunks present, at the sizes the manifest records.
134
+ This does not download them, so it cannot prove their contents.
135
+ ```
136
+
137
+ It costs one request per hundred chunks and no bandwidth, so it is cheap enough to run on a
138
+ schedule. What it proves is that a restore would find everything it needs โ€” every chunk still
139
+ there, under the file name telstore wrote, at the length the manifest records. It does not
140
+ read the chunks, so it cannot speak for what is inside them; only a restore does that, and a
141
+ restore checks every sha256 before it renames anything into place.
142
+
143
+ A backup missing chunks is named line by line, and the exit code is 1:
144
+
145
+ ```
146
+ Chunk 3/12 is gone: message 1042 is no longer in @my_backups.
147
+
148
+ 12 chunks checked, 1 damaged. This backup cannot be restored.
149
+ ```
150
+
120
151
  ## Several backups at once
121
152
 
122
- `restore` and `delete` take a list of ids the same way, over one connection, with a summary
123
- and a non-zero exit code if any of them failed:
153
+ `restore`, `verify` and `delete` take a list of ids the same way, over one connection, with a
154
+ summary and a non-zero exit code if any of them failed:
124
155
 
125
156
  ```bash
126
157
  npx telstore restore telstore-20260905-7f3a91 telstore-20260901-9de447
158
+ npx telstore verify telstore-20260905-7f3a91 telstore-20260901-9de447
127
159
  npx telstore delete telstore-20260905-7f3a91 telstore-20260901-9de447
128
160
  ```
129
161
 
@@ -180,7 +212,8 @@ There is no expiry and no revocation: to end a session for good, terminate it un
180
212
  protects your login rather than your files.
181
213
  - A chunk cannot exceed 1950MB: Telegram accepts at most 4000 parts of 512KB per file.
182
214
  - Deleting a chunk message in the Telegram app destroys the backup, and keeping the
183
- `backupId` is what saves you hunting for its manifest in the chat by hand.
215
+ `backupId` is what saves you hunting for its manifest in the chat by hand. `verify` is how
216
+ you find that out before you need the file rather than after.
184
217
 
185
218
  Settings and credentials live in `~/.telstore/config.json`, mode 600 โ€” `apiId`, `apiHash` and
186
219
  the session at the top level (or a single `sealed` blob after `login --token`), everything
package/bin/telstore.js CHANGED
@@ -134,6 +134,21 @@ async function main() {
134
134
  return
135
135
  }
136
136
 
137
+ case 'verify': {
138
+ if (!parsed.args[0]) {
139
+ throw new Error('Missing backup id. Example: npx telstore verify telstore-20260905-7f3a91')
140
+ }
141
+
142
+ const { runVerifies } = await import('../src/commands/verify.js')
143
+
144
+ const { failed } = await runVerifies(parsed.args, parsed.options)
145
+
146
+ // A backup that is damaged, and one telstore could not look up at all, both mean the
147
+ // run did not find what it was asked to check. Whatever runs telstore learns that here.
148
+ if (failed > 0) process.exitCode = 1
149
+ return
150
+ }
151
+
137
152
  case 'delete': {
138
153
  if (!parsed.args[0]) {
139
154
  throw new Error('Missing backup id. Example: npx telstore delete telstore-20260905-7f3a91')
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "telstore",
3
- "version": "0.1.8",
3
+ "version": "0.1.9",
4
4
  "description": "Split large files into chunks and store them on Telegram",
5
5
  "keywords": [
6
6
  "telegram",
package/src/cli.js CHANGED
@@ -6,6 +6,7 @@ const SUBCOMMANDS = new Set([
6
6
  'logout',
7
7
  'list',
8
8
  'restore',
9
+ 'verify',
9
10
  'delete',
10
11
  'status',
11
12
  'config',
@@ -21,6 +22,7 @@ const OPTIONS = {
21
22
  out: { type: 'string' },
22
23
  note: { type: 'string' },
23
24
  limit: { type: 'string' },
25
+ search: { type: 'string' },
24
26
  verbose: { type: 'boolean' },
25
27
  unset: { type: 'boolean' },
26
28
  yes: { type: 'boolean' },
@@ -34,7 +36,9 @@ Usage:
34
36
  npx telstore login Log in to Telegram, only needed once
35
37
  npx telstore <file|folder|pattern>... Split files and upload them to Telegram
36
38
  npx telstore list List the backups stored in the destination
39
+ npx telstore list --search <text> List only the backups that text appears in
37
40
  npx telstore restore <backup-id>... Download the chunks and reassemble the files
41
+ npx telstore verify <backup-id>... Check that a backup's chunks are all still in the chat
38
42
  npx telstore delete <backup-id>... Remove backups' chunks and manifests from the chat
39
43
  npx telstore status Show the account, the destination and unfinished uploads and restores
40
44
  npx telstore config Show every setting and where its value comes from
@@ -50,8 +54,10 @@ backup. A folder means the files one level inside it, and a pattern means the na
50
54
  one file is listed and confirmed before the first byte goes out. Run telstore again with only
51
55
  the files that are left to carry on after an interruption.
52
56
 
53
- restore and delete take several ids the same way: one connection, one line each, and an exit
54
- code that reports any that failed. delete shows everything it is about to destroy and asks once.
57
+ restore, verify and delete take several ids the same way: one connection, one line each, and
58
+ an exit code that reports any that failed. delete shows everything it is about to destroy and
59
+ asks once. verify downloads nothing: it asks the chat whether every chunk message is still
60
+ there at the length the manifest records, which is what restore would need.
55
61
 
56
62
  Settings:
57
63
  npx telstore config <name> Print one setting's value
@@ -80,6 +86,12 @@ Options apply to one run and are never saved. Use config to change a setting for
80
86
  shell hands the words after the first to telstore as more files
81
87
  to upload.
82
88
  --limit <n> How many backups list shows this run.
89
+ --search <text> List only the backups whose file name, note, backup id or
90
+ creation day contains this text. Telegram's own index does
91
+ the looking, so the whole chat is reached without reading it
92
+ message by message โ€” but it matches whole words only:
93
+ "projex" finds projex.zip and "proj" finds nothing. A term
94
+ with spaces has to be quoted, as --note does.
83
95
  --token Log in by pasting a session token. It takes no value on
84
96
  purpose: a token written on the command line would sit in
85
97
  "ps" for the whole life of the command, and stay in that
package/src/client.js CHANGED
@@ -1,8 +1,10 @@
1
1
  import { Api, TelegramClient } from 'teleproto'
2
2
  import { Logger } from 'teleproto/extensions/index.js'
3
3
  import { LogLevel } from 'teleproto/extensions/Logger.js'
4
+ import { returnBigInt } from 'teleproto/Helpers.js'
4
5
  import { StringSession } from 'teleproto/sessions/index.js'
5
6
 
7
+ import { MANIFEST_TAG } from './caption.js'
6
8
  import { manifestFileName } from './manifest.js'
7
9
  import { withRetry } from './retry.js'
8
10
  import { assertLoggedIn, unlockConfig } from './session.js'
@@ -23,6 +25,26 @@ export function documentFileName(message) {
23
25
  return named?.fileName ?? null
24
26
  }
25
27
 
28
+ // Telegram records a document's length as a BigInteger, and comparing that to the plain
29
+ // number a manifest carries with === is false for every size there is.
30
+ export function documentSize(message) {
31
+ const size = message?.media?.document?.size
32
+
33
+ return size === undefined || size === null ? null : returnBigInt(size).toJSNumber()
34
+ }
35
+
36
+ // The flat shape both readers of a chat hand back. The raw message is kept alongside it
37
+ // because downloading needs it whole.
38
+ function toDocument(message) {
39
+ return {
40
+ id: message.id,
41
+ fileName: documentFileName(message),
42
+ caption: message.message ?? '',
43
+ date: message.date,
44
+ message,
45
+ }
46
+ }
47
+
26
48
  // The one place telstore searches a chat. Both callers want documents and nothing else,
27
49
  // and getMessages is preferred over a raw Api.messages.Search because it handles offsets,
28
50
  // hashes and pagination itself, so we don't hand-build easily mistyped fields. The raw
@@ -34,13 +56,93 @@ export async function searchDocuments(client, peer, { search, limit }) {
34
56
  limit,
35
57
  })
36
58
 
37
- return messages.map((message) => ({
38
- id: message.id,
39
- fileName: documentFileName(message),
40
- caption: message.message ?? '',
41
- date: message.date,
42
- message,
43
- }))
59
+ return messages.map(toDocument)
60
+ }
61
+
62
+ // The paging both readers share. It is ours rather than iterMessages', for the reasons
63
+ // deleteMessages does not use teleproto's: every page then carries the retry policy and the
64
+ // stall deadline, and a page that fails is retried by itself instead of restarting from the
65
+ // newest message. offsetId is the id of the last message of the page before, and Telegram
66
+ // answers with the messages older than it โ€” a reader that forgot to advance it would fetch
67
+ // the newest page over and over and never reach an older backup.
68
+ //
69
+ // A generator because the caller stops when it has what it wants: a chat of ten thousand
70
+ // chunks costs one request to list the backups at the top of it.
71
+ export const DOCUMENT_PAGE_SIZE = 100
72
+
73
+ async function* iterMessagePages(client, peer, { search, what, options }) {
74
+ const {
75
+ pageSize = DOCUMENT_PAGE_SIZE,
76
+ max = Infinity,
77
+ retryOptions = {},
78
+ stallMs = DEFAULT_STALL_MS,
79
+ } = options
80
+
81
+ let offsetId = 0
82
+ let read = 0
83
+
84
+ while (read < max) {
85
+ const limit = Math.min(pageSize, max - read)
86
+
87
+ const messages = await withRetry(
88
+ () =>
89
+ withStallTimeout(
90
+ client.getMessages(peer, {
91
+ ...(search === undefined ? {} : { search }),
92
+ filter: new Api.InputMessagesFilterDocument(),
93
+ limit,
94
+ offsetId,
95
+ }),
96
+ stallMs,
97
+ () =>
98
+ `Telegram stopped answering while reading the ${what} older than message ` +
99
+ `${offsetId}: nothing back for ${Math.round(stallMs / 1000)}s.`,
100
+ ),
101
+ retryOptions,
102
+ )
103
+
104
+ if (!messages || messages.length === 0) return
105
+
106
+ for (const message of messages) yield toDocument(message)
107
+
108
+ read += messages.length
109
+ offsetId = messages[messages.length - 1].id
110
+
111
+ // A short page is the end of the results. Asking again would cost a request to be told
112
+ // the same thing.
113
+ if (messages.length < limit) return
114
+ }
115
+ }
116
+
117
+ // What `list` reads the chat with. searchDocuments asks Telegram's text index a question;
118
+ // this asks for the documents themselves, newest first, which is the only answer that was
119
+ // right every time it was measured โ€” a chat's text index can come back empty while the chat
120
+ // is full of backups, and did for a whole day in a channel that had just been created
121
+ // (docs/design/captions.md carries the measurements).
122
+ export async function* iterDocuments(client, peer, options = {}) {
123
+ yield* iterMessagePages(client, peer, { what: 'documents', options })
124
+ }
125
+
126
+ // What `list --search` reads the chat with, and the one place that knows how to ask the index
127
+ // for backups. Two things about the query are load-bearing, both measured against a real chat
128
+ // on 2026-09-08 (docs/design/captions.md carries the numbers):
129
+ //
130
+ // The tag is ANDed in because a term alone brings the backup's chunk messages back too โ€” a
131
+ // chunk caption carries the id โ€” and on a backup of a few hundred chunks those would fill
132
+ // every page before a single manifest appeared. `#telstore` rides on the manifest alone, so
133
+ // the results come back manifests only: searching one backup id returned 1 manifest and 1
134
+ // chunk, and the same id with the tag returned the manifest by itself.
135
+ //
136
+ // The tag goes *after* the term, never before. A query that starts with the hash is read as
137
+ // a hashtag lookup and stops ANDing the rest: "#telstore projex" came back empty while
138
+ // "projex #telstore" returned the one manifest. Leading it would turn every search into
139
+ // "no backups found", which is the sentence this project must never say wrongly.
140
+ export async function* iterManifestSearch(client, peer, term, options = {}) {
141
+ yield* iterMessagePages(client, peer, {
142
+ search: `${term} ${MANIFEST_TAG}`,
143
+ what: 'search results',
144
+ options,
145
+ })
44
146
  }
45
147
 
46
148
  // How telstore finds a backup's manifest, in one place because restore and delete must not
@@ -70,11 +172,15 @@ export async function readMessageBytes(client, message) {
70
172
  //
71
173
  // Telegram does not complain about an id that is no longer there, so sending a batch twice
72
174
  // costs nothing: a delete interrupted halfway is finished by running it again.
73
- export const DELETE_BATCH_SIZE = 100
175
+ //
176
+ // The hundred is Telegram's own limit on how many message ids one request may name, and it
177
+ // is the same limit whether the request removes them or asks about them โ€” so getDocuments
178
+ // below counts in the same batches rather than keeping a second opinion about one number.
179
+ export const MESSAGE_BATCH_SIZE = 100
74
180
 
75
181
  export async function deleteMessages(client, peer, ids, options = {}) {
76
182
  const {
77
- batchSize = DELETE_BATCH_SIZE,
183
+ batchSize = MESSAGE_BATCH_SIZE,
78
184
  retryOptions = {},
79
185
  stallMs = DEFAULT_STALL_MS,
80
186
  onBatch,
@@ -110,6 +216,53 @@ export async function deleteMessages(client, peer, ids, options = {}) {
110
216
  return deleted
111
217
  }
112
218
 
219
+ // What verify asks the chat, and the read-only mirror of deleteMessages above: our own
220
+ // batching, one request in flight at a time, under the same retry policy and the same stall
221
+ // deadline as every other network wait in telstore.
222
+ //
223
+ // The answer is a Map rather than a list because the question is "which of these are still
224
+ // there". Telegram reports a message that is gone as MessageEmpty โ€” an object carrying the
225
+ // id it was asked about โ€” so anything that is not a real message is left out here, where the
226
+ // shape is understood, rather than passed on to a caller that would read an empty as a chunk
227
+ // still sitting in the chat.
228
+ export async function getDocuments(client, peer, ids, options = {}) {
229
+ const {
230
+ batchSize = MESSAGE_BATCH_SIZE,
231
+ retryOptions = {},
232
+ stallMs = DEFAULT_STALL_MS,
233
+ onBatch,
234
+ } = options
235
+
236
+ const found = new Map()
237
+
238
+ for (let start = 0; start < ids.length; start += batchSize) {
239
+ const batch = ids.slice(start, start + batchSize)
240
+
241
+ const messages = await withRetry(
242
+ () =>
243
+ withStallTimeout(
244
+ client.getMessages(peer, { ids: batch }),
245
+ stallMs,
246
+ () =>
247
+ `Telegram stopped answering while looking up messages ${start + 1}-` +
248
+ `${start + batch.length} of ${ids.length}: nothing back for ` +
249
+ `${Math.round(stallMs / 1000)}s.`,
250
+ ),
251
+ retryOptions,
252
+ )
253
+
254
+ for (const message of messages ?? []) {
255
+ if (!message || message instanceof Api.MessageEmpty) continue
256
+
257
+ found.set(message.id, message)
258
+ }
259
+
260
+ onBatch?.(Math.min(start + batch.length, ids.length), ids.length)
261
+ }
262
+
263
+ return found
264
+ }
265
+
113
266
  // Every command ends by putting the connection down, and a failure there must never
114
267
  // swallow the real error already on its way up. Commands that print progress hand in an
115
268
  // onWarn to say so; the quieter ones let it pass, because a connection that will not
@@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'
2
2
 
3
3
  import { chatName, describeChat } from '../chat.js'
4
4
  import {
5
- DELETE_BATCH_SIZE,
5
+ MESSAGE_BATCH_SIZE,
6
6
  closeQuietly,
7
7
  connect as realConnect,
8
8
  deleteMessages as realDeleteMessages,
@@ -181,7 +181,7 @@ export async function runDelete(backupId, options = {}, deps = {}) {
181
181
  throw new Error('Cancelled on request.')
182
182
  }
183
183
 
184
- const loud = chunkIds.length > DELETE_BATCH_SIZE
184
+ const loud = chunkIds.length > MESSAGE_BATCH_SIZE
185
185
  let removed = 0
186
186
 
187
187
  try {
@@ -1,11 +1,13 @@
1
- import { MANIFEST_TAG, parseManifestCaption } from '../caption.js'
1
+ import { parseManifestCaption } from '../caption.js'
2
2
  import { chatName, describeChat } from '../chat.js'
3
3
  import {
4
4
  closeQuietly,
5
5
  connect as realConnect,
6
- searchDocuments,
6
+ iterDocuments,
7
+ iterManifestSearch,
7
8
  } from '../client.js'
8
9
  import { configFile, defaultConfigDir, loadConfig } from '../config.js'
10
+ import { MANIFEST_SUFFIX } from '../manifest.js'
9
11
  import { assertLoggedIn } from '../session.js'
10
12
  import { requireChat, resolveSettings } from '../settings.js'
11
13
 
@@ -30,15 +32,48 @@ function shorten(note) {
30
32
  return note.length > NOTE_WIDTH ? `${note.slice(0, NOTE_WIDTH - 1)}โ€ฆ` : note
31
33
  }
32
34
 
33
- // Telegram indexes the tag the manifest caption carries, so one search returns one hit
34
- // per backup instead of one per chunk. What comes back is still whatever the server
35
- // decided to match, which is why the caller filters on the file name afterwards.
36
- async function realSearchManifests(client, peer, limit) {
37
- return await searchDocuments(client, peer, { search: MANIFEST_TAG, limit })
35
+ // What list has to read through depends on how big the backups are, not how many there are:
36
+ // it walks from the newest message down and stops the moment it has --limit manifests, so the
37
+ // three thousandth backup in a chat costs nothing because it is never reached. What costs is
38
+ // the chunks in between โ€” one message each โ€” which is why the ceiling is a budget per backup
39
+ // asked for rather than one number for every chat.
40
+ //
41
+ // 60 documents per backup covers a backup of about 105GB at the default chunk size. A fixed
42
+ // 1000 was both too tight and too loose at once: twenty backups of 100GB need 1140 documents
43
+ // and got 1000 of them, while `--limit 5` never needed more than 300.
44
+ export const DOCUMENTS_PER_BACKUP = 60
45
+
46
+ // A search result is already a manifest, so it buys far more backups per document read. Its
47
+ // budget only has to cover what matchesTerm throws away โ€” measured 2026-09-08, a term like
48
+ // "2026-09" comes back matching everything and is then cut down to the month asked for.
49
+ export const RESULTS_PER_BACKUP = 20
50
+
51
+ // --limit takes any whole number, so the budget needs an end of its own: without one,
52
+ // `--limit 100000` would ask for six million documents and sixty thousand requests.
53
+ export const MAX_LIST_DOCUMENTS = 10000
54
+
55
+ export function documentBudget(limit, perBackup) {
56
+ return Math.min(limit * perBackup, MAX_LIST_DOCUMENTS)
38
57
  }
39
58
 
59
+ // Stopping at the ceiling used to leave the reader standing there: "there may be older backups
60
+ // further back" is true and offers nothing to do about it. --search reaches them without
61
+ // reading the chunks in between, which is the whole reason it exists.
62
+ const DEEPER_HINT =
63
+ '"npx telstore list --search <text>" reaches older ones without reading every chunk ' +
64
+ 'in between.'
65
+
66
+ // Telegram matches whole words and nothing shorter: measured 2026-09-08, "projex" found
67
+ // projex.zip while "proj", "pro" and "pr" each found nothing at all. That is the one way a
68
+ // search can come back empty over a backup that is plainly there, so the empty answer says
69
+ // it, and points at the listing that never asks the index.
70
+ const SEARCH_MISS_HELP =
71
+ 'Telegram matches whole words: "projex" finds projex.zip, "proj" does not. ' +
72
+ 'Run "npx telstore list" without --search to see every backup without going through ' +
73
+ 'the search index.'
74
+
40
75
  function backupIdFromFileName(fileName) {
41
- return fileName.replace(/\.manifest\.json$/, '')
76
+ return fileName.slice(0, -MANIFEST_SUFFIX.length)
42
77
  }
43
78
 
44
79
  function utcDay(unixSeconds) {
@@ -74,6 +109,76 @@ function toRow(message) {
74
109
  }
75
110
  }
76
111
 
112
+ // A search term is a question about one run, and an empty one is not a question: answering
113
+ // it with every backup would look exactly like a search that matched everything.
114
+ function parseSearchTerm(raw) {
115
+ if (raw === undefined || raw === null) return null
116
+
117
+ const term = String(raw).trim()
118
+
119
+ if (term === '') {
120
+ throw new Error(
121
+ '--search is empty. Write the word to look for, or leave the flag off โ€” "list" ' +
122
+ 'without it shows every backup.',
123
+ )
124
+ }
125
+
126
+ return term
127
+ }
128
+
129
+ // The four fields a person remembers about a backup, and the whole of what --search compares
130
+ // against. The note is matched entire rather than the 40 characters the table has room for:
131
+ // a word that fell off the end of the column is still a word they typed. A card that cannot
132
+ // be read back leaves only what the message itself knows.
133
+ function searchableFields(message) {
134
+ const id = backupIdFromFileName(message.fileName)
135
+ const card = parseManifestCaption(message.caption)
136
+
137
+ if (!card) return [id, utcDay(message.date)]
138
+
139
+ return [id, card.name, card.note ?? '', card.createdAt.slice(0, 10)]
140
+ }
141
+
142
+ // Telegram decides what comes back; this decides what is true. The index answers a term the
143
+ // way it wants to โ€” measured 2026-09-08, "2026-09" returned every document in the chat โ€” so
144
+ // a hit is shown only if the term really is in one of the fields above. Without this pass a
145
+ // search for a month would list backups from every other month, which is the plausible wrong
146
+ // answer this project exists to refuse.
147
+ function matchesTerm(message, term) {
148
+ return searchableFields(message).some((field) => field.toLowerCase().includes(term))
149
+ }
150
+
151
+ // A walk of one page is over in about the time it takes to notice โ€” 165ms against a real
152
+ // chat โ€” and that is the usual case, so nothing is drawn for the first stretch: a line that
153
+ // appears and is wiped in the same breath is a flicker, not information. Past that the read
154
+ // is long enough that silence reads as the hang this project refuses everywhere else.
155
+ //
156
+ // \r only moves the cursor home, so every line is padded to the widest one drawn and the last
157
+ // write wipes the row: the table that follows must never land on half a progress line.
158
+ const NOTICE_QUIET_MS = 400
159
+ const NOTICE_INTERVAL_MS = 200
160
+
161
+ function createWalkNotice({ write, now, quietMs = NOTICE_QUIET_MS, intervalMs = NOTICE_INTERVAL_MS }) {
162
+ const startedAt = now()
163
+ let lastDrawnAt = 0
164
+ let widest = 0
165
+
166
+ return {
167
+ tick(text) {
168
+ if (now() - startedAt < quietMs) return
169
+ if (lastDrawnAt !== 0 && now() - lastDrawnAt < intervalMs) return
170
+
171
+ lastDrawnAt = now()
172
+ widest = Math.max(widest, text.length)
173
+ write(`\r${text.padEnd(widest)}`)
174
+ },
175
+ clear() {
176
+ if (widest === 0) return
177
+ write(`\r${' '.repeat(widest)}\r`)
178
+ },
179
+ }
180
+ }
181
+
77
182
  function renderTable(rows) {
78
183
  // Most people never write a note, and a column of dashes tells them nothing they did not
79
184
  // already know while costing every other column the width it takes.
@@ -102,12 +207,21 @@ export async function runList(options = {}, deps = {}) {
102
207
  configDir = defaultConfigDir(),
103
208
  connect = realConnect,
104
209
  disconnect = (client) => client.destroy(),
105
- searchManifests = realSearchManifests,
210
+ readDocuments = iterDocuments,
211
+ searchManifests = iterManifestSearch,
106
212
  log = (line) => console.log(line),
213
+ // The notice is drawn on stderr, and only onto a terminal: unlike an upload's progress
214
+ // bar, `list` is a command people pipe into grep, and a carriage return in a log file is
215
+ // rubbish. Null means draw nothing at all.
216
+ writeProgress = process.stderr.isTTY ? (text) => process.stderr.write(text) : null,
217
+ now = () => Date.now(),
107
218
  } = deps
108
219
 
109
220
  const config = await loadConfig(configDir)
110
221
  const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
222
+ // Before the login gate: a bad term is the user's own typing, and telling them to log in
223
+ // first would send them off after the wrong thing.
224
+ const term = parseSearchTerm(options.search)
111
225
  // Ask about the login before the destination: telling someone who has never logged in
112
226
  // to pick a chat sends them off after the wrong thing.
113
227
  assertLoggedIn(config)
@@ -115,29 +229,99 @@ export async function runList(options = {}, deps = {}) {
115
229
 
116
230
  const client = await connect(config, { verbose: settings.verbose })
117
231
 
118
- let found
232
+ // Walked rather than searched, unless a term was given. Telegram's text index can answer
233
+ // nothing at all about a chat that is full of backups โ€” it did for a whole day in a channel
234
+ // that had just been created โ€” and "No backups found" is a sentence someone acts on. The
235
+ // documents themselves were right every time they were asked for.
236
+ //
237
+ // --search is the one place worth paying the index for: a term matches a backup that may be
238
+ // ten thousand messages back, and walking to it would cost a request per hundred documents
239
+ // in between, every time, for as long as the chat keeps growing. So the search narrows and
240
+ // matchesTerm decides โ€” the index is asked where to look, never what is true.
241
+ const found = []
242
+ let read = 0
243
+
244
+ const unit = term ? 'search results' : 'documents'
245
+ const budget = documentBudget(settings.limit, term ? RESULTS_PER_BACKUP : DOCUMENTS_PER_BACKUP)
246
+
247
+ const results = term
248
+ ? searchManifests(client, chat, term, { max: budget })
249
+ : readDocuments(client, chat, { max: budget })
250
+
251
+ const wanted = term === null ? null : term.toLowerCase()
252
+ const notice = writeProgress ? createWalkNotice({ write: writeProgress, now }) : null
253
+
119
254
  try {
120
- found = await searchManifests(client, chat, settings.limit)
255
+ for await (const document of results) {
256
+ read += 1
257
+
258
+ notice?.tick(
259
+ `Reading ${chatName(chat)}โ€ฆ ${read} ${unit}, ${found.length} backup` +
260
+ `${found.length === 1 ? '' : 's'}`,
261
+ )
262
+
263
+ if (!document.fileName?.endsWith(MANIFEST_SUFFIX)) continue
264
+ if (wanted !== null && !matchesTerm(document, wanted)) continue
265
+
266
+ found.push(document)
267
+
268
+ // Everything past here is older than the twentieth newest backup, and nobody asked
269
+ // for it. In a chat of ten thousand chunks this is the difference between one
270
+ // request and ten.
271
+ if (found.length >= settings.limit) break
272
+ }
121
273
  } finally {
274
+ notice?.clear()
122
275
  await closeQuietly(client, disconnect)
123
276
  }
124
277
 
278
+ // The one thing either reader cannot see is what lies past its own ceiling, so anything it
279
+ // says about the whole chat has to stop at the edge of what it read.
280
+ const capped = found.length < settings.limit && read >= budget
281
+
125
282
  log(`Destination ${describeChat(chat)}`)
283
+ if (term) log(`Search ${JSON.stringify(term)}`)
126
284
  log('')
127
285
 
128
- const rows = found
129
- .filter((message) => message.fileName?.endsWith('.manifest.json'))
130
- .map(toRow)
286
+ const rows = found.map(toRow)
131
287
 
132
288
  if (rows.length === 0) {
133
- log(`No backups found in ${chatName(chat)}. Upload one with: npx telstore <file>`)
289
+ if (term) {
290
+ log(
291
+ capped
292
+ ? `No backups matching ${JSON.stringify(term)} in the newest ${budget} ` +
293
+ `${unit} from ${chatName(chat)}. There may be older ones further back.`
294
+ : `No backups matching ${JSON.stringify(term)} in ${chatName(chat)}.`,
295
+ )
296
+ log(SEARCH_MISS_HELP)
297
+ return rows
298
+ }
299
+
300
+ log(
301
+ capped
302
+ ? `No backups in the newest ${budget} ${unit} of ${chatName(chat)}. ` +
303
+ `There may be older ones further back. ${DEEPER_HINT}`
304
+ : `No backups found in ${chatName(chat)}. Upload one with: npx telstore <file>`,
305
+ )
134
306
  return rows
135
307
  }
136
308
 
137
309
  for (const line of renderTable(rows)) log(line)
138
310
 
139
311
  log('')
140
- log(`${rows.length} backup${rows.length === 1 ? '' : 's'}. Restore with: npx telstore restore <backup-id>`)
312
+ log(
313
+ `${rows.length} backup${rows.length === 1 ? '' : 's'}` +
314
+ `${term ? ` matching ${JSON.stringify(term)}` : ''}. ` +
315
+ 'Restore with: npx telstore restore <backup-id>',
316
+ )
317
+
318
+ if (capped) {
319
+ log(
320
+ `Read the newest ${budget} ${unit} in ${chatName(chat)} to find them โ€” ` +
321
+ `there may be older ${term ? 'matches' : 'backups'} further back.` +
322
+ `${term ? '' : ` ${DEEPER_HINT}`}`,
323
+ )
324
+ }
141
325
 
142
326
  return rows
143
327
  }
@@ -0,0 +1,298 @@
1
+ import { chatName, describeChat } from '../chat.js'
2
+ import {
3
+ MESSAGE_BATCH_SIZE,
4
+ closeQuietly,
5
+ connect as realConnect,
6
+ documentFileName,
7
+ documentSize,
8
+ findManifestMessage,
9
+ getDocuments as realGetDocuments,
10
+ readMessageBytes as realReadMessageBytes,
11
+ } from '../client.js'
12
+ import { configFile, defaultConfigDir, loadConfig } from '../config.js'
13
+ import { chunkFileName, manifestFileName, parseManifest } from '../manifest.js'
14
+ import { formatBytes, formatDuration } from '../progress.js'
15
+ import { assertLoggedIn } from '../session.js'
16
+ import { requireChat, resolveSettings } from '../settings.js'
17
+
18
+ // parseManifest guarantees every number in a manifest, but not the file name โ€” it is
19
+ // decoration, and nothing verifies differently because of it. It is still text off a chat.
20
+ function describeName(name) {
21
+ return typeof name === 'string' && name.trim() !== '' ? name : 'โ€”'
22
+ }
23
+
24
+ function plural(n, word) {
25
+ return `${n} ${word}${n === 1 ? '' : 's'}`
26
+ }
27
+
28
+ // What is wrong with one chunk, or null when nothing is. The first failing check wins: a
29
+ // chunk is damaged or it is not, and listing three complaints about one message would make
30
+ // "2 damaged" mean something other than two chunks.
31
+ //
32
+ // Every question here is one the chat can answer without sending a byte of the file. What
33
+ // this cannot ask is whether the bytes inside are the bytes that went up โ€” only downloading
34
+ // them answers that, which is why the closing line says so rather than leaving it implied.
35
+ function inspect(chunk, message, { backupId, total, chat }) {
36
+ const at = `Chunk ${chunk.i + 1}/${total}`
37
+
38
+ if (!message) {
39
+ return `${at} is gone: message ${chunk.msgId} is no longer in ${chatName(chat)}.`
40
+ }
41
+
42
+ const size = documentSize(message)
43
+
44
+ if (size === null) {
45
+ return `${at} is message ${chunk.msgId}, which has no file attached.`
46
+ }
47
+
48
+ const wanted = chunkFileName(backupId, chunk.i)
49
+ const fileName = documentFileName(message)
50
+
51
+ if (fileName !== wanted) {
52
+ return fileName === null
53
+ ? `${at} is message ${chunk.msgId}, whose file carries no name; the manifest expects ` +
54
+ `"${wanted}".`
55
+ : `${at} is message ${chunk.msgId}, which carries the file name ` +
56
+ `${JSON.stringify(fileName)} rather than "${wanted}".`
57
+ }
58
+
59
+ if (size !== chunk.size) {
60
+ return `${at} is ${size} bytes in the chat, the manifest records ${chunk.size}.`
61
+ }
62
+
63
+ return null
64
+ }
65
+
66
+ export async function runVerify(backupId, options = {}, deps = {}) {
67
+ const {
68
+ connect = realConnect,
69
+ disconnect = (client) => client.destroy(),
70
+ configDir = defaultConfigDir(),
71
+ searchManifest = findManifestMessage,
72
+ readMessageBytes = realReadMessageBytes,
73
+ getDocuments = realGetDocuments,
74
+ retryOptions = {},
75
+ writeErr = (line) => process.stderr.write(line),
76
+ log: writeLog = (line) => console.log(line),
77
+ silent = false,
78
+ } = deps
79
+
80
+ const config = await loadConfig(configDir)
81
+ const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
82
+ // Ask about the login before the destination, as list does: telling someone who has never
83
+ // logged in to pick a chat sends them off after the wrong thing.
84
+ assertLoggedIn(config)
85
+ const chat = requireChat(settings)
86
+
87
+ const log = silent ? () => {} : writeLog
88
+ const warn = silent ? () => {} : writeErr
89
+
90
+ // Upload and restore stay quiet until the third retry so a handful of -503s do not bury
91
+ // the progress bar. There is no bar here to bury โ€” this command sends a handful of small
92
+ // requests and prints one verdict โ€” so a wait long enough to notice is announced at once.
93
+ function onRetry(err, attempt, delayMs) {
94
+ warn(
95
+ `\nTemporary error (${err.message}), retry ${attempt} in ` +
96
+ `${formatDuration(delayMs / 1000)}.\n`,
97
+ )
98
+ }
99
+
100
+ const client = await connect(config, { verbose: settings.verbose })
101
+
102
+ try {
103
+ const manifestMessage = await searchManifest(client, chat, backupId)
104
+
105
+ if (!manifestMessage) {
106
+ throw new Error(
107
+ `No backup ${backupId} found in ${chatName(chat)}. Check the id with ` +
108
+ '"npx telstore list", or use --chat to point at the right chat.',
109
+ )
110
+ }
111
+
112
+ // The full layout checks, not the lenient path delete takes. delete reads a manifest to
113
+ // destroy what it names, so a broken one is exactly what somebody is there to remove;
114
+ // verify reads it to answer whether restore would work, and restore would refuse this
115
+ // one. Saying so in parseManifest's own words keeps one description of one fault.
116
+ const manifest = parseManifest(await readMessageBytes(client, manifestMessage))
117
+
118
+ // The same refusal delete makes, for the same reason from the other side: the manifest
119
+ // was found by the file name telstore wrote, so a body naming another backup is a file
120
+ // that was renamed or replaced, and its message ids describe somebody else's chunks.
121
+ // Reporting those as this backup's health is the one wrong answer this command can give.
122
+ if (manifest.id !== undefined && manifest.id !== backupId) {
123
+ throw new Error(
124
+ `The manifest named ${manifestFileName(backupId)} describes backup ` +
125
+ `${JSON.stringify(manifest.id)}, not ${backupId}. Its message ids point at another ` +
126
+ `backup's chunks, so telstore cannot say whether ${backupId} is still there.`,
127
+ )
128
+ }
129
+
130
+ const total = manifest.chunks.length
131
+
132
+ log(`Backup ${backupId}`)
133
+ log(
134
+ `File ${describeName(manifest.name)} ` +
135
+ `(${formatBytes(manifest.size)}, ${plural(total, 'chunk')})`,
136
+ )
137
+ log(`In ${describeChat(chat)}`)
138
+ log('')
139
+
140
+ // A backup at the 10000-chunk ceiling is a hundred requests, sent one at a time. Silence
141
+ // for that long reads as a hang, which is the one thing no wait in this project may look
142
+ // like โ€” and below a single request there is no progress worth a line.
143
+ const loud = total > MESSAGE_BATCH_SIZE
144
+
145
+ const found = await getDocuments(
146
+ client,
147
+ chat,
148
+ manifest.chunks.map((chunk) => chunk.msgId),
149
+ {
150
+ retryOptions: { ...retryOptions, onRetry },
151
+ onBatch: (done, all) => {
152
+ if (loud) warn(`\rChecking chunk messages ${done}/${all}โ€ฆ`)
153
+ },
154
+ },
155
+ )
156
+
157
+ if (loud) warn('\n')
158
+
159
+ const damaged = []
160
+
161
+ for (const chunk of manifest.chunks) {
162
+ const fault = inspect(chunk, found.get(chunk.msgId), { backupId, total, chat })
163
+
164
+ if (fault) {
165
+ damaged.push(fault)
166
+ log(fault)
167
+ }
168
+ }
169
+
170
+ if (damaged.length > 0) {
171
+ log('')
172
+ log(
173
+ `${plural(total, 'chunk')} checked, ${damaged.length} damaged. ` +
174
+ 'This backup cannot be restored.',
175
+ )
176
+ } else {
177
+ log(
178
+ `${plural(total, 'chunk')} present, at the ` +
179
+ `${total === 1 ? 'size' : 'sizes'} the manifest records.`,
180
+ )
181
+ log('This does not download them, so it cannot prove their contents.')
182
+ log('')
183
+ log(`Restore with: npx telstore restore ${backupId}`)
184
+ }
185
+
186
+ return { id: backupId, name: manifest.name, chunks: total, damaged }
187
+ } finally {
188
+ await closeQuietly(client, disconnect)
189
+ }
190
+ }
191
+
192
+ // A backup is a failure whether telstore could not look it up or looked and found it broken.
193
+ // The command worked either way โ€” but "did the run find everything it was asked to check"
194
+ // is the question the exit code answers, and both answers to that are no.
195
+ function isFailure(result) {
196
+ return Boolean(result.error) || result.damaged.length > 0
197
+ }
198
+
199
+ // The same shape as runDeletes and runRestores, minus the question: verify removes nothing,
200
+ // so there is nothing to authorise. What is knowable before the connection is refused up
201
+ // front; what only the chat can answer is per id, named when it happens and again at the end.
202
+ export async function runVerifies(backupIds, options = {}, deps = {}) {
203
+ const {
204
+ connect = realConnect,
205
+ disconnect = (client) => client.destroy(),
206
+ configDir = defaultConfigDir(),
207
+ writeErr = (line) => process.stderr.write(line),
208
+ log: writeLog = (line) => console.log(line),
209
+ silent = false,
210
+ } = deps
211
+
212
+ // One id keeps its own wording and its own thrown error. A summary about one backup only
213
+ // repeats the lines above it.
214
+ if (backupIds.length === 1) {
215
+ const result = await runVerify(backupIds[0], options, deps)
216
+ return { results: [result], failed: isFailure(result) ? 1 : 0 }
217
+ }
218
+
219
+ const duplicate = backupIds.find((id, index) => backupIds.indexOf(id) !== index)
220
+
221
+ if (duplicate) {
222
+ throw new Error(
223
+ `${duplicate} is named twice. Checking one backup twice asks the chat the same ` +
224
+ 'question again โ€” name it once.',
225
+ )
226
+ }
227
+
228
+ const config = await loadConfig(configDir)
229
+
230
+ assertLoggedIn(config)
231
+
232
+ const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
233
+ const chat = requireChat(settings)
234
+
235
+ const log = silent ? () => {} : writeLog
236
+ const warn = silent ? () => {} : writeErr
237
+
238
+ let shared = null
239
+ const perId = {
240
+ ...deps,
241
+ connect: async (theirConfig, connectOptions) =>
242
+ (shared ??= await connect(theirConfig, connectOptions)),
243
+ disconnect: async () => {},
244
+ }
245
+
246
+ const results = []
247
+
248
+ try {
249
+ await perId.connect(config, { verbose: settings.verbose })
250
+
251
+ for (const [index, backupId] of backupIds.entries()) {
252
+ if (index > 0) log('')
253
+ log(`[${index + 1}/${backupIds.length}] ${backupId}`)
254
+
255
+ try {
256
+ results.push(await runVerify(backupId, options, perId))
257
+ } catch (err) {
258
+ // Unlike delete, an id nothing knows about does not stop the run: nothing here is
259
+ // destroyed, and the other ids are exactly the ones somebody is checking on.
260
+ results.push({ id: backupId, error: err.message, damaged: [] })
261
+ warn(`\n${backupId} failed: ${err.message}\n`)
262
+ }
263
+ }
264
+ } finally {
265
+ if (shared) {
266
+ await closeQuietly(shared, disconnect, (err) =>
267
+ warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
268
+ )
269
+ }
270
+ }
271
+
272
+ const failed = results.filter(isFailure).length
273
+
274
+ log('')
275
+ for (const line of summaryLines(results, failed)) log(line)
276
+
277
+ return { results, failed }
278
+ }
279
+
280
+ // Every id gets a line whether it checked out or not: one missing from this list would be a
281
+ // backup nobody could tell the state of, which is the whole reason this command exists.
282
+ function summaryLines(results, failed) {
283
+ const width = Math.max(...results.map((result) => result.id.length))
284
+
285
+ return [
286
+ `${results.length} backups: ${results.length - failed} verified, ${failed} failed.`,
287
+ '',
288
+ ...results.map((result) => {
289
+ const id = result.id.padEnd(width)
290
+
291
+ if (result.error) return ` ${id} failed: ${result.error}`
292
+
293
+ return result.damaged.length > 0
294
+ ? ` ${id} ${plural(result.damaged.length, 'chunk')} damaged`
295
+ : ` ${id} ${plural(result.chunks, 'chunk')} present`
296
+ }),
297
+ ]
298
+ }
package/src/manifest.js CHANGED
@@ -15,8 +15,13 @@ export function chunkFileName(id, i) {
15
15
  return `${id}.part${String(i + 1).padStart(4, '0')}`
16
16
  }
17
17
 
18
+ // The suffix telstore has written on every manifest since version 1, and what `list` picks
19
+ // a manifest out of a chat by. One definition, because a reader that disagrees with the
20
+ // writer by one character finds nothing at all.
21
+ export const MANIFEST_SUFFIX = '.manifest.json'
22
+
18
23
  export function manifestFileName(id) {
19
- return `${id}.manifest.json`
24
+ return `${id}${MANIFEST_SUFFIX}`
20
25
  }
21
26
 
22
27
  export function buildManifest({